Use parallel ENC maps data loading in ENC atlases

src/common/downloader.cpp

@@ -37,6 +37,80 @@
 
 #define TMP_SUFFIX ".download"
 
+// QNetworkReply::errorString() returns bullshit, use our own reporting
+static const char *errorString(QNetworkReply::NetworkError error)
+{
+	switch (error) {
+		case QNetworkReply::ConnectionRefusedError:
+			return "Connection refused";
+		case QNetworkReply::RemoteHostClosedError:
+			return "Connection closed";
+		case QNetworkReply::HostNotFoundError:
+			return "Host not found";
+		case QNetworkReply::TimeoutError:
+			return "Connection timeout";
+		case QNetworkReply::OperationCanceledError:
+			return "Operation canceled";
+		case QNetworkReply::SslHandshakeFailedError:
+			return "SSL handshake failed";
+		case QNetworkReply::TemporaryNetworkFailureError:
+			return "Temporary network failure";
+		case QNetworkReply::NetworkSessionFailedError:
+			return "Network session failed";
+		case QNetworkReply::BackgroundRequestNotAllowedError:
+			return "Background request not allowed";
+		case QNetworkReply::TooManyRedirectsError:
+			return "Too many redirects";
+		case QNetworkReply::InsecureRedirectError:
+			return "Insecure redirect";
+		case QNetworkReply::ProxyConnectionRefusedError:
+			return "Proxy connection refused";
+		case QNetworkReply::ProxyConnectionClosedError:
+			return "Proxy connection closed";
+		case QNetworkReply::ProxyNotFoundError:
+			return "Proxy not found";
+		case QNetworkReply::ProxyTimeoutError:
+			return "Proxy timeout error";
+		case QNetworkReply::ProxyAuthenticationRequiredError:
+			return "Proxy authentication required";
+		case QNetworkReply::ContentAccessDenied:
+			return "Content access denied";
+		case QNetworkReply::ContentOperationNotPermittedError:
+			return "Content operation not permitted";
+		case QNetworkReply::ContentNotFoundError:
+			return "Content not found";
+		case QNetworkReply::AuthenticationRequiredError:
+			return "Authentication required";
+		case QNetworkReply::ContentReSendError:
+			return "Content re-send error";
+		case QNetworkReply::ContentConflictError:
+			return "Content conflict";
+		case QNetworkReply::ContentGoneError:
+			return "Content gone";
+		case QNetworkReply::InternalServerError:
+			return "Internal server error";
+		case QNetworkReply::OperationNotImplementedError:
+			return "Operation not implemented";
+		case QNetworkReply::ServiceUnavailableError:
+			return "Service unavailable";
+		case QNetworkReply::ProtocolUnknownError:
+			return "Protocol unknown";
+		case QNetworkReply::ProtocolInvalidOperationError:
+			return "Protocol invalid operation";
+		case QNetworkReply::UnknownNetworkError:
+			return "Unknown network error";
+		case QNetworkReply::UnknownProxyError:
+			return "Unknown proxy error";
+		case QNetworkReply::UnknownContentError:
+			return "Unknown content error";
+		case QNetworkReply::ProtocolFailure:
+			return "Protocol failure";
+		case QNetworkReply::UnknownServerError:
+			return "Unknown server error";
+		default:
+			return "Unknown error";
+	}
+}
 
 static QString tmpName(const QString &origName)
 {
@@ -158,11 +232,17 @@ void Downloader::emitReadReady()
 
 void Downloader::insertError(const QUrl &url, QNetworkReply::NetworkError error)
 {
-	if (error == QNetworkReply::OperationCanceledError)
+	switch (error) {
+		case QNetworkReply::OperationCanceledError:
+		case QNetworkReply::TimeoutError:
+		case QNetworkReply::RemoteHostClosedError:
+		case QNetworkReply::ConnectionRefusedError:
 			_errorDownloads.insert(url, _errorDownloads.value(url) + 1);
-	else
+			break;
+		default:
 			_errorDownloads.insert(url, RETRIES);
 	}
+}
 
 void Downloader::readData(QNetworkReply *reply)
 {
@@ -179,8 +259,7 @@ void Downloader::downloadFinished(QNetworkReply *reply)
 	QFile *file = _currentDownloads.value(reply->request().url());
 	if (error) {
 		insertError(url, error);
-		qWarning("%s: %s", url.toEncoded().constData(),
+		qWarning("%s: %s", url.toEncoded().constData(), errorString(error));
-		  qPrintable(reply->errorString()));
 		file->remove();
 	} else {
 		file->close();

src/map/ENC/atlasdata.cpp

@@ -2,45 +2,71 @@
 
 using namespace ENC;
 
-bool AtlasData::pointCb(const QString *map, void *context)
+bool AtlasData::pointCb(MapEntry *map, void *context)
 {
 	PointCTX *ctx = (PointCTX*)context;
 
-	ctx->lock.lock();
+start:
+	ctx->cacheLock.lock();
 
-	MapData *cached = ctx->cache.object(map);
+	MapData *cached = ctx->cache.object(map->path);
 
 	if (!cached) {
-		MapData *data = new MapData(*map);
+		ctx->cacheLock.unlock();
+
+		if (map->lock.tryLock()) {
+			MapData *data = new MapData(map->path);
 			data->points(ctx->rect, ctx->points);
-		ctx->cache.insert(map, data);
+
+			ctx->cacheLock.lock();
+			ctx->cache.insert(map->path, data);
+
+			map->lock.unlock();
+		} else {
+			map->lock.lock();
+			map->lock.unlock();
+			goto start;
+		}
 	} else
 		cached->points(ctx->rect, ctx->points);
 
-	ctx->lock.unlock();
+	ctx->cacheLock.unlock();
 
 	return true;
 }
 
-bool AtlasData::polyCb(const QString *map, void *context)
+bool AtlasData::polyCb(MapEntry *map, void *context)
 {
 	PolyCTX *ctx = (PolyCTX*)context;
 
-	ctx->lock.lock();
+start:
+	ctx->cacheLock.lock();
 
-	MapData *cached = ctx->cache.object(map);
+	MapData *cached = ctx->cache.object(map->path);
 
 	if (!cached) {
-		MapData *data = new MapData(*map);
+		ctx->cacheLock.unlock();
+
+		if (map->lock.tryLock()) {
+			MapData *data = new MapData(map->path);
 			data->polygons(ctx->rect, ctx->polygons);
 			data->lines(ctx->rect, ctx->lines);
-		ctx->cache.insert(map, data);
+
+			ctx->cacheLock.lock();
+			ctx->cache.insert(map->path, data);
+
+			map->lock.unlock();
+		} else {
+			map->lock.lock();
+			map->lock.unlock();
+			goto start;
+		}
 	} else {
 		cached->polygons(ctx->rect, ctx->polygons);
 		cached->lines(ctx->rect, ctx->lines);
 	}
 
-	ctx->lock.unlock();
+	ctx->cacheLock.unlock();
 
 	return true;
 }
@@ -61,14 +87,14 @@ void AtlasData::addMap(const RectC &bounds, const QString &path)
 	max[0] = bounds.right();
 	max[1] = bounds.top();
 
-	_tree.Insert(min, max, new QString(path));
+	_tree.Insert(min, max, new MapEntry(path));
 }
 
 void AtlasData::polys(const RectC &rect, QList<MapData::Poly> *polygons,
   QList<MapData::Line> *lines)
 {
 	double min[2], max[2];
-	PolyCTX polyCtx(rect, polygons, lines, _cache, _lock);
+	PolyCTX polyCtx(rect, polygons, lines, _cache, _cacheLock);
 
 	min[0] = rect.left();
 	min[1] = rect.bottom();
@@ -81,7 +107,7 @@ void AtlasData::polys(const RectC &rect, QList<MapData::Poly> *polygons,
 void AtlasData::points(const RectC &rect, QList<MapData::Point> *points)
 {
 	double min[2], max[2];
-	PointCTX pointCtx(rect, points, _cache, _lock);
+	PointCTX pointCtx(rect, points, _cache, _cacheLock);
 
 	min[0] = rect.left();
 	min[1] = rect.bottom();

src/map/ENC/atlasdata.h

@@ -8,13 +8,13 @@
 
 namespace ENC {
 
-typedef QCache<const QString*, MapData> MapCache;
+typedef QCache<QString, MapData> MapCache;
 
 class AtlasData
 {
 public:
-	AtlasData(MapCache &cache, QMutex &lock)
+	AtlasData(MapCache &cache, QMutex &cacheLock)
-	  : _cache(cache), _lock(lock) {}
+	  : _cache(cache), _cacheLock(cacheLock) {}
 	~AtlasData();
 
 	void addMap(const RectC &bounds, const QString &path);
@@ -24,40 +24,47 @@ public:
 	void points(const RectC &rect, QList<MapData::Point> *points);
 
 private:
-	typedef RTree<const QString*, double, 2> MapTree;
+	struct MapEntry {
+		MapEntry(const QString &path) : path(path) {}
+
+		QString path;
+		QMutex lock;
+	};
+
+	typedef RTree<MapEntry*, double, 2> MapTree;
 
 	struct PolyCTX
 	{
 		PolyCTX(const RectC &rect,  QList<MapData::Poly> *polygons,
-		  QList<MapData::Line> *lines, MapCache &cache, QMutex &lock)
+		  QList<MapData::Line> *lines, MapCache &cache, QMutex &cacheLock)
 		  : rect(rect), polygons(polygons), lines(lines), cache(cache),
-		  lock(lock) {}
+		  cacheLock(cacheLock) {}
 
 		const RectC &rect;
 		QList<MapData::Poly> *polygons;
 		QList<MapData::Line> *lines;
 		MapCache &cache;
-		QMutex &lock;
+		QMutex &cacheLock;
 	};
 
 	struct PointCTX
 	{
 		PointCTX(const RectC &rect, QList<MapData::Point> *points,
-		  MapCache &cache, QMutex &lock) : rect(rect), points(points),
+		  MapCache &cache, QMutex &cacheLock)
-		  cache(cache), lock(lock) {}
+		  : rect(rect), points(points), cache(cache), cacheLock(cacheLock) {}
 
 		const RectC &rect;
 		QList<MapData::Point> *points;
 		MapCache &cache;
-		QMutex &lock;
+		QMutex &cacheLock;
 	};
 
-	static bool polyCb(const QString *map, void *context);
+	static bool polyCb(MapEntry *map, void *context);
-	static bool pointCb(const QString *map, void *context);
+	static bool pointCb(MapEntry *map, void *context);
 
 	MapTree _tree;
 	MapCache &_cache;
-	QMutex &_lock;
+	QMutex &_cacheLock;
 };
 
 }

src/map/encatlas.cpp

@@ -113,7 +113,7 @@ void ENCAtlas::addMap(const QDir &dir, const QByteArray &file,
 	IntendedUsage iu = usage(path);
 	auto it = _data.find(iu);
 	if (it == _data.end())
-		it = _data.insert(iu, new AtlasData(_cache, _lock));
+		it = _data.insert(iu, new AtlasData(_cache, _cacheLock));
 
 	it.value()->addMap(bounds, path);
 

src/map/encatlas.h

@@ -89,7 +89,7 @@ private:
 	QMap<IntendedUsage, ENC::AtlasData*> _data;
 	ENC::Style *_style;
 	ENC::MapCache _cache;
-	QMutex _lock;
+	QMutex _cacheLock;
 	IntendedUsage _usage;
 	int _zoom;