Improved tile loading parallelism

src/map/mapsforge/mapdata.cpp

@@ -219,10 +219,10 @@ bool MapData::readTags(SubFile &subfile, int count,
 	return true;
 }
 
-bool MapData::readSubFiles()
+bool MapData::readSubFiles(QFile &file)
 {
 	/* both _pointFile and _pathFile can be used here */
-	QDataStream stream(&_pointFile);
+	QDataStream stream(&file);
 
 	for (int i = 0; i < _subFiles.size(); i++) {
 		const SubFileInfo &f = _subFiles.at(i);
@@ -422,8 +422,7 @@ bool MapData::readHeader(QFile &file)
 	return true;
 }
 
-MapData::MapData(const QString &fileName)
+MapData::MapData(const QString &fileName) : _fileName(fileName), _valid(false)
-  : _pointFile(fileName), _pathFile(fileName), _valid(false)
 {
 	QFile file(fileName);
 
@@ -460,17 +459,13 @@ RectC MapData::bounds() const
 
 void MapData::load()
 {
-	_pointFile.open(QIODevice::ReadOnly | QIODevice::Unbuffered);
+	QFile file(_fileName);
-	_pathFile.open(QIODevice::ReadOnly | QIODevice::Unbuffered);
+	if (file.open(QIODevice::ReadOnly | QIODevice::Unbuffered))
-
+		readSubFiles(file);
-	readSubFiles();
 }
 
 void MapData::clear()
 {
-	_pointFile.close();
-	_pathFile.close();
-
 	_pathCache.clear();
 	_pointCache.clear();
 
@@ -494,14 +489,14 @@ void MapData::clearTiles()
 bool MapData::pathCb(VectorTile *tile, void *context)
 {
 	PathCTX *ctx = (PathCTX*)context;
-	ctx->data->paths(tile, ctx->rect, ctx->zoom, ctx->list);
+	ctx->data->paths(ctx->file, tile, ctx->rect, ctx->zoom, ctx->list);
 	return true;
 }
 
 bool MapData::pointCb(VectorTile *tile, void *context)
 {
 	PointCTX *ctx = (PointCTX*)context;
-	ctx->data->points(tile, ctx->rect, ctx->zoom, ctx->list);
+	ctx->data->points(ctx->file, tile, ctx->rect, ctx->zoom, ctx->list);
 	return true;
 }
 
@@ -514,13 +509,14 @@ int MapData::level(int zoom) const
 	return _subFiles.size() - 1;
 }
 
-void MapData::points(const RectC &rect, int zoom, QList<Point> *list)
+void MapData::points(QFile &file, const RectC &rect, int zoom,
+  QList<Point> *list)
 {
 	if (!rect.isValid())
 		return;
 
 	int l(level(zoom));
-	PointCTX ctx(this, rect, zoom, list);
+	PointCTX ctx(file, this, rect, zoom, list);
 	double min[2], max[2];
 
 	min[0] = rect.left();
@@ -531,53 +527,58 @@ void MapData::points(const RectC &rect, int zoom, QList<Point> *list)
 	_tiles.at(l)->Search(min, max, pointCb, &ctx);
 }
 
-void MapData::points(const VectorTile *tile, const RectC &rect, int zoom,
+void MapData::points(QFile &file, VectorTile *tile, const RectC &rect,
-  QList<Point> *list)
+  int zoom, QList<Point> *list)
 {
 	Key key(tile, zoom);
 
-	_pointLock.lock();
+	tile->lock.lock();
 
+	_pointCacheLock.lock();
 	QList<Point> *tilePoints = _pointCache.object(key);
-
 	if (!tilePoints) {
+		_pointCacheLock.unlock();
 		QList<Point> *p = new QList<Point>();
-		if (readPoints(tile, zoom, p)) {
+		if (readPoints(file, tile, zoom, p)) {
 			copyPoints(rect, p, list);
+			_pointCacheLock.lock();
 			_pointCache.insert(key, p);
+			_pointCacheLock.unlock();
 		} else
 			delete p;
-	} else
+	} else {
 		copyPoints(rect, tilePoints, list);
-
+		_pointCacheLock.unlock();
-	_pointLock.unlock();
-
-
-	_pathLock.lock();
-
-	QList<Path> *tilePaths = _pathCache.object(key);
-
-	if (!tilePaths) {
-		QList<Path> *p = new QList<Path>();
-		if (readPaths(tile, zoom, p)) {
-			copyPoints(rect, p, list);
-			_pathCache.insert(key, p);
-		} else
-			delete p;
-	} else
-		copyPoints(rect, tilePaths, list);
-
-	_pathLock.unlock();
 	}
 
-void MapData::paths(const RectC &searchRect, const RectC &boundsRect, int zoom,
+	_pathCacheLock.lock();
-  QList<Path> *list)
+	QList<Path> *tilePaths = _pathCache.object(key);
+	if (!tilePaths) {
+		_pathCacheLock.unlock();
+		QList<Path> *p = new QList<Path>();
+		if (readPaths(file, tile, zoom, p)) {
+			copyPoints(rect, p, list);
+			_pathCacheLock.lock();
+			_pathCache.insert(key, p);
+			_pathCacheLock.unlock();
+		} else
+			delete p;
+	} else {
+		copyPoints(rect, tilePaths, list);
+		_pathCacheLock.unlock();
+	}
+
+	tile->lock.unlock();
+}
+
+void MapData::paths(QFile &file, const RectC &searchRect,
+  const RectC &boundsRect, int zoom, QList<Path> *list)
 {
 	if (!searchRect.isValid())
 		return;
 
 	int l(level(zoom));
-	PathCTX ctx(this, boundsRect, zoom, list);
+	PathCTX ctx(file, this, boundsRect, zoom, list);
 	double min[2], max[2];
 
 	min[0] = searchRect.left();
@@ -588,32 +589,38 @@ void MapData::paths(const RectC &searchRect, const RectC &boundsRect, int zoom,
 	_tiles.at(l)->Search(min, max, pathCb, &ctx);
 }
 
-void MapData::paths(const VectorTile *tile, const RectC &rect, int zoom,
+void MapData::paths(QFile &file, VectorTile *tile, const RectC &rect, int zoom,
   QList<Path> *list)
 {
 	Key key(tile, zoom);
 
-	_pathLock.lock();
+	tile->lock.lock();
 
+	_pathCacheLock.lock();
 	QList<Path> *cached = _pathCache.object(key);
-
 	if (!cached) {
+		_pathCacheLock.unlock();
 		QList<Path> *p = new QList<Path>();
-		if (readPaths(tile, zoom, p)) {
+		if (readPaths(file, tile, zoom, p)) {
 			copyPaths(rect, p, list);
+			_pathCacheLock.lock();
 			_pathCache.insert(key, p);
+			_pathCacheLock.unlock();
 		} else
 			delete p;
-	} else
+	} else {
 		copyPaths(rect, cached, list);
-
+		_pathCacheLock.unlock();
-	_pathLock.unlock();
 	}
 
-bool MapData::readPaths(const VectorTile *tile, int zoom, QList<Path> *list)
+	tile->lock.unlock();
+}
+
+bool MapData::readPaths(QFile &file, const VectorTile *tile, int zoom,
+  QList<Path> *list)
 {
 	const SubFileInfo &info = _subFiles.at(level(zoom));
-	SubFile subfile(_pathFile, info.offset, info.size);
+	SubFile subfile(file, info.offset, info.size);
 	int rows = info.max - info.min + 1;
 	QVector<unsigned> paths(rows);
 	quint32 blocks, unused, val, cnt = 0;
@@ -698,10 +705,11 @@ bool MapData::readPaths(const VectorTile *tile, int zoom, QList<Path> *list)
 	return true;
 }
 
-bool MapData::readPoints(const VectorTile *tile, int zoom, QList<Point> *list)
+bool MapData::readPoints(QFile &file, const VectorTile *tile, int zoom,
+  QList<Point> *list)
 {
 	const SubFileInfo &info = _subFiles.at(level(zoom));
-	SubFile subfile(_pointFile, info.offset, info.size);
+	SubFile subfile(file, info.offset, info.size);
 	int rows = info.max - info.min + 1;
 	QVector<unsigned> points(rows);
 	quint32 val, unused, cnt = 0;

src/map/mapsforge/mapdata.h

@@ -58,14 +58,15 @@ public:
 		  {return point.layer < other.point.layer;}
 	};
 
+	const QString &fileName() const {return _fileName;}
 	RectC bounds() const;
 	Range zooms() const
 	  {return Range(_subFiles.first().min, _subFiles.last().max);}
 	int tileSize() const {return _tileSize;}
 
-	void points(const RectC &rect, int zoom, QList<Point> *list);
+	void points(QFile &file, const RectC &rect, int zoom, QList<Point> *list);
-	void paths(const RectC &searchRect, const RectC &boundsRect, int zoom,
+	void paths(QFile &file, const RectC &searchRect, const RectC &boundsRect,
-	  QList<Path> *set);
+	  int zoom, QList<Path> *set);
 	unsigned tagId(const QByteArray &name) const {return _keys.value(name);}
 
 	void load();
@@ -89,12 +90,15 @@ private:
 
 		size_t offset;
 		Coordinates pos;
+		QMutex lock;
 	};
 
 	struct PathCTX {
-		PathCTX(MapData *data, const RectC &rect, int zoom, QList<Path> *list)
+		PathCTX(QFile &file, MapData *data, const RectC &rect, int zoom,
-		  : data(data), rect(rect), zoom(zoom), list(list) {}
+		  QList<Path> *list)
+		  : file(file), data(data), rect(rect), zoom(zoom), list(list) {}
 
+		QFile &file;
 		MapData *data;
 		const RectC &rect;
 		int zoom;
@@ -102,9 +106,11 @@ private:
 	};
 
 	struct PointCTX {
-		PointCTX(MapData *data, const RectC &rect, int zoom, QList<Point> *list)
+		PointCTX(QFile &file, MapData *data, const RectC &rect, int zoom,
-		  : data(data), rect(rect), zoom(zoom), list(list) {}
+		  QList<Point> *list)
+		  : file(file), data(data), rect(rect), zoom(zoom), list(list) {}
 
+		QFile &file;
 		MapData *data;
 		const RectC &rect;
 		int zoom;
@@ -143,16 +149,18 @@ private:
 	bool readTagInfo(SubFile &hdr, QVector<TagSource> &tags);
 	bool readMapInfo(SubFile &hdr, QByteArray &projection, bool &debugMap);
 	bool readHeader(QFile &file);
-	bool readSubFiles();
+	bool readSubFiles(QFile &file);
 	void clearTiles();
 
 	int level(int zoom) const;
-	void paths(const VectorTile *tile, const RectC &rect, int zoom,
+	void paths(QFile &file, VectorTile *tile, const RectC &rect, int zoom,
 	  QList<Path> *list);
-	void points(const VectorTile *tile, const RectC &rect, int zoom,
+	void points(QFile &file, VectorTile *tile, const RectC &rect, int zoom,
+	  QList<Point> *list);
+	bool readPaths(QFile &file, const VectorTile *tile, int zoom,
+	  QList<Path> *list);
+	bool readPoints(QFile &file, const VectorTile *tile, int zoom,
 	  QList<Point> *list);
-	bool readPaths(const VectorTile *tile, int zoom, QList<Path> *list);
-	bool readPoints(const VectorTile *tile, int zoom, QList<Point> *list);
 
 	static bool readTags(SubFile &subfile, int count,
 	  const QVector<TagSource> &tags, QVector<Tag> &list);
@@ -161,7 +169,7 @@ private:
 
 	friend HASH_T qHash(const MapData::Key &key);
 
-	QFile _pointFile, _pathFile;
+	QString _fileName;
 	RectC _bounds;
 	quint16 _tileSize;
 	QVector<SubFileInfo> _subFiles;
@@ -171,7 +179,7 @@ private:
 
 	QCache<Key, QList<Path> > _pathCache;
 	QCache<Key, QList<Point> > _pointCache;
-	QMutex _pathLock, _pointLock;
+	QMutex _pathCacheLock, _pointCacheLock;
 
 	bool _valid;
 	QString _errorString;

src/map/mapsforge/rastertile.cpp

@@ -408,6 +408,10 @@ void RasterTile::fetchData(QList<MapData::Path> &paths,
   QList<MapData::Point> &points) const
 {
 	QPoint ttl(_rect.topLeft());
+	QFile file(_data->fileName());
+
+	if (!file.open(QIODevice::ReadOnly | QIODevice::Unbuffered))
+		return;
 
 	QRectF pathRect(QPointF(ttl.x() - PATHS_EXTENT, ttl.y() - PATHS_EXTENT),
 	  QPointF(ttl.x() + _rect.width() + PATHS_EXTENT, ttl.y() + _rect.height()
@@ -419,15 +423,15 @@ void RasterTile::fetchData(QList<MapData::Path> &paths,
 	  _transform.img2proj(pathRect.bottomRight()));
 	RectD searchRectD(_transform.img2proj(searchRect.topLeft()),
 	  _transform.img2proj(searchRect.bottomRight()));
-	_data->paths(searchRectD.toRectC(_proj, 20), pathRectD.toRectC(_proj, 20),
+	_data->paths(file, searchRectD.toRectC(_proj, 20),
-	  _zoom, &paths);
+	  pathRectD.toRectC(_proj, 20), _zoom, &paths);
 
 	QRectF pointRect(QPointF(ttl.x() - TEXT_EXTENT, ttl.y() - TEXT_EXTENT),
 	  QPointF(ttl.x() + _rect.width() + TEXT_EXTENT, ttl.y() + _rect.height()
 	  + TEXT_EXTENT));
 	RectD pointRectD(_transform.img2proj(pointRect.topLeft()),
 	  _transform.img2proj(pointRect.bottomRight()));
-	_data->points(pointRectD.toRectC(_proj, 20), _zoom, &points);
+	_data->points(file, pointRectD.toRectC(_proj, 20), _zoom, &points);
 }
 
 MatrixD RasterTile::elevation(int extend) const