diff --git a/gpxsee.pro b/gpxsee.pro index 5fb182b2..7551367f 100644 --- a/gpxsee.pro +++ b/gpxsee.pro @@ -43,6 +43,7 @@ HEADERS += src/common/config.h \ src/common/greatcircle.h \ src/common/programpaths.h \ src/common/tifffile.h \ + src/common/downloader.h \ src/GUI/app.h \ src/GUI/icons.h \ src/GUI/gui.h \ @@ -135,7 +136,6 @@ HEADERS += src/common/config.h \ src/map/map.h \ src/map/maplist.h \ src/map/onlinemap.h \ - src/map/downloader.h \ src/map/tile.h \ src/map/emptymap.h \ src/map/ozimap.h \ @@ -247,6 +247,7 @@ SOURCES += src/main.cpp \ src/common/greatcircle.cpp \ src/common/programpaths.cpp \ src/common/tifffile.cpp \ + src/common/downloader.cpp \ src/GUI/app.cpp \ src/GUI/gui.cpp \ src/GUI/axisitem.cpp \ @@ -315,7 +316,6 @@ SOURCES += src/main.cpp \ src/map/kmzmap.cpp \ src/map/maplist.cpp \ src/map/onlinemap.cpp \ - src/map/downloader.cpp \ src/map/emptymap.cpp \ src/map/ozimap.cpp \ src/map/polyconic.cpp \ diff --git a/src/GUI/app.cpp b/src/GUI/app.cpp index 37e9d3f5..bdc23571 100644 --- a/src/GUI/app.cpp +++ b/src/GUI/app.cpp @@ -9,7 +9,7 @@ #include #include "common/programpaths.h" #include "common/config.h" -#include "map/downloader.h" +#include "common/downloader.h" #include "map/ellipsoid.h" #include "map/gcs.h" #include "map/pcs.h" diff --git a/src/GUI/gui.cpp b/src/GUI/gui.cpp index e8195733..36254308 100644 --- a/src/GUI/gui.cpp +++ b/src/GUI/gui.cpp @@ -26,11 +26,11 @@ #include #include #include "common/programpaths.h" +#include "common/downloader.h" #include "data/data.h" #include "data/poi.h" #include "map/maplist.h" #include "map/emptymap.h" -#include "map/downloader.h" #include "map/crs.h" #include "icons.h" #include "keys.h" diff --git a/src/map/downloader.cpp b/src/common/downloader.cpp similarity index 50% rename from src/map/downloader.cpp rename to src/common/downloader.cpp index 7a1dccfa..cb83b415 100644 --- a/src/map/downloader.cpp +++ b/src/common/downloader.cpp @@ -21,16 +21,28 @@ #define USER_AGENT \ APP_NAME "/" APP_VERSION " (" PLATFORM_STR "; Qt " QT_VERSION_STR ")" -#define ATTR_REDIRECT QNetworkRequest::RedirectionTargetAttribute -#define ATTR_FILE QNetworkRequest::User -#define ATTR_ORIGIN \ - static_cast(QNetworkRequest::User + 1) -#define ATTR_LEVEL \ - static_cast(QNetworkRequest::User + 2) - #define MAX_REDIRECT_LEVEL 5 #define RETRIES 3 +#define ATTR_REDIRECT_POLICY QNetworkRequest::RedirectPolicyAttribute +#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) +#define ATTR_HTTP2_ALLOWED QNetworkRequest::HTTP2AllowedAttribute +#else // QT 5.15 +#define ATTR_HTTP2_ALLOWED QNetworkRequest::Http2AllowedAttribute +#endif // QT 5.15 + +#define TMP_SUFFIX ".download" + + +static QString tmpName(const QString &origName) +{ + return origName + TMP_SUFFIX; +} + +static QString origName(const QString &tmpName) +{ + return tmpName.left(tmpName.size() - (sizeof(TMP_SUFFIX) - 1)); +} Authorization::Authorization(const QString &username, const QString &password) { @@ -67,93 +79,67 @@ private: QBasicTimer _timer; }; -class Downloader::Redirect -{ -public: - Redirect() : _level(0) {} - Redirect(const QUrl &origin, int level) : - _origin(origin), _level(level) {} - - const QUrl &origin() const {return _origin;} - int level() const {return _level;} - -private: - QUrl _origin; - int _level; -}; - QNetworkAccessManager *Downloader::_manager = 0; int Downloader::_timeout = 30; bool Downloader::_http2 = true; -bool Downloader::doDownload(const Download &dl, - const QByteArray &authorization, const Redirect *redirect) +bool Downloader::doDownload(const Download &dl, const QByteArray &authorization) { const QUrl &url = dl.url(); if (!url.isValid() || !(url.scheme() == QLatin1String("http") || url.scheme() == QLatin1String("https"))) { qWarning("%s: Invalid URL", qPrintable(url.toString())); - if (redirect) - _errorDownloads.insert(redirect->origin(), RETRIES); return false; } if (_errorDownloads.value(url) >= RETRIES) return false; - if (_currentDownloads.contains(url) && !redirect) + if (_currentDownloads.contains(url)) return false; QNetworkRequest request(url); - request.setAttribute(ATTR_FILE, QVariant(dl.file())); - if (redirect) { - request.setAttribute(ATTR_ORIGIN, QVariant(redirect->origin())); - request.setAttribute(ATTR_LEVEL, QVariant(redirect->level())); - } + request.setMaximumRedirectsAllowed(MAX_REDIRECT_LEVEL); + request.setAttribute(ATTR_REDIRECT_POLICY, + QNetworkRequest::NoLessSafeRedirectPolicy); + request.setAttribute(ATTR_HTTP2_ALLOWED, QVariant(_http2)); request.setRawHeader("User-Agent", USER_AGENT); if (!authorization.isNull()) request.setRawHeader("Authorization", authorization); -#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0) - request.setAttribute(QNetworkRequest::HTTP2AllowedAttribute, - QVariant(_http2)); -#else // QT 5.15 - request.setAttribute(QNetworkRequest::Http2AllowedAttribute, - QVariant(_http2)); -#endif // QT 5.15 + + QFile *file = new QFile(tmpName(dl.file())); + if (!file->open(QIODevice::ReadWrite)) { + qWarning("%s: %s", qPrintable(file->fileName()), + qPrintable(file->errorString())); + _errorDownloads.insert(url, RETRIES); + return false; + } Q_ASSERT(_manager); QNetworkReply *reply = _manager->get(request); - if (reply && reply->isRunning()) { - _currentDownloads.insert(url); + file->setParent(reply); + _currentDownloads.insert(url, file); + + if (reply->isRunning()) { ReplyTimeout::setTimeout(reply, _timeout); + connect(reply, &QIODevice::readyRead, this, &Downloader::emitReadReady); connect(reply, &QNetworkReply::finished, this, &Downloader::emitFinished); - } else if (reply) + } else { + readData(reply); downloadFinished(reply); - else - return false; + } return true; } void Downloader::emitFinished() { - downloadFinished(static_cast(sender())); + downloadFinished(qobject_cast(sender())); } -bool Downloader::saveToDisk(const QString &filename, QIODevice *data) +void Downloader::emitReadReady() { - QFile file(filename); - - if (!file.open(QIODevice::WriteOnly)) { - qWarning("Error writing file: %s: %s", - qPrintable(filename), qPrintable(file.errorString())); - return false; - } - - file.write(data->readAll()); - file.close(); - - return true; + readData(qobject_cast(sender())); } void Downloader::insertError(const QUrl &url, QNetworkReply::NetworkError error) @@ -164,54 +150,27 @@ void Downloader::insertError(const QUrl &url, QNetworkReply::NetworkError error) _errorDownloads.insert(url, RETRIES); } +void Downloader::readData(QNetworkReply *reply) +{ + QFile *file = _currentDownloads.value(reply->request().url()); + Q_ASSERT(file); + file->write(reply->readAll()); +} + void Downloader::downloadFinished(QNetworkReply *reply) { QUrl url(reply->request().url()); QNetworkReply::NetworkError error = reply->error(); + QFile *file = _currentDownloads.value(reply->request().url()); if (error) { - QUrl origin(reply->request().attribute(ATTR_ORIGIN).toUrl()); - if (origin.isEmpty()) { - insertError(url, error); - qWarning("Error downloading file: %s: %s", - url.toEncoded().constData(), qPrintable(reply->errorString())); - } else { - insertError(origin, error); - qWarning("Error downloading file: %s -> %s: %s", - origin.toEncoded().constData(), url.toEncoded().constData(), - qPrintable(reply->errorString())); - } + insertError(url, error); + qWarning("%s: %s", url.toEncoded().constData(), + qPrintable(reply->errorString())); + file->remove(); } else { - QUrl location(reply->attribute(ATTR_REDIRECT).toUrl()); - QString filename(reply->request().attribute(ATTR_FILE).toString()); - - if (!location.isEmpty()) { - QUrl origin(reply->request().attribute(ATTR_ORIGIN).toUrl()); - int level = reply->request().attribute(ATTR_LEVEL).toInt(); - - if (level >= MAX_REDIRECT_LEVEL) { - _errorDownloads.insert(origin, RETRIES); - qWarning("Error downloading file: %s: " - "redirect level limit reached (redirect loop?)", - origin.toEncoded().constData()); - } else { - QUrl redirectUrl; - if (location.isRelative()) { - QString path = QDir::isAbsolutePath(location.path()) - ? location.path() : "/" + location.path(); - redirectUrl = QUrl(url.scheme() + "://" + url.host() + path); - } else - redirectUrl = location; - - Redirect redirect(origin.isEmpty() ? url : origin, level + 1); - Download dl(redirectUrl, filename); - doDownload(dl, reply->request().rawHeader("Authorization"), - &redirect); - } - } else { - if (!saveToDisk(filename, reply)) - _errorDownloads.insert(url, RETRIES); - } + file->close(); + file->rename(origName(file->fileName())); } _currentDownloads.remove(url); @@ -238,3 +197,12 @@ void Downloader::enableHTTP2(bool enable) _http2 = enable; _manager->clearConnectionCache(); } + +#ifndef QT_NO_DEBUG +QDebug operator<<(QDebug dbg, const Download &download) +{ + dbg.nospace() << "Download(" << download.url() << "," << download.file() + << ")"; + return dbg.space(); +} +#endif // QT_NO_DEBUG diff --git a/src/map/downloader.h b/src/common/downloader.h similarity index 86% rename from src/map/downloader.h rename to src/common/downloader.h index a447cf63..aed215fa 100644 --- a/src/map/downloader.h +++ b/src/common/downloader.h @@ -5,9 +5,9 @@ #include #include #include -#include #include +class QFile; class Download { @@ -55,18 +55,17 @@ signals: private slots: void emitFinished(); - void downloadFinished(QNetworkReply *reply); + void emitReadReady(); private: - class Redirect; class ReplyTimeout; void insertError(const QUrl &url, QNetworkReply::NetworkError error); - bool doDownload(const Download &dl, const QByteArray &authorization, - const Redirect *redirect = 0); - bool saveToDisk(const QString &filename, QIODevice *data); + bool doDownload(const Download &dl, const QByteArray &authorization); + void downloadFinished(QNetworkReply *reply); + void readData(QNetworkReply *reply); - QSet _currentDownloads; + QHash _currentDownloads; QHash _errorDownloads; static QNetworkAccessManager *_manager; @@ -74,4 +73,8 @@ private: static bool _http2; }; +#ifndef QT_NO_DEBUG +QDebug operator<<(QDebug dbg, const Download &download); +#endif // QT_NO_DEBUG + #endif // DOWNLOADER_H diff --git a/src/map/mapsource.h b/src/map/mapsource.h index 564e9dc6..29ba96b4 100644 --- a/src/map/mapsource.h +++ b/src/map/mapsource.h @@ -5,7 +5,7 @@ #include "common/range.h" #include "common/rectc.h" #include "common/kv.h" -#include "downloader.h" +#include "common/downloader.h" #include "coordinatesystem.h" class Map; diff --git a/src/map/onlinemap.cpp b/src/map/onlinemap.cpp index b6dac165..9b72ef57 100644 --- a/src/map/onlinemap.cpp +++ b/src/map/onlinemap.cpp @@ -3,7 +3,7 @@ #include #include "common/rectc.h" #include "common/programpaths.h" -#include "downloader.h" +#include "common/downloader.h" #include "osm.h" #include "onlinemap.h" diff --git a/src/map/tileloader.h b/src/map/tileloader.h index d12ba6c4..b0953809 100644 --- a/src/map/tileloader.h +++ b/src/map/tileloader.h @@ -3,8 +3,8 @@ #include #include +#include "common/downloader.h" #include "tile.h" -#include "downloader.h" class TileLoader : public QObject { @@ -38,4 +38,4 @@ private: bool _quadTiles; }; -#endif // TILELOADER_Honlinemap +#endif // TILELOADER_H diff --git a/src/map/wms.cpp b/src/map/wms.cpp index b65a3e8f..1860015b 100644 --- a/src/map/wms.cpp +++ b/src/map/wms.cpp @@ -3,7 +3,6 @@ #include #include #include -#include "downloader.h" #include "crs.h" #include "wms.h" @@ -320,20 +319,6 @@ bool WMS::parseCapabilities() return true; } -bool WMS::downloadCapabilities(const QString &url) -{ - if (!_downloader) { - _downloader = new Downloader(this); - connect(_downloader, &Downloader::finished, this, - &WMS::capabilitiesReady); - } - - QList dl; - dl.append(Download(url, _path)); - - return _downloader->get(dl, _setup.authorization()); -} - void WMS::capabilitiesReady() { if (!QFileInfo(_path).exists()) { @@ -348,15 +333,20 @@ void WMS::capabilitiesReady() } WMS::WMS(const QString &file, const WMS::Setup &setup, QObject *parent) - : QObject(parent), _setup(setup), _path(file), _downloader(0), _valid(false), - _ready(false) + : QObject(parent), _setup(setup), _path(file), _valid(false), _ready(false) { QString url = QString("%1%2service=WMS&request=GetCapabilities") .arg(setup.url(), setup.url().contains('?') ? "&" : "?"); - if (!QFileInfo(file).exists()) - _valid = downloadCapabilities(url); - else { + if (!QFileInfo(file).exists()) { + Downloader *downloader = new Downloader(this); + connect(downloader, &Downloader::finished, this, + &WMS::capabilitiesReady); + + QList dl; + dl.append(Download(url, _path)); + _valid = downloader->get(dl, _setup.authorization()); + } else { _ready = true; _valid = parseCapabilities(); } diff --git a/src/map/wms.h b/src/map/wms.h index 430c3d9e..6bfc6828 100644 --- a/src/map/wms.h +++ b/src/map/wms.h @@ -6,8 +6,8 @@ #include "common/range.h" #include "common/rectc.h" #include "common/kv.h" +#include "common/downloader.h" #include "projection.h" -#include "downloader.h" #include "coordinatesystem.h" class QXmlStreamReader; @@ -109,11 +109,9 @@ private: void capability(QXmlStreamReader &reader, CTX &ctx); void capabilities(QXmlStreamReader &reader, CTX &ctx); bool parseCapabilities(); - bool downloadCapabilities(const QString &url); WMS::Setup _setup; QString _path; - Downloader *_downloader; Projection _projection; RangeF _scaleDenominator; RectC _bbox; diff --git a/src/map/wmsmap.cpp b/src/map/wmsmap.cpp index 4c75f469..b49230ca 100644 --- a/src/map/wmsmap.cpp +++ b/src/map/wmsmap.cpp @@ -4,7 +4,6 @@ #include "common/wgs84.h" #include "common/rectc.h" #include "common/programpaths.h" -#include "downloader.h" #include "tileloader.h" #include "wmsmap.h" diff --git a/src/map/wmts.cpp b/src/map/wmts.cpp index ecc69c8a..1b7af8c3 100644 --- a/src/map/wmts.cpp +++ b/src/map/wmts.cpp @@ -6,7 +6,6 @@ #include #include #include -#include "downloader.h" #include "pcs.h" #include "crs.h" #include "wmts.h" @@ -302,20 +301,6 @@ bool WMTS::parseCapabilities(CTX &ctx) return true; } -bool WMTS::downloadCapabilities(const QString &url) -{ - if (!_downloader) { - _downloader = new Downloader(this); - connect(_downloader, &Downloader::finished, this, - &WMTS::capabilitiesReady); - } - - QList dl; - dl.append(Download(url, _path)); - - return _downloader->get(dl, _setup.authorization()); -} - void WMTS::capabilitiesReady() { if (!QFileInfo(_path).exists()) { @@ -363,7 +348,7 @@ bool WMTS::init() } WMTS::WMTS(const QString &file, const WMTS::Setup &setup, QObject *parent) - : QObject(parent), _setup(setup), _downloader(0), _valid(false), _ready(false) + : QObject(parent), _setup(setup), _valid(false), _ready(false) { QUrl url(setup.rest() ? setup.url() : QString( "%1%2service=WMTS&Version=1.0.0&request=GetCapabilities").arg(setup.url(), @@ -371,9 +356,15 @@ WMTS::WMTS(const QString &file, const WMTS::Setup &setup, QObject *parent) _path = url.isLocalFile() ? url.toLocalFile() : file; - if (!url.isLocalFile() && !QFileInfo(file).exists()) - _valid = downloadCapabilities(url.toString()); - else { + if (!url.isLocalFile() && !QFileInfo(file).exists()) { + Downloader *downloader = new Downloader(this); + connect(downloader, &Downloader::finished, this, + &WMTS::capabilitiesReady); + + QList dl; + dl.append(Download(url.toString(), _path)); + _valid = downloader->get(dl, _setup.authorization()); + } else { _ready = true; _valid = init(); } diff --git a/src/map/wmts.h b/src/map/wmts.h index 237b034b..af2ee140 100644 --- a/src/map/wmts.h +++ b/src/map/wmts.h @@ -9,8 +9,8 @@ #include "common/config.h" #include "common/rectc.h" #include "common/kv.h" +#include "common/downloader.h" #include "projection.h" -#include "downloader.h" #include "coordinatesystem.h" class QXmlStreamReader; @@ -154,13 +154,11 @@ private: void contents(QXmlStreamReader &reader, CTX &ctx); void capabilities(QXmlStreamReader &reader, CTX &ctx); bool parseCapabilities(CTX &ctx); - bool downloadCapabilities(const QString &url); void createZooms(const CTX &ctx); bool init(); WMTS::Setup _setup; QString _path; - Downloader *_downloader; RectC _bbox; QList _zooms; Projection _projection;