root/GPXSee
1
0
Fork 0
mirror of https://github.com/tumic0/GPXSee.git synced 2024-11-27 21:24:47 +01:00
Code Issues Projects Releases Activity

Redesigned HTTP downloader

Browse Source 
- Save the data as they come rather than at once
- + some related refactoring
This commit is contained in:
Martin Tůma 2021-08-26 22:22:18 +02:00
parent d5a472ddc0
commit 018d0ba085
13 changed files with 107 additions and 160 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
gpxsee.pro
View File

@ -43,6 +43,7 @@ HEADERS += src/common/config.h \
src/common/greatcircle.h \
src/common/programpaths.h \
src/common/tifffile.h \
src/common/downloader.h \
src/GUI/app.h \
src/GUI/icons.h \
src/GUI/gui.h \
@ -135,7 +136,6 @@ HEADERS += src/common/config.h \
src/map/map.h \
src/map/maplist.h \
src/map/onlinemap.h \
src/map/downloader.h \
src/map/tile.h \
src/map/emptymap.h \
src/map/ozimap.h \
@ -247,6 +247,7 @@ SOURCES += src/main.cpp \
src/common/greatcircle.cpp \
src/common/programpaths.cpp \
src/common/tifffile.cpp \
src/common/downloader.cpp \
src/GUI/app.cpp \
src/GUI/gui.cpp \
src/GUI/axisitem.cpp \
@ -315,7 +316,6 @@ SOURCES += src/main.cpp \
src/map/kmzmap.cpp \
src/map/maplist.cpp \
src/map/onlinemap.cpp \
src/map/downloader.cpp \
src/map/emptymap.cpp \
src/map/ozimap.cpp \
src/map/polyconic.cpp \

2
src/GUI/app.cpp
View File

@ -9,7 +9,7 @@
#include <QSurfaceFormat>
#include "common/programpaths.h"
#include "common/config.h"
#include "map/downloader.h"
#include "common/downloader.h"
#include "map/ellipsoid.h"
#include "map/gcs.h"
#include "map/pcs.h"

2
src/GUI/gui.cpp
View File

@ -26,11 +26,11 @@
#include <QStyle>
#include <QTabBar>
#include "common/programpaths.h"
#include "common/downloader.h"
#include "data/data.h"
#include "data/poi.h"
#include "map/maplist.h"
#include "map/emptymap.h"
#include "map/downloader.h"
#include "map/crs.h"
#include "icons.h"
#include "keys.h"

166
src/map/downloader.cpp → src/common/downloader.cpp
View File

@ -21,16 +21,28 @@
#define USER_AGENT \
APP_NAME "/" APP_VERSION " (" PLATFORM_STR "; Qt " QT_VERSION_STR ")"
#define ATTR_REDIRECT QNetworkRequest::RedirectionTargetAttribute
#define ATTR_FILE QNetworkRequest::User
#define ATTR_ORIGIN \
static_cast<QNetworkRequest::Attribute>(QNetworkRequest::User + 1)
#define ATTR_LEVEL \
static_cast<QNetworkRequest::Attribute>(QNetworkRequest::User + 2)
#define MAX_REDIRECT_LEVEL 5
#define RETRIES 3
#define ATTR_REDIRECT_POLICY QNetworkRequest::RedirectPolicyAttribute
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
#define ATTR_HTTP2_ALLOWED QNetworkRequest::HTTP2AllowedAttribute
#else // QT 5.15
#define ATTR_HTTP2_ALLOWED QNetworkRequest::Http2AllowedAttribute
#endif // QT 5.15
#define TMP_SUFFIX ".download"
static QString tmpName(const QString &origName)
{
return origName + TMP_SUFFIX;
}
static QString origName(const QString &tmpName)
{
return tmpName.left(tmpName.size() - (sizeof(TMP_SUFFIX) - 1));
}
Authorization::Authorization(const QString &username, const QString &password)
{
@ -67,93 +79,67 @@ private:
QBasicTimer _timer;
};
class Downloader::Redirect
{
public:
Redirect() : _level(0) {}
Redirect(const QUrl &origin, int level) :
_origin(origin), _level(level) {}
const QUrl &origin() const {return _origin;}
int level() const {return _level;}
private:
QUrl _origin;
int _level;
};
QNetworkAccessManager *Downloader::_manager = 0;
int Downloader::_timeout = 30;
bool Downloader::_http2 = true;
bool Downloader::doDownload(const Download &dl,
const QByteArray &authorization, const Redirect *redirect)
bool Downloader::doDownload(const Download &dl, const QByteArray &authorization)
{
const QUrl &url = dl.url();
if (!url.isValid() || !(url.scheme() == QLatin1String("http")
|| url.scheme() == QLatin1String("https"))) {
qWarning("%s: Invalid URL", qPrintable(url.toString()));
if (redirect)
_errorDownloads.insert(redirect->origin(), RETRIES);
return false;
}
if (_errorDownloads.value(url) >= RETRIES)
return false;
if (_currentDownloads.contains(url) && !redirect)
if (_currentDownloads.contains(url))
return false;
QNetworkRequest request(url);
request.setAttribute(ATTR_FILE, QVariant(dl.file()));
if (redirect) {
request.setAttribute(ATTR_ORIGIN, QVariant(redirect->origin()));
request.setAttribute(ATTR_LEVEL, QVariant(redirect->level()));
}
request.setMaximumRedirectsAllowed(MAX_REDIRECT_LEVEL);
request.setAttribute(ATTR_REDIRECT_POLICY,
QNetworkRequest::NoLessSafeRedirectPolicy);
request.setAttribute(ATTR_HTTP2_ALLOWED, QVariant(_http2));
request.setRawHeader("User-Agent", USER_AGENT);
if (!authorization.isNull())
request.setRawHeader("Authorization", authorization);
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
request.setAttribute(QNetworkRequest::HTTP2AllowedAttribute,
QVariant(_http2));
#else // QT 5.15
request.setAttribute(QNetworkRequest::Http2AllowedAttribute,
QVariant(_http2));
#endif // QT 5.15
QFile *file = new QFile(tmpName(dl.file()));
if (!file->open(QIODevice::ReadWrite)) {
qWarning("%s: %s", qPrintable(file->fileName()),
qPrintable(file->errorString()));
_errorDownloads.insert(url, RETRIES);
return false;
}
Q_ASSERT(_manager);
QNetworkReply *reply = _manager->get(request);
if (reply && reply->isRunning()) {
_currentDownloads.insert(url);
file->setParent(reply);
_currentDownloads.insert(url, file);
if (reply->isRunning()) {
ReplyTimeout::setTimeout(reply, _timeout);
connect(reply, &QIODevice::readyRead, this, &Downloader::emitReadReady);
connect(reply, &QNetworkReply::finished, this, &Downloader::emitFinished);
} else if (reply)
} else {
readData(reply);
downloadFinished(reply);
else
return false;
}
return true;
}
void Downloader::emitFinished()
{
downloadFinished(static_cast<QNetworkReply*>(sender()));
downloadFinished(qobject_cast<QNetworkReply*>(sender()));
}
bool Downloader::saveToDisk(const QString &filename, QIODevice *data)
void Downloader::emitReadReady()
{
QFile file(filename);
if (!file.open(QIODevice::WriteOnly)) {
qWarning("Error writing file: %s: %s",
qPrintable(filename), qPrintable(file.errorString()));
return false;
}
file.write(data->readAll());
file.close();
return true;
readData(qobject_cast<QNetworkReply*>(sender()));
}
void Downloader::insertError(const QUrl &url, QNetworkReply::NetworkError error)
@ -164,54 +150,27 @@ void Downloader::insertError(const QUrl &url, QNetworkReply::NetworkError error)
_errorDownloads.insert(url, RETRIES);
}
void Downloader::readData(QNetworkReply *reply)
{
QFile *file = _currentDownloads.value(reply->request().url());
Q_ASSERT(file);
file->write(reply->readAll());
}
void Downloader::downloadFinished(QNetworkReply *reply)
{
QUrl url(reply->request().url());
QNetworkReply::NetworkError error = reply->error();
QFile *file = _currentDownloads.value(reply->request().url());
if (error) {
QUrl origin(reply->request().attribute(ATTR_ORIGIN).toUrl());
if (origin.isEmpty()) {
insertError(url, error);
qWarning("Error downloading file: %s: %s",
url.toEncoded().constData(), qPrintable(reply->errorString()));
} else {
insertError(origin, error);
qWarning("Error downloading file: %s -> %s: %s",
origin.toEncoded().constData(), url.toEncoded().constData(),
qPrintable(reply->errorString()));
}
insertError(url, error);
qWarning("%s: %s", url.toEncoded().constData(),
qPrintable(reply->errorString()));
file->remove();
} else {
QUrl location(reply->attribute(ATTR_REDIRECT).toUrl());
QString filename(reply->request().attribute(ATTR_FILE).toString());
if (!location.isEmpty()) {
QUrl origin(reply->request().attribute(ATTR_ORIGIN).toUrl());
int level = reply->request().attribute(ATTR_LEVEL).toInt();
if (level >= MAX_REDIRECT_LEVEL) {
_errorDownloads.insert(origin, RETRIES);
qWarning("Error downloading file: %s: "
"redirect level limit reached (redirect loop?)",
origin.toEncoded().constData());
} else {
QUrl redirectUrl;
if (location.isRelative()) {
QString path = QDir::isAbsolutePath(location.path())
? location.path() : "/" + location.path();
redirectUrl = QUrl(url.scheme() + "://" + url.host() + path);
} else
redirectUrl = location;
Redirect redirect(origin.isEmpty() ? url : origin, level + 1);
Download dl(redirectUrl, filename);
doDownload(dl, reply->request().rawHeader("Authorization"),
&redirect);
}
} else {
if (!saveToDisk(filename, reply))
_errorDownloads.insert(url, RETRIES);
}
file->close();
file->rename(origName(file->fileName()));
}
_currentDownloads.remove(url);
@ -238,3 +197,12 @@ void Downloader::enableHTTP2(bool enable)
_http2 = enable;
_manager->clearConnectionCache();
}
#ifndef QT_NO_DEBUG
QDebug operator<<(QDebug dbg, const Download &download)
{
dbg.nospace() << "Download(" << download.url() << "," << download.file()
<< ")";
return dbg.space();
}
#endif // QT_NO_DEBUG

17
src/map/downloader.h → src/common/downloader.h
View File

@ -5,9 +5,9 @@
#include <QNetworkReply>
#include <QUrl>
#include <QList>
#include <QSet>
#include <QHash>
class QFile;
class Download
{
@ -55,18 +55,17 @@ signals:
private slots:
void emitFinished();
void downloadFinished(QNetworkReply *reply);
void emitReadReady();
private:
class Redirect;
class ReplyTimeout;
void insertError(const QUrl &url, QNetworkReply::NetworkError error);
bool doDownload(const Download &dl, const QByteArray &authorization,
const Redirect *redirect = 0);
bool saveToDisk(const QString &filename, QIODevice *data);
bool doDownload(const Download &dl, const QByteArray &authorization);
void downloadFinished(QNetworkReply *reply);
void readData(QNetworkReply *reply);
QSet<QUrl> _currentDownloads;
QHash<QUrl, QFile*> _currentDownloads;
QHash<QUrl, int> _errorDownloads;
static QNetworkAccessManager *_manager;
@ -74,4 +73,8 @@ private:
static bool _http2;
};
#ifndef QT_NO_DEBUG
QDebug operator<<(QDebug dbg, const Download &download);
#endif // QT_NO_DEBUG
#endif // DOWNLOADER_H

2
src/map/mapsource.h
View File

@ -5,7 +5,7 @@
#include "common/range.h"
#include "common/rectc.h"
#include "common/kv.h"
#include "downloader.h"
#include "common/downloader.h"
#include "coordinatesystem.h"
class Map;

2
src/map/onlinemap.cpp
View File

@ -3,7 +3,7 @@
#include <QDir>
#include "common/rectc.h"
#include "common/programpaths.h"
#include "downloader.h"
#include "common/downloader.h"
#include "osm.h"
#include "onlinemap.h"

4
src/map/tileloader.h
View File

@ -3,8 +3,8 @@
#include <QObject>
#include <QString>
#include "common/downloader.h"
#include "tile.h"
#include "downloader.h"
class TileLoader : public QObject
{
@ -38,4 +38,4 @@ private:
bool _quadTiles;
};
#endif // TILELOADER_Honlinemap
#endif // TILELOADER_H

30
src/map/wms.cpp
View File

@ -3,7 +3,6 @@
#include <QEventLoop>
#include <QXmlStreamReader>
#include <QStringList>
#include "downloader.h"
#include "crs.h"
#include "wms.h"
@ -320,20 +319,6 @@ bool WMS::parseCapabilities()
return true;
}
bool WMS::downloadCapabilities(const QString &url)
{
if (!_downloader) {
_downloader = new Downloader(this);
connect(_downloader, &Downloader::finished, this,
&WMS::capabilitiesReady);
}
QList<Download> dl;
dl.append(Download(url, _path));
return _downloader->get(dl, _setup.authorization());
}
void WMS::capabilitiesReady()
{
if (!QFileInfo(_path).exists()) {
@ -348,15 +333,20 @@ void WMS::capabilitiesReady()
}
WMS::WMS(const QString &file, const WMS::Setup &setup, QObject *parent)
: QObject(parent), _setup(setup), _path(file), _downloader(0), _valid(false),
_ready(false)
: QObject(parent), _setup(setup), _path(file), _valid(false), _ready(false)
{
QString url = QString("%1%2service=WMS&request=GetCapabilities")
.arg(setup.url(), setup.url().contains('?') ? "&" : "?");
if (!QFileInfo(file).exists())
_valid = downloadCapabilities(url);
else {
if (!QFileInfo(file).exists()) {
Downloader *downloader = new Downloader(this);
connect(downloader, &Downloader::finished, this,
&WMS::capabilitiesReady);
QList<Download> dl;
dl.append(Download(url, _path));
_valid = downloader->get(dl, _setup.authorization());
} else {
_ready = true;
_valid = parseCapabilities();
}

4
src/map/wms.h
View File

@ -6,8 +6,8 @@
#include "common/range.h"
#include "common/rectc.h"
#include "common/kv.h"
#include "common/downloader.h"
#include "projection.h"
#include "downloader.h"
#include "coordinatesystem.h"
class QXmlStreamReader;
@ -109,11 +109,9 @@ private:
void capability(QXmlStreamReader &reader, CTX &ctx);
void capabilities(QXmlStreamReader &reader, CTX &ctx);
bool parseCapabilities();
bool downloadCapabilities(const QString &url);
WMS::Setup _setup;
QString _path;
Downloader *_downloader;
Projection _projection;
RangeF _scaleDenominator;
RectC _bbox;

1
src/map/wmsmap.cpp
View File

@ -4,7 +4,6 @@
#include "common/wgs84.h"
#include "common/rectc.h"
#include "common/programpaths.h"
#include "downloader.h"
#include "tileloader.h"
#include "wmsmap.h"

29
src/map/wmts.cpp
View File

@ -6,7 +6,6 @@
#include <QStringList>
#include <QtAlgorithms>
#include <QXmlStreamReader>
#include "downloader.h"
#include "pcs.h"
#include "crs.h"
#include "wmts.h"
@ -302,20 +301,6 @@ bool WMTS::parseCapabilities(CTX &ctx)
return true;
}
bool WMTS::downloadCapabilities(const QString &url)
{
if (!_downloader) {
_downloader = new Downloader(this);
connect(_downloader, &Downloader::finished, this,
&WMTS::capabilitiesReady);
}
QList<Download> dl;
dl.append(Download(url, _path));
return _downloader->get(dl, _setup.authorization());
}
void WMTS::capabilitiesReady()
{
if (!QFileInfo(_path).exists()) {
@ -363,7 +348,7 @@ bool WMTS::init()
}
WMTS::WMTS(const QString &file, const WMTS::Setup &setup, QObject *parent)
: QObject(parent), _setup(setup), _downloader(0), _valid(false), _ready(false)
: QObject(parent), _setup(setup), _valid(false), _ready(false)
{
QUrl url(setup.rest() ? setup.url() : QString(
"%1%2service=WMTS&Version=1.0.0&request=GetCapabilities").arg(setup.url(),
@ -371,9 +356,15 @@ WMTS::WMTS(const QString &file, const WMTS::Setup &setup, QObject *parent)
_path = url.isLocalFile() ? url.toLocalFile() : file;
if (!url.isLocalFile() && !QFileInfo(file).exists())
_valid = downloadCapabilities(url.toString());
else {
if (!url.isLocalFile() && !QFileInfo(file).exists()) {
Downloader *downloader = new Downloader(this);
connect(downloader, &Downloader::finished, this,
&WMTS::capabilitiesReady);
QList<Download> dl;
dl.append(Download(url.toString(), _path));
_valid = downloader->get(dl, _setup.authorization());
} else {
_ready = true;
_valid = init();
}

4
src/map/wmts.h
View File

@ -9,8 +9,8 @@
#include "common/config.h"
#include "common/rectc.h"
#include "common/kv.h"
#include "common/downloader.h"
#include "projection.h"
#include "downloader.h"
#include "coordinatesystem.h"
class QXmlStreamReader;
@ -154,13 +154,11 @@ private:
void contents(QXmlStreamReader &reader, CTX &ctx);
void capabilities(QXmlStreamReader &reader, CTX &ctx);
bool parseCapabilities(CTX &ctx);
bool downloadCapabilities(const QString &url);
void createZooms(const CTX &ctx);
bool init();
WMTS::Setup _setup;
QString _path;
Downloader *_downloader;
RectC _bbox;
QList<Zoom> _zooms;
Projection _projection;