root/GPXSee
1
0
Fork 0
mirror of https://github.com/tumic0/GPXSee.git synced 2025-06-27 19:49:15 +02:00
Code Issues Projects Releases Activity

Redesigned HTTP downloader

Browse Source 
- Save the data as they come rather than at once
- + some related refactoring
This commit is contained in:
Martin Tůma
2021-08-26 22:22:18 +02:00
parent d5a472ddc0
commit 018d0ba085
13 changed files with 107 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
src/map/downloader.cpp
View File

@ -1,240 +0,0 @@
#include <QFile>
#include <QFileInfo>
#include <QNetworkRequest>
#include <QBasicTimer>
#include <QDir>
#include <QTimerEvent>
#include "common/config.h"
#include "downloader.h"
#if defined(Q_OS_LINUX)
#define PLATFORM_STR "Linux"
#elif defined(Q_OS_WIN32)
#define PLATFORM_STR "Windows"
#elif defined(Q_OS_MAC)
#define PLATFORM_STR "OS X"
#else
#define PLATFORM_STR "Unknown"
#endif
#define USER_AGENT \
APP_NAME "/" APP_VERSION " (" PLATFORM_STR "; Qt " QT_VERSION_STR ")"
#define ATTR_REDIRECT QNetworkRequest::RedirectionTargetAttribute
#define ATTR_FILE QNetworkRequest::User
#define ATTR_ORIGIN \
static_cast<QNetworkRequest::Attribute>(QNetworkRequest::User + 1)
#define ATTR_LEVEL \
static_cast<QNetworkRequest::Attribute>(QNetworkRequest::User + 2)
#define MAX_REDIRECT_LEVEL 5
#define RETRIES 3
Authorization::Authorization(const QString &username, const QString &password)
{
QString concatenated = username + ":" + password;
QByteArray data = concatenated.toLocal8Bit().toBase64();
_header = "Basic " + data;
}
class Downloader::ReplyTimeout : public QObject
{
public:
static void setTimeout(QNetworkReply *reply, int timeout)
{
Q_ASSERT(reply);
new ReplyTimeout(reply, timeout);
}
private:
ReplyTimeout(QNetworkReply *reply, int timeout) : QObject(reply)
{
_timer.start(timeout * 1000, this);
}
void timerEvent(QTimerEvent *ev)
{
if (!_timer.isActive() || ev->timerId() != _timer.timerId())
return;
QNetworkReply *reply = static_cast<QNetworkReply*>(parent());
if (reply->isRunning())
reply->close();
_timer.stop();
}
QBasicTimer _timer;
};
class Downloader::Redirect
{
public:
Redirect() : _level(0) {}
Redirect(const QUrl &origin, int level) :
_origin(origin), _level(level) {}
const QUrl &origin() const {return _origin;}
int level() const {return _level;}
private:
QUrl _origin;
int _level;
};
QNetworkAccessManager *Downloader::_manager = 0;
int Downloader::_timeout = 30;
bool Downloader::_http2 = true;
bool Downloader::doDownload(const Download &dl,
const QByteArray &authorization, const Redirect *redirect)
{
const QUrl &url = dl.url();
if (!url.isValid() || !(url.scheme() == QLatin1String("http")
|| url.scheme() == QLatin1String("https"))) {
qWarning("%s: Invalid URL", qPrintable(url.toString()));
if (redirect)
_errorDownloads.insert(redirect->origin(), RETRIES);
return false;
}
if (_errorDownloads.value(url) >= RETRIES)
return false;
if (_currentDownloads.contains(url) && !redirect)
return false;
QNetworkRequest request(url);
request.setAttribute(ATTR_FILE, QVariant(dl.file()));
if (redirect) {
request.setAttribute(ATTR_ORIGIN, QVariant(redirect->origin()));
request.setAttribute(ATTR_LEVEL, QVariant(redirect->level()));
}
request.setRawHeader("User-Agent", USER_AGENT);
if (!authorization.isNull())
request.setRawHeader("Authorization", authorization);
#if QT_VERSION < QT_VERSION_CHECK(5, 15, 0)
request.setAttribute(QNetworkRequest::HTTP2AllowedAttribute,
QVariant(_http2));
#else // QT 5.15
request.setAttribute(QNetworkRequest::Http2AllowedAttribute,
QVariant(_http2));
#endif // QT 5.15
Q_ASSERT(_manager);
QNetworkReply *reply = _manager->get(request);
if (reply && reply->isRunning()) {
_currentDownloads.insert(url);
ReplyTimeout::setTimeout(reply, _timeout);
connect(reply, &QNetworkReply::finished, this, &Downloader::emitFinished);
} else if (reply)
downloadFinished(reply);
else
return false;
return true;
}
void Downloader::emitFinished()
{
downloadFinished(static_cast<QNetworkReply*>(sender()));
}
bool Downloader::saveToDisk(const QString &filename, QIODevice *data)
{
QFile file(filename);
if (!file.open(QIODevice::WriteOnly)) {
qWarning("Error writing file: %s: %s",
qPrintable(filename), qPrintable(file.errorString()));
return false;
}
file.write(data->readAll());
file.close();
return true;
}
void Downloader::insertError(const QUrl &url, QNetworkReply::NetworkError error)
{
if (error == QNetworkReply::OperationCanceledError)
_errorDownloads.insert(url, _errorDownloads.value(url) + 1);
else
_errorDownloads.insert(url, RETRIES);
}
void Downloader::downloadFinished(QNetworkReply *reply)
{
QUrl url(reply->request().url());
QNetworkReply::NetworkError error = reply->error();
if (error) {
QUrl origin(reply->request().attribute(ATTR_ORIGIN).toUrl());
if (origin.isEmpty()) {
insertError(url, error);
qWarning("Error downloading file: %s: %s",
url.toEncoded().constData(), qPrintable(reply->errorString()));
} else {
insertError(origin, error);
qWarning("Error downloading file: %s -> %s: %s",
origin.toEncoded().constData(), url.toEncoded().constData(),
qPrintable(reply->errorString()));
}
} else {
QUrl location(reply->attribute(ATTR_REDIRECT).toUrl());
QString filename(reply->request().attribute(ATTR_FILE).toString());
if (!location.isEmpty()) {
QUrl origin(reply->request().attribute(ATTR_ORIGIN).toUrl());
int level = reply->request().attribute(ATTR_LEVEL).toInt();
if (level >= MAX_REDIRECT_LEVEL) {
_errorDownloads.insert(origin, RETRIES);
qWarning("Error downloading file: %s: "
"redirect level limit reached (redirect loop?)",
origin.toEncoded().constData());
} else {
QUrl redirectUrl;
if (location.isRelative()) {
QString path = QDir::isAbsolutePath(location.path())
? location.path() : "/" + location.path();
redirectUrl = QUrl(url.scheme() + "://" + url.host() + path);
} else
redirectUrl = location;
Redirect redirect(origin.isEmpty() ? url : origin, level + 1);
Download dl(redirectUrl, filename);
doDownload(dl, reply->request().rawHeader("Authorization"),
&redirect);
}
} else {
if (!saveToDisk(filename, reply))
_errorDownloads.insert(url, RETRIES);
}
}
_currentDownloads.remove(url);
reply->deleteLater();
if (_currentDownloads.isEmpty())
emit finished();
}
bool Downloader::get(const QList<Download> &list,
const Authorization &authorization)
{
bool finishEmitted = false;
for (int i = 0; i < list.count(); i++)
finishEmitted |= doDownload(list.at(i), authorization.header());
return finishEmitted;
}
void Downloader::enableHTTP2(bool enable)
{
Q_ASSERT(_manager);
_http2 = enable;
_manager->clearConnectionCache();
}

77
src/map/downloader.h
View File

@ -1,77 +0,0 @@
#ifndef DOWNLOADER_H
#define DOWNLOADER_H
#include <QNetworkAccessManager>
#include <QNetworkReply>
#include <QUrl>
#include <QList>
#include <QSet>
#include <QHash>
class Download
{
public:
Download(const QUrl &url, const QString &file) : _url(url), _file(file) {}
const QUrl &url() const {return _url;}
const QString &file() const {return _file;}
private:
QUrl _url;
QString _file;
};
class Authorization
{
public:
Authorization() {}
Authorization(const QString &username, const QString &password);
const QByteArray &header() const {return _header;}
private:
QByteArray _header;
};
class Downloader : public QObject
{
Q_OBJECT
public:
Downloader(QObject *parent = 0) : QObject(parent) {}
bool get(const QList<Download> &list, const Authorization &authorization
= Authorization());
void clearErrors() {_errorDownloads.clear();}
static void setNetworkManager(QNetworkAccessManager *manager)
{_manager = manager;}
static void setTimeout(int timeout) {_timeout = timeout;}
static void enableHTTP2(bool enable);
signals:
void finished();
private slots:
void emitFinished();
void downloadFinished(QNetworkReply *reply);
private:
class Redirect;
class ReplyTimeout;
void insertError(const QUrl &url, QNetworkReply::NetworkError error);
bool doDownload(const Download &dl, const QByteArray &authorization,
const Redirect *redirect = 0);
bool saveToDisk(const QString &filename, QIODevice *data);
QSet<QUrl> _currentDownloads;
QHash<QUrl, int> _errorDownloads;
static QNetworkAccessManager *_manager;
static int _timeout;
static bool _http2;
};
#endif // DOWNLOADER_H

2
src/map/mapsource.h
View File

@ -5,7 +5,7 @@
#include "common/range.h"
#include "common/rectc.h"
#include "common/kv.h"
#include "downloader.h"
#include "common/downloader.h"
#include "coordinatesystem.h"
class Map;

2
src/map/onlinemap.cpp
View File

@ -3,7 +3,7 @@
#include <QDir>
#include "common/rectc.h"
#include "common/programpaths.h"
#include "downloader.h"
#include "common/downloader.h"
#include "osm.h"
#include "onlinemap.h"

4
src/map/tileloader.h
View File

@ -3,8 +3,8 @@
#include <QObject>
#include <QString>
#include "common/downloader.h"
#include "tile.h"
#include "downloader.h"
class TileLoader : public QObject
{
@ -38,4 +38,4 @@ private:
bool _quadTiles;
};
#endif // TILELOADER_Honlinemap
#endif // TILELOADER_H

30
src/map/wms.cpp
View File

@ -3,7 +3,6 @@
#include <QEventLoop>
#include <QXmlStreamReader>
#include <QStringList>
#include "downloader.h"
#include "crs.h"
#include "wms.h"
@ -320,20 +319,6 @@ bool WMS::parseCapabilities()
return true;
}
bool WMS::downloadCapabilities(const QString &url)
{
if (!_downloader) {
_downloader = new Downloader(this);
connect(_downloader, &Downloader::finished, this,
&WMS::capabilitiesReady);
}
QList<Download> dl;
dl.append(Download(url, _path));
return _downloader->get(dl, _setup.authorization());
}
void WMS::capabilitiesReady()
{
if (!QFileInfo(_path).exists()) {
@ -348,15 +333,20 @@ void WMS::capabilitiesReady()
}
WMS::WMS(const QString &file, const WMS::Setup &setup, QObject *parent)
: QObject(parent), _setup(setup), _path(file), _downloader(0), _valid(false),
_ready(false)
: QObject(parent), _setup(setup), _path(file), _valid(false), _ready(false)
{
QString url = QString("%1%2service=WMS&request=GetCapabilities")
.arg(setup.url(), setup.url().contains('?') ? "&" : "?");
if (!QFileInfo(file).exists())
_valid = downloadCapabilities(url);
else {
if (!QFileInfo(file).exists()) {
Downloader *downloader = new Downloader(this);
connect(downloader, &Downloader::finished, this,
&WMS::capabilitiesReady);
QList<Download> dl;
dl.append(Download(url, _path));
_valid = downloader->get(dl, _setup.authorization());
} else {
_ready = true;
_valid = parseCapabilities();
}

4
src/map/wms.h
View File

@ -6,8 +6,8 @@
#include "common/range.h"
#include "common/rectc.h"
#include "common/kv.h"
#include "common/downloader.h"
#include "projection.h"
#include "downloader.h"
#include "coordinatesystem.h"
class QXmlStreamReader;
@ -109,11 +109,9 @@ private:
void capability(QXmlStreamReader &reader, CTX &ctx);
void capabilities(QXmlStreamReader &reader, CTX &ctx);
bool parseCapabilities();
bool downloadCapabilities(const QString &url);
WMS::Setup _setup;
QString _path;
Downloader *_downloader;
Projection _projection;
RangeF _scaleDenominator;
RectC _bbox;

1
src/map/wmsmap.cpp
View File

@ -4,7 +4,6 @@
#include "common/wgs84.h"
#include "common/rectc.h"
#include "common/programpaths.h"
#include "downloader.h"
#include "tileloader.h"
#include "wmsmap.h"

29
src/map/wmts.cpp
View File

@ -6,7 +6,6 @@
#include <QStringList>
#include <QtAlgorithms>
#include <QXmlStreamReader>
#include "downloader.h"
#include "pcs.h"
#include "crs.h"
#include "wmts.h"
@ -302,20 +301,6 @@ bool WMTS::parseCapabilities(CTX &ctx)
return true;
}
bool WMTS::downloadCapabilities(const QString &url)
{
if (!_downloader) {
_downloader = new Downloader(this);
connect(_downloader, &Downloader::finished, this,
&WMTS::capabilitiesReady);
}
QList<Download> dl;
dl.append(Download(url, _path));
return _downloader->get(dl, _setup.authorization());
}
void WMTS::capabilitiesReady()
{
if (!QFileInfo(_path).exists()) {
@ -363,7 +348,7 @@ bool WMTS::init()
}
WMTS::WMTS(const QString &file, const WMTS::Setup &setup, QObject *parent)
: QObject(parent), _setup(setup), _downloader(0), _valid(false), _ready(false)
: QObject(parent), _setup(setup), _valid(false), _ready(false)
{
QUrl url(setup.rest() ? setup.url() : QString(
"%1%2service=WMTS&Version=1.0.0&request=GetCapabilities").arg(setup.url(),
@ -371,9 +356,15 @@ WMTS::WMTS(const QString &file, const WMTS::Setup &setup, QObject *parent)
_path = url.isLocalFile() ? url.toLocalFile() : file;
if (!url.isLocalFile() && !QFileInfo(file).exists())
_valid = downloadCapabilities(url.toString());
else {
if (!url.isLocalFile() && !QFileInfo(file).exists()) {
Downloader *downloader = new Downloader(this);
connect(downloader, &Downloader::finished, this,
&WMTS::capabilitiesReady);
QList<Download> dl;
dl.append(Download(url.toString(), _path));
_valid = downloader->get(dl, _setup.authorization());
} else {
_ready = true;
_valid = init();
}

4
src/map/wmts.h
View File

@ -9,8 +9,8 @@
#include "common/config.h"
#include "common/rectc.h"
#include "common/kv.h"
#include "common/downloader.h"
#include "projection.h"
#include "downloader.h"
#include "coordinatesystem.h"
class QXmlStreamReader;
@ -154,13 +154,11 @@ private:
void contents(QXmlStreamReader &reader, CTX &ctx);
void capabilities(QXmlStreamReader &reader, CTX &ctx);
bool parseCapabilities(CTX &ctx);
bool downloadCapabilities(const QString &url);
void createZooms(const CTX &ctx);
bool init();
WMTS::Setup _setup;
QString _path;
Downloader *_downloader;
RectC _bbox;
QList<Zoom> _zooms;
Projection _projection;