CactiChameleon9/PrismLauncher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

NOISSUE refactor *Download into more, smaller pieces

Browse Source 
* Download is now Download.
* Download uses Sink subclasses to process various events.
* Validators can be used to further customize the Sink behaviour.
This commit is contained in:
Petr Mrázek
2016-05-28 19:54:17 +02:00
parent a750f6e63c
commit a1abbd9e05
51 changed files with 824 additions and 765 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
api/logic/net/ByteArrayDownload.cpp
View File

@ -1,105 +0,0 @@
/* Copyright 2013-2015 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ByteArrayDownload.h"
#include "Env.h"
#include <QDebug>
ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction()
{
m_url = url;
m_status = Job_NotStarted;
}
void ByteArrayDownload::start()
{
qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request(m_url);
request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
auto worker = ENV.qnam();
QNetworkReply *rep = worker->get(request);
m_reply.reset(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
}
void ByteArrayDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
m_total_progress = bytesTotal;
m_progress = bytesReceived;
emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
}
void ByteArrayDownload::downloadError(QNetworkReply::NetworkError error)
{
// error happened during download.
qCritical() << "Error getting URL:" << m_url.toString().toLocal8Bit()
<< "Network error: " << error;
m_status = Job_Failed;
m_errorString = m_reply->errorString();
}
void ByteArrayDownload::downloadFinished()
{
QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader);
QString redirectURL;
if(redirect.isValid())
{
redirectURL = redirect.toString();
}
// FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061
else if(m_reply->hasRawHeader("Location"))
{
auto data = m_reply->rawHeader("Location");
if(data.size() > 2 && data[0] == '/' && data[1] == '/')
redirectURL = m_reply->url().scheme() + ":" + data;
}
if (!redirectURL.isEmpty())
{
m_url = QUrl(redirect.toString());
qDebug() << "Following redirect to " << m_url.toString();
start();
return;
}
// if the download succeeded
if (m_status != Job_Failed)
{
// nothing went wrong...
m_status = Job_Finished;
m_data = m_reply->readAll();
m_content_type = m_reply->header(QNetworkRequest::ContentTypeHeader).toString();
m_reply.reset();
emit succeeded(m_index_within_job);
return;
}
// else the download failed
else
{
m_reply.reset();
emit failed(m_index_within_job);
return;
}
}
void ByteArrayDownload::downloadReadyRead()
{
// ~_~
}

48
api/logic/net/ByteArrayDownload.h
View File

@ -1,48 +0,0 @@
/* Copyright 2013-2015 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "NetAction.h"
#include "multimc_logic_export.h"
typedef std::shared_ptr<class ByteArrayDownload> ByteArrayDownloadPtr;
class MULTIMC_LOGIC_EXPORT ByteArrayDownload : public NetAction
{
Q_OBJECT
public:
ByteArrayDownload(QUrl url);
static ByteArrayDownloadPtr make(QUrl url)
{
return ByteArrayDownloadPtr(new ByteArrayDownload(url));
}
virtual ~ByteArrayDownload() {};
public:
/// if not saving to file, downloaded data is placed here
QByteArray m_data;
QString m_errorString;
public
slots:
virtual void start();
protected
slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
};

57
api/logic/net/ByteArraySink.h Normal file
View File

@ -0,0 +1,57 @@
#pragma once
#include "Sink.h"
namespace Net {
/*
* Sink object for downloads that uses an external QByteArray it doesn't own as a target.
*/
class ByteArraySink : public Sink
{
public:
ByteArraySink(QByteArray *output)
:m_output(output)
{
// nil
};
virtual ~ByteArraySink()
{
// nil
}
public:
JobStatus init(QNetworkRequest & request) override
{
m_output->clear();
if(initAllValidators(request))
return Job_InProgress;
return Job_Failed;
};
JobStatus write(QByteArray & data) override
{
m_output->append(data);
if(writeAllValidators(data))
return Job_InProgress;
return Job_Failed;
}
JobStatus abort() override
{
m_output->clear();
failAllValidators();
return Job_Failed;
}
JobStatus finalize(QNetworkReply &reply) override
{
if(finalizeAllValidators(reply))
return Job_Finished;
return Job_Failed;
}
private:
QByteArray * m_output;
};
}

192
api/logic/net/CacheDownload.cpp
View File

@ -1,192 +0,0 @@
/* Copyright 2013-2015 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "CacheDownload.h"
#include <QCryptographicHash>
#include <QFileInfo>
#include <QDateTime>
#include <QDebug>
#include "Env.h"
#include <FileSystem.h>
CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
: NetAction(), md5sum(QCryptographicHash::Md5)
{
m_url = url;
m_entry = entry;
m_target_path = entry->getFullPath();
m_status = Job_NotStarted;
}
void CacheDownload::start()
{
m_status = Job_InProgress;
if (!m_entry->isStale())
{
m_status = Job_Finished;
emit succeeded(m_index_within_job);
return;
}
// create a new save file
m_output_file.reset(new QSaveFile(m_target_path));
// if there already is a file and md5 checking is in effect and it can be opened
if (!FS::ensureFilePathExists(m_target_path))
{
qCritical() << "Could not create folder for " + m_target_path;
m_status = Job_Failed;
emit failed(m_index_within_job);
return;
}
if (!m_output_file->open(QIODevice::WriteOnly))
{
qCritical() << "Could not open " + m_target_path + " for writing";
m_status = Job_Failed;
emit failed(m_index_within_job);
return;
}
qDebug() << "Downloading " << m_url.toString();
QNetworkRequest request(m_url);
// check file consistency first.
QFile current(m_target_path);
if(current.exists() && current.size() != 0)
{
if (m_entry->getRemoteChangedTimestamp().size())
request.setRawHeader(QString("If-Modified-Since").toLatin1(),
m_entry->getRemoteChangedTimestamp().toLatin1());
if (m_entry->getETag().size())
request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->getETag().toLatin1());
}
request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Cached)");
auto worker = ENV.qnam();
QNetworkReply *rep = worker->get(request);
m_reply.reset(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
}
void CacheDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
m_total_progress = bytesTotal;
m_progress = bytesReceived;
emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
}
void CacheDownload::downloadError(QNetworkReply::NetworkError error)
{
// error happened during download.
qCritical() << "Failed " << m_url.toString() << " with reason " << error;
m_status = Job_Failed;
}
void CacheDownload::downloadFinished()
{
QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader);
QString redirectURL;
if(redirect.isValid())
{
redirectURL = redirect.toString();
}
// FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061
else if(m_reply->hasRawHeader("Location"))
{
auto data = m_reply->rawHeader("Location");
if(data.size() > 2 && data[0] == '/' && data[1] == '/')
redirectURL = m_reply->url().scheme() + ":" + data;
}
if (!redirectURL.isEmpty())
{
m_url = QUrl(redirect.toString());
qDebug() << "Following redirect to " << m_url.toString();
start();
return;
}
// if the download succeeded
if (m_status == Job_Failed)
{
m_output_file->cancelWriting();
m_reply.reset();
emit failed(m_index_within_job);
return;
}
// if we wrote any data to the save file, we try to commit the data to the real file.
if (wroteAnyData)
{
// nothing went wrong...
if (m_output_file->commit())
{
m_status = Job_Finished;
m_entry->setMD5Sum(md5sum.result().toHex().constData());
}
else
{
qCritical() << "Failed to commit changes to " << m_target_path;
m_output_file->cancelWriting();
m_reply.reset();
m_status = Job_Failed;
emit failed(m_index_within_job);
return;
}
}
else
{
m_status = Job_Finished;
}
// then get rid of the save file
m_output_file.reset();
QFileInfo output_file_info(m_target_path);
m_entry->setETag(m_reply->rawHeader("ETag").constData());
if (m_reply->hasRawHeader("Last-Modified"))
{
m_entry->setRemoteChangedTimestamp(m_reply->rawHeader("Last-Modified").constData());
}
m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch());
m_entry->setStale(false);
ENV.metacache()->updateEntry(m_entry);
m_reply.reset();
emit succeeded(m_index_within_job);
return;
}
void CacheDownload::downloadReadyRead()
{
QByteArray ba = m_reply->readAll();
md5sum.addData(ba);
if (m_output_file->write(ba) != ba.size())
{
qCritical() << "Failed writing into " + m_target_path;
m_status = Job_Failed;
m_output_file->cancelWriting();
m_output_file.reset();
emit failed(m_index_within_job);
wroteAnyData = false;
return;
}
wroteAnyData = true;
}

57
api/logic/net/ChecksumValidator.h Normal file
View File

@ -0,0 +1,57 @@
#pragma once
#include "Validator.h"
#include <QCryptographicHash>
#include <memory>
#include <QFile>
namespace Net {
class ChecksumValidator: public Validator
{
public: /* con/des */
ChecksumValidator(QCryptographicHash::Algorithm algorithm, QByteArray expected = QByteArray())
:m_checksum(algorithm), m_expected(expected)
{
};
virtual ~ChecksumValidator() {};
public: /* methods */
bool init(QNetworkRequest &) override
{
m_checksum.reset();
return true;
}
bool write(QByteArray & data) override
{
m_checksum.addData(data);
this->data.append(data);
return true;
}
bool abort() override
{
return true;
}
bool validate(QNetworkReply &) override
{
if(m_expected.size() && m_expected != hash())
{
qWarning() << "Checksum mismatch, download is bad.";
return false;
}
return true;
}
QByteArray hash()
{
return m_checksum.result();
}
void setExpected(QByteArray expected)
{
m_expected = expected;
}
private: /* data */
QByteArray data;
QCryptographicHash m_checksum;
QByteArray m_expected;
};
}

199
api/logic/net/Download.cpp Normal file
View File

@ -0,0 +1,199 @@
/* Copyright 2013-2015 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Download.h"
#include <QFileInfo>
#include <QDateTime>
#include <QDebug>
#include "Env.h"
#include <FileSystem.h>
#include "ChecksumValidator.h"
#include "MetaCacheSink.h"
#include "ByteArraySink.h"
namespace Net {
Download::Download():NetAction()
{
m_status = Job_NotStarted;
}
Download::Ptr Download::makeCached(QUrl url, MetaEntryPtr entry)
{
Download * dl = new Download();
dl->m_url = url;
auto md5Node = new ChecksumValidator(QCryptographicHash::Md5);
auto cachedNode = new MetaCacheSink(entry, md5Node);
dl->m_sink.reset(cachedNode);
dl->m_target_path = entry->getFullPath();
return std::shared_ptr<Download>(dl);
}
Download::Ptr Download::makeByteArray(QUrl url, QByteArray *output)
{
Download * dl = new Download();
dl->m_url = url;
dl->m_sink.reset(new ByteArraySink(output));
return std::shared_ptr<Download>(dl);
}
Download::Ptr Download::makeFile(QUrl url, QString path)
{
Download * dl = new Download();
dl->m_url = url;
dl->m_sink.reset(new FileSink(path));
return std::shared_ptr<Download>(dl);
}
void Download::addValidator(Validator * v)
{
m_sink->addValidator(v);
}
void Download::start()
{
QNetworkRequest request(m_url);
m_status = m_sink->init(request);
switch(m_status)
{
case Job_Finished:
emit succeeded(m_index_within_job);
qDebug() << "Download cache hit " << m_url.toString();
return;
case Job_InProgress:
qDebug() << "Downloading " << m_url.toString();
break;
case Job_NotStarted:
case Job_Failed:
emit failed(m_index_within_job);
return;
}
request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0");
auto worker = ENV.qnam();
QNetworkReply *rep = worker->get(request);
m_reply.reset(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)), SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
}
void Download::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
m_total_progress = bytesTotal;
m_progress = bytesReceived;
emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
}
void Download::downloadError(QNetworkReply::NetworkError error)
{
// error happened during download.
qCritical() << "Failed " << m_url.toString() << " with reason " << error;
m_status = Job_Failed;
}
bool Download::handleRedirect()
{
QVariant redirect = m_reply->header(QNetworkRequest::LocationHeader);
QString redirectURL;
if(redirect.isValid())
{
redirectURL = redirect.toString();
}
// FIXME: This is a hack for https://bugreports.qt-project.org/browse/QTBUG-41061
else if(m_reply->hasRawHeader("Location"))
{
auto data = m_reply->rawHeader("Location");
if(data.size() > 2 && data[0] == '/' && data[1] == '/')
{
redirectURL = m_reply->url().scheme() + ":" + data;
}
}
if (!redirectURL.isEmpty())
{
m_url = QUrl(redirect.toString());
qDebug() << "Following redirect to " << m_url.toString();
start();
return true;
}
return false;
}
void Download::downloadFinished()
{
// handle HTTP redirection first
if(handleRedirect())
{
qDebug() << "Download redirected:" << m_url.toString();
return;
}
// if the download failed before this point ...
if (m_status == Job_Failed)
{
qDebug() << "Download failed in previous step:" << m_url.toString();
m_sink->abort();
m_reply.reset();
emit failed(m_index_within_job);
return;
}
// make sure we got all the remaining data, if any
auto data = m_reply->readAll();
if(data.size())
{
qDebug() << "Writing extra" << data.size() << "bytes to" << m_target_path;
m_status = m_sink->write(data);
}
// otherwise, finalize the whole graph
m_status = m_sink->finalize(*m_reply.get());
if (m_status != Job_Finished)
{
qDebug() << "Download failed to finalize:" << m_url.toString();
m_sink->abort();
m_reply.reset();
emit failed(m_index_within_job);
return;
}
m_reply.reset();
qDebug() << "Download succeeded:" << m_url.toString();
emit succeeded(m_index_within_job);
}
void Download::downloadReadyRead()
{
if(m_status == Job_InProgress)
{
auto data = m_reply->readAll();
m_status = m_sink->write(data);
if(m_status == Job_Failed)
{
qCritical() << "Failed to process response chunk for " << m_target_path;
}
// qDebug() << "Download" << m_url.toString() << "gained" << data.size() << "bytes";
}
else
{
qCritical() << "Cannot write to " << m_target_path << ", illegal status" << m_status;
}
}
}

55
api/logic/net/CacheDownload.h → api/logic/net/Download.h
View File

@ -17,47 +17,50 @@
#include "NetAction.h"
#include "HttpMetaCache.h"
#include <QCryptographicHash>
#include <QSaveFile>
#include "Validator.h"
#include "Sink.h"
#include "multimc_logic_export.h"
typedef std::shared_ptr<class CacheDownload> CacheDownloadPtr;
class MULTIMC_LOGIC_EXPORT CacheDownload : public NetAction
namespace Net {
class MULTIMC_LOGIC_EXPORT Download : public NetAction
{
Q_OBJECT
private:
MetaEntryPtr m_entry;
/// if saving to file, use the one specified in this string
QString m_target_path;
/// this is the output file, if any
std::unique_ptr<QSaveFile> m_output_file;
/// the hash-as-you-download
QCryptographicHash md5sum;
bool wroteAnyData = false;
public: /* types */
typedef std::shared_ptr<class Download> Ptr;
protected: /* con/des */
explicit Download();
public:
explicit CacheDownload(QUrl url, MetaEntryPtr entry);
static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry)
{
return CacheDownloadPtr(new CacheDownload(url, entry));
}
virtual ~CacheDownload(){};
virtual ~Download(){};
static Download::Ptr makeCached(QUrl url, MetaEntryPtr entry);
static Download::Ptr makeByteArray(QUrl url, QByteArray *output);
static Download::Ptr makeFile(QUrl url, QString path);
public: /* methods */
// FIXME: remove this
QString getTargetFilepath()
{
return m_target_path;
}
protected
slots:
// FIXME: remove this
void addValidator(Validator * v);
private: /* methods */
bool handleRedirect();
protected slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public
slots:
public slots:
virtual void start();
private: /* data */
// FIXME: remove this, it has no business being here.
QString m_target_path;
std::unique_ptr<Sink> m_sink;
};
}

99
api/logic/net/FileSink.cpp Normal file
View File

@ -0,0 +1,99 @@
#include "FileSink.h"
#include <QFile>
#include <QFileInfo>
#include "Env.h"
#include "FileSystem.h"
namespace Net {
FileSink::FileSink(QString filename)
:m_filename(filename)
{
// nil
};
FileSink::~FileSink()
{
// nil
};
JobStatus FileSink::init(QNetworkRequest& request)
{
auto result = initCache(request);
if(result != Job_InProgress)
{
return result;
}
// create a new save file and open it for writing
if (!FS::ensureFilePathExists(m_filename))
{
qCritical() << "Could not create folder for " + m_filename;
return Job_Failed;
}
m_output_file.reset(new QSaveFile(m_filename));
if (!m_output_file->open(QIODevice::WriteOnly))
{
qCritical() << "Could not open " + m_filename + " for writing";
return Job_Failed;
}
if(initAllValidators(request))
return Job_InProgress;
return Job_Failed;
}
JobStatus FileSink::initCache(QNetworkRequest &)
{
return Job_InProgress;
}
JobStatus FileSink::write(QByteArray& data)
{
if (!writeAllValidators(data) || m_output_file->write(data) != data.size())
{
qCritical() << "Failed writing into " + m_filename;
m_output_file->cancelWriting();
m_output_file.reset();
wroteAnyData = false;
return Job_Failed;
}
wroteAnyData = true;
return Job_InProgress;
}
JobStatus FileSink::abort()
{
m_output_file->cancelWriting();
failAllValidators();
return Job_Failed;
}
JobStatus FileSink::finalize(QNetworkReply& reply)
{
// if we wrote any data to the save file, we try to commit the data to the real file.
if (wroteAnyData)
{
// ask validators for data consistency
// we only do this for actual downloads, not 'your data is still the same' cache hits
if(!finalizeAllValidators(reply))
return Job_Failed;
// nothing went wrong...
if (!m_output_file->commit())
{
qCritical() << "Failed to commit changes to " << m_filename;
m_output_file->cancelWriting();
return Job_Failed;
}
}
// then get rid of the save file
m_output_file.reset();
return finalizeCache(reply);
}
JobStatus FileSink::finalizeCache(QNetworkReply &)
{
return Job_Finished;
}
}

27
api/logic/net/FileSink.h Normal file
View File

@ -0,0 +1,27 @@
#pragma once
#include "Sink.h"
#include <QSaveFile>
namespace Net {
class FileSink : public Sink
{
public: /* con/des */
FileSink(QString filename);
virtual ~FileSink();
public: /* methods */
JobStatus init(QNetworkRequest & request) override;
JobStatus write(QByteArray & data) override;
JobStatus abort() override;
JobStatus finalize(QNetworkReply & reply) override;
protected: /* methods */
virtual JobStatus initCache(QNetworkRequest &);
virtual JobStatus finalizeCache(QNetworkReply &reply);
protected: /* data */
QString m_filename;
bool wroteAnyData = false;
std::unique_ptr<QSaveFile> m_output_file;
};
}

155
api/logic/net/MD5EtagDownload.cpp
View File

@ -1,155 +0,0 @@
/* Copyright 2013-2015 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Env.h"
#include "MD5EtagDownload.h"
#include <FileSystem.h>
#include <QCryptographicHash>
#include <QDebug>
MD5EtagDownload::MD5EtagDownload(QUrl url, QString target_path) : NetAction()
{
m_url = url;
m_target_path = target_path;
m_status = Job_NotStarted;
}
void MD5EtagDownload::start()
{
QString filename = m_target_path;
m_output_file.setFileName(filename);
// if there already is a file and md5 checking is in effect and it can be opened
if (m_output_file.exists() && m_output_file.open(QIODevice::ReadOnly))
{
// get the md5 of the local file.
m_local_md5 =
QCryptographicHash::hash(m_output_file.readAll(), QCryptographicHash::Md5)
.toHex()
.constData();
m_output_file.close();
// if we are expecting some md5sum, compare it with the local one
if (!m_expected_md5.isEmpty())
{
// skip if they match
if(m_local_md5 == m_expected_md5)
{
qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
emit succeeded(m_index_within_job);
return;
}
}
else
{
// no expected md5. we use the local md5sum as an ETag
}
}
if (!FS::ensureFilePathExists(filename))
{
emit failed(m_index_within_job);
return;
}
QNetworkRequest request(m_url);
qDebug() << "Downloading " << m_url.toString() << " local MD5: " << m_local_md5;
if(!m_local_md5.isEmpty())
{
request.setRawHeader(QString("If-None-Match").toLatin1(), m_local_md5.toLatin1());
}
if(!m_expected_md5.isEmpty())
qDebug() << "Expecting " << m_expected_md5;
request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
// Go ahead and try to open the file.
// If we don't do this, empty files won't be created, which breaks the updater.
// Plus, this way, we don't end up starting a download for a file we can't open.
if (!m_output_file.open(QIODevice::WriteOnly))
{
emit failed(m_index_within_job);
return;
}
auto worker = ENV.qnam();
QNetworkReply *rep = worker->get(request);
m_reply.reset(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
}
void MD5EtagDownload::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
m_total_progress = bytesTotal;
m_progress = bytesReceived;
emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
}
void MD5EtagDownload::downloadError(QNetworkReply::NetworkError error)
{
qCritical() << "Error" << error << ":" << m_reply->errorString() << "while downloading"
<< m_reply->url();
m_status = Job_Failed;
}
void MD5EtagDownload::downloadFinished()
{
// if the download succeeded
if (m_status != Job_Failed)
{
// nothing went wrong...
m_status = Job_Finished;
m_output_file.close();
// FIXME: compare with the real written data md5sum
// this is just an ETag
qDebug() << "Finished " << m_url.toString() << " got " << m_reply->rawHeader("ETag").constData();
m_reply.reset();
emit succeeded(m_index_within_job);
return;
}
// else the download failed
else
{
m_output_file.close();
m_output_file.remove();
m_reply.reset();
emit failed(m_index_within_job);
return;
}
}
void MD5EtagDownload::downloadReadyRead()
{
if (!m_output_file.isOpen())
{
if (!m_output_file.open(QIODevice::WriteOnly))
{
/*
* Can't open the file... the job failed
*/
m_reply->abort();
emit failed(m_index_within_job);
return;
}
}
m_output_file.write(m_reply->readAll());
}

52
api/logic/net/MD5EtagDownload.h
View File

@ -1,52 +0,0 @@
/* Copyright 2013-2015 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "NetAction.h"
#include <QFile>
typedef std::shared_ptr<class MD5EtagDownload> Md5EtagDownloadPtr;
class MD5EtagDownload : public NetAction
{
Q_OBJECT
public:
/// the expected md5 checksum. Only set from outside
QString m_expected_md5;
/// the md5 checksum of a file that already exists.
QString m_local_md5;
/// if saving to file, use the one specified in this string
QString m_target_path;
/// this is the output file, if any
QFile m_output_file;
public:
explicit MD5EtagDownload(QUrl url, QString target_path);
static Md5EtagDownloadPtr make(QUrl url, QString target_path)
{
return Md5EtagDownloadPtr(new MD5EtagDownload(url, target_path));
}
virtual ~MD5EtagDownload(){};
protected
slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
virtual void downloadError(QNetworkReply::NetworkError error);
virtual void downloadFinished();
virtual void downloadReadyRead();
public
slots:
virtual void start();
};

59
api/logic/net/MetaCacheSink.cpp Normal file
View File

@ -0,0 +1,59 @@
#include "MetaCacheSink.h"
#include <QFile>
#include <QFileInfo>
#include "Env.h"
#include "FileSystem.h"
namespace Net {
MetaCacheSink::MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum)
:Net::FileSink(entry->getFullPath()), m_entry(entry), m_md5Node(md5sum)
{
addValidator(md5sum);
};
MetaCacheSink::~MetaCacheSink()
{
// nil
};
JobStatus MetaCacheSink::initCache(QNetworkRequest& request)
{
if (!m_entry->isStale())
{
return Job_Finished;
}
// check if file exists, if it does, use its information for the request
QFile current(m_filename);
if(current.exists() && current.size() != 0)
{
if (m_entry->getRemoteChangedTimestamp().size())
{
request.setRawHeader(QString("If-Modified-Since").toLatin1(), m_entry->getRemoteChangedTimestamp().toLatin1());
}
if (m_entry->getETag().size())
{
request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->getETag().toLatin1());
}
}
return Job_InProgress;
}
JobStatus MetaCacheSink::finalizeCache(QNetworkReply & reply)
{
QFileInfo output_file_info(m_filename);
if(wroteAnyData)
{
m_entry->setMD5Sum(m_md5Node->hash().toHex().constData());
}
m_entry->setETag(reply.rawHeader("ETag").constData());
if (reply.hasRawHeader("Last-Modified"))
{
m_entry->setRemoteChangedTimestamp(reply.rawHeader("Last-Modified").constData());
}
m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch());
m_entry->setStale(false);
ENV.metacache()->updateEntry(m_entry);
return Job_Finished;
}
}

21
api/logic/net/MetaCacheSink.h Normal file
View File

@ -0,0 +1,21 @@
#pragma once
#include "FileSink.h"
#include "ChecksumValidator.h"
#include "net/HttpMetaCache.h"
namespace Net {
class MetaCacheSink : public FileSink
{
public: /* con/des */
MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum);
virtual ~MetaCacheSink();
protected: /* methods */
JobStatus initCache(QNetworkRequest & request) override;
JobStatus finalizeCache(QNetworkReply & reply) override;
private: /* data */
MetaEntryPtr m_entry;
ChecksumValidator * m_md5Node;
};
}

3
api/logic/net/NetAction.h
View File

@ -59,9 +59,6 @@ public:
/// the network reply
unique_qobject_ptr<QNetworkReply> m_reply;
/// the content of the content-type header
QString m_content_type;
/// source URL
QUrl m_url;

4
api/logic/net/NetJob.cpp
View File

@ -14,9 +14,7 @@
*/
#include "NetJob.h"
#include "MD5EtagDownload.h"
#include "ByteArrayDownload.h"
#include "CacheDownload.h"
#include "Download.h"
#include <QDebug>

4
api/logic/net/NetJob.h
View File

@ -16,9 +16,7 @@
#pragma once
#include <QtNetwork>
#include "NetAction.h"
#include "ByteArrayDownload.h"
#include "MD5EtagDownload.h"
#include "CacheDownload.h"
#include "Download.h"
#include "HttpMetaCache.h"
#include "tasks/Task.h"
#include "QObjectPtr.h"

70
api/logic/net/Sink.h Normal file
View File

@ -0,0 +1,70 @@
#pragma once
#include "net/NetAction.h"
#include "multimc_logic_export.h"
#include "Validator.h"
namespace Net {
class MULTIMC_LOGIC_EXPORT Sink
{
public: /* con/des */
Sink() {};
virtual ~Sink() {};
public: /* methods */
virtual JobStatus init(QNetworkRequest & request) = 0;
virtual JobStatus write(QByteArray & data) = 0;
virtual JobStatus abort() = 0;
virtual JobStatus finalize(QNetworkReply & reply) = 0;
void addValidator(Validator * validator)
{
if(validator)
{
validators.push_back(std::shared_ptr<Validator>(validator));
}
}
protected: /* methods */
bool finalizeAllValidators(QNetworkReply & reply)
{
for(auto & validator: validators)
{
if(!validator->validate(reply))
return false;
}
return true;
}
bool failAllValidators()
{
bool success = true;
for(auto & validator: validators)
{
success &= validator->abort();
}
return success;
}
bool initAllValidators(QNetworkRequest & request)
{
for(auto & validator: validators)
{
if(!validator->init(request))
return false;
}
return true;
}
bool writeAllValidators(QByteArray & data)
{
for(auto & validator: validators)
{
if(!validator->write(data))
return false;
}
return true;
}
protected: /* data */
std::vector<std::shared_ptr<Validator>> validators;
};
}

20
api/logic/net/Validator.h Normal file
View File

@ -0,0 +1,20 @@
#pragma once
#include "net/NetAction.h"
#include "multimc_logic_export.h"
namespace Net {
class MULTIMC_LOGIC_EXPORT Validator
{
public: /* con/des */
Validator() {};
virtual ~Validator() {};
public: /* methods */
virtual bool init(QNetworkRequest & request) = 0;
virtual bool write(QByteArray & data) = 0;
virtual bool abort() = 0;
virtual bool validate(QNetworkReply & reply) = 0;
};
}