CactiChameleon9/PrismLauncher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

NOISSUE Flatten gui and logic libraries into MultiMC

Browse Source
This commit is contained in:
Petr Mrázek
2021-07-25 19:11:59 +02:00
parent dd13368085
commit 20b9f2b42a
1113 changed files with 1228 additions and 1401 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
launcher/net/ByteArraySink.h Normal file
View File

@ -0,0 +1,62 @@
#pragma once
#include "Sink.h"
namespace Net {
/*
* Sink object for downloads that uses an external QByteArray it doesn't own as a target.
*/
class ByteArraySink : public Sink
{
public:
ByteArraySink(QByteArray *output)
:m_output(output)
{
// nil
};
virtual ~ByteArraySink()
{
// nil
}
public:
JobStatus init(QNetworkRequest & request) override
{
m_output->clear();
if(initAllValidators(request))
return Job_InProgress;
return Job_Failed;
};
JobStatus write(QByteArray & data) override
{
m_output->append(data);
if(writeAllValidators(data))
return Job_InProgress;
return Job_Failed;
}
JobStatus abort() override
{
m_output->clear();
failAllValidators();
return Job_Failed;
}
JobStatus finalize(QNetworkReply &reply) override
{
if(finalizeAllValidators(reply))
return Job_Finished;
return Job_Failed;
}
bool hasLocalData() override
{
return false;
}
private:
QByteArray * m_output;
};
}

55
launcher/net/ChecksumValidator.h Normal file
View File

@ -0,0 +1,55 @@
#pragma once
#include "Validator.h"
#include <QCryptographicHash>
#include <memory>
#include <QFile>
namespace Net {
class ChecksumValidator: public Validator
{
public: /* con/des */
ChecksumValidator(QCryptographicHash::Algorithm algorithm, QByteArray expected = QByteArray())
:m_checksum(algorithm), m_expected(expected)
{
};
virtual ~ChecksumValidator() {};
public: /* methods */
bool init(QNetworkRequest &) override
{
m_checksum.reset();
return true;
}
bool write(QByteArray & data) override
{
m_checksum.addData(data);
return true;
}
bool abort() override
{
return true;
}
bool validate(QNetworkReply &) override
{
if(m_expected.size() && m_expected != hash())
{
qWarning() << "Checksum mismatch, download is bad.";
return false;
}
return true;
}
QByteArray hash()
{
return m_checksum.result();
}
void setExpected(QByteArray expected)
{
m_expected = expected;
}
private: /* data */
QCryptographicHash m_checksum;
QByteArray m_expected;
};
}

309
launcher/net/Download.cpp Normal file
View File

@ -0,0 +1,309 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Download.h"
#include "BuildConfig.h"
#include <QFileInfo>
#include <QDateTime>
#include <QDebug>
#include "Env.h"
#include <FileSystem.h>
#include "ChecksumValidator.h"
#include "MetaCacheSink.h"
#include "ByteArraySink.h"
namespace Net {
Download::Download():NetAction()
{
m_status = Job_NotStarted;
}
Download::Ptr Download::makeCached(QUrl url, MetaEntryPtr entry, Options options)
{
Download * dl = new Download();
dl->m_url = url;
dl->m_options = options;
auto md5Node = new ChecksumValidator(QCryptographicHash::Md5);
auto cachedNode = new MetaCacheSink(entry, md5Node);
dl->m_sink.reset(cachedNode);
dl->m_target_path = entry->getFullPath();
return std::shared_ptr<Download>(dl);
}
Download::Ptr Download::makeByteArray(QUrl url, QByteArray *output, Options options)
{
Download * dl = new Download();
dl->m_url = url;
dl->m_options = options;
dl->m_sink.reset(new ByteArraySink(output));
return std::shared_ptr<Download>(dl);
}
Download::Ptr Download::makeFile(QUrl url, QString path, Options options)
{
Download * dl = new Download();
dl->m_url = url;
dl->m_options = options;
dl->m_sink.reset(new FileSink(path));
return std::shared_ptr<Download>(dl);
}
void Download::addValidator(Validator * v)
{
m_sink->addValidator(v);
}
void Download::start()
{
if(m_status == Job_Aborted)
{
qWarning() << "Attempt to start an aborted Download:" << m_url.toString();
emit aborted(m_index_within_job);
return;
}
QNetworkRequest request(m_url);
m_status = m_sink->init(request);
switch(m_status)
{
case Job_Finished:
emit succeeded(m_index_within_job);
qDebug() << "Download cache hit " << m_url.toString();
return;
case Job_InProgress:
qDebug() << "Downloading " << m_url.toString();
break;
case Job_Failed_Proceed: // this is meaningless in this context. We do need a sink.
case Job_NotStarted:
case Job_Failed:
emit failed(m_index_within_job);
return;
case Job_Aborted:
return;
}
request.setHeader(QNetworkRequest::UserAgentHeader, BuildConfig.USER_AGENT);
QNetworkReply *rep = ENV.qnam().get(request);
m_reply.reset(rep);
connect(rep, SIGNAL(downloadProgress(qint64, qint64)), SLOT(downloadProgress(qint64, qint64)));
connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, &QNetworkReply::sslErrors, this, &Download::sslErrors);
connect(rep, &QNetworkReply::readyRead, this, &Download::downloadReadyRead);
}
void Download::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
{
m_total_progress = bytesTotal;
m_progress = bytesReceived;
emit netActionProgress(m_index_within_job, bytesReceived, bytesTotal);
}
void Download::downloadError(QNetworkReply::NetworkError error)
{
if(error == QNetworkReply::OperationCanceledError)
{
qCritical() << "Aborted " << m_url.toString();
m_status = Job_Aborted;
}
else
{
if(m_options & Option::AcceptLocalFiles)
{
if(m_sink->hasLocalData())
{
m_status = Job_Failed_Proceed;
return;
}
}
// error happened during download.
qCritical() << "Failed " << m_url.toString() << " with reason " << error;
m_status = Job_Failed;
}
}
void Download::sslErrors(const QList<QSslError> & errors)
{
int i = 1;
for (auto error : errors)
{
qCritical() << "Download" << m_url.toString() << "SSL Error #" << i << " : " << error.errorString();
auto cert = error.certificate();
qCritical() << "Certificate in question:\n" << cert.toText();
i++;
}
}
bool Download::handleRedirect()
{
QUrl redirect = m_reply->header(QNetworkRequest::LocationHeader).toUrl();
if(!redirect.isValid())
{
if(!m_reply->hasRawHeader("Location"))
{
// no redirect -> it's fine to continue
return false;
}
// there is a Location header, but it's not correct. we need to apply some workarounds...
QByteArray redirectBA = m_reply->rawHeader("Location");
if(redirectBA.size() == 0)
{
// empty, yet present redirect header? WTF?
return false;
}
QString redirectStr = QString::fromUtf8(redirectBA);
if(redirectStr.startsWith("//"))
{
/*
* IF the URL begins with //, we need to insert the URL scheme.
* See: https://bugreports.qt.io/browse/QTBUG-41061
* See: http://tools.ietf.org/html/rfc3986#section-4.2
*/
redirectStr = m_reply->url().scheme() + ":" + redirectStr;
}
else if(redirectStr.startsWith("/"))
{
/*
* IF the URL begins with /, we need to process it as a relative URL
*/
auto url = m_reply->url();
url.setPath(redirectStr, QUrl::TolerantMode);
redirectStr = url.toString();
}
/*
* Next, make sure the URL is parsed in tolerant mode. Qt doesn't parse the location header in tolerant mode, which causes issues.
* FIXME: report Qt bug for this
*/
redirect = QUrl(redirectStr, QUrl::TolerantMode);
if(!redirect.isValid())
{
qWarning() << "Failed to parse redirect URL:" << redirectStr;
downloadError(QNetworkReply::ProtocolFailure);
return false;
}
qDebug() << "Fixed location header:" << redirect;
}
else
{
qDebug() << "Location header:" << redirect;
}
m_url = QUrl(redirect.toString());
qDebug() << "Following redirect to " << m_url.toString();
start();
return true;
}
void Download::downloadFinished()
{
// handle HTTP redirection first
if(handleRedirect())
{
qDebug() << "Download redirected:" << m_url.toString();
return;
}
// if the download failed before this point ...
if (m_status == Job_Failed_Proceed)
{
qDebug() << "Download failed but we are allowed to proceed:" << m_url.toString();
m_sink->abort();
m_reply.reset();
emit succeeded(m_index_within_job);
return;
}
else if (m_status == Job_Failed)
{
qDebug() << "Download failed in previous step:" << m_url.toString();
m_sink->abort();
m_reply.reset();
emit failed(m_index_within_job);
return;
}
else if(m_status == Job_Aborted)
{
qDebug() << "Download aborted in previous step:" << m_url.toString();
m_sink->abort();
m_reply.reset();
emit aborted(m_index_within_job);
return;
}
// make sure we got all the remaining data, if any
auto data = m_reply->readAll();
if(data.size())
{
qDebug() << "Writing extra" << data.size() << "bytes to" << m_target_path;
m_status = m_sink->write(data);
}
// otherwise, finalize the whole graph
m_status = m_sink->finalize(*m_reply.get());
if (m_status != Job_Finished)
{
qDebug() << "Download failed to finalize:" << m_url.toString();
m_sink->abort();
m_reply.reset();
emit failed(m_index_within_job);
return;
}
m_reply.reset();
qDebug() << "Download succeeded:" << m_url.toString();
emit succeeded(m_index_within_job);
}
void Download::downloadReadyRead()
{
if(m_status == Job_InProgress)
{
auto data = m_reply->readAll();
m_status = m_sink->write(data);
if(m_status == Job_Failed)
{
qCritical() << "Failed to process response chunk for " << m_target_path;
}
// qDebug() << "Download" << m_url.toString() << "gained" << data.size() << "bytes";
}
else
{
qCritical() << "Cannot write to " << m_target_path << ", illegal status" << m_status;
}
}
}
bool Net::Download::abort()
{
if(m_reply)
{
m_reply->abort();
}
else
{
m_status = Job_Aborted;
}
return true;
}
bool Net::Download::canAbort()
{
return true;
}

75
launcher/net/Download.h Normal file
View File

@ -0,0 +1,75 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include "NetAction.h"
#include "HttpMetaCache.h"
#include "Validator.h"
#include "Sink.h"
namespace Net {
class Download : public NetAction
{
Q_OBJECT
public: /* types */
typedef std::shared_ptr<class Download> Ptr;
enum class Option
{
NoOptions = 0,
AcceptLocalFiles = 1
};
Q_DECLARE_FLAGS(Options, Option)
protected: /* con/des */
explicit Download();
public:
virtual ~Download(){};
static Download::Ptr makeCached(QUrl url, MetaEntryPtr entry, Options options = Option::NoOptions);
static Download::Ptr makeByteArray(QUrl url, QByteArray *output, Options options = Option::NoOptions);
static Download::Ptr makeFile(QUrl url, QString path, Options options = Option::NoOptions);
public: /* methods */
QString getTargetFilepath()
{
return m_target_path;
}
void addValidator(Validator * v);
bool abort() override;
bool canAbort() override;
private: /* methods */
bool handleRedirect();
protected slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
void downloadError(QNetworkReply::NetworkError error) override;
void sslErrors(const QList<QSslError> & errors);
void downloadFinished() override;
void downloadReadyRead() override;
public slots:
void start() override;
private: /* data */
// FIXME: remove this, it has no business being here.
QString m_target_path;
std::unique_ptr<Sink> m_sink;
Options m_options;
};
}
Q_DECLARE_OPERATORS_FOR_FLAGS(Net::Download::Options)

115
launcher/net/FileSink.cpp Normal file
View File

@ -0,0 +1,115 @@
#include "FileSink.h"
#include <QFile>
#include <QFileInfo>
#include "Env.h"
#include "FileSystem.h"
namespace Net {
FileSink::FileSink(QString filename)
:m_filename(filename)
{
// nil
}
FileSink::~FileSink()
{
// nil
}
JobStatus FileSink::init(QNetworkRequest& request)
{
auto result = initCache(request);
if(result != Job_InProgress)
{
return result;
}
// create a new save file and open it for writing
if (!FS::ensureFilePathExists(m_filename))
{
qCritical() << "Could not create folder for " + m_filename;
return Job_Failed;
}
wroteAnyData = false;
m_output_file.reset(new QSaveFile(m_filename));
if (!m_output_file->open(QIODevice::WriteOnly))
{
qCritical() << "Could not open " + m_filename + " for writing";
return Job_Failed;
}
if(initAllValidators(request))
return Job_InProgress;
return Job_Failed;
}
JobStatus FileSink::initCache(QNetworkRequest &)
{
return Job_InProgress;
}
JobStatus FileSink::write(QByteArray& data)
{
if (!writeAllValidators(data) || m_output_file->write(data) != data.size())
{
qCritical() << "Failed writing into " + m_filename;
m_output_file->cancelWriting();
m_output_file.reset();
wroteAnyData = false;
return Job_Failed;
}
wroteAnyData = true;
return Job_InProgress;
}
JobStatus FileSink::abort()
{
m_output_file->cancelWriting();
failAllValidators();
return Job_Failed;
}
JobStatus FileSink::finalize(QNetworkReply& reply)
{
bool gotFile = false;
QVariant statusCodeV = reply.attribute(QNetworkRequest::HttpStatusCodeAttribute);
bool validStatus = false;
int statusCode = statusCodeV.toInt(&validStatus);
if(validStatus)
{
// this leaves out 304 Not Modified
gotFile = statusCode == 200 || statusCode == 203;
}
// if we wrote any data to the save file, we try to commit the data to the real file.
// if it actually got a proper file, we write it even if it was empty
if (gotFile || wroteAnyData)
{
// ask validators for data consistency
// we only do this for actual downloads, not 'your data is still the same' cache hits
if(!finalizeAllValidators(reply))
return Job_Failed;
// nothing went wrong...
if (!m_output_file->commit())
{
qCritical() << "Failed to commit changes to " << m_filename;
m_output_file->cancelWriting();
return Job_Failed;
}
}
// then get rid of the save file
m_output_file.reset();
return finalizeCache(reply);
}
JobStatus FileSink::finalizeCache(QNetworkReply &)
{
return Job_Finished;
}
bool FileSink::hasLocalData()
{
QFileInfo info(m_filename);
return info.exists() && info.size() != 0;
}
}

28
launcher/net/FileSink.h Normal file
View File

@ -0,0 +1,28 @@
#pragma once
#include "Sink.h"
#include <QSaveFile>
namespace Net {
class FileSink : public Sink
{
public: /* con/des */
FileSink(QString filename);
virtual ~FileSink();
public: /* methods */
JobStatus init(QNetworkRequest & request) override;
JobStatus write(QByteArray & data) override;
JobStatus abort() override;
JobStatus finalize(QNetworkReply & reply) override;
bool hasLocalData() override;
protected: /* methods */
virtual JobStatus initCache(QNetworkRequest &);
virtual JobStatus finalizeCache(QNetworkReply &reply);
protected: /* data */
QString m_filename;
bool wroteAnyData = false;
std::unique_ptr<QSaveFile> m_output_file;
};
}

273
launcher/net/HttpMetaCache.cpp Normal file
View File

@ -0,0 +1,273 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "Env.h"
#include "HttpMetaCache.h"
#include "FileSystem.h"
#include <QFileInfo>
#include <QFile>
#include <QDateTime>
#include <QCryptographicHash>
#include <QDebug>
#include <QJsonDocument>
#include <QJsonArray>
#include <QJsonObject>
QString MetaEntry::getFullPath()
{
// FIXME: make local?
return FS::PathCombine(basePath, relativePath);
}
HttpMetaCache::HttpMetaCache(QString path) : QObject()
{
m_index_file = path;
saveBatchingTimer.setSingleShot(true);
saveBatchingTimer.setTimerType(Qt::VeryCoarseTimer);
connect(&saveBatchingTimer, SIGNAL(timeout()), SLOT(SaveNow()));
}
HttpMetaCache::~HttpMetaCache()
{
saveBatchingTimer.stop();
SaveNow();
}
MetaEntryPtr HttpMetaCache::getEntry(QString base, QString resource_path)
{
// no base. no base path. can't store
if (!m_entries.contains(base))
{
// TODO: log problem
return MetaEntryPtr();
}
EntryMap &map = m_entries[base];
if (map.entry_list.contains(resource_path))
{
return map.entry_list[resource_path];
}
return MetaEntryPtr();
}
MetaEntryPtr HttpMetaCache::resolveEntry(QString base, QString resource_path, QString expected_etag)
{
auto entry = getEntry(base, resource_path);
// it's not present? generate a default stale entry
if (!entry)
{
return staleEntry(base, resource_path);
}
auto &selected_base = m_entries[base];
QString real_path = FS::PathCombine(selected_base.base_path, resource_path);
QFileInfo finfo(real_path);
// is the file really there? if not -> stale
if (!finfo.isFile() || !finfo.isReadable())
{
// if the file doesn't exist, we disown the entry
selected_base.entry_list.remove(resource_path);
return staleEntry(base, resource_path);
}
if (!expected_etag.isEmpty() && expected_etag != entry->etag)
{
// if the etag doesn't match expected, we disown the entry
selected_base.entry_list.remove(resource_path);
return staleEntry(base, resource_path);
}
// if the file changed, check md5sum
qint64 file_last_changed = finfo.lastModified().toUTC().toMSecsSinceEpoch();
if (file_last_changed != entry->local_changed_timestamp)
{
QFile input(real_path);
input.open(QIODevice::ReadOnly);
QString md5sum = QCryptographicHash::hash(input.readAll(), QCryptographicHash::Md5)
.toHex()
.constData();
if (entry->md5sum != md5sum)
{
selected_base.entry_list.remove(resource_path);
return staleEntry(base, resource_path);
}
// md5sums matched... keep entry and save the new state to file
entry->local_changed_timestamp = file_last_changed;
SaveEventually();
}
// entry passed all the checks we cared about.
entry->basePath = getBasePath(base);
return entry;
}
bool HttpMetaCache::updateEntry(MetaEntryPtr stale_entry)
{
if (!m_entries.contains(stale_entry->baseId))
{
qCritical() << "Cannot add entry with unknown base: "
<< stale_entry->baseId.toLocal8Bit();
return false;
}
if (stale_entry->stale)
{
qCritical() << "Cannot add stale entry: " << stale_entry->getFullPath().toLocal8Bit();
return false;
}
m_entries[stale_entry->baseId].entry_list[stale_entry->relativePath] = stale_entry;
SaveEventually();
return true;
}
bool HttpMetaCache::evictEntry(MetaEntryPtr entry)
{
if(entry)
{
entry->stale = true;
SaveEventually();
return true;
}
return false;
}
MetaEntryPtr HttpMetaCache::staleEntry(QString base, QString resource_path)
{
auto foo = new MetaEntry();
foo->baseId = base;
foo->basePath = getBasePath(base);
foo->relativePath = resource_path;
foo->stale = true;
return MetaEntryPtr(foo);
}
void HttpMetaCache::addBase(QString base, QString base_root)
{
// TODO: report error
if (m_entries.contains(base))
return;
// TODO: check if the base path is valid
EntryMap foo;
foo.base_path = base_root;
m_entries[base] = foo;
}
QString HttpMetaCache::getBasePath(QString base)
{
if (m_entries.contains(base))
{
return m_entries[base].base_path;
}
return QString();
}
void HttpMetaCache::Load()
{
if(m_index_file.isNull())
return;
QFile index(m_index_file);
if (!index.open(QIODevice::ReadOnly))
return;
QJsonDocument json = QJsonDocument::fromJson(index.readAll());
if (!json.isObject())
return;
auto root = json.object();
// check file version first
auto version_val = root.value("version");
if (!version_val.isString())
return;
if (version_val.toString() != "1")
return;
// read the entry array
auto entries_val = root.value("entries");
if (!entries_val.isArray())
return;
QJsonArray array = entries_val.toArray();
for (auto element : array)
{
if (!element.isObject())
return;
auto element_obj = element.toObject();
QString base = element_obj.value("base").toString();
if (!m_entries.contains(base))
continue;
auto &entrymap = m_entries[base];
auto foo = new MetaEntry();
foo->baseId = base;
QString path = foo->relativePath = element_obj.value("path").toString();
foo->md5sum = element_obj.value("md5sum").toString();
foo->etag = element_obj.value("etag").toString();
foo->local_changed_timestamp = element_obj.value("last_changed_timestamp").toDouble();
foo->remote_changed_timestamp =
element_obj.value("remote_changed_timestamp").toString();
// presumed innocent until closer examination
foo->stale = false;
entrymap.entry_list[path] = MetaEntryPtr(foo);
}
}
void HttpMetaCache::SaveEventually()
{
// reset the save timer
saveBatchingTimer.stop();
saveBatchingTimer.start(30000);
}
void HttpMetaCache::SaveNow()
{
if(m_index_file.isNull())
return;
QJsonObject toplevel;
toplevel.insert("version", QJsonValue(QString("1")));
QJsonArray entriesArr;
for (auto group : m_entries)
{
for (auto entry : group.entry_list)
{
// do not save stale entries. they are dead.
if(entry->stale)
{
continue;
}
QJsonObject entryObj;
entryObj.insert("base", QJsonValue(entry->baseId));
entryObj.insert("path", QJsonValue(entry->relativePath));
entryObj.insert("md5sum", QJsonValue(entry->md5sum));
entryObj.insert("etag", QJsonValue(entry->etag));
entryObj.insert("last_changed_timestamp",
QJsonValue(double(entry->local_changed_timestamp)));
if (!entry->remote_changed_timestamp.isEmpty())
entryObj.insert("remote_changed_timestamp",
QJsonValue(entry->remote_changed_timestamp));
entriesArr.append(entryObj);
}
}
toplevel.insert("entries", entriesArr);
QJsonDocument doc(toplevel);
try
{
FS::write(m_index_file, doc.toJson());
}
catch (const Exception &e)
{
qWarning() << e.what();
}
}

123
launcher/net/HttpMetaCache.h Normal file
View File

@ -0,0 +1,123 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <QString>
#include <QMap>
#include <qtimer.h>
#include <memory>
class HttpMetaCache;
class MetaEntry
{
friend class HttpMetaCache;
protected:
MetaEntry() {}
public:
bool isStale()
{
return stale;
}
void setStale(bool stale)
{
this->stale = stale;
}
QString getFullPath();
QString getRemoteChangedTimestamp()
{
return remote_changed_timestamp;
}
void setRemoteChangedTimestamp(QString remote_changed_timestamp)
{
this->remote_changed_timestamp = remote_changed_timestamp;
}
void setLocalChangedTimestamp(qint64 timestamp)
{
local_changed_timestamp = timestamp;
}
QString getETag()
{
return etag;
}
void setETag(QString etag)
{
this->etag = etag;
}
QString getMD5Sum()
{
return md5sum;
}
void setMD5Sum(QString md5sum)
{
this->md5sum = md5sum;
}
protected:
QString baseId;
QString basePath;
QString relativePath;
QString md5sum;
QString etag;
qint64 local_changed_timestamp = 0;
QString remote_changed_timestamp; // QString for now, RFC 2822 encoded time
bool stale = true;
};
typedef std::shared_ptr<MetaEntry> MetaEntryPtr;
class HttpMetaCache : public QObject
{
Q_OBJECT
public:
// supply path to the cache index file
HttpMetaCache(QString path = QString());
~HttpMetaCache();
// get the entry solely from the cache
// you probably don't want this, unless you have some specific caching needs.
MetaEntryPtr getEntry(QString base, QString resource_path);
// get the entry from cache and verify that it isn't stale (within reason)
MetaEntryPtr resolveEntry(QString base, QString resource_path,
QString expected_etag = QString());
// add a previously resolved stale entry
bool updateEntry(MetaEntryPtr stale_entry);
// evict selected entry from cache
bool evictEntry(MetaEntryPtr entry);
void addBase(QString base, QString base_root);
// (re)start a timer that calls SaveNow later.
void SaveEventually();
void Load();
QString getBasePath(QString base);
public
slots:
void SaveNow();
private:
// create a new stale entry, given the parameters
MetaEntryPtr staleEntry(QString base, QString resource_path);
struct EntryMap
{
QString base_path;
QMap<QString, MetaEntryPtr> entry_list;
};
QMap<QString, EntryMap> m_entries;
QString m_index_file;
QTimer saveBatchingTimer;
};

65
launcher/net/MetaCacheSink.cpp Normal file
View File

@ -0,0 +1,65 @@
#include "MetaCacheSink.h"
#include <QFile>
#include <QFileInfo>
#include "Env.h"
#include "FileSystem.h"
namespace Net {
MetaCacheSink::MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum)
:Net::FileSink(entry->getFullPath()), m_entry(entry), m_md5Node(md5sum)
{
addValidator(md5sum);
}
MetaCacheSink::~MetaCacheSink()
{
// nil
}
JobStatus MetaCacheSink::initCache(QNetworkRequest& request)
{
if (!m_entry->isStale())
{
return Job_Finished;
}
// check if file exists, if it does, use its information for the request
QFile current(m_filename);
if(current.exists() && current.size() != 0)
{
if (m_entry->getRemoteChangedTimestamp().size())
{
request.setRawHeader(QString("If-Modified-Since").toLatin1(), m_entry->getRemoteChangedTimestamp().toLatin1());
}
if (m_entry->getETag().size())
{
request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->getETag().toLatin1());
}
}
return Job_InProgress;
}
JobStatus MetaCacheSink::finalizeCache(QNetworkReply & reply)
{
QFileInfo output_file_info(m_filename);
if(wroteAnyData)
{
m_entry->setMD5Sum(m_md5Node->hash().toHex().constData());
}
m_entry->setETag(reply.rawHeader("ETag").constData());
if (reply.hasRawHeader("Last-Modified"))
{
m_entry->setRemoteChangedTimestamp(reply.rawHeader("Last-Modified").constData());
}
m_entry->setLocalChangedTimestamp(output_file_info.lastModified().toUTC().toMSecsSinceEpoch());
m_entry->setStale(false);
ENV.metacache()->updateEntry(m_entry);
return Job_Finished;
}
bool MetaCacheSink::hasLocalData()
{
QFileInfo info(m_filename);
return info.exists() && info.size() != 0;
}
}

22
launcher/net/MetaCacheSink.h Normal file
View File

@ -0,0 +1,22 @@
#pragma once
#include "FileSink.h"
#include "ChecksumValidator.h"
#include "net/HttpMetaCache.h"
namespace Net {
class MetaCacheSink : public FileSink
{
public: /* con/des */
MetaCacheSink(MetaEntryPtr entry, ChecksumValidator * md5sum);
virtual ~MetaCacheSink();
bool hasLocalData() override;
protected: /* methods */
JobStatus initCache(QNetworkRequest & request) override;
JobStatus finalizeCache(QNetworkReply & reply) override;
private: /* data */
MetaEntryPtr m_entry;
ChecksumValidator * m_md5Node;
};
}

10
launcher/net/Mode.h Normal file
View File

@ -0,0 +1,10 @@
#pragma once
namespace Net
{
enum class Mode
{
Offline,
Online
};
}

113
launcher/net/NetAction.h Normal file
View File

@ -0,0 +1,113 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <QObject>
#include <QUrl>
#include <memory>
#include <QNetworkReply>
#include <QObjectPtr.h>
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed,
Job_Aborted,
/*
* FIXME: @NUKE this confuses the task failing with us having a fallback in the form of local data. Clear up the confusion.
* Same could be true for aborted task - the presence of pre-existing result is a separate concern
*/
Job_Failed_Proceed
};
typedef std::shared_ptr<class NetAction> NetActionPtr;
class NetAction : public QObject
{
Q_OBJECT
protected:
explicit NetAction() : QObject(0) {};
public:
virtual ~NetAction() {};
bool isRunning() const
{
return m_status == Job_InProgress;
}
bool isFinished() const
{
return m_status >= Job_Finished;
}
bool wasSuccessful() const
{
return m_status == Job_Finished || m_status == Job_Failed_Proceed;
}
qint64 totalProgress() const
{
return m_total_progress;
}
qint64 currentProgress() const
{
return m_progress;
}
virtual bool abort()
{
return false;
}
virtual bool canAbort()
{
return false;
}
QUrl url()
{
return m_url;
}
signals:
void started(int index);
void netActionProgress(int index, qint64 current, qint64 total);
void succeeded(int index);
void failed(int index);
void aborted(int index);
protected slots:
virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) = 0;
virtual void downloadError(QNetworkReply::NetworkError error) = 0;
virtual void downloadFinished() = 0;
virtual void downloadReadyRead() = 0;
public slots:
virtual void start() = 0;
public:
/// index within the parent job, FIXME: nuke
int m_index_within_job = 0;
/// the network reply
unique_qobject_ptr<QNetworkReply> m_reply;
/// source URL
QUrl m_url;
qint64 m_progress = 0;
qint64 m_total_progress = 1;
protected:
JobStatus m_status = Job_NotStarted;
};

218
launcher/net/NetJob.cpp Normal file
View File

@ -0,0 +1,218 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "NetJob.h"
#include "Download.h"
#include <QDebug>
void NetJob::partSucceeded(int index)
{
// do progress. all slots are 1 in size at least
auto &slot = parts_progress[index];
partProgress(index, slot.total_progress, slot.total_progress);
m_doing.remove(index);
m_done.insert(index);
downloads[index].get()->disconnect(this);
startMoreParts();
}
void NetJob::partFailed(int index)
{
m_doing.remove(index);
auto &slot = parts_progress[index];
if (slot.failures == 3)
{
m_failed.insert(index);
}
else
{
slot.failures++;
m_todo.enqueue(index);
}
downloads[index].get()->disconnect(this);
startMoreParts();
}
void NetJob::partAborted(int index)
{
m_aborted = true;
m_doing.remove(index);
m_failed.insert(index);
downloads[index].get()->disconnect(this);
startMoreParts();
}
void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
{
auto &slot = parts_progress[index];
slot.current_progress = bytesReceived;
slot.total_progress = bytesTotal;
int done = m_done.size();
int doing = m_doing.size();
int all = parts_progress.size();
qint64 bytesAll = 0;
qint64 bytesTotalAll = 0;
for(auto & partIdx: m_doing)
{
auto part = parts_progress[partIdx];
// do not count parts with unknown/nonsensical total size
if(part.total_progress <= 0)
{
continue;
}
bytesAll += part.current_progress;
bytesTotalAll += part.total_progress;
}
qint64 inprogress = (bytesTotalAll == 0) ? 0 : (bytesAll * 1000) / bytesTotalAll;
auto current = done * 1000 + doing * inprogress;
auto current_total = all * 1000;
// HACK: make sure it never jumps backwards.
// FAIL: This breaks if the size is not known (or is it something else?) and jumps to 1000, so if it is 1000 reset it to inprogress
if(m_current_progress == 1000) {
m_current_progress = inprogress;
}
if(m_current_progress > current)
{
current = m_current_progress;
}
m_current_progress = current;
setProgress(current, current_total);
}
void NetJob::executeTask()
{
// hack that delays early failures so they can be caught easier
QMetaObject::invokeMethod(this, "startMoreParts", Qt::QueuedConnection);
}
void NetJob::startMoreParts()
{
if(!isRunning())
{
// this actually makes sense. You can put running downloads into a NetJob and then not start it until much later.
return;
}
// OK. We are actively processing tasks, proceed.
// Check for final conditions if there's nothing in the queue.
if(!m_todo.size())
{
if(!m_doing.size())
{
if(!m_failed.size())
{
emitSucceeded();
}
else if(m_aborted)
{
emitAborted();
}
else
{
emitFailed(tr("Job '%1' failed to process:\n%2").arg(objectName()).arg(getFailedFiles().join("\n")));
}
}
return;
}
// There's work to do, try to start more parts.
while (m_doing.size() < 6)
{
if(!m_todo.size())
return;
int doThis = m_todo.dequeue();
m_doing.insert(doThis);
auto part = downloads[doThis];
// connect signals :D
connect(part.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
connect(part.get(), SIGNAL(failed(int)), SLOT(partFailed(int)));
connect(part.get(), SIGNAL(aborted(int)), SLOT(partAborted(int)));
connect(part.get(), SIGNAL(netActionProgress(int, qint64, qint64)),
SLOT(partProgress(int, qint64, qint64)));
part->start();
}
}
QStringList NetJob::getFailedFiles()
{
QStringList failed;
for (auto index: m_failed)
{
failed.push_back(downloads[index]->url().toString());
}
failed.sort();
return failed;
}
bool NetJob::canAbort() const
{
bool canFullyAbort = true;
// can abort the waiting?
for(auto index: m_todo)
{
auto part = downloads[index];
canFullyAbort &= part->canAbort();
}
// can abort the active?
for(auto index: m_doing)
{
auto part = downloads[index];
canFullyAbort &= part->canAbort();
}
return canFullyAbort;
}
bool NetJob::abort()
{
bool fullyAborted = true;
// fail all waiting
m_failed.unite(m_todo.toSet());
m_todo.clear();
// abort active
auto toKill = m_doing.toList();
for(auto index: toKill)
{
auto part = downloads[index];
fullyAborted &= part->abort();
}
return fullyAborted;
}
bool NetJob::addNetAction(NetActionPtr action)
{
action->m_index_within_job = downloads.size();
downloads.append(action);
part_info pi;
parts_progress.append(pi);
partProgress(parts_progress.count() - 1, action->currentProgress(), action->totalProgress());
if(action->isRunning())
{
connect(action.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
connect(action.get(), SIGNAL(failed(int)), SLOT(partFailed(int)));
connect(action.get(), SIGNAL(netActionProgress(int, qint64, qint64)), SLOT(partProgress(int, qint64, qint64)));
}
else
{
m_todo.append(parts_progress.size() - 1);
}
return true;
}
NetJob::~NetJob() = default;

89
launcher/net/NetJob.h Normal file
View File

@ -0,0 +1,89 @@
/* Copyright 2013-2021 MultiMC Contributors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#pragma once
#include <QtNetwork>
#include "NetAction.h"
#include "Download.h"
#include "HttpMetaCache.h"
#include "tasks/Task.h"
#include "QObjectPtr.h"
class NetJob;
typedef shared_qobject_ptr<NetJob> NetJobPtr;
class NetJob : public Task
{
Q_OBJECT
public:
explicit NetJob(QString job_name) : Task()
{
setObjectName(job_name);
}
virtual ~NetJob();
bool addNetAction(NetActionPtr action);
NetActionPtr operator[](int index)
{
return downloads[index];
}
const NetActionPtr at(const int index)
{
return downloads.at(index);
}
NetActionPtr first()
{
if (downloads.size())
return downloads[0];
return NetActionPtr();
}
int size() const
{
return downloads.size();
}
QStringList getFailedFiles();
bool canAbort() const override;
private slots:
void startMoreParts();
public slots:
virtual void executeTask() override;
virtual bool abort() override;
private slots:
void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
void partSucceeded(int index);
void partFailed(int index);
void partAborted(int index);
private:
struct part_info
{
qint64 current_progress = 0;
qint64 total_progress = 1;
int failures = 0;
};
QList<NetActionPtr> downloads;
QList<part_info> parts_progress;
QQueue<int> m_todo;
QSet<int> m_doing;
QSet<int> m_done;
QSet<int> m_failed;
qint64 m_current_progress = 0;
bool m_aborted = false;
};

104
launcher/net/PasteUpload.cpp Normal file
View File

@ -0,0 +1,104 @@
#include "PasteUpload.h"
#include "Env.h"
#include <QDebug>
#include <QJsonObject>
#include <QJsonArray>
#include <QJsonDocument>
#include <QFile>
#include <BuildConfig.h>
PasteUpload::PasteUpload(QWidget *window, QString text, QString key) : m_window(window)
{
m_key = key;
QByteArray temp;
QJsonObject topLevelObj;
QJsonObject sectionObject;
sectionObject.insert("contents", text);
QJsonArray sectionArray;
sectionArray.append(sectionObject);
topLevelObj.insert("description", "MultiMC Log Upload");
topLevelObj.insert("sections", sectionArray);
QJsonDocument docOut;
docOut.setObject(topLevelObj);
m_jsonContent = docOut.toJson();
}
PasteUpload::~PasteUpload()
{
}
bool PasteUpload::validateText()
{
return m_jsonContent.size() <= maxSize();
}
void PasteUpload::executeTask()
{
QNetworkRequest request(QUrl("https://api.paste.ee/v1/pastes"));
request.setHeader(QNetworkRequest::UserAgentHeader, BuildConfig.USER_AGENT_UNCACHED);
request.setRawHeader("Content-Type", "application/json");
request.setRawHeader("Content-Length", QByteArray::number(m_jsonContent.size()));
request.setRawHeader("X-Auth-Token", m_key.toStdString().c_str());
QNetworkReply *rep = ENV.qnam().post(request, m_jsonContent);
m_reply = std::shared_ptr<QNetworkReply>(rep);
setStatus(tr("Uploading to paste.ee"));
connect(rep, &QNetworkReply::uploadProgress, this, &Task::setProgress);
connect(rep, SIGNAL(error(QNetworkReply::NetworkError)), this, SLOT(downloadError(QNetworkReply::NetworkError)));
connect(rep, SIGNAL(finished()), this, SLOT(downloadFinished()));
}
void PasteUpload::downloadError(QNetworkReply::NetworkError error)
{
// error happened during download.
qCritical() << "Network error: " << error;
emitFailed(m_reply->errorString());
}
void PasteUpload::downloadFinished()
{
QByteArray data = m_reply->readAll();
// if the download succeeded
if (m_reply->error() == QNetworkReply::NetworkError::NoError)
{
m_reply.reset();
QJsonParseError jsonError;
QJsonDocument doc = QJsonDocument::fromJson(data, &jsonError);
if (jsonError.error != QJsonParseError::NoError)
{
emitFailed(jsonError.errorString());
return;
}
if (!parseResult(doc))
{
emitFailed(tr("paste.ee returned an error. Please consult the logs for more information"));
return;
}
}
// else the download failed
else
{
emitFailed(QString("Network error: %1").arg(m_reply->errorString()));
m_reply.reset();
return;
}
emitSucceeded();
}
bool PasteUpload::parseResult(QJsonDocument doc)
{
auto object = doc.object();
auto status = object.value("success").toBool();
if (!status)
{
qCritical() << "paste.ee reported error:" << QString(object.value("error").toString());
return false;
}
m_pasteLink = object.value("link").toString();
m_pasteID = object.value("id").toString();
qDebug() << m_pasteLink;
return true;
}

47
launcher/net/PasteUpload.h Normal file
View File

@ -0,0 +1,47 @@
#pragma once
#include "tasks/Task.h"
#include <QNetworkReply>
#include <QBuffer>
#include <memory>
class PasteUpload : public Task
{
Q_OBJECT
public:
PasteUpload(QWidget *window, QString text, QString key = "public");
virtual ~PasteUpload();
QString pasteLink()
{
return m_pasteLink;
}
QString pasteID()
{
return m_pasteID;
}
int maxSize()
{
// 2MB for paste.ee - public
if(m_key == "public")
return 1024*1024*2;
// 12MB for paste.ee - with actual key
return 1024*1024*12;
}
bool validateText();
protected:
virtual void executeTask();
private:
bool parseResult(QJsonDocument doc);
QString m_error;
QWidget *m_window;
QString m_pasteID;
QString m_pasteLink;
QString m_key;
QByteArray m_jsonContent;
std::shared_ptr<QNetworkReply> m_reply;
public
slots:
void downloadError(QNetworkReply::NetworkError);
void downloadFinished();
};

70
launcher/net/Sink.h Normal file
View File

@ -0,0 +1,70 @@
#pragma once
#include "net/NetAction.h"
#include "Validator.h"
namespace Net {
class Sink
{
public: /* con/des */
Sink() {};
virtual ~Sink() {};
public: /* methods */
virtual JobStatus init(QNetworkRequest & request) = 0;
virtual JobStatus write(QByteArray & data) = 0;
virtual JobStatus abort() = 0;
virtual JobStatus finalize(QNetworkReply & reply) = 0;
virtual bool hasLocalData() = 0;
void addValidator(Validator * validator)
{
if(validator)
{
validators.push_back(std::shared_ptr<Validator>(validator));
}
}
protected: /* methods */
bool finalizeAllValidators(QNetworkReply & reply)
{
for(auto & validator: validators)
{
if(!validator->validate(reply))
return false;
}
return true;
}
bool failAllValidators()
{
bool success = true;
for(auto & validator: validators)
{
success &= validator->abort();
}
return success;
}
bool initAllValidators(QNetworkRequest & request)
{
for(auto & validator: validators)
{
if(!validator->init(request))
return false;
}
return true;
}
bool writeAllValidators(QByteArray & data)
{
for(auto & validator: validators)
{
if(!validator->write(data))
return false;
}
return true;
}
protected: /* data */
std::vector<std::shared_ptr<Validator>> validators;
};
}

18
launcher/net/Validator.h Normal file
View File

@ -0,0 +1,18 @@
#pragma once
#include "net/NetAction.h"
namespace Net {
class Validator
{
public: /* con/des */
Validator() {};
virtual ~Validator() {};
public: /* methods */
virtual bool init(QNetworkRequest & request) = 0;
virtual bool write(QByteArray & data) = 0;
virtual bool abort() = 0;
virtual bool validate(QNetworkReply & reply) = 0;
};
}