CactiChameleon9/PrismLauncher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework curseforge download (#611)

Browse Source 
* Use the bulk endpoint on mod resolution for faster download
* Search on modrinth for api blocked mods
* Display a dialog for manually downloading blocked mods
This commit is contained in:
timoreo22
2022-05-28 21:53:12 +02:00
committed by GitHub
parent fcbe233fdb
commit 699ad316f0
14 changed files with 633 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

126
launcher/modplatform/flame/FileResolvingTask.cpp
View File

@ -1,7 +1,9 @@
#include "FileResolvingTask.h"
#include "Json.h"
Flame::FileResolvingTask::FileResolvingTask(shared_qobject_ptr<QNetworkAccessManager> network, Flame::Manifest& toProcess)
#include "Json.h"
#include "net/Upload.h"
Flame::FileResolvingTask::FileResolvingTask(const shared_qobject_ptr<QNetworkAccessManager>& network, Flame::Manifest& toProcess)
: m_network(network), m_toProcess(toProcess)
{}
@ -10,40 +12,116 @@ void Flame::FileResolvingTask::executeTask()
setStatus(tr("Resolving mod IDs..."));
setProgress(0, m_toProcess.files.size());
m_dljob = new NetJob("Mod id resolver", m_network);
results.resize(m_toProcess.files.size());
int index = 0;
for (auto& file : m_toProcess.files) {
auto projectIdStr = QString::number(file.projectId);
auto fileIdStr = QString::number(file.fileId);
QString metaurl = QString("https://api.curseforge.com/v1/mods/%1/files/%2").arg(projectIdStr, fileIdStr);
auto dl = Net::Download::makeByteArray(QUrl(metaurl), &results[index]);
m_dljob->addNetAction(dl);
index++;
}
result.reset(new QByteArray());
//build json data to send
QJsonObject object;
object["fileIds"] = QJsonArray::fromVariantList(std::accumulate(m_toProcess.files.begin(), m_toProcess.files.end(), QVariantList(), [](QVariantList& l, const File& s) {
l.push_back(s.fileId);
return l;
}));
QByteArray data = Json::toText(object);
auto dl = Net::Upload::makeByteArray(QUrl("https://api.curseforge.com/v1/mods/files"), result.get(), data);
m_dljob->addNetAction(dl);
connect(m_dljob.get(), &NetJob::finished, this, &Flame::FileResolvingTask::netJobFinished);
m_dljob->start();
}
void Flame::FileResolvingTask::netJobFinished()
{
bool failed = false;
int index = 0;
for (auto& bytes : results) {
auto& out = m_toProcess.files[index];
// job to check modrinth for blocked projects
auto job = new NetJob("Modrinth check", m_network);
blockedProjects = QMap<File *,QByteArray *>();
auto doc = Json::requireDocument(*result);
auto array = Json::requireArray(doc.object()["data"]);
for (QJsonValueRef file : array) {
auto fileid = Json::requireInteger(Json::requireObject(file)["id"]);
auto& out = m_toProcess.files[fileid];
try {
failed &= (!out.parseFromBytes(bytes));
out.parseFromObject(Json::requireObject(file));
} catch (const JSONValidationError& e) {
qCritical() << "Resolving of" << out.projectId << out.fileId << "failed because of a parsing error:";
qCritical() << e.cause();
qCritical() << "JSON:";
qCritical() << bytes;
failed = true;
qDebug() << "Blocked mod on curseforge" << out.fileName;
auto hash = out.hash;
if(!hash.isEmpty()) {
auto url = QString("https://api.modrinth.com/v2/version_file/%1?algorithm=sha1").arg(hash);
auto output = new QByteArray();
auto dl = Net::Download::makeByteArray(QUrl(url), output);
QObject::connect(dl.get(), &Net::Download::succeeded, [&out]() {
out.resolved = true;
});
job->addNetAction(dl);
blockedProjects.insert(&out, output);
}
}
index++;
}
if (!failed) {
emitSucceeded();
connect(job, &NetJob::finished, this, &Flame::FileResolvingTask::modrinthCheckFinished);
job->start();
}
void Flame::FileResolvingTask::modrinthCheckFinished() {
qDebug() << "Finished with blocked mods : " << blockedProjects.size();
for (auto it = blockedProjects.keyBegin(); it != blockedProjects.keyEnd(); it++) {
auto &out = *it;
auto bytes = blockedProjects[out];
if (!out->resolved) {
delete bytes;
continue;
}
QJsonDocument doc = QJsonDocument::fromJson(*bytes);
auto obj = doc.object();
auto array = Json::requireArray(obj,"files");
for (auto file: array) {
auto fileObj = Json::requireObject(file);
auto primary = Json::requireBoolean(fileObj,"primary");
if (primary) {
out->url = Json::requireUrl(fileObj,"url");
qDebug() << "Found alternative on modrinth " << out->fileName;
break;
}
}
delete bytes;
}
//copy to an output list and filter out projects found on modrinth
auto block = new QList<File *>();
auto it = blockedProjects.keys();
std::copy_if(it.begin(), it.end(), std::back_inserter(*block), [](File *f) {
return !f->resolved;
});
//Display not found mods early
if (!block->empty()) {
//blocked mods found, we need the slug for displaying.... we need another job :D !
auto slugJob = new NetJob("Slug Job", m_network);
auto slugs = QVector<QByteArray>(block->size());
auto index = 0;
for (auto fileInfo: *block) {
auto projectId = fileInfo->projectId;
slugs[index] = QByteArray();
auto url = QString("https://api.curseforge.com/v1/mods/%1").arg(projectId);
auto dl = Net::Download::makeByteArray(url, &slugs[index]);
slugJob->addNetAction(dl);
index++;
}
connect(slugJob, &NetJob::succeeded, this, [slugs, this, slugJob, block]() {
slugJob->deleteLater();
auto index = 0;
for (const auto &slugResult: slugs) {
auto json = QJsonDocument::fromJson(slugResult);
auto base = Json::requireString(Json::requireObject(Json::requireObject(Json::requireObject(json),"data"),"links"),
"websiteUrl");
auto mod = block->at(index);
auto link = QString("%1/download/%2").arg(base, QString::number(mod->fileId));
mod->websiteUrl = link;
index++;
}
emitSucceeded();
});
slugJob->start();
} else {
emitFailed(tr("Some mod ID resolving tasks failed."));
emitSucceeded();
}
}

8
launcher/modplatform/flame/FileResolvingTask.h
View File

@ -10,7 +10,7 @@ class FileResolvingTask : public Task
{
Q_OBJECT
public:
explicit FileResolvingTask(shared_qobject_ptr<QNetworkAccessManager> network, Flame::Manifest &toProcess);
explicit FileResolvingTask(const shared_qobject_ptr<QNetworkAccessManager>& network, Flame::Manifest &toProcess);
virtual ~FileResolvingTask() {};
const Flame::Manifest &getResults() const
@ -27,7 +27,11 @@ protected slots:
private: /* data */
shared_qobject_ptr<QNetworkAccessManager> m_network;
Flame::Manifest m_toProcess;
QVector<QByteArray> results;
std::shared_ptr<QByteArray> result;
NetJob::Ptr m_dljob;
void modrinthCheckFinished();
QMap<File *, QByteArray *> blockedProjects;
};
}

35
launcher/modplatform/flame/PackManifest.cpp
View File

@ -41,7 +41,7 @@ static void loadManifestV1(Flame::Manifest& m, QJsonObject& manifest)
auto obj = Json::requireObject(item);
Flame::File file;
loadFileV1(file, obj);
m.files.append(file);
m.files.insert(file.fileId,file);
}
m.overrides = Json::ensureString(manifest, "overrides", "overrides");
}
@ -61,21 +61,9 @@ void Flame::loadManifest(Flame::Manifest& m, const QString& filepath)
loadManifestV1(m, obj);
}
bool Flame::File::parseFromBytes(const QByteArray& bytes)
bool Flame::File::parseFromObject(const QJsonObject& obj)
{
auto doc = Json::requireDocument(bytes);
if (!doc.isObject()) {
throw JSONValidationError(QString("data is not an object? that's not supposed to happen"));
}
auto obj = Json::ensureObject(doc.object(), "data");
fileName = Json::requireString(obj, "fileName");
QString rawUrl = Json::requireString(obj, "downloadUrl");
url = QUrl(rawUrl, QUrl::TolerantMode);
if (!url.isValid()) {
throw JSONValidationError(QString("Invalid URL: %1").arg(rawUrl));
}
// This is a piece of a Flame project JSON pulled out into the file metadata (here) for convenience
// It is also optional
type = File::Type::SingleFile;
@ -87,6 +75,25 @@ bool Flame::File::parseFromBytes(const QByteArray& bytes)
// this is probably a mod, dunno what else could modpacks download
targetFolder = "mods";
}
// get the hash
hash = QString();
auto hashes = Json::ensureArray(obj, "hashes");
for(QJsonValueRef item : hashes) {
auto hobj = Json::requireObject(item);
auto algo = Json::requireInteger(hobj, "algo");
auto value = Json::requireString(hobj, "value");
if (algo == 1) {
hash = value;
}
}
// may throw, if the project is blocked
QString rawUrl = Json::ensureString(obj, "downloadUrl");
url = QUrl(rawUrl, QUrl::TolerantMode);
if (!url.isValid()) {
throw JSONValidationError(QString("Invalid URL: %1").arg(rawUrl));
}
resolved = true;
return true;

10
launcher/modplatform/flame/PackManifest.h
View File

@ -2,19 +2,24 @@
#include <QString>
#include <QVector>
#include <QMap>
#include <QUrl>
#include <QJsonObject>
namespace Flame
{
struct File
{
// NOTE: throws JSONValidationError
bool parseFromBytes(const QByteArray &bytes);
bool parseFromObject(const QJsonObject& object);
int projectId = 0;
int fileId = 0;
// NOTE: the opposite to 'optional'. This is at the time of writing unused.
bool required = true;
QString hash;
// NOTE: only set on blocked files ! Empty otherwise.
QString websiteUrl;
// our
bool resolved = false;
@ -54,7 +59,8 @@ struct Manifest
QString name;
QString version;
QString author;
QVector<Flame::File> files;
//File id -> File
QMap<int,Flame::File> files;
QString overrides;
};