feat: improve metadata gen. networking and performance

This makes the metadata generation code a lot messier and harder to use,
but there's not really much else that can be done about it while
preserving all it's capabilities :(

At least we now have speed

Signed-off-by: flow <flowlnlnln@gmail.com>
This commit is contained in:
flow 2022-06-16 11:45:29 -03:00
parent 91a5c4bdcb
commit 4e6978ff6f
No known key found for this signature in database
GPG Key ID: 8D0F221F0A59F469
7 changed files with 641 additions and 335 deletions

View File

@ -13,7 +13,7 @@ class CheckUpdateTask : public Task {
public:
CheckUpdateTask(std::list<Mod>& mods, std::list<Version>& mcVersions, ModAPI::ModLoaderTypes loaders, std::shared_ptr<ModFolderModel> mods_folder)
: m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {};
: Task(nullptr), m_mods(mods), m_game_versions(mcVersions), m_loaders(loaders), m_mods_folder(mods_folder) {};
struct UpdatableMod {
QString name;

View File

@ -19,226 +19,502 @@ static ModPlatform::ProviderCapabilities ProviderCaps;
static ModrinthAPI modrinth_api;
static FlameAPI flame_api;
EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir& dir, bool try_all, ModPlatform::Provider prov)
: m_mod(mod), m_index_dir(dir), m_provider(prov), m_try_all(try_all)
{}
EnsureMetadataTask::EnsureMetadataTask(Mod& mod, QDir dir, ModPlatform::Provider prov) : Task(nullptr), m_index_dir(dir), m_provider(prov)
{
auto hash = getHash(mod);
if (hash.isEmpty())
emitFail(mod);
else
m_mods.insert(hash, mod);
}
EnsureMetadataTask::EnsureMetadataTask(std::list<Mod>& mods, QDir dir, ModPlatform::Provider prov)
: Task(nullptr), m_index_dir(dir), m_provider(prov)
{
for (auto& mod : mods) {
if (!mod.valid()) {
emitFail(mod);
continue;
}
auto hash = getHash(mod);
if (hash.isEmpty()) {
emitFail(mod);
continue;
}
m_mods.insert(hash, mod);
}
}
QString EnsureMetadataTask::getHash(Mod& mod)
{
/* Here we create a mapping hash -> mod, because we need that relationship to parse the API routes */
QByteArray jar_data;
try {
jar_data = FS::read(mod.fileinfo().absoluteFilePath());
} catch (FS::FileSystemException& e) {
qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name());
qCritical() << QString("Reason: ") << e.cause();
return {};
}
switch (m_provider) {
case ModPlatform::Provider::MODRINTH: {
auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first();
return QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex());
}
case ModPlatform::Provider::FLAME: {
QByteArray jar_data_treated;
for (char c : jar_data) {
// CF-specific
if (!(c == 9 || c == 10 || c == 13 || c == 32))
jar_data_treated.push_back(c);
}
return QString::number(MurmurHash2(jar_data_treated, jar_data_treated.length()));
}
}
return {};
}
bool EnsureMetadataTask::abort()
{
return m_task_handler->abort();
// Prevent sending signals to a dead object
disconnect(this, 0, 0, 0);
if (m_current_task)
return m_current_task->abort();
return true;
}
void EnsureMetadataTask::executeTask()
{
// They already have the right metadata :o
if (m_mod.status() != ModStatus::NoMetadata && m_mod.metadata() && m_mod.metadata()->provider == m_provider) {
emitReady();
return;
setStatus(tr("Checking if mods have metadata..."));
for (auto mod : m_mods) {
if (!mod.valid())
continue;
// They already have the right metadata :o
if (mod.status() != ModStatus::NoMetadata && mod.metadata() && mod.metadata()->provider == m_provider) {
qDebug() << "Mod" << mod.name() << "already has metadata!";
emitReady(mod);
return;
}
// Folders don't have metadata
if (mod.type() == Mod::MOD_FOLDER) {
emitReady(mod);
return;
}
}
// Folders don't have metadata
if (m_mod.type() == Mod::MOD_FOLDER) {
emitReady();
return;
}
setStatus(tr("Generating %1's metadata...").arg(m_mod.name()));
qDebug() << QString("Generating %1's metadata...").arg(m_mod.name());
QByteArray jar_data;
try {
jar_data = FS::read(m_mod.fileinfo().absoluteFilePath());
} catch (FS::FileSystemException& e) {
qCritical() << QString("Failed to open / read JAR file of %1").arg(m_mod.name());
qCritical() << QString("Reason: ") << e.cause();
emitFail();
return;
}
auto tsk = new MultipleOptionsTask(nullptr, "GetMetadataTask");
NetJob::Ptr version_task;
switch (m_provider) {
case (ModPlatform::Provider::MODRINTH):
modrinthEnsureMetadata(*tsk, jar_data);
if (m_try_all)
flameEnsureMetadata(*tsk, jar_data);
version_task = modrinthVersionsTask();
break;
case (ModPlatform::Provider::FLAME):
flameEnsureMetadata(*tsk, jar_data);
if (m_try_all)
modrinthEnsureMetadata(*tsk, jar_data);
version_task = flameVersionsTask();
break;
}
connect(tsk, &MultipleOptionsTask::finished, this, [tsk] { tsk->deleteLater(); });
connect(tsk, &MultipleOptionsTask::failed, [this] {
qCritical() << QString("Download of %1's metadata failed").arg(m_mod.name());
auto invalidade_leftover = [this] {
QMutableHashIterator<QString, Mod> mods_iter(m_mods);
while (mods_iter.hasNext()) {
auto mod = mods_iter.next();
emitFail(mod.value());
}
emitFail();
});
connect(tsk, &MultipleOptionsTask::succeeded, this, &EnsureMetadataTask::emitReady);
emitSucceeded();
};
m_task_handler = tsk;
connect(version_task.get(), &Task::finished, this, [this, invalidade_leftover] {
NetJob::Ptr project_task;
tsk->start();
}
switch (m_provider) {
case (ModPlatform::Provider::MODRINTH):
project_task = modrinthProjectsTask();
break;
case (ModPlatform::Provider::FLAME):
project_task = flameProjectsTask();
break;
}
void EnsureMetadataTask::emitReady()
{
emit metadataReady();
emitSucceeded();
}
void EnsureMetadataTask::emitFail()
{
qDebug() << QString("Failed to generate metadata for %1").arg(m_mod.name());
emit metadataFailed();
//emitFailed(tr("Failed to generate metadata for %1").arg(m_mod.name()));
emitSucceeded();
}
void EnsureMetadataTask::modrinthEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data)
{
// Modrinth currently garantees that some hash types will always be present.
// But let's be sure and cover all cases anyways :)
for (auto hash_type : ProviderCaps.hashType(ModPlatform::Provider::MODRINTH)) {
auto* response = new QByteArray();
auto hash = QString(ProviderCaps.hash(ModPlatform::Provider::MODRINTH, jar_data, hash_type).toHex());
auto ver_task = modrinth_api.currentVersion(hash, hash_type, response);
// Prevents unfortunate timings when aborting the task
if (!ver_task)
if (!project_task) {
invalidade_leftover();
return;
}
connect(ver_task.get(), &NetJob::succeeded, this, [this, ver_task, response] {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
ver_task->failed(parse_error.errorString());
return;
}
auto doc_obj = Json::requireObject(doc);
auto ver = Modrinth::loadIndexedPackVersion(doc_obj, {}, m_mod.fileinfo().fileName());
// Minimal IndexedPack to create the metadata
ModPlatform::IndexedPack pack;
pack.name = m_mod.name();
pack.provider = ModPlatform::Provider::MODRINTH;
pack.addonId = ver.addonId;
// Prevent file name mismatch
ver.fileName = m_mod.fileinfo().fileName();
QDir tmp_index_dir(m_index_dir);
{
LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
QEventLoop loop;
QTimer timeout;
QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit);
update_metadata.start();
timeout.start(100);
loop.exec();
}
auto mod_name = m_mod.name();
auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name));
m_mod.setMetadata(meta);
connect(project_task.get(), &Task::finished, this, [=] {
invalidade_leftover();
project_task->deleteLater();
m_current_task = nullptr;
});
tsk.addTask(ver_task);
}
m_current_task = project_task.get();
project_task->start();
});
connect(version_task.get(), &Task::finished, [=] {
version_task->deleteLater();
m_current_task = nullptr;
});
if (m_mods.size() > 1)
setStatus(tr("Requesting metadata information from %1...").arg(ProviderCaps.readableName(m_provider)));
else if (!m_mods.empty())
setStatus(tr("Requesting metadata information from %1 for '%2'...")
.arg(ProviderCaps.readableName(m_provider), m_mods.begin().value().name()));
m_current_task = version_task.get();
version_task->start();
}
void EnsureMetadataTask::flameEnsureMetadata(SequentialTask& tsk, QByteArray& jar_data)
void EnsureMetadataTask::emitReady(Mod& m)
{
QByteArray jar_data_treated;
for (char c : jar_data) {
// CF-specific
if (!(c == 9 || c == 10 || c == 13 || c == 32))
jar_data_treated.push_back(c);
}
qDebug() << QString("Generated metadata for %1").arg(m.name());
emit metadataReady(m);
m_mods.remove(getHash(m));
}
void EnsureMetadataTask::emitFail(Mod& m)
{
qDebug() << QString("Failed to generate metadata for %1").arg(m.name());
emit metadataFailed(m);
m_mods.remove(getHash(m));
}
// Modrinth
NetJob::Ptr EnsureMetadataTask::modrinthVersionsTask()
{
auto hash_type = ProviderCaps.hashType(ModPlatform::Provider::MODRINTH).first();
auto* response = new QByteArray();
auto ver_task = modrinth_api.currentVersions(m_mods.keys(), hash_type, response);
std::list<uint> fingerprints;
auto murmur = MurmurHash2(jar_data_treated, jar_data_treated.length());
fingerprints.push_back(murmur);
auto ver_task = flame_api.matchFingerprints(fingerprints, response);
connect(ver_task.get(), &Task::succeeded, this, [this, ver_task, response] {
QDir tmp_index_dir(m_index_dir);
// Prevents unfortunate timings when aborting the task
if (!ver_task)
return {};
connect(ver_task.get(), &NetJob::succeeded, this, [this, response] {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from " << m_mod.name() << " at " << parse_error.offset
qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
ver_task->failed(parse_error.errorString());
failed(parse_error.errorString());
return;
}
try {
auto entries = Json::requireObject(doc);
for (auto& hash : m_mods.keys()) {
auto mod = m_mods.find(hash).value();
try {
auto entry = Json::requireObject(entries, hash);
setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name()));
qDebug() << "Getting version for" << mod.name() << "from Modrinth";
m_temp_versions.insert(hash, Modrinth::loadIndexedPackVersion(entry));
} catch (Json::JsonException& e) {
qDebug() << e.cause();
qDebug() << entries;
emitFail(mod);
}
}
} catch (Json::JsonException& e) {
qDebug() << e.cause();
qDebug() << doc;
}
});
return ver_task;
}
NetJob::Ptr EnsureMetadataTask::modrinthProjectsTask()
{
QHash<QString, QString> addonIds;
for (auto const& data : m_temp_versions)
addonIds.insert(data.addonId.toString(), data.hash);
auto response = new QByteArray();
NetJob::Ptr proj_task;
if (addonIds.isEmpty()) {
qWarning() << "No addonId found!";
} else if (addonIds.size() == 1) {
proj_task = modrinth_api.getProject(*addonIds.keyBegin(), response);
} else {
proj_task = modrinth_api.getProjects(addonIds.keys(), response);
}
// Prevents unfortunate timings when aborting the task
if (!proj_task)
return {};
connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] {
QJsonParseError parse_error{};
auto doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
return;
}
try {
QJsonArray entries;
if (addonIds.size() == 1)
entries = { doc.object() };
else
entries = Json::requireArray(doc);
for (auto entry : entries) {
auto entry_obj = Json::requireObject(entry);
auto entry_id = Json::requireString(entry_obj, "id");
auto hash = addonIds.find(entry_id).value();
auto mod = m_mods.find(hash).value();
try {
setStatus(tr("Parsing API response from Modrinth for '%1'...").arg(mod.name()));
ModPlatform::IndexedPack pack;
Modrinth::loadIndexedPack(pack, entry_obj);
modrinthCallback(pack, m_temp_versions.find(hash).value(), mod);
} catch (Json::JsonException& e) {
qDebug() << e.cause();
qDebug() << entries;
emitFail(mod);
}
}
} catch (Json::JsonException& e) {
qDebug() << e.cause();
qDebug() << doc;
}
});
return proj_task;
}
// Flame
NetJob::Ptr EnsureMetadataTask::flameVersionsTask()
{
auto* response = new QByteArray();
std::list<uint> fingerprints;
for (auto& murmur : m_mods.keys()) {
fingerprints.push_back(murmur.toUInt());
}
auto ver_task = flame_api.matchFingerprints(fingerprints, response);
connect(ver_task.get(), &Task::succeeded, this, [this, response] {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from Modrinth::CurrentVersions at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
failed(parse_error.errorString());
return;
}
try {
auto doc_obj = Json::requireObject(doc);
auto data_obj = Json::ensureObject(doc_obj, "data");
auto match_obj = Json::ensureObject(Json::ensureArray(data_obj, "exactMatches")[0], {});
if (match_obj.isEmpty()) {
qCritical() << "Fingerprint match is empty!";
auto data_obj = Json::requireObject(doc_obj, "data");
auto data_arr = Json::requireArray(data_obj, "exactMatches");
if (data_arr.isEmpty()) {
qWarning() << "No matches found for fingerprint search!";
ver_task->failed(parse_error.errorString());
return;
}
auto file_obj = Json::ensureObject(match_obj, "file");
for (auto match : data_arr) {
auto match_obj = Json::ensureObject(match, {});
auto file_obj = Json::ensureObject(match_obj, "file", {});
ModPlatform::IndexedPack pack;
pack.name = m_mod.name();
pack.provider = ModPlatform::Provider::FLAME;
pack.addonId = Json::requireInteger(file_obj, "modId");
if (match_obj.isEmpty() || file_obj.isEmpty()) {
qWarning() << "Fingerprint match is empty!";
ModPlatform::IndexedVersion ver = FlameMod::loadIndexedPackVersion(file_obj);
return;
}
// Prevent file name mismatch
ver.fileName = m_mod.fileinfo().fileName();
auto fingerprint = QString::number(Json::ensureVariant(file_obj, "fileFingerprint").toUInt());
auto mod = m_mods.find(fingerprint);
if (mod == m_mods.end()) {
qWarning() << "Invalid fingerprint from the API response.";
continue;
}
{
LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
QEventLoop loop;
QTimer timeout;
setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod->name()));
QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
QObject::connect(&timeout, &QTimer::timeout, &loop, &QEventLoop::quit);
update_metadata.start();
timeout.start(100);
loop.exec();
m_temp_versions.insert(fingerprint, FlameMod::loadIndexedPackVersion(file_obj));
}
auto mod_name = m_mod.name();
auto meta = new Metadata::ModStruct(Metadata::get(tmp_index_dir, mod_name));
m_mod.setMetadata(meta);
} catch (Json::JsonException& e) {
emitFailed(e.cause() + " : " + e.what());
qDebug() << e.cause();
qDebug() << doc;
}
});
tsk.addTask(ver_task);
return ver_task;
}
NetJob::Ptr EnsureMetadataTask::flameProjectsTask()
{
QHash<QString, QString> addonIds;
for (auto const& hash : m_mods.keys()) {
if (m_temp_versions.contains(hash)) {
auto const& data = m_temp_versions.find(hash).value();
addonIds.insert(data.addonId.toString(), hash);
}
}
auto response = new QByteArray();
NetJob::Ptr proj_task;
if (addonIds.isEmpty()) {
qWarning() << "No addonId found!";
} else if (addonIds.size() == 1) {
proj_task = flame_api.getProject(*addonIds.keyBegin(), response);
} else {
proj_task = flame_api.getProjects(addonIds.keys(), response);
}
// Prevents unfortunate timings when aborting the task
if (!proj_task)
return {};
connect(proj_task.get(), &NetJob::succeeded, this, [this, response, addonIds] {
QJsonParseError parse_error{};
auto doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from Modrinth projects task at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
return;
}
try {
QJsonArray entries;
if (addonIds.size() == 1)
entries = { Json::requireObject(Json::requireObject(doc), "data") };
else
entries = Json::requireArray(Json::requireObject(doc), "data");
for (auto entry : entries) {
auto entry_obj = Json::requireObject(entry);
auto id = QString::number(Json::requireInteger(entry_obj, "id"));
auto hash = addonIds.find(id).value();
auto mod = m_mods.find(hash).value();
try {
setStatus(tr("Parsing API response from CurseForge for '%1'...").arg(mod.name()));
ModPlatform::IndexedPack pack;
FlameMod::loadIndexedPack(pack, entry_obj);
flameCallback(pack, m_temp_versions.find(hash).value(), mod);
} catch (Json::JsonException& e) {
qDebug() << e.cause();
qDebug() << entries;
emitFail(mod);
}
}
} catch (Json::JsonException& e) {
qDebug() << e.cause();
qDebug() << doc;
}
});
return proj_task;
}
void EnsureMetadataTask::modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod)
{
// Prevent file name mismatch
ver.fileName = mod.fileinfo().fileName();
QDir tmp_index_dir(m_index_dir);
{
LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
QEventLoop loop;
QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
update_metadata.start();
if (!update_metadata.isFinished())
loop.exec();
}
auto metadata = Metadata::get(tmp_index_dir, pack.slug);
if (!metadata.isValid()) {
qCritical() << "Failed to generate metadata at last step!";
emitFail(mod);
return;
}
mod.setMetadata(metadata);
emitReady(mod);
}
void EnsureMetadataTask::flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod& mod)
{
try {
// Prevent file name mismatch
ver.fileName = mod.fileinfo().fileName();
QDir tmp_index_dir(m_index_dir);
{
LocalModUpdateTask update_metadata(m_index_dir, pack, ver);
QEventLoop loop;
QObject::connect(&update_metadata, &Task::finished, &loop, &QEventLoop::quit);
update_metadata.start();
if (!update_metadata.isFinished())
loop.exec();
}
auto metadata = Metadata::get(tmp_index_dir, pack.slug);
if (!metadata.isValid()) {
qCritical() << "Failed to generate metadata at last step!";
emitFail(mod);
return;
}
mod.setMetadata(metadata);
emitReady(mod);
} catch (Json::JsonException& e) {
qDebug() << e.cause();
emitFail(mod);
}
}

View File

@ -2,6 +2,7 @@
#include "ModIndex.h"
#include "tasks/SequentialTask.h"
#include "net/NetJob.h"
class Mod;
class QDir;
@ -11,7 +12,10 @@ class EnsureMetadataTask : public Task {
Q_OBJECT
public:
EnsureMetadataTask(Mod&, QDir&, bool try_all, ModPlatform::Provider = ModPlatform::Provider::MODRINTH);
EnsureMetadataTask(Mod&, QDir, ModPlatform::Provider = ModPlatform::Provider::MODRINTH);
EnsureMetadataTask(std::list<Mod>&, QDir, ModPlatform::Provider = ModPlatform::Provider::MODRINTH);
~EnsureMetadataTask() = default;
public slots:
bool abort() override;
@ -20,22 +24,31 @@ class EnsureMetadataTask : public Task {
private:
// FIXME: Move to their own namespace
void modrinthEnsureMetadata(SequentialTask&, QByteArray&);
void flameEnsureMetadata(SequentialTask&, QByteArray&);
auto modrinthVersionsTask() -> NetJob::Ptr;
auto modrinthProjectsTask() -> NetJob::Ptr;
auto flameVersionsTask() -> NetJob::Ptr;
auto flameProjectsTask() -> NetJob::Ptr;
// Helpers
void emitReady();
void emitFail();
void emitReady(Mod&);
void emitFail(Mod&);
auto getHash(Mod&) -> QString;
private slots:
void modrinthCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod&);
void flameCallback(ModPlatform::IndexedPack& pack, ModPlatform::IndexedVersion& ver, Mod&);
signals:
void metadataReady();
void metadataFailed();
void metadataReady(Mod&);
void metadataFailed(Mod&);
private:
Mod& m_mod;
QDir& m_index_dir;
QHash<QString, Mod> m_mods;
QDir m_index_dir;
ModPlatform::Provider m_provider;
bool m_try_all;
MultipleOptionsTask* m_task_handler = nullptr;
QHash<QString, ModPlatform::IndexedVersion> m_temp_versions;
NetJob* m_current_task;
};

View File

@ -10,7 +10,6 @@
#include "ModDownloadTask.h"
static FlameAPI api;
static ModPlatform::ProviderCapabilities ProviderCaps;
bool FlameCheckUpdate::abort()
{
@ -64,6 +63,50 @@ ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info)
return pack;
}
ModPlatform::IndexedVersion getFileInfo(int addonId, int fileId)
{
ModPlatform::IndexedVersion ver;
QEventLoop loop;
auto get_file_info_job = new NetJob("Flame::GetFileInfoJob", APPLICATION->network());
auto response = new QByteArray();
auto url = QString("https://api.curseforge.com/v1/mods/%1/files/%2").arg(QString::number(addonId), QString::number(fileId));
auto dl = Net::Download::makeByteArray(url, response);
get_file_info_job->addNetAction(dl);
QObject::connect(get_file_info_job, &NetJob::succeeded, [response, &ver]() {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
return;
}
try {
auto doc_obj = Json::requireObject(doc);
auto data_obj = Json::requireObject(doc_obj, "data");
ver = FlameMod::loadIndexedPackVersion(data_obj);
} catch (Json::JsonException& e) {
qWarning() << e.cause();
qDebug() << doc;
}
});
QObject::connect(get_file_info_job, &NetJob::finished, [&loop, get_file_info_job] {
get_file_info_job->deleteLater();
loop.quit();
});
get_file_info_job->start();
loop.exec();
return ver;
}
/* Check for update:
* - Get latest version available
* - Compare hash of the latest version with the current hash
@ -72,148 +115,60 @@ ModPlatform::IndexedPack getProjectInfo(ModPlatform::IndexedVersion& ver_info)
void FlameCheckUpdate::executeTask()
{
setStatus(tr("Preparing mods for CurseForge..."));
setProgress(0, 5);
QHash<int, Mod> mappings;
// Create all hashes
std::list<uint> murmur_hashes;
auto best_hash_type = ProviderCaps.hashType(ModPlatform::Provider::FLAME).first();
int i = 0;
for (auto mod : m_mods) {
QByteArray jar_data;
setStatus(tr("Getting API response from CurseForge for '%1'").arg(mod.name()));
setProgress(i++, m_mods.size());
try {
jar_data = FS::read(mod.fileinfo().absoluteFilePath());
} catch (FS::FileSystemException& e) {
qCritical() << QString("Failed to open / read JAR file of %1").arg(mod.name());
qCritical() << QString("Reason: ") << e.cause();
auto latest_ver = api.getLatestVersion({ mod.metadata()->project_id.toString(), m_game_versions, m_loaders });
failed(e.what());
// Check if we were aborted while getting the latest version
if (m_was_aborted) {
aborted();
return;
}
QByteArray jar_data_treated;
for (char c : jar_data) {
// CF-specific
if (!(c == 9 || c == 10 || c == 13 || c == 32))
jar_data_treated.push_back(c);
setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name()));
if (!latest_ver.addonId.isValid()) {
emit checkFailed(mod, tr("No valid version found for this mod. It's probably unavailable for the current game "
"version / mod loader."));
continue;
}
auto murmur_hash = MurmurHash2(jar_data_treated, jar_data_treated.length());
murmur_hashes.emplace_back(murmur_hash);
if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != mod.metadata()->file_id) {
auto pack = getProjectInfo(latest_ver);
auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString());
emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url);
mappings.insert(mod.metadata()->mod_id().toInt(), mod);
}
auto* response = new QByteArray();
auto job = api.matchFingerprints(murmur_hashes, response);
QEventLoop lock;
connect(job.get(), &Task::succeeded, this, [this, response, &mappings] {
QJsonParseError parse_error{};
QJsonDocument doc = QJsonDocument::fromJson(*response, &parse_error);
if (parse_error.error != QJsonParseError::NoError) {
qWarning() << "Error while parsing JSON response from FlameCheckUpdate at " << parse_error.offset
<< " reason: " << parse_error.errorString();
qWarning() << *response;
failed(parse_error.errorString());
return;
continue;
}
setStatus(tr("Parsing the first API response from CurseForge..."));
setProgress(2, 5);
if (!latest_ver.hash.isEmpty() && (mod.metadata()->hash != latest_ver.hash || mod.status() == ModStatus::NotInstalled)) {
// Fake pack with the necessary info to pass to the download task :)
ModPlatform::IndexedPack pack;
pack.name = mod.name();
pack.slug = mod.metadata()->slug;
pack.addonId = mod.metadata()->project_id;
pack.websiteUrl = mod.homeurl();
for (auto& author : mod.authors())
pack.authors.append({ author });
pack.description = mod.description();
pack.provider = ModPlatform::Provider::FLAME;
try {
auto doc_obj = Json::requireObject(doc);
auto data_obj = Json::ensureObject(doc_obj, "data");
auto match_arr = Json::ensureArray(data_obj, "exactMatches");
for (auto match : match_arr) {
auto match_obj = Json::ensureObject(match);
ModPlatform::IndexedVersion current_ver;
try {
auto file_obj = Json::requireObject(match_obj, "file");
current_ver = FlameMod::loadIndexedPackVersion(file_obj);
} catch (Json::JsonException& e) {
qCritical() << "Error while parsing Flame indexed version";
qCritical() << e.what();
failed(tr("An error occured while parsing a CurseForge indexed version!"));
return;
}
auto mod_iter = mappings.find(current_ver.addonId.toInt());
if (mod_iter == mappings.end()) {
qCritical() << "Failed to remap mod from Flame!";
qDebug() << match_obj;
continue;
}
auto mod = mod_iter.value();
setStatus(tr("Waiting for the API response from CurseForge for '%1'...").arg(mod.name()));
setProgress(3, 5);
auto latest_ver = api.getLatestVersion({ current_ver.addonId.toString(), m_game_versions, m_loaders });
// Check if we were aborted while getting the latest version
if (m_was_aborted) {
aborted();
return;
}
setStatus(tr("Parsing the API response from CurseForge for '%1'...").arg(mod.name()));
setProgress(4, 5);
if (!latest_ver.addonId.isValid()) {
emit checkFailed(
mod,
tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader."));
continue;
}
if (latest_ver.downloadUrl.isEmpty() && latest_ver.fileId != current_ver.fileId) {
auto pack = getProjectInfo(latest_ver);
auto recover_url = QString("%1/download/%2").arg(pack.websiteUrl, latest_ver.fileId.toString());
emit checkFailed(mod, tr("Mod has a new update available, but is opted-out on CurseForge"), recover_url);
continue;
}
if (!latest_ver.hash.isEmpty() && (current_ver.hash != latest_ver.hash || mod.status() == ModStatus::NotInstalled)) {
// Fake pack with the necessary info to pass to the download task :)
ModPlatform::IndexedPack pack;
pack.name = mod.name();
pack.addonId = mod.metadata()->project_id;
pack.websiteUrl = mod.homeurl();
for (auto& author : mod.authors())
pack.authors.append({ author });
pack.description = mod.description();
pack.provider = ModPlatform::Provider::FLAME;
auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder);
m_updatable.emplace_back(mod.name(), current_ver.hash, current_ver.version, latest_ver.version,
api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()),
ModPlatform::Provider::FLAME, download_task);
}
auto old_version = mod.version();
if (old_version.isEmpty() && mod.status() != ModStatus::NotInstalled) {
auto current_ver = getFileInfo(latest_ver.addonId.toInt(), mod.metadata()->file_id.toInt());
old_version = current_ver.version;
}
} catch (Json::JsonException& e) {
failed(e.cause() + " : " + e.what());
auto download_task = new ModDownloadTask(pack, latest_ver, m_mods_folder);
m_updatable.emplace_back(mod.name(), mod.metadata()->hash, old_version, latest_ver.version,
api.getModFileChangelog(latest_ver.addonId.toInt(), latest_ver.fileId.toInt()),
ModPlatform::Provider::FLAME, download_task);
}
});
connect(job.get(), &Task::finished, &lock, &QEventLoop::quit);
setStatus(tr("Waiting for the first API response from CurseForge..."));
setProgress(1, 5);
m_net_job = job.get();
job->start();
lock.exec();
}
emitSucceeded();
}

View File

@ -88,7 +88,9 @@ void ModrinthCheckUpdate::executeTask()
qDebug() << "Mod " << mappings.find(hash).value().name() << " got an empty response.";
qDebug() << "Hash: " << hash;
emit checkFailed(mappings.find(hash).value(), tr("Couldn't find the latest version of this mod with the correct mod loader and game version."));
emit checkFailed(
mappings.find(hash).value(),
tr("No valid version found for this mod. It's probably unavailable for the current game version / mod loader."));
continue;
}
@ -134,6 +136,7 @@ void ModrinthCheckUpdate::executeTask()
// Fake pack with the necessary info to pass to the download task :)
ModPlatform::IndexedPack pack;
pack.name = mod.name();
pack.slug = mod.metadata()->slug;
pack.addonId = mod.metadata()->project_id;
pack.websiteUrl = mod.homeurl();
for (auto& author : mod.authors())

View File

@ -34,12 +34,13 @@ static ModAPI::ModLoaderTypes mcLoaders(BaseInstance* inst)
ModUpdateDialog::ModUpdateDialog(QWidget* parent,
BaseInstance* instance,
const std::shared_ptr<ModFolderModel>& mods,
const std::shared_ptr<ModFolderModel> mods,
std::list<Mod>& search_for)
: ReviewMessageBox(parent, tr("Confirm mods to update"), "")
, m_parent(parent)
, m_mod_model(mods)
, m_candidates(search_for)
, m_second_try_metadata(new SequentialTask())
, m_instance(instance)
{
ReviewMessageBox::setGeometry(0, 0, 800, 600);
@ -47,15 +48,6 @@ ModUpdateDialog::ModUpdateDialog(QWidget* parent,
ui->explainLabel->setText(tr("You're about to update the following mods:"));
ui->onlyCheckedLabel->setText(tr("Only mods with a check will be updated!"));
connect(&m_check_task, &Task::failed, this,
[&](QString reason) { CustomMessageBox::selectable(this, tr("Error"), reason, QMessageBox::Critical)->exec(); });
connect(&m_check_task, &Task::succeeded, this, [&]() {
QStringList warnings = m_check_task.warnings();
if (warnings.count()) {
CustomMessageBox::selectable(this, tr("Warnings"), warnings.join('\n'), QMessageBox::Warning)->exec();
}
});
}
void ModUpdateDialog::checkCandidates()
@ -89,26 +81,39 @@ void ModUpdateDialog::checkCandidates()
auto versions = mcVersions(m_instance);
auto loaders = mcLoaders(m_instance);
SequentialTask check_task (m_parent, tr("Checking for updates"));
if (!m_modrinth_to_update.empty()) {
m_modrinth_check_task = new ModrinthCheckUpdate(m_modrinth_to_update, versions, loaders, m_mod_model);
connect(m_modrinth_check_task, &CheckUpdateTask::checkFailed, this,
[this](Mod mod, QString reason, QUrl recover_url) { m_failed_check_update.emplace_back(mod, reason, recover_url); });
m_check_task.addTask(m_modrinth_check_task);
check_task.addTask(m_modrinth_check_task);
}
if (!m_flame_to_update.empty()) {
m_flame_check_task = new FlameCheckUpdate(m_flame_to_update, versions, loaders, m_mod_model);
connect(m_flame_check_task, &CheckUpdateTask::checkFailed, this,
[this](Mod mod, QString reason, QUrl recover_url) { m_failed_check_update.emplace_back(mod, reason, recover_url); });
m_check_task.addTask(m_flame_check_task);
check_task.addTask(m_flame_check_task);
}
connect(&check_task, &Task::failed, this,
[&](QString reason) { CustomMessageBox::selectable(this, tr("Error"), reason, QMessageBox::Critical)->exec(); });
connect(&check_task, &Task::succeeded, this, [&]() {
QStringList warnings = check_task.warnings();
if (warnings.count()) {
CustomMessageBox::selectable(this, tr("Warnings"), warnings.join('\n'), QMessageBox::Warning)->exec();
}
});
// Check for updates
// FIXME: SOMEHOW THIS IS NOT MODAL???????
ProgressDialog progress_dialog(m_parent);
progress_dialog.setSkipButton(true, tr("Abort"));
progress_dialog.setVisible(true);
progress_dialog.setWindowTitle(tr("Checking for updates..."));
auto ret = progress_dialog.execWithTask(&m_check_task);
auto ret = progress_dialog.execWithTask(&check_task);
// If the dialog was skipped / some download error happened
if (ret == QDialog::DialogCode::Rejected) {
@ -183,13 +188,29 @@ auto ModUpdateDialog::ensureMetadata() -> bool
{
auto index_dir = indexDir();
auto* seq = new SequentialTask(m_parent, tr("Looking for metadata"));
SequentialTask seq(m_parent, tr("Looking for metadata"));
// A better use of data structures here could remove the need for this QHash
QHash<QString, bool> should_try_others;
std::list<Mod> modrinth_tmp;
std::list<Mod> flame_tmp;
bool confirm_rest = false;
bool try_others_rest = false;
bool skip_rest = false;
ModPlatform::Provider provider_rest = ModPlatform::Provider::MODRINTH;
auto addToTmp = [&](Mod& m, ModPlatform::Provider p) {
switch (p) {
case ModPlatform::Provider::MODRINTH:
modrinth_tmp.push_back(m);
break;
case ModPlatform::Provider::FLAME:
flame_tmp.push_back(m);
break;
}
};
for (auto& candidate : m_candidates) {
if (candidate.status() != ModStatus::NoMetadata) {
onMetadataEnsured(candidate);
@ -200,10 +221,8 @@ auto ModUpdateDialog::ensureMetadata() -> bool
continue;
if (confirm_rest) {
auto* task = new EnsureMetadataTask(candidate, index_dir, try_others_rest, provider_rest);
connect(task, &EnsureMetadataTask::metadataReady, [this, &candidate] { onMetadataEnsured(candidate); });
connect(task, &EnsureMetadataTask::metadataFailed, [this, &candidate] { onMetadataFailed(candidate); });
seq->addTask(task);
addToTmp(candidate, provider_rest);
should_try_others.insert(candidate.internal_id(), try_others_rest);
continue;
}
@ -224,18 +243,36 @@ auto ModUpdateDialog::ensureMetadata() -> bool
try_others_rest = response.try_others;
}
if (confirmed) {
auto* task = new EnsureMetadataTask(candidate, index_dir, response.try_others, response.chosen);
connect(task, &EnsureMetadataTask::metadataReady, [this, &candidate] { onMetadataEnsured(candidate); });
connect(task, &EnsureMetadataTask::metadataFailed, [this, &candidate] { onMetadataFailed(candidate); });
seq->addTask(task);
}
should_try_others.insert(candidate.internal_id(), response.try_others);
if (confirmed)
addToTmp(candidate, response.chosen);
}
if (!modrinth_tmp.empty()) {
auto* modrinth_task = new EnsureMetadataTask(modrinth_tmp, index_dir, ModPlatform::Provider::MODRINTH);
connect(modrinth_task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); });
connect(modrinth_task, &EnsureMetadataTask::metadataFailed, [this, &should_try_others](Mod& candidate) {
onMetadataFailed(candidate, should_try_others.find(candidate.internal_id()).value(), ModPlatform::Provider::MODRINTH);
});
seq.addTask(modrinth_task);
}
if (!flame_tmp.empty()) {
auto* flame_task = new EnsureMetadataTask(flame_tmp, index_dir, ModPlatform::Provider::FLAME);
connect(flame_task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); });
connect(flame_task, &EnsureMetadataTask::metadataFailed, [this, &should_try_others](Mod& candidate) {
onMetadataFailed(candidate, should_try_others.find(candidate.internal_id()).value(), ModPlatform::Provider::FLAME);
});
seq.addTask(flame_task);
}
seq.addTask(m_second_try_metadata);
ProgressDialog checking_dialog(m_parent);
checking_dialog.setSkipButton(true, tr("Abort"));
checking_dialog.setWindowTitle(tr("Generating metadata..."));
auto ret_metadata = checking_dialog.execWithTask(seq);
auto ret_metadata = checking_dialog.execWithTask(&seq);
return (ret_metadata != QDialog::DialogCode::Rejected);
}
@ -256,9 +293,31 @@ void ModUpdateDialog::onMetadataEnsured(Mod& mod)
}
}
void ModUpdateDialog::onMetadataFailed(Mod& mod)
ModPlatform::Provider next(ModPlatform::Provider p)
{
m_failed_metadata.push_back(mod);
switch (p) {
case ModPlatform::Provider::MODRINTH:
return ModPlatform::Provider::FLAME;
case ModPlatform::Provider::FLAME:
return ModPlatform::Provider::MODRINTH;
}
return ModPlatform::Provider::FLAME;
}
void ModUpdateDialog::onMetadataFailed(Mod& mod, bool try_others, ModPlatform::Provider first_choice)
{
if (try_others) {
auto index_dir = indexDir();
auto* task = new EnsureMetadataTask(mod, index_dir, next(first_choice));
connect(task, &EnsureMetadataTask::metadataReady, [this](Mod& candidate) { onMetadataEnsured(candidate); });
connect(task, &EnsureMetadataTask::metadataFailed, [this](Mod& candidate) { onMetadataFailed(candidate, false); });
m_second_try_metadata->addTask(task);
} else {
m_failed_metadata.push_back(mod);
}
}
void ModUpdateDialog::appendMod(CheckUpdateTask::UpdatableMod const& info)

View File

@ -17,7 +17,7 @@ class ModUpdateDialog final : public ReviewMessageBox {
public:
explicit ModUpdateDialog(QWidget* parent,
BaseInstance* instance,
const std::shared_ptr<ModFolderModel>& mod_model,
const std::shared_ptr<ModFolderModel> mod_model,
std::list<Mod>& search_for);
void checkCandidates();
@ -35,21 +35,21 @@ class ModUpdateDialog final : public ReviewMessageBox {
private slots:
void onMetadataEnsured(Mod&);
void onMetadataFailed(Mod&);
void onMetadataFailed(Mod&, bool try_others = false, ModPlatform::Provider first_choice = ModPlatform::Provider::MODRINTH);
private:
QWidget* m_parent;
SequentialTask m_check_task;
ModrinthCheckUpdate* m_modrinth_check_task = nullptr;
FlameCheckUpdate* m_flame_check_task = nullptr;
const std::shared_ptr<ModFolderModel>& m_mod_model;
const std::shared_ptr<ModFolderModel> m_mod_model;
std::list<Mod>& m_candidates;
std::list<Mod> m_modrinth_to_update;
std::list<Mod> m_flame_to_update;
SequentialTask* m_second_try_metadata;
std::list<Mod> m_failed_metadata;
std::list<std::tuple<Mod, QString, QUrl>> m_failed_check_update;