Use the forge mirrors for downloading forge libraries

Let's hope we never, ever see a forge download error again.

CMakeLists.txt

@@ -256,6 +256,8 @@ logic/net/ByteArrayDownload.h
 logic/net/ByteArrayDownload.cpp
 logic/net/CacheDownload.h
 logic/net/CacheDownload.cpp
+logic/net/ForgeMirrors.h
+logic/net/ForgeMirrors.cpp
 logic/net/ForgeXzDownload.h
 logic/net/ForgeXzDownload.cpp
 logic/net/NetJob.h

gui/MainWindow.cpp

@@ -180,6 +180,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi
 		{
 			MMC->lwjgllist()->loadList();
 		}
+		/*
 		assets_downloader = new OneSixAssets();
 		connect(assets_downloader, SIGNAL(indexStarted()), SLOT(assetsIndexStarted()));
 		connect(assets_downloader, SIGNAL(filesStarted()), SLOT(assetsFilesStarted()));
@@ -188,6 +189,7 @@ MainWindow::MainWindow(QWidget *parent) : QMainWindow(parent), ui(new Ui::MainWi
 		connect(assets_downloader, SIGNAL(failed()), SLOT(assetsFailed()));
 		connect(assets_downloader, SIGNAL(finished()), SLOT(assetsFinished()));
 		assets_downloader->start();
+		*/
 	}
 }
 

logic/OneSixUpdate.cpp

@@ -28,6 +28,7 @@
 #include "OneSixVersion.h"
 #include "OneSixLibrary.h"
 #include "OneSixInstance.h"
+#include "net/ForgeMirrors.h"
 
 #include "pathutils.h"
 
@@ -163,20 +164,33 @@ void OneSixUpdate::jarlibStart()
 	libs.append(version->getActiveNormalLibs());
 
 	auto metacache = MMC->metacache();
+	QList<ForgeXzDownloadPtr> ForgeLibs;
+	bool already_forge_xz = false;
 	for (auto lib : libs)
 	{
 		if (lib->hint() == "local")
 			continue;
-		QString download_path = lib->downloadUrl();
 		auto entry = metacache->resolveEntry("libraries", lib->storagePath());
 		if (entry->stale)
 		{
 			if (lib->hint() == "forge-pack-xz")
-				jarlibDownloadJob->addNetAction(ForgeXzDownload::make(download_path, entry));
+			{
+				ForgeLibs.append(ForgeXzDownload::make(lib->storagePath(), entry));
+			}
 			else
-				jarlibDownloadJob->addNetAction(CacheDownload::make(download_path, entry));
+			{
+				jarlibDownloadJob->addNetAction(CacheDownload::make(lib->downloadUrl(), entry));
 			}
 		}
+	}
+	// TODO: think about how to propagate this from the original json file... or IF AT ALL
+	QString forgeMirrorList = "http://files.minecraftforge.net/mirror-brand.list";
+	if (!ForgeLibs.empty())
+	{
+		jarlibDownloadJob->addNetAction(
+			ForgeMirrors::make(ForgeLibs, jarlibDownloadJob, forgeMirrorList));
+	}
+
 	connect(jarlibDownloadJob.get(), SIGNAL(succeeded()), SLOT(jarlibFinished()));
 	connect(jarlibDownloadJob.get(), SIGNAL(failed()), SLOT(jarlibFailed()));
 	connect(jarlibDownloadJob.get(), SIGNAL(progress(qint64, qint64)),

logic/net/CacheDownload.cpp

@@ -29,7 +29,6 @@ CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
 	m_entry = entry;
 	m_target_path = entry->getFullPath();
 	m_status = Job_NotStarted;
-	m_opened_for_saving = false;
 }
 
 void CacheDownload::start()
@@ -87,7 +86,7 @@ void CacheDownload::downloadFinished()
 
 		// nothing went wrong...
 		m_status = Job_Finished;
-		if (m_opened_for_saving)
+		if (m_output_file.isOpen())
 		{
 			// save the data to the downloadable if we aren't saving to file
 			m_output_file.close();
@@ -133,7 +132,7 @@ void CacheDownload::downloadFinished()
 
 void CacheDownload::downloadReadyRead()
 {
-	if (!m_opened_for_saving)
+	if (!m_output_file.isOpen())
 	{
 		if (!m_output_file.open(QIODevice::WriteOnly))
 		{
@@ -144,7 +143,6 @@ void CacheDownload::downloadReadyRead()
 			emit failed(index_within_job);
 			return;
 		}
-		m_opened_for_saving = true;
 	}
 	QByteArray ba = m_reply->readAll();
 	md5sum.addData(ba);

logic/net/CacheDownload.h

@@ -26,8 +26,6 @@ class CacheDownload : public NetAction
 	Q_OBJECT
 public:
 	MetaEntryPtr m_entry;
-	/// is the saving file already open?
-	bool m_opened_for_saving;
 	/// if saving to file, use the one specified in this string
 	QString m_target_path;
 	/// this is the output file, if any

logic/net/FileDownload.cpp

@@ -25,7 +25,6 @@ FileDownload::FileDownload(QUrl url, QString target_path) : NetAction()
 	m_target_path = target_path;
 	m_check_md5 = false;
 	m_status = Job_NotStarted;
-	m_opened_for_saving = false;
 }
 
 void FileDownload::start()
@@ -113,7 +112,7 @@ void FileDownload::downloadFinished()
 
 void FileDownload::downloadReadyRead()
 {
-	if (!m_opened_for_saving)
+	if (!m_output_file.isOpen())
 	{
 		if (!m_output_file.open(QIODevice::WriteOnly))
 		{
@@ -124,7 +123,6 @@ void FileDownload::downloadReadyRead()
 			emit failed(index_within_job);
 			return;
 		}
-		m_opened_for_saving = true;
 	}
 	m_output_file.write(m_reply->readAll());
 }

logic/net/FileDownload.h

@@ -29,8 +29,6 @@ public:
 	bool m_check_md5;
 	/// the expected md5 checksum
 	QString m_expected_md5;
-	/// is the saving file already open?
-	bool m_opened_for_saving;
 	/// if saving to file, use the one specified in this string
 	QString m_target_path;
 	/// this is the output file, if any

logic/net/ForgeMirror.h

@@ -0,0 +1,10 @@
+#pragma once
+#include <QString>
+
+struct ForgeMirror
+{
+	QString name;
+	QString logo_url;
+	QString website_url;
+	QString mirror_url;
+};

logic/net/ForgeMirrors.cpp

@@ -0,0 +1,116 @@
+#include "MultiMC.h"
+#include "ForgeMirrors.h"
+#include "logger/QsLog.h"
+#include <algorithm>
+#include <random>
+
+ForgeMirrors::ForgeMirrors(QList<ForgeXzDownloadPtr> &libs, NetJobPtr parent_job,
+						   QString mirrorlist)
+{
+	m_libs = libs;
+	m_parent_job = parent_job;
+	m_url = QUrl(mirrorlist);
+	m_status = Job_NotStarted;
+}
+
+void ForgeMirrors::start()
+{
+	QLOG_INFO() << "Downloading " << m_url.toString();
+	QNetworkRequest request(m_url);
+	request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
+	auto worker = MMC->qnam();
+	QNetworkReply *rep = worker->get(request);
+
+	m_reply = std::shared_ptr<QNetworkReply>(rep);
+	connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
+			SLOT(downloadProgress(qint64, qint64)));
+	connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
+	connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
+			SLOT(downloadError(QNetworkReply::NetworkError)));
+	connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
+}
+
+void ForgeMirrors::downloadError(QNetworkReply::NetworkError error)
+{
+	// error happened during download.
+	QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
+				 << "Network error: " << error;
+	m_status = Job_Failed;
+}
+
+void ForgeMirrors::downloadFinished()
+{
+	// if the download succeeded
+	if (m_status != Job_Failed)
+	{
+		// nothing went wrong... ?
+		parseMirrorList();
+		return;
+	}
+	// else the download failed, we use a fixed list
+	else
+	{
+		m_status = Job_Finished;
+		m_reply.reset();
+		deferToFixedList();
+		return;
+	}
+}
+
+void ForgeMirrors::deferToFixedList()
+{
+	m_mirrors.clear();
+	m_mirrors.append(
+		{"Minecraft Forge",				  "http://files.minecraftforge.net/forge_logo.png",
+		 "http://files.minecraftforge.net/", "http://files.minecraftforge.net/maven/"});
+	m_mirrors.append({"Creeper Host",
+					  "http://files.minecraftforge.net/forge_logo.png",
+					  "https://www.creeperhost.net/link.php?id=1",
+					  "http://new.creeperrepo.net/forge/maven/"});
+	injectDownloads();
+	emit succeeded(index_within_job);
+}
+
+void ForgeMirrors::parseMirrorList()
+{
+	m_status = Job_Finished;
+	auto data = m_reply->readAll();
+	m_reply.reset();
+	auto dataLines = data.split('\n');
+	for(auto line: dataLines)
+	{
+		auto elements = line.split('!');
+		if (elements.size() == 4)
+		{
+			m_mirrors.append({elements[0],elements[1],elements[2],elements[3]});
+		}
+	}
+	if(!m_mirrors.size())
+		deferToFixedList();
+	injectDownloads();
+	emit succeeded(index_within_job);
+}
+
+void ForgeMirrors::injectDownloads()
+{
+	// shuffle the mirrors randomly
+	std::random_device rd;
+	std::mt19937 rng(rd());
+	std::shuffle(m_mirrors.begin(), m_mirrors.end(), rng);
+
+	// tell parent to download the libs
+	for(auto lib: m_libs)
+	{
+		lib->setMirrors(m_mirrors);
+		m_parent_job->addNetAction(lib);
+	}
+}
+
+void ForgeMirrors::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
+{
+	emit progress(index_within_job, bytesReceived, bytesTotal);
+}
+
+void ForgeMirrors::downloadReadyRead()
+{
+}

logic/net/ForgeMirrors.h

@@ -0,0 +1,58 @@
+/* Copyright 2013 MultiMC Contributors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include "NetAction.h"
+#include "HttpMetaCache.h"
+#include "ForgeXzDownload.h"
+#include "NetJob.h"
+#include <QFile>
+#include <QTemporaryFile>
+typedef std::shared_ptr<class ForgeMirrors> ForgeMirrorsPtr;
+
+class ForgeMirrors : public NetAction
+{
+	Q_OBJECT
+public:
+	QList<ForgeXzDownloadPtr> m_libs;
+	NetJobPtr m_parent_job;
+	QList<ForgeMirror> m_mirrors;
+
+public:
+	explicit ForgeMirrors(QList<ForgeXzDownloadPtr> &libs, NetJobPtr parent_job,
+						  QString mirrorlist);
+	static ForgeMirrorsPtr make(QList<ForgeXzDownloadPtr> &libs, NetJobPtr parent_job,
+								QString mirrorlist)
+	{
+		return ForgeMirrorsPtr(new ForgeMirrors(libs, parent_job, mirrorlist));
+	}
+
+protected
+slots:
+	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
+	virtual void downloadError(QNetworkReply::NetworkError error);
+	virtual void downloadFinished();
+	virtual void downloadReadyRead();
+
+private:
+	void parseMirrorList();
+	void deferToFixedList();
+	void injectDownloads();
+
+public
+slots:
+	virtual void start();
+};

logic/net/ForgeXzDownload.cpp

@@ -22,30 +22,44 @@
 #include <QDateTime>
 #include "logger/QsLog.h"
 
-ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry) : NetAction()
+ForgeXzDownload::ForgeXzDownload(QString relative_path, MetaEntryPtr entry) : NetAction()
 {
-	QString urlstr = url.toString();
-	urlstr.append(".pack.xz");
-	m_url = QUrl(urlstr);
 	m_entry = entry;
 	m_target_path = entry->getFullPath();
 	m_status = Job_NotStarted;
-	m_opened_for_saving = false;
+	m_url_path = relative_path;
+}
+
+void ForgeXzDownload::setMirrors(QList<ForgeMirror> &mirrors)
+{
+	m_mirror_index = 0;
+	m_mirrors = mirrors;
+	updateUrl();
 }
 
 void ForgeXzDownload::start()
 {
+	m_status = Job_InProgress;
 	if (!m_entry->stale)
 	{
+		m_status = Job_Finished;
 		emit succeeded(index_within_job);
 		return;
 	}
 	// can we actually create the real, final file?
 	if (!ensureFilePathExists(m_target_path))
 	{
+		m_status = Job_Failed;
 		emit failed(index_within_job);
 		return;
 	}
+	if (m_mirrors.empty())
+	{
+		m_status = Job_Failed;
+		emit failed(index_within_job);
+		return;
+	}
+
 	QLOG_INFO() << "Downloading " << m_url.toString();
 	QNetworkRequest request(m_url);
 	request.setRawHeader(QString("If-None-Match").toLatin1(), m_entry->etag.toLatin1());
@@ -75,14 +89,53 @@ void ForgeXzDownload::downloadError(QNetworkReply::NetworkError error)
 	m_status = Job_Failed;
 }
 
+void ForgeXzDownload::failAndTryNextMirror()
+{
+	m_status = Job_Failed;
+	int next = m_mirror_index + 1;
+	if(m_mirrors.size() == next)
+		m_mirror_index = 0;
+	else
+		m_mirror_index = next;
+
+	updateUrl();
+	emit failed(index_within_job);
+}
+
+void ForgeXzDownload::updateUrl()
+{
+	QLOG_INFO() << "Updating URL for " << m_url_path;
+	for (auto possible : m_mirrors)
+	{
+		QLOG_INFO() << "Possible: " << possible.name << " : " << possible.mirror_url;
+	}
+	QString aggregate = m_mirrors[m_mirror_index].mirror_url + m_url_path + ".pack.xz";
+	m_url = QUrl(aggregate);
+}
+
 void ForgeXzDownload::downloadFinished()
 {
+	//TEST: defer to other possible mirrors (autofail the first one)
+	/*
+	QLOG_INFO() <<"dl " << index_within_job << " mirror " << m_mirror_index;
+	if( m_mirror_index == 0)
+	{
+		QLOG_INFO() <<"dl " << index_within_job << " AUTOFAIL";
+		m_status = Job_Failed;
+		m_pack200_xz_file.close();
+		m_pack200_xz_file.remove();
+		m_reply.reset();
+		failAndTryNextMirror();
+		return;
+	}
+	*/
+
 	// if the download succeeded
 	if (m_status != Job_Failed)
 	{
 		// nothing went wrong...
 		m_status = Job_Finished;
-		if (m_opened_for_saving)
+		if (m_pack200_xz_file.isOpen())
 		{
 			// we actually downloaded something! process and isntall it
 			decompressAndInstall();
@@ -90,7 +143,7 @@ void ForgeXzDownload::downloadFinished()
 		}
 		else
 		{
-			// something bad happened
+			// something bad happened -- on the local machine!
 			m_status = Job_Failed;
 			m_pack200_xz_file.remove();
 			m_reply.reset();
@@ -101,10 +154,11 @@ void ForgeXzDownload::downloadFinished()
 	// else the download failed
 	else
 	{
+		m_status = Job_Failed;
 		m_pack200_xz_file.close();
 		m_pack200_xz_file.remove();
 		m_reply.reset();
-		emit failed(index_within_job);
+		failAndTryNextMirror();
 		return;
 	}
 }
@@ -112,7 +166,7 @@ void ForgeXzDownload::downloadFinished()
 void ForgeXzDownload::downloadReadyRead()
 {
 
-	if (!m_opened_for_saving)
+	if (!m_pack200_xz_file.isOpen())
 	{
 		if (!m_pack200_xz_file.open())
 		{
@@ -123,7 +177,6 @@ void ForgeXzDownload::downloadReadyRead()
 			emit failed(index_within_job);
 			return;
 		}
-		m_opened_for_saving = true;
 	}
 	m_pack200_xz_file.write(m_reply->readAll());
 }
@@ -156,8 +209,7 @@ void ForgeXzDownload::decompressAndInstall()
 		if (s == nullptr)
 		{
 			xz_dec_end(s);
-			m_status = Job_Failed;
+			failAndTryNextMirror();
-			emit failed(index_within_job);
 			return;
 		}
 		b.in = in;
@@ -182,8 +234,7 @@ void ForgeXzDownload::decompressAndInstall()
 				{
 					// msg = "Write error\n";
 					xz_dec_end(s);
-					m_status = Job_Failed;
+					failAndTryNextMirror();
-					emit failed(index_within_job);
 					return;
 				}
 
@@ -217,44 +268,38 @@ void ForgeXzDownload::decompressAndInstall()
 			case XZ_MEM_ERROR:
 				QLOG_ERROR() << "Memory allocation failed\n";
 				xz_dec_end(s);
-				m_status = Job_Failed;
+				failAndTryNextMirror();
-				emit failed(index_within_job);
 				return;
 
 			case XZ_MEMLIMIT_ERROR:
 				QLOG_ERROR() << "Memory usage limit reached\n";
 				xz_dec_end(s);
-				m_status = Job_Failed;
+				failAndTryNextMirror();
-				emit failed(index_within_job);
 				return;
 
 			case XZ_FORMAT_ERROR:
 				QLOG_ERROR() << "Not a .xz file\n";
 				xz_dec_end(s);
-				m_status = Job_Failed;
+				failAndTryNextMirror();
-				emit failed(index_within_job);
 				return;
 
 			case XZ_OPTIONS_ERROR:
 				QLOG_ERROR() << "Unsupported options in the .xz headers\n";
 				xz_dec_end(s);
-				m_status = Job_Failed;
+				failAndTryNextMirror();
-				emit failed(index_within_job);
 				return;
 
 			case XZ_DATA_ERROR:
 			case XZ_BUF_ERROR:
 				QLOG_ERROR() << "File is corrupt\n";
 				xz_dec_end(s);
-				m_status = Job_Failed;
+				failAndTryNextMirror();
-				emit failed(index_within_job);
 				return;
 
 			default:
 				QLOG_ERROR() << "Bug!\n";
 				xz_dec_end(s);
-				m_status = Job_Failed;
+				failAndTryNextMirror();
-				emit failed(index_within_job);
 				return;
 			}
 		}
@@ -274,7 +319,7 @@ void ForgeXzDownload::decompressAndInstall()
 		QFile f(m_target_path);
 		if (f.exists())
 			f.remove();
-		emit failed(index_within_job);
+		failAndTryNextMirror();
 		return;
 	}
 
@@ -283,7 +328,7 @@ void ForgeXzDownload::decompressAndInstall()
 	if (!jar_file.open(QIODevice::ReadOnly))
 	{
 		jar_file.remove();
-		emit failed(index_within_job);
+		failAndTryNextMirror();
 		return;
 	}
 	m_entry->md5sum = QCryptographicHash::hash(jar_file.readAll(), QCryptographicHash::Md5)

logic/net/ForgeXzDownload.h

@@ -19,6 +19,8 @@
 #include "HttpMetaCache.h"
 #include <QFile>
 #include <QTemporaryFile>
+#include "ForgeMirror.h"
+
 typedef std::shared_ptr<class ForgeXzDownload> ForgeXzDownloadPtr;
 
 class ForgeXzDownload : public NetAction
@@ -26,19 +28,24 @@ class ForgeXzDownload : public NetAction
 	Q_OBJECT
 public:
 	MetaEntryPtr m_entry;
-	/// is the saving file already open?
-	bool m_opened_for_saving;
 	/// if saving to file, use the one specified in this string
 	QString m_target_path;
 	/// this is the output file, if any
 	QTemporaryFile m_pack200_xz_file;
+	/// mirror index (NOT OPTICS, I SWEAR)
+	int m_mirror_index = 0;
+	/// list of mirrors to use. Mirror has the url base
+	QList<ForgeMirror> m_mirrors;
+	/// path relative to the mirror base
+	QString m_url_path;
 
 public:
-	explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry);
+	explicit ForgeXzDownload(QString relative_path, MetaEntryPtr entry);
-	static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry)
+	static ForgeXzDownloadPtr make(QString relative_path, MetaEntryPtr entry)
 	{
-		return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry));
+		return ForgeXzDownloadPtr(new ForgeXzDownload(relative_path, entry));
 	}
+	void setMirrors(QList<ForgeMirror> & mirrors);
 
 protected
 slots:
@@ -53,4 +60,6 @@ slots:
 
 private:
 	void decompressAndInstall();
+	void failAndTryNextMirror();
+	void updateUrl();
 };

logic/net/NetJob.cpp

@@ -89,6 +89,7 @@ void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
 void NetJob::start()
 {
 	QLOG_INFO() << m_job_name.toLocal8Bit() << " started.";
+	m_running = true;
 	for (auto iter : downloads)
 	{
 		connect(iter.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));

logic/net/NetJob.h

@@ -40,6 +40,16 @@ public:
 		downloads.append(action);
 		parts_progress.append(part_info());
 		total_progress++;
+		// if this is already running, the action needs to be started right away!
+		if (isRunning())
+		{
+			emit progress(current_progress, total_progress);
+			connect(base.get(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
+			connect(base.get(), SIGNAL(failed(int)), SLOT(partFailed(int)));
+			connect(base.get(), SIGNAL(progress(int, qint64, qint64)),
+					SLOT(partProgress(int, qint64, qint64)));
+			base->start();
+		}
 		return true;
 	}