S3 bucket listing support and network code refactors.

* Adds support for listing all objects in an S3 bucket.
* Renames a bunch of network related classes (Download->Action)
* Net actions now have static constructors

logic/net/ByteArrayDownload.cpp

@@ -2,7 +2,7 @@
 #include "MultiMC.h"
 #include <logger/QsLog.h>
 
-ByteArrayDownload::ByteArrayDownload(QUrl url) : Download()
+ByteArrayDownload::ByteArrayDownload(QUrl url) : NetAction()
 {
 	m_url = url;
 	m_status = Job_NotStarted;

logic/net/ByteArrayDownload.h

@@ -1,24 +1,29 @@
 #pragma once
-#include "Download.h"
+#include "NetAction.h"
 
-class ByteArrayDownload: public Download
+typedef std::shared_ptr<class ByteArrayDownload> ByteArrayDownloadPtr;
+class ByteArrayDownload : public NetAction
 {
 	Q_OBJECT
 public:
 	ByteArrayDownload(QUrl url);
-	
+	static ByteArrayDownloadPtr make(QUrl url)
+	{
+		return ByteArrayDownloadPtr(new ByteArrayDownload(url));
+	}
+
 public:
 	/// if not saving to file, downloaded data is placed here
 	QByteArray m_data;
-	
-public slots:
+
+public
+slots:
 	virtual void start();
-	
-protected slots:
+
+protected
+slots:
 	void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	void downloadError(QNetworkReply::NetworkError error);
 	void downloadFinished();
 	void downloadReadyRead();
 };
-
-typedef std::shared_ptr<ByteArrayDownload> ByteArrayDownloadPtr;

logic/net/CacheDownload.cpp

@@ -8,7 +8,7 @@
 #include <logger/QsLog.h>
 
 CacheDownload::CacheDownload(QUrl url, MetaEntryPtr entry)
-	: Download(), md5sum(QCryptographicHash::Md5)
+	: NetAction(), md5sum(QCryptographicHash::Md5)
 {
 	m_url = url;
 	m_entry = entry;

logic/net/CacheDownload.h

@@ -1,11 +1,12 @@
 #pragma once
 
-#include "Download.h"
+#include "NetAction.h"
 #include "HttpMetaCache.h"
 #include <QFile>
 #include <qcryptographichash.h>
 
-class CacheDownload : public Download
+typedef std::shared_ptr<class CacheDownload> CacheDownloadPtr;
+class CacheDownload : public NetAction
 {
 	Q_OBJECT
 public:
@@ -18,17 +19,22 @@ public:
 	QFile m_output_file;
 	/// the hash-as-you-download
 	QCryptographicHash md5sum;
+
 public:
 	explicit CacheDownload(QUrl url, MetaEntryPtr entry);
-	
-protected slots:
+	static CacheDownloadPtr make(QUrl url, MetaEntryPtr entry)
+	{
+		return CacheDownloadPtr(new CacheDownload(url, entry));
+	}
+
+protected
+slots:
 	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	virtual void downloadError(QNetworkReply::NetworkError error);
 	virtual void downloadFinished();
 	virtual void downloadReadyRead();
-	
-public slots:
+
+public
+slots:
 	virtual void start();
 };
-
-typedef std::shared_ptr<CacheDownload> CacheDownloadPtr;

logic/net/FileDownload.cpp

@@ -6,7 +6,7 @@
 
 
 FileDownload::FileDownload ( QUrl url, QString target_path )
-	:Download()
+	:NetAction()
 {
 	m_url = url;
 	m_target_path = target_path;

logic/net/FileDownload.h

@@ -1,14 +1,16 @@
 #pragma once
 
-#include "Download.h"
+#include "NetAction.h"
 #include <QFile>
 
-class FileDownload : public Download
+typedef std::shared_ptr<class FileDownload> FileDownloadPtr;
+class FileDownload : public NetAction
 {
 	Q_OBJECT
 public:
 	/// if true, check the md5sum against a provided md5sum
-	/// also, if a file exists, perform an md5sum first and don't download only if they don't match
+	/// also, if a file exists, perform an md5sum first and don't download only if they don't
+	/// match
 	bool m_check_md5;
 	/// the expected md5 checksum
 	QString m_expected_md5;
@@ -18,18 +20,21 @@ public:
 	QString m_target_path;
 	/// this is the output file, if any
 	QFile m_output_file;
-	
+
 public:
 	explicit FileDownload(QUrl url, QString target_path);
-	
-protected slots:
+	static FileDownloadPtr make(QUrl url, QString target_path)
+	{
+		return FileDownloadPtr(new FileDownload(url, target_path));
+	}
+protected
+slots:
 	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	virtual void downloadError(QNetworkReply::NetworkError error);
 	virtual void downloadFinished();
 	virtual void downloadReadyRead();
-	
-public slots:
+
+public
+slots:
 	virtual void start();
 };
-
-typedef std::shared_ptr<FileDownload> FileDownloadPtr;

logic/net/ForgeXzDownload.cpp

@@ -8,7 +8,7 @@
 #include <logger/QsLog.h>
 
 ForgeXzDownload::ForgeXzDownload(QUrl url, MetaEntryPtr entry)
-	: Download()
+	: NetAction()
 {
 	QString urlstr = url.toString();
 	urlstr.append(".pack.xz");

logic/net/ForgeXzDownload.h

@@ -1,11 +1,12 @@
 #pragma once
 
-#include "Download.h"
+#include "NetAction.h"
 #include "HttpMetaCache.h"
 #include <QFile>
 #include <QTemporaryFile>
+typedef std::shared_ptr<class ForgeXzDownload> ForgeXzDownloadPtr;
 
-class ForgeXzDownload : public Download
+class ForgeXzDownload : public NetAction
 {
 	Q_OBJECT
 public:
@@ -19,17 +20,22 @@ public:
 
 public:
 	explicit ForgeXzDownload(QUrl url, MetaEntryPtr entry);
-	
-protected slots:
+	static ForgeXzDownloadPtr make(QUrl url, MetaEntryPtr entry)
+	{
+		return ForgeXzDownloadPtr(new ForgeXzDownload(url, entry));
+	}
+
+protected
+slots:
 	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);
 	virtual void downloadError(QNetworkReply::NetworkError error);
 	virtual void downloadFinished();
 	virtual void downloadReadyRead();
-	
-public slots:
+
+public
+slots:
 	virtual void start();
+
 private:
 	void decompressAndInstall();
 };
-
-typedef std::shared_ptr<ForgeXzDownload> ForgeXzDownloadPtr;

logic/net/NetAction.h

@@ -13,14 +13,15 @@ enum JobStatus
 	Job_Failed
 };
 
-class Download : public QObject
+typedef std::shared_ptr<class NetAction> NetActionPtr;
+class NetAction : public QObject
 {
 	Q_OBJECT
 protected:
-	explicit Download() : QObject(0) {};
+	explicit NetAction() : QObject(0) {};
 
 public:
-	virtual ~Download() {};
+	virtual ~NetAction() {};
 
 public:
 	/// the network reply
@@ -50,5 +51,3 @@ protected slots:
 public slots:
 	virtual void start() = 0;
 };
-
-typedef std::shared_ptr<Download> DownloadPtr;

logic/net/NetJob.cpp

@@ -1,4 +1,4 @@
-#include "DownloadJob.h"
+#include "NetJob.h"
 #include "pathutils.h"
 #include "MultiMC.h"
 #include "FileDownload.h"
@@ -7,47 +7,7 @@
 
 #include <logger/QsLog.h>
 
-ByteArrayDownloadPtr DownloadJob::addByteArrayDownload(QUrl url)
-{
-	ByteArrayDownloadPtr ptr(new ByteArrayDownload(url));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-FileDownloadPtr DownloadJob::addFileDownload(QUrl url, QString rel_target_path)
-{
-	FileDownloadPtr ptr(new FileDownload(url, rel_target_path));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-CacheDownloadPtr DownloadJob::addCacheDownload(QUrl url, MetaEntryPtr entry)
-{
-	CacheDownloadPtr ptr(new CacheDownload(url, entry));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-ForgeXzDownloadPtr DownloadJob::addForgeXzDownload(QUrl url, MetaEntryPtr entry)
-{
-	ForgeXzDownloadPtr ptr(new ForgeXzDownload(url, entry));
-	ptr->index_within_job = downloads.size();
-	downloads.append(ptr);
-	parts_progress.append(part_info());
-	total_progress++;
-	return ptr;
-}
-
-void DownloadJob::partSucceeded(int index)
+void NetJob::partSucceeded(int index)
 {
 	// do progress. all slots are 1 in size at least
 	auto &slot = parts_progress[index];
@@ -73,7 +33,7 @@ void DownloadJob::partSucceeded(int index)
 	}
 }
 
-void DownloadJob::partFailed(int index)
+void NetJob::partFailed(int index)
 {
 	auto &slot = parts_progress[index];
 	if (slot.failures == 3)
@@ -97,7 +57,7 @@ void DownloadJob::partFailed(int index)
 	}
 }
 
-void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
+void NetJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTotal)
 {
 	auto &slot = parts_progress[index];
 
@@ -111,7 +71,7 @@ void DownloadJob::partProgress(int index, qint64 bytesReceived, qint64 bytesTota
 	emit progress(current_progress, total_progress);
 }
 
-void DownloadJob::start()
+void NetJob::start()
 {
 	QLOG_INFO() << m_job_name.toLocal8Bit() << " started.";
 	for (auto iter : downloads)
@@ -124,7 +84,7 @@ void DownloadJob::start()
 	}
 }
 
-QStringList DownloadJob::getFailedFiles()
+QStringList NetJob::getFailedFiles()
 {
 	QStringList failed;
 	for (auto download : downloads)

logic/net/NetJob.h

@@ -1,7 +1,7 @@
 #pragma once
 #include <QtNetwork>
 #include <QLabel>
-#include "Download.h"
+#include "NetAction.h"
 #include "ByteArrayDownload.h"
 #include "FileDownload.h"
 #include "CacheDownload.h"
@@ -9,51 +9,57 @@
 #include "ForgeXzDownload.h"
 #include "logic/tasks/ProgressProvider.h"
 
-class DownloadJob;
-typedef std::shared_ptr<DownloadJob> DownloadJobPtr;
+class NetJob;
+typedef std::shared_ptr<NetJob> NetJobPtr;
 
-/**
- * A single file for the downloader/cache to process.
- */
-class DownloadJob : public ProgressProvider
+class NetJob : public ProgressProvider
 {
 	Q_OBJECT
 public:
-	explicit DownloadJob(QString job_name)
-		:ProgressProvider(), m_job_name(job_name){};
-	
-	ByteArrayDownloadPtr addByteArrayDownload(QUrl url);
-	FileDownloadPtr      addFileDownload(QUrl url, QString rel_target_path);
-	CacheDownloadPtr     addCacheDownload(QUrl url, MetaEntryPtr entry);
-	ForgeXzDownloadPtr   addForgeXzDownload(QUrl url, MetaEntryPtr entry);
-	
-	DownloadPtr operator[](int index)
+	explicit NetJob(QString job_name) : ProgressProvider(), m_job_name(job_name) {};
+
+	template <typename T>
+	bool addNetAction(T action)
+	{
+		NetActionPtr base = std::static_pointer_cast<NetAction>(action); 
+		base->index_within_job = downloads.size();
+		downloads.append(action);
+		parts_progress.append(part_info());
+		total_progress++;
+		return true;
+	}
+
+	NetActionPtr operator[](int index)
 	{
 		return downloads[index];
-	};
-	DownloadPtr first()
+	}
+	;
+	NetActionPtr first()
 	{
-		if(downloads.size())
+		if (downloads.size())
 			return downloads[0];
-		return DownloadPtr();
+		return NetActionPtr();
 	}
 	int size() const
 	{
 		return downloads.size();
 	}
-	virtual void getProgress(qint64& current, qint64& total)
+	virtual void getProgress(qint64 &current, qint64 &total)
 	{
 		current = current_progress;
 		total = total_progress;
-	};
+	}
+	;
 	virtual QString getStatus() const
 	{
 		return m_job_name;
-	};
+	}
+	;
 	virtual bool isRunning() const
 	{
 		return m_running;
-	};
+	}
+	;
 	QStringList getFailedFiles();
 signals:
 	void started();
@@ -61,12 +67,15 @@ signals:
 	void filesProgress(int, int, int);
 	void succeeded();
 	void failed();
-public slots:
+public
+slots:
 	virtual void start();
-private slots:
+private
+slots:
 	void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);
 	void partSucceeded(int index);
 	void partFailed(int index);
+
 private:
 	struct part_info
 	{
@@ -75,7 +84,7 @@ private:
 		int failures = 0;
 	};
 	QString m_job_name;
-	QList<DownloadPtr> downloads;
+	QList<NetActionPtr> downloads;
 	QList<part_info> parts_progress;
 	qint64 current_progress = 0;
 	qint64 total_progress = 0;
@@ -83,4 +92,3 @@ private:
 	int num_failed = 0;
 	bool m_running = false;
 };
-

logic/net/S3ListBucket.cpp

@@ -0,0 +1,161 @@
+#include "S3ListBucket.h"
+#include "MultiMC.h"
+#include <logger/QsLog.h>
+#include <QUrlQuery>
+#include <qxmlstream.h>
+#include <QDomDocument>
+
+inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
+{
+	QDomNodeList elementList = parent.elementsByTagName(tagname);
+	if (elementList.count())
+		return elementList.at(0).toElement();
+	else
+		return QDomElement();
+}
+
+S3ListBucket::S3ListBucket(QUrl url) : NetAction()
+{
+	m_url = url;
+	m_status = Job_NotStarted;
+}
+
+void S3ListBucket::start()
+{
+	QUrl finalUrl = m_url;
+	if (current_marker.size())
+	{
+		QUrlQuery query;
+		query.addQueryItem("marker", current_marker);
+		finalUrl.setQuery(query);
+	}
+	QNetworkRequest request(finalUrl);
+	request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
+	auto worker = MMC->qnam();
+	QNetworkReply *rep = worker->get(request);
+
+	m_reply = std::shared_ptr<QNetworkReply>(rep);
+	connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
+			SLOT(downloadProgress(qint64, qint64)));
+	connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
+	connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
+			SLOT(downloadError(QNetworkReply::NetworkError)));
+	connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
+}
+
+void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
+{
+	emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal);
+}
+
+void S3ListBucket::downloadError(QNetworkReply::NetworkError error)
+{
+	// error happened during download.
+	QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
+				 << "Network error: " << error;
+	m_status = Job_Failed;
+}
+
+void S3ListBucket::processValidReply()
+{
+	QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker;
+	auto readContents = [&](QXmlStreamReader & xml)
+	{
+		QString Key, ETag, Size;
+		while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents"))
+		{
+			if (xml.tokenType() == QXmlStreamReader::StartElement)
+			{
+				if (xml.name() == "Key")
+				{
+					Key = xml.readElementText();
+				}
+				if (xml.name() == "ETag")
+				{
+					ETag = xml.readElementText();
+				}
+				if (xml.name() == "Size")
+				{
+					Size = xml.readElementText();
+				}
+			}
+			xml.readNext();
+		}
+		if (xml.error() != QXmlStreamReader::NoError)
+			return;
+		objects.append({Key, ETag, Size.toLongLong()});
+	};
+
+	// nothing went wrong...
+	QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
+	QByteArray ba = m_reply->readAll();
+
+	QString xmlErrorMsg;
+
+	bool is_truncated = false;
+	QXmlStreamReader xml(ba);
+	while (!xml.atEnd() && !xml.hasError())
+	{
+		/* Read next element.*/
+		QXmlStreamReader::TokenType token = xml.readNext();
+		/* If token is just StartDocument, we'll go to next.*/
+		if (token == QXmlStreamReader::StartDocument)
+		{
+			continue;
+		}
+		if (token == QXmlStreamReader::StartElement)
+		{
+			/* If it's named person, we'll dig the information from there.*/
+			if (xml.name() == "Contents")
+			{
+				readContents(xml);
+			}
+			else if (xml.name() == "IsTruncated")
+			{
+				is_truncated = (xml.readElementText() == "true");
+			}
+		}
+	}
+	if (xml.hasError())
+	{
+		QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:"
+					 << xml.errorString() << ba;
+		emit failed(index_within_job);
+		return;
+	}
+	if(is_truncated)
+	{
+		current_marker = objects.last().Key;
+		bytesSoFar += m_reply->size();
+		m_reply.reset();
+		start();
+	}
+	else
+	{
+		m_status = Job_Finished;
+		m_reply.reset();
+		emit succeeded(index_within_job);
+	}
+	return;
+}
+
+void S3ListBucket::downloadFinished()
+{
+	// if the download succeeded
+	if (m_status != Job_Failed)
+	{
+		processValidReply();
+	}
+	// else the download failed
+	else
+	{
+		m_reply.reset();
+		emit failed(index_within_job);
+		return;
+	}
+}
+
+void S3ListBucket::downloadReadyRead()
+{
+	// ~_~
+}

logic/net/S3ListBucket.h

@@ -0,0 +1,42 @@
+#pragma once
+#include "NetAction.h"
+
+struct S3Object
+{
+	QString Key;
+	QString ETag;
+	qlonglong size;
+};
+
+typedef std::shared_ptr<class S3ListBucket> S3ListBucketPtr;
+class S3ListBucket : public NetAction
+{
+	Q_OBJECT
+public:
+	S3ListBucket(QUrl url);
+	static S3ListBucketPtr make(QUrl url)
+	{
+		return S3ListBucketPtr(new S3ListBucket(url));
+	}
+
+public:
+	QList<S3Object> objects;
+
+public
+slots:
+	virtual void start() override;
+
+protected
+slots:
+	virtual void downloadProgress(qint64 bytesReceived, qint64 bytesTotal) override;
+	virtual void downloadError(QNetworkReply::NetworkError error) override;
+	virtual void downloadFinished() override;
+	virtual void downloadReadyRead() override;
+
+private:
+	void processValidReply();
+
+private:
+	qint64 bytesSoFar = 0;
+	QString current_marker;
+};