S3 bucket listing support and network code refactors.

* Adds support for listing all objects in an S3 bucket. * Renames a bunch of network related classes (Download->Action) * Net actions now have static constructors
2013-10-26 19:55:48 +02:00
parent c467ebf132
commit 9233477295
27 changed files with 405 additions and 243 deletions
--- a/logic/net/S3ListBucket.cpp
+++ b/logic/net/S3ListBucket.cpp
@ -0,0 +1,161 @@
+#include "S3ListBucket.h"
+#include "MultiMC.h"
+#include <logger/QsLog.h>
+#include <QUrlQuery>
+#include <qxmlstream.h>
+#include <QDomDocument>
+
+inline QDomElement getDomElementByTagName(QDomElement parent, QString tagname)
+{
+	QDomNodeList elementList = parent.elementsByTagName(tagname);
+	if (elementList.count())
+		return elementList.at(0).toElement();
+	else
+		return QDomElement();
+}
+
+S3ListBucket::S3ListBucket(QUrl url) : NetAction()
+{
+	m_url = url;
+	m_status = Job_NotStarted;
+}
+
+void S3ListBucket::start()
+{
+	QUrl finalUrl = m_url;
+	if (current_marker.size())
+	{
+		QUrlQuery query;
+		query.addQueryItem("marker", current_marker);
+		finalUrl.setQuery(query);
+	}
+	QNetworkRequest request(finalUrl);
+	request.setHeader(QNetworkRequest::UserAgentHeader, "MultiMC/5.0 (Uncached)");
+	auto worker = MMC->qnam();
+	QNetworkReply *rep = worker->get(request);
+
+	m_reply = std::shared_ptr<QNetworkReply>(rep);
+	connect(rep, SIGNAL(downloadProgress(qint64, qint64)),
+			SLOT(downloadProgress(qint64, qint64)));
+	connect(rep, SIGNAL(finished()), SLOT(downloadFinished()));
+	connect(rep, SIGNAL(error(QNetworkReply::NetworkError)),
+			SLOT(downloadError(QNetworkReply::NetworkError)));
+	connect(rep, SIGNAL(readyRead()), SLOT(downloadReadyRead()));
+}
+
+void S3ListBucket::downloadProgress(qint64 bytesReceived, qint64 bytesTotal)
+{
+	emit progress(index_within_job, bytesSoFar + bytesReceived, bytesSoFar + bytesTotal);
+}
+
+void S3ListBucket::downloadError(QNetworkReply::NetworkError error)
+{
+	// error happened during download.
+	QLOG_ERROR() << "Error getting URL:" << m_url.toString().toLocal8Bit()
+				 << "Network error: " << error;
+	m_status = Job_Failed;
+}
+
+void S3ListBucket::processValidReply()
+{
+	QLOG_TRACE() << "GOT: " << m_url.toString() << " marker:" << current_marker;
+	auto readContents = [&](QXmlStreamReader & xml)
+	{
+		QString Key, ETag, Size;
+		while (!(xml.tokenType() == QXmlStreamReader::EndElement && xml.name() == "Contents"))
+		{
+			if (xml.tokenType() == QXmlStreamReader::StartElement)
+			{
+				if (xml.name() == "Key")
+				{
+					Key = xml.readElementText();
+				}
+				if (xml.name() == "ETag")
+				{
+					ETag = xml.readElementText();
+				}
+				if (xml.name() == "Size")
+				{
+					Size = xml.readElementText();
+				}
+			}
+			xml.readNext();
+		}
+		if (xml.error() != QXmlStreamReader::NoError)
+			return;
+		objects.append({Key, ETag, Size.toLongLong()});
+	};
+
+	// nothing went wrong...
+	QString prefix("http://s3.amazonaws.com/Minecraft.Resources/");
+	QByteArray ba = m_reply->readAll();
+
+	QString xmlErrorMsg;
+
+	bool is_truncated = false;
+	QXmlStreamReader xml(ba);
+	while (!xml.atEnd() && !xml.hasError())
+	{
+		/* Read next element.*/
+		QXmlStreamReader::TokenType token = xml.readNext();
+		/* If token is just StartDocument, we'll go to next.*/
+		if (token == QXmlStreamReader::StartDocument)
+		{
+			continue;
+		}
+		if (token == QXmlStreamReader::StartElement)
+		{
+			/* If it's named person, we'll dig the information from there.*/
+			if (xml.name() == "Contents")
+			{
+				readContents(xml);
+			}
+			else if (xml.name() == "IsTruncated")
+			{
+				is_truncated = (xml.readElementText() == "true");
+			}
+		}
+	}
+	if (xml.hasError())
+	{
+		QLOG_ERROR() << "Failed to process s3.amazonaws.com/Minecraft.Resources. XML error:"
+					 << xml.errorString() << ba;
+		emit failed(index_within_job);
+		return;
+	}
+	if(is_truncated)
+	{
+		current_marker = objects.last().Key;
+		bytesSoFar += m_reply->size();
+		m_reply.reset();
+		start();
+	}
+	else
+	{
+		m_status = Job_Finished;
+		m_reply.reset();
+		emit succeeded(index_within_job);
+	}
+	return;
+}
+
+void S3ListBucket::downloadFinished()
+{
+	// if the download succeeded
+	if (m_status != Job_Failed)
+	{
+		processValidReply();
+	}
+	// else the download failed
+	else
+	{
+		m_reply.reset();
+		emit failed(index_within_job);
+		return;
+	}
+}
+
+void S3ListBucket::downloadReadyRead()
+{
+	// ~_~
+}