CactiChameleon9/PrismLauncher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Speed up the downloads. Massively.

Browse Source
This commit is contained in:
Petr Mrázek
2013-09-02 00:25:40 +02:00
parent 78e278c1e3
commit f8e8414d54
18 changed files with 264 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
logic/net/DownloadJob.cpp
View File

@ -2,9 +2,7 @@
#include "pathutils.h"
#include "NetWorker.h"
DownloadJob::DownloadJob (QUrl url,
QString target_path,
QString expected_md5 )
Download::Download (QUrl url, QString target_path, QString expected_md5 )
:Job()
{
m_url = url;
@ -17,14 +15,7 @@ DownloadJob::DownloadJob (QUrl url,
m_opened_for_saving = false;
}
JobPtr DownloadJob::create (QUrl url,
QString target_path,
QString expected_md5 )
{
return JobPtr ( new DownloadJob ( url, target_path, expected_md5 ) );
}
void DownloadJob::start()
void Download::start()
{
if ( m_save_to_file )
{
@ -40,7 +31,7 @@ void DownloadJob::start()
if ( m_check_md5 && hash == m_expected_md5 )
{
qDebug() << "Skipping " << m_url.toString() << ": md5 match.";
emit finish();
emit succeeded(index_within_job);
return;
}
else
@ -50,7 +41,7 @@ void DownloadJob::start()
}
if(!ensureFilePathExists(filename))
{
emit fail();
emit failed(index_within_job);
return;
}
}
@ -58,7 +49,7 @@ void DownloadJob::start()
QNetworkRequest request ( m_url );
request.setRawHeader(QString("If-None-Match").toLatin1(), m_expected_md5.toLatin1());
auto &worker = NetWorker::spawn();
auto &worker = NetWorker::qnam();
QNetworkReply * rep = worker.get ( request );
m_reply = QSharedPointer<QNetworkReply> ( rep, &QObject::deleteLater );
@ -68,19 +59,19 @@ void DownloadJob::start()
connect ( rep, SIGNAL ( readyRead() ), SLOT ( downloadReadyRead() ) );
}
void DownloadJob::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
void Download::downloadProgress ( qint64 bytesReceived, qint64 bytesTotal )
{
emit progress ( bytesReceived, bytesTotal );
emit progress (index_within_job, bytesReceived, bytesTotal );
}
void DownloadJob::downloadError ( QNetworkReply::NetworkError error )
void Download::downloadError ( QNetworkReply::NetworkError error )
{
// error happened during download.
// TODO: log the reason why
m_status = Job_Failed;
}
void DownloadJob::downloadFinished()
void Download::downloadFinished()
{
// if the download succeeded
if ( m_status != Job_Failed )
@ -99,7 +90,7 @@ void DownloadJob::downloadFinished()
//TODO: check md5 here!
m_reply.clear();
emit finish();
emit succeeded(index_within_job);
return;
}
// else the download failed
@ -111,12 +102,12 @@ void DownloadJob::downloadFinished()
m_output_file.remove();
}
m_reply.clear();
emit fail();
emit failed(index_within_job);
return;
}
}
void DownloadJob::downloadReadyRead()
void Download::downloadReadyRead()
{
if( m_save_to_file )
{
@ -128,7 +119,7 @@ void DownloadJob::downloadReadyRead()
* Can't open the file... the job failed
*/
m_reply->abort();
emit fail();
emit failed(index_within_job);
return;
}
m_opened_for_saving = true;
@ -136,3 +127,58 @@ void DownloadJob::downloadReadyRead()
m_output_file.write ( m_reply->readAll() );
}
}
DownloadPtr DownloadJob::add ( QUrl url, QString rel_target_path, QString expected_md5 )
{
DownloadPtr ptr (new Download(url, rel_target_path, expected_md5));
ptr->index_within_job = downloads.size();
downloads.append(ptr);
return ptr;
}
void DownloadJob::partSucceeded ( int index )
{
num_succeeded++;
if(num_failed + num_succeeded == downloads.size())
{
if(num_failed)
{
emit failed();
}
else
{
emit succeeded();
}
}
}
void DownloadJob::partFailed ( int index )
{
num_failed++;
if(num_failed + num_succeeded == downloads.size())
{
if(num_failed)
{
emit failed();
}
else
{
emit succeeded();
}
}
}
void DownloadJob::partProgress ( int index, qint64 bytesReceived, qint64 bytesTotal )
{
// PROGRESS? DENIED!
}
void DownloadJob::start()
{
for(auto iter: downloads)
{
connect(iter.data(), SIGNAL(succeeded(int)), SLOT(partSucceeded(int)));
iter->start();
}
}

111
logic/net/DownloadJob.h
View File

@ -1,28 +1,37 @@
#pragma once
#include "JobQueue.h"
#include <QtNetwork>
/**
* A single file for the downloader/cache to process.
*/
class DownloadJob : public Job
class DownloadJob;
class Download;
typedef QSharedPointer<DownloadJob> DownloadJobPtr;
typedef QSharedPointer<Download> DownloadPtr;
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed
};
class Job : public QObject
{
Q_OBJECT
protected:
explicit Job(): QObject(0){};
public:
DownloadJob(QUrl url,
QString rel_target_path = QString(),
QString expected_md5 = QString()
);
static JobPtr create(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
virtual ~Job() {};
public slots:
virtual void start();
private slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
virtual void start() = 0;
};
class Download: public Job
{
friend class DownloadJob;
Q_OBJECT
protected:
Download(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
public:
/// the network reply
QSharedPointer<QNetworkReply> m_reply;
@ -46,6 +55,72 @@ public:
/// if not saving to file, downloaded data is placed here
QByteArray m_data;
int currentProgress = 0;
int totalProgress = 0;
/// The file's status
JobStatus m_status;
int index_within_job = 0;
signals:
void started(int index);
void progress(int index, qint64 current, qint64 total);
void succeeded(int index);
void failed(int index);
public slots:
virtual void start();
private slots:
void downloadProgress(qint64 bytesReceived, qint64 bytesTotal);;
void downloadError(QNetworkReply::NetworkError error);
void downloadFinished();
void downloadReadyRead();
};
/**
* A single file for the downloader/cache to process.
*/
class DownloadJob : public Job
{
Q_OBJECT
public:
explicit DownloadJob()
:Job(){};
explicit DownloadJob(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString())
:Job()
{
add(url, rel_target_path, expected_md5);
};
DownloadPtr add(QUrl url, QString rel_target_path = QString(), QString expected_md5 = QString());
DownloadPtr operator[](int index)
{
return downloads[index];
};
DownloadPtr first()
{
if(downloads.size())
return downloads[0];
return DownloadPtr();
}
int size() const
{
return downloads.size();
}
signals:
void started();
void progress(qint64 current, qint64 total);
void succeeded();
void failed();
public slots:
virtual void start();
private slots:
void partProgress(int index, qint64 bytesReceived, qint64 bytesTotal);;
void partSucceeded(int index);
void partFailed(int index);
private:
QList<DownloadPtr> downloads;
int num_succeeded = 0;
int num_failed = 0;
};

180
logic/net/JobQueue.h
View File

@ -1,180 +0,0 @@
#pragma once
#include <QtCore>
#include "libutil_config.h"
enum JobStatus
{
Job_NotStarted,
Job_InProgress,
Job_Finished,
Job_Failed
};
class JobList;
class Job : public QObject
{
Q_OBJECT
protected:
explicit Job(): QObject(0){};
public:
virtual ~Job() {};
signals:
void finish();
void fail();
void progress(qint64 current, qint64 total);
public slots:
virtual void start() = 0;
};
typedef QSharedPointer<Job> JobPtr;
/**
* A list of jobs, to be processed one by one.
*/
class JobList : public QObject
{
friend class JobListQueue;
Q_OBJECT
public:
JobList() : QObject(0)
{
m_status = Job_NotStarted;
current_job_idx = 0;
}
JobStatus getStatus()
{
return m_status;
}
void add(JobPtr dlable)
{
if(m_status == Job_NotStarted)
m_jobs.append(dlable);
//else there's a bug. TODO: catch the bugs
}
JobPtr getFirstJob()
{
if(m_jobs.size())
return m_jobs[0];
else
return JobPtr();
}
void start()
{
current_job_idx = 0;
auto job = m_jobs[current_job_idx];
connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
job->start();
emit started();
}
private slots:
void currentJobFinished()
{
if(current_job_idx == m_jobs.size() - 1)
{
m_status = Job_Finished;
emit finished();
}
else
{
current_job_idx++;
auto job = m_jobs[current_job_idx];
connect(job.data(), SIGNAL(progress(qint64,qint64)), SLOT(currentJobProgress(qint64,qint64)));
connect(job.data(), SIGNAL(finish()), SLOT(currentJobFinished()));
connect(job.data(), SIGNAL(fail()), SLOT(currentJobFailed()));
job->start();
}
}
void currentJobFailed()
{
m_status = Job_Failed;
emit failed();
}
void currentJobProgress(qint64 current, qint64 total)
{
if(!total)
return;
int total_jobs = m_jobs.size();
if(!total_jobs)
return;
float job_chunk = 1000.0 / float(total_jobs);
float cur = current;
float tot = total;
float last_chunk = (cur / tot) * job_chunk;
float list_total = job_chunk * current_job_idx + last_chunk;
emit progress(qint64(list_total), 1000LL);
}
private:
QVector<JobPtr> m_jobs;
/// The overall status of this job list
JobStatus m_status;
int current_job_idx;
signals:
void progress(qint64 current, qint64 total);
void started();
void finished();
void failed();
};
typedef QSharedPointer<JobList> JobListPtr;
/**
* A queue of job lists! The job lists fail or finish as units.
*/
class JobListQueue : public QObject
{
Q_OBJECT
public:
JobListQueue(QObject *p = 0):
QObject(p),
currentIndex(0),
is_running(false){}
void enqueue(JobListPtr job)
{
jobs.enqueue(job);
// finish or fail, we should catch that and start the next one
connect(job.data(),SIGNAL(finished()), SLOT(startNextJob()));
connect(job.data(),SIGNAL(failed()), SLOT(startNextJob()));
if(!is_running)
{
QTimer::singleShot(0, this, SLOT(startNextJob()));
}
}
private slots:
void startNextJob()
{
if (jobs.isEmpty())
{
currentJobList.clear();
currentIndex = 0;
is_running = false;
emit finishedAllJobs();
return;
}
currentJobList = jobs.dequeue();
is_running = true;
currentIndex = 0;
currentJobList->start();
}
signals:
void finishedAllJobs();
private:
JobListPtr currentJobList;
QQueue<JobListPtr> jobs;
unsigned currentIndex;
bool is_running;
};

20
logic/net/NetWorker.cpp
View File

@ -1,7 +1,25 @@
#include "NetWorker.h"
#include <QThreadStorage>
NetWorker& NetWorker::spawn()
class NetWorker::Private
{
public:
QNetworkAccessManager manager;
};
NetWorker::NetWorker ( QObject* parent ) : QObject ( parent )
{
d = new Private();
}
QNetworkAccessManager& NetWorker::qnam()
{
auto & w = worker();
return w.d->manager;
}
NetWorker& NetWorker::worker()
{
static QThreadStorage<NetWorker *> storage;
if (!storage.hasLocalData())

17
logic/net/NetWorker.h
View File

@ -9,12 +9,23 @@
'OOOOOY _.'
'""""''
*/
#pragma once
#include <QNetworkAccessManager>
class NetWorker : public QNetworkAccessManager
#include <QUrl>
class NetWorker : public QObject
{
Q_OBJECT
public:
static NetWorker &spawn();
// for high level access to the sevices (preferred)
static NetWorker &worker();
// for low-level access to the network manager object
static QNetworkAccessManager &qnam();
public:
private:
explicit NetWorker ( QObject* parent = 0 );
class Private;
Private * d;
};