From 0264af9b73182a30dc7df23efdd51bfa4303ae8b Mon Sep 17 00:00:00 2001 From: Floris Bos Date: Tue, 10 Mar 2020 23:22:20 +0100 Subject: [PATCH] Performance improvements - Use larger buffer size when writing uncompressed files (was libcurl's default of 16 kb, change it to 128 kb) - Uncompress the next MB of data, while it is hashing/writing to disk in seperate thread. --- config.h | 3 +++ debian/changelog | 1 + downloadextractthread.cpp | 32 ++++++++++++++++++++++---------- downloadextractthread.h | 6 +++++- downloadthread.cpp | 25 +++++++++++++++++-------- downloadthread.h | 9 ++++++++- imagewriter.cpp | 1 + 7 files changed, 57 insertions(+), 20 deletions(-) diff --git a/config.h b/config.h index b4c21f0..b81922c 100644 --- a/config.h +++ b/config.h @@ -22,6 +22,9 @@ /* Block size used for writes (currently used when using .zip images only) */ #define IMAGEWRITER_BLOCKSIZE 1*1024*1024 +/* Block size used with uncompressed images */ +#define IMAGEWRITER_UNCOMPRESSED_BLOCKSIZE 128*1024 + /* Block size used when reading during verify stage */ #define IMAGEWRITER_VERIFY_BLOCKSIZE 128*1024 diff --git a/debian/changelog b/debian/changelog index 62e9546..4fd3bce 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ rpi-imager (1.2) unstable; urgency=medium * Mention version number in title bar. + * Performance improvements -- Floris Bos Tue, 10 Mar 2020 17:08:11 +0100 diff --git a/downloadextractthread.cpp b/downloadextractthread.cpp index acf2766..1e69036 100644 --- a/downloadextractthread.cpp +++ b/downloadextractthread.cpp @@ -45,10 +45,11 @@ protected: DownloadExtractThread::DownloadExtractThread(const QByteArray &url, const QByteArray &localfilename, const QByteArray &expectedHash, QObject *parent) : DownloadThread(url, localfilename, expectedHash, parent), _abufsize(IMAGEWRITER_BLOCKSIZE), _ethreadStarted(false), - _isImage(true), _inputHash(OSLIST_HASH_ALGORITHM) + _isImage(true), _inputHash(OSLIST_HASH_ALGORITHM), _activeBuf(0), _writeThreadStarted(false) { _extractThread = new _extractThreadClass(this); - _abuf = (char *) qMallocAligned(_abufsize, 4096); + _abuf[0] = (char *) qMallocAligned(_abufsize, 4096); + _abuf[1] = (char *) qMallocAligned(_abufsize, 4096); } DownloadExtractThread::~DownloadExtractThread() @@ -59,7 +60,8 @@ DownloadExtractThread::~DownloadExtractThread() } _queue.clear(); _cv.notify_one(); - qFreeAligned(_abuf); + qFreeAligned(_abuf[0]); + qFreeAligned(_abuf[1]); } size_t DownloadExtractThread::_writeData(const char *buf, size_t len) @@ -147,24 +149,34 @@ void DownloadExtractThread::extractImageRun() while (true) { - ssize_t size = archive_read_data(a, _abuf, _abufsize); + ssize_t size = archive_read_data(a, _abuf[_activeBuf], _abufsize); if (size < 0) throw runtime_error(archive_error_string(a)); if (size == 0) break; - if (_writeFile(_abuf, size) != (size_t) size) + if (_writeThreadStarted) { - if (!_cancelled) + //if (_writeFile(_abuf, size) != (size_t) size) + if (!_writeFuture.result()) { - DownloadThread::cancelDownload(); - emit error(tr("Error writing to storage")); + if (!_cancelled) + { + DownloadThread::cancelDownload(); + emit error(tr("Error writing to storage")); + } + archive_read_free(a); + return; } - archive_read_free(a); - return; } + + _writeFuture = QtConcurrent::run(static_cast(this), &DownloadThread::_writeFile, _abuf[_activeBuf], size); + _activeBuf = _activeBuf ? 0 : 1; + _writeThreadStarted = true; } + if (_writeThreadStarted) + _writeFuture.waitForFinished(); _writeComplete(); } catch (exception &e) diff --git a/downloadextractthread.h b/downloadextractthread.h index ec54085..0fc7a07 100644 --- a/downloadextractthread.h +++ b/downloadextractthread.h @@ -9,6 +9,7 @@ #include "downloadthread.h" #include #include +#include class _extractThreadClass; @@ -32,7 +33,7 @@ public: virtual void enableMultipleFileExtraction(); protected: - char *_abuf; + char *_abuf[2]; size_t _abufsize; _extractThreadClass *_extractThread; std::deque _queue; @@ -41,6 +42,9 @@ protected: std::condition_variable _cv; bool _ethreadStarted, _isImage; QCryptographicHash _inputHash; + int _activeBuf; + bool _writeThreadStarted; + QFuture _writeFuture; QByteArray _popQueue(); void _pushQueue(const char *data, size_t len); diff --git a/downloadthread.cpp b/downloadthread.cpp index 4726c80..2ed035b 100644 --- a/downloadthread.cpp +++ b/downloadthread.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #ifdef Q_OS_LINUX #include @@ -33,7 +34,7 @@ int DownloadThread::_curlCount = 0; DownloadThread::DownloadThread(const QByteArray &url, const QByteArray &localfilename, const QByteArray &expectedHash, QObject *parent) : QThread(parent), _startOffset(0), _lastDlTotal(0), _lastDlNow(0), _verifyTotal(0), _lastVerifyNow(0), _bytesWritten(0), _url(url), _filename(localfilename), _expectedHash(expectedHash), _firstBlock(nullptr), _cancelled(false), _successful(false), _verifyEnabled(false), _cacheEnabled(false), _lastModified(0), _serverTime(0), _lastFailureTime(0), - _file(NULL), _writehash(OSLIST_HASH_ALGORITHM), _verifyhash(OSLIST_HASH_ALGORITHM) + _file(NULL), _writehash(OSLIST_HASH_ALGORITHM), _verifyhash(OSLIST_HASH_ALGORITHM), _inputBufferSize(0) { if (!_curlCount) curl_global_init(CURL_GLOBAL_DEFAULT); @@ -279,6 +280,8 @@ void DownloadThread::run() curl_easy_setopt(_c, CURLOPT_FAILONERROR, 1); curl_easy_setopt(_c, CURLOPT_HEADERFUNCTION, &DownloadThread::_curl_header_callback); curl_easy_setopt(_c, CURLOPT_HEADERDATA, this); + if (_inputBufferSize) + curl_easy_setopt(_c, CURLOPT_BUFFERSIZE, _inputBufferSize); if (!_useragent.isEmpty()) curl_easy_setopt(_c, CURLOPT_USERAGENT, _useragent.constData()); @@ -379,22 +382,26 @@ void DownloadThread::setCacheFile(const QString &filename, qint64 filesize) } } +void DownloadThread::_hashData(const char *buf, size_t len) +{ + _writehash.addData(buf, len); +} + size_t DownloadThread::_writeFile(const char *buf, size_t len) { if (_cancelled) return len; - _writehash.addData(buf, len); - if (!_firstBlock) { + _writehash.addData(buf, len); _firstBlock = (char *) qMallocAligned(len, 4096); _firstBlockSize = len; ::memcpy(_firstBlock, buf, len); - //_firstBlock = QByteArray(buf, len); return _file.seek(len) ? len : 0; } + QFuture wh = QtConcurrent::run(this, &DownloadThread::_hashData, buf, len); qint64 written = _file.write(buf, len); _bytesWritten += written; @@ -402,12 +409,9 @@ size_t DownloadThread::_writeFile(const char *buf, size_t len) if ((size_t) written != len) { qDebug() << "Write error:" << _file.errorString() << "while writing len:" << len; - if (len % 512) - qDebug() << "NOT SECTOR SIZE write len:" << len; - if (((uintptr_t)buf) % 512) - qDebug() << "Memory not aligned to sector size"; } + wh.waitForFinished(); return (written < 0) ? 0 : written; } @@ -646,3 +650,8 @@ bool DownloadThread::isImage() { return true; } + +void DownloadThread::setInputBufferSize(int len) +{ + _inputBufferSize = len; +} diff --git a/downloadthread.h b/downloadthread.h index c8457e7..98b3931 100644 --- a/downloadthread.h +++ b/downloadthread.h @@ -104,6 +104,11 @@ public: */ void setCacheFile(const QString &filename, qint64 filesize = 0); + /* + * Set input buffer size + */ + void setInputBufferSize(int len); + /* * Thread safe download progress query functions */ @@ -114,6 +119,7 @@ public: uint64_t bytesWritten(); virtual bool isImage(); + size_t _writeFile(const char *buf, size_t len); signals: void success(); @@ -126,7 +132,7 @@ protected: virtual void _onDownloadSuccess(); virtual void _onDownloadError(const QString &msg); - size_t _writeFile(const char *buf, size_t len); + void _hashData(const char *buf, size_t len); void _writeComplete(); bool _verify(); int _authopen(const QByteArray &filename); @@ -155,6 +161,7 @@ protected: bool _cancelled, _successful, _verifyEnabled, _cacheEnabled; time_t _lastModified, _serverTime, _lastFailureTime; QElapsedTimer _timer; + int _inputBufferSize; #ifdef Q_OS_WIN WinFile _file, _volumeFile; diff --git a/imagewriter.cpp b/imagewriter.cpp index 2ea7473..6d1a8d5 100644 --- a/imagewriter.cpp +++ b/imagewriter.cpp @@ -172,6 +172,7 @@ void ImageWriter::startWrite() else { _thread = new DownloadThread(urlstr, _dst.toLatin1(), _expectedHash, this); + _thread->setInputBufferSize(IMAGEWRITER_UNCOMPRESSED_BLOCKSIZE); } _powersave.applyBlock(tr("Downloading and writing image"));