Performance improvements

- Use larger buffer size when writing uncompressed files
  (was libcurl's default of 16 kb, change it to 128 kb)
- Uncompress the next MB of data, while it is hashing/writing to
  disk in seperate thread.

config.h

@@ -22,6 +22,9 @@
 /* Block size used for writes (currently used when using .zip images only) */
 #define IMAGEWRITER_BLOCKSIZE             1*1024*1024
 
+/* Block size used with uncompressed images */
+#define IMAGEWRITER_UNCOMPRESSED_BLOCKSIZE 128*1024
+
 /* Block size used when reading during verify stage */
 #define IMAGEWRITER_VERIFY_BLOCKSIZE      128*1024
 

debian/changelog

@@ -1,6 +1,7 @@
 rpi-imager (1.2) unstable; urgency=medium
 
   * Mention version number in title bar.
+  * Performance improvements
 
  -- Floris Bos <bos@je-eigen-domein.nl>  Tue, 10 Mar 2020 17:08:11 +0100
 

downloadextractthread.cpp

@@ -45,10 +45,11 @@ protected:
 
 DownloadExtractThread::DownloadExtractThread(const QByteArray &url, const QByteArray &localfilename, const QByteArray &expectedHash, QObject *parent)
     : DownloadThread(url, localfilename, expectedHash, parent), _abufsize(IMAGEWRITER_BLOCKSIZE), _ethreadStarted(false),
-      _isImage(true), _inputHash(OSLIST_HASH_ALGORITHM)
+      _isImage(true), _inputHash(OSLIST_HASH_ALGORITHM), _activeBuf(0), _writeThreadStarted(false)
 {
     _extractThread = new _extractThreadClass(this);
-    _abuf = (char *) qMallocAligned(_abufsize, 4096);
+    _abuf[0] = (char *) qMallocAligned(_abufsize, 4096);
+    _abuf[1] = (char *) qMallocAligned(_abufsize, 4096);
 }
 
 DownloadExtractThread::~DownloadExtractThread()
@@ -59,7 +60,8 @@ DownloadExtractThread::~DownloadExtractThread()
     }
     _queue.clear();
     _cv.notify_one();
-    qFreeAligned(_abuf);
+    qFreeAligned(_abuf[0]);
+    qFreeAligned(_abuf[1]);
 }
 
 size_t DownloadExtractThread::_writeData(const char *buf, size_t len)
@@ -147,13 +149,16 @@ void DownloadExtractThread::extractImageRun()
 
         while (true)
         {
-            ssize_t size = archive_read_data(a, _abuf, _abufsize);
+            ssize_t size = archive_read_data(a, _abuf[_activeBuf], _abufsize);
             if (size < 0)
                 throw runtime_error(archive_error_string(a));
             if (size == 0)
                 break;
 
-            if (_writeFile(_abuf, size) != (size_t) size)
+            if (_writeThreadStarted)
+            {
+                //if (_writeFile(_abuf, size) != (size_t) size)
+                if (!_writeFuture.result())
                 {
                     if (!_cancelled)
                     {
@@ -165,6 +170,13 @@ void DownloadExtractThread::extractImageRun()
                 }
             }
 
+            _writeFuture = QtConcurrent::run(static_cast<DownloadThread *>(this), &DownloadThread::_writeFile, _abuf[_activeBuf], size);
+            _activeBuf = _activeBuf ? 0 : 1;
+            _writeThreadStarted = true;
+        }
+
+        if (_writeThreadStarted)
+            _writeFuture.waitForFinished();
         _writeComplete();
     }
     catch (exception &e)

downloadextractthread.h

@@ -9,6 +9,7 @@
 #include "downloadthread.h"
 #include <deque>
 #include <condition_variable>
+#include <QtConcurrent/QtConcurrent>
 
 class _extractThreadClass;
 
@@ -32,7 +33,7 @@ public:
     virtual void enableMultipleFileExtraction();
 
 protected:
-    char *_abuf;
+    char *_abuf[2];
     size_t _abufsize;
     _extractThreadClass *_extractThread;
     std::deque<QByteArray> _queue;
@@ -41,6 +42,9 @@ protected:
     std::condition_variable _cv;
     bool _ethreadStarted, _isImage;
     QCryptographicHash _inputHash;
+    int _activeBuf;
+    bool _writeThreadStarted;
+    QFuture<size_t> _writeFuture;
 
     QByteArray _popQueue();
     void _pushQueue(const char *data, size_t len);

downloadthread.cpp

@@ -18,6 +18,7 @@
 #include <regex>
 #include <QDebug>
 #include <QProcess>
+#include <QtConcurrent/QtConcurrent>
 
 #ifdef Q_OS_LINUX
 #include <sys/ioctl.h>
@@ -33,7 +34,7 @@ int DownloadThread::_curlCount = 0;
 DownloadThread::DownloadThread(const QByteArray &url, const QByteArray &localfilename, const QByteArray &expectedHash, QObject *parent) :
     QThread(parent), _startOffset(0), _lastDlTotal(0), _lastDlNow(0), _verifyTotal(0), _lastVerifyNow(0), _bytesWritten(0), _url(url), _filename(localfilename), _expectedHash(expectedHash),
     _firstBlock(nullptr), _cancelled(false), _successful(false), _verifyEnabled(false), _cacheEnabled(false), _lastModified(0), _serverTime(0),  _lastFailureTime(0),
-    _file(NULL), _writehash(OSLIST_HASH_ALGORITHM), _verifyhash(OSLIST_HASH_ALGORITHM)
+    _file(NULL), _writehash(OSLIST_HASH_ALGORITHM), _verifyhash(OSLIST_HASH_ALGORITHM), _inputBufferSize(0)
 {
     if (!_curlCount)
         curl_global_init(CURL_GLOBAL_DEFAULT);
@@ -279,6 +280,8 @@ void DownloadThread::run()
     curl_easy_setopt(_c, CURLOPT_FAILONERROR, 1);
     curl_easy_setopt(_c, CURLOPT_HEADERFUNCTION, &DownloadThread::_curl_header_callback);
     curl_easy_setopt(_c, CURLOPT_HEADERDATA, this);
+    if (_inputBufferSize)
+        curl_easy_setopt(_c, CURLOPT_BUFFERSIZE, _inputBufferSize);
 
     if (!_useragent.isEmpty())
         curl_easy_setopt(_c, CURLOPT_USERAGENT, _useragent.constData());
@@ -379,22 +382,26 @@ void DownloadThread::setCacheFile(const QString &filename, qint64 filesize)
     }
 }
 
+void DownloadThread::_hashData(const char *buf, size_t len)
+{
+    _writehash.addData(buf, len);
+}
+
 size_t DownloadThread::_writeFile(const char *buf, size_t len)
 {
     if (_cancelled)
         return len;
 
-    _writehash.addData(buf, len);
-
     if (!_firstBlock)
     {
+        _writehash.addData(buf, len);
         _firstBlock = (char *) qMallocAligned(len, 4096);
         _firstBlockSize = len;
         ::memcpy(_firstBlock, buf, len);
-        //_firstBlock = QByteArray(buf, len);
 
         return _file.seek(len) ? len : 0;
     }
+    QFuture<void> wh = QtConcurrent::run(this, &DownloadThread::_hashData, buf, len);
 
     qint64 written = _file.write(buf, len);
     _bytesWritten += written;
@@ -402,12 +409,9 @@ size_t DownloadThread::_writeFile(const char *buf, size_t len)
     if ((size_t) written != len)
     {
         qDebug() << "Write error:" << _file.errorString() << "while writing len:" << len;
-        if (len % 512)
-            qDebug() << "NOT SECTOR SIZE write len:" << len;
-        if (((uintptr_t)buf) % 512)
-            qDebug() << "Memory not aligned to sector size";
     }
 
+    wh.waitForFinished();
     return (written < 0) ? 0 : written;
 }
 
@@ -646,3 +650,8 @@ bool DownloadThread::isImage()
 {
     return true;
 }
+
+void DownloadThread::setInputBufferSize(int len)
+{
+    _inputBufferSize = len;
+}

downloadthread.h

@@ -104,6 +104,11 @@ public:
      */
     void setCacheFile(const QString &filename, qint64 filesize = 0);
 
+    /*
+     * Set input buffer size
+     */
+    void setInputBufferSize(int len);
+
     /*
      * Thread safe download progress query functions
      */
@@ -114,6 +119,7 @@ public:
     uint64_t bytesWritten();
 
     virtual bool isImage();
+    size_t _writeFile(const char *buf, size_t len);
 
 signals:
     void success();
@@ -126,7 +132,7 @@ protected:
     virtual void _onDownloadSuccess();
     virtual void _onDownloadError(const QString &msg);
 
-    size_t _writeFile(const char *buf, size_t len);
+    void _hashData(const char *buf, size_t len);
     void _writeComplete();
     bool _verify();
     int _authopen(const QByteArray &filename);
@@ -155,6 +161,7 @@ protected:
     bool _cancelled, _successful, _verifyEnabled, _cacheEnabled;
     time_t _lastModified, _serverTime, _lastFailureTime;
     QElapsedTimer _timer;
+    int _inputBufferSize;
 
 #ifdef Q_OS_WIN
     WinFile _file, _volumeFile;

imagewriter.cpp

@@ -172,6 +172,7 @@ void ImageWriter::startWrite()
     else
     {
         _thread = new DownloadThread(urlstr, _dst.toLatin1(), _expectedHash, this);
+        _thread->setInputBufferSize(IMAGEWRITER_UNCOMPRESSED_BLOCKSIZE);
     }
 
     _powersave.applyBlock(tr("Downloading and writing image"));