qml: move OS list fetch to backend

- Simple implementation of OS list fetching in backend
- Replace frontend OS list fetching by calls to backend
- OS list updates are brought in asynchronously, avoiding excessive UI
  blockage.
- "Erase" and "Custom" OS list options are always present, even in a
  no-internet scenario

Based-On: cillian64/rpi-imager/oslist_backend

src/imagewriter.cpp

@@ -53,6 +53,10 @@
 #include <QtPlatformHeaders/QEglFSFunctions>
 #endif
 
+namespace {
+    constexpr uint MAX_SUBITEMS_DEPTH = 16;
+} // namespace anonymous
+
 ImageWriter::ImageWriter(QObject *parent)
     : QObject(parent), _repo(QUrl(QString(OSLIST_URL))), _dlnow(0), _verifynow(0),
       _engine(nullptr), _thread(nullptr), _verifyEnabled(false), _cachingEnabled(false),
@@ -167,6 +171,10 @@ ImageWriter::ImageWriter(QObject *parent)
         }
     }
     //_currentKeyboard = "us";
+
+    // Centralised network manager, for fetching OS lists
+    _networkManager = std::make_unique<QNetworkAccessManager>(this);
+    connect(_networkManager.get(), SIGNAL(finished(QNetworkReply *)), this, SLOT(handleNetworkRequestFinished(QNetworkReply *)));
 }
 
 ImageWriter::~ImageWriter()
@@ -416,6 +424,166 @@ void ImageWriter::setCustomOsListUrl(const QUrl &url)
     _repo = url;
 }
 
+namespace {
+    QJsonArray findAndInsertJsonResult(QJsonArray parent_list, QJsonArray incomingBody, QUrl referenceUrl, uint8_t count = 0) {
+        if (count > MAX_SUBITEMS_DEPTH) {
+            qDebug() << "Aborting insertion of subitems, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels.";
+            return {};
+        }
+
+        QJsonArray returnArray = {};
+
+        for (auto ositem : parent_list) {
+            auto ositemObject = ositem.toObject();
+
+            if (ositemObject.contains("subitems")) {
+                // Recurse!
+                ositemObject["subitems"] = findAndInsertJsonResult(ositemObject["subitems"].toArray(), incomingBody, referenceUrl, count++);
+            } else if (ositemObject.contains("subitems_url")) {
+                if ( !ositemObject["subitems_url"].toString().compare(referenceUrl.toString())) {
+                   // qDebug() << "Replacing URL [" << ositemObject["subitems_url"] << "] with body";
+                    ositemObject.insert("subitems", incomingBody);
+                    ositemObject.remove("subitems_url");
+                } else {
+                    //qDebug() << "Moving to next match for " << referenceUrl << " as we didn't match with " << ositemObject["subitems_url"].toString();
+                }
+            }
+
+            returnArray += ositemObject;
+        }
+
+        return returnArray;
+    }
+
+    void findAndQueueUnresolvedSubitemsJson(QJsonArray incoming, QNetworkAccessManager *manager, uint8_t count = 0) {
+        // Step 2: Queue the other downloads.
+        if (count > MAX_SUBITEMS_DEPTH) {
+            qDebug() << "Aborting fetch of subitems JSON, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels.";
+            return;
+        }
+
+        for (auto entry : incoming) {
+            auto entryObject = entry.toObject();
+            if (entryObject.contains("subitems")) {
+                // No need to handle a return - this isn't processing a list, it's searching and queuing downloads.
+                findAndQueueUnresolvedSubitemsJson(entryObject["subitems"].toArray(), manager, count++);
+            } else if (entryObject.contains("subitems_url")) {
+                auto url = entryObject["subitems_url"].toString();
+                manager->get(QNetworkRequest(url));
+            }
+        }
+    }
+} // namespace anonymous
+
+
+void ImageWriter::setHWFilterList(const QByteArray &json) {
+    QJsonDocument json_document = QJsonDocument::fromJson(json);
+
+    _deviceFilter = json_document.array();
+}
+
+void ImageWriter::handleNetworkRequestFinished(QNetworkReply *data) {
+    // Defer deletion
+    //data->deleteLater();
+
+    if (data->error() == QNetworkReply::NoError) {
+        auto httpStatusCode = data->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
+
+        if (httpStatusCode >= 200 && httpStatusCode < 300) {
+           // TODO: Indicate download complete, hand back for re-assembly. Userdata to point to qJsonDocument?
+
+            auto responseDoc = QJsonDocument::fromJson(data->readAll()).object();
+
+            if (responseDoc.contains("os_list")) {
+                // Step 1: Insert the items into the canonical JSON document.
+                //         It doesn't matter that these may still contain subitems_url items
+                //         As these will be fixed up as the subitems_url instances are blinked in
+                if (_completeOsList.isEmpty()) {
+                    std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
+                    _completeOsList = QJsonDocument(responseDoc);
+                } else {
+                    // TODO: Insert into current graph
+                    std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
+                    auto new_list = findAndInsertJsonResult(_completeOsList["os_list"].toArray(), responseDoc["os_list"].toArray(), data->url());
+                    auto imager_meta = _completeOsList["imager"].toObject();
+                    _completeOsList = QJsonDocument(QJsonObject({
+                        {"imager", imager_meta},
+                        {"os_list", new_list}
+                    }));
+                }
+
+                // TODO: Find and queue subitem downloads. Recursively?
+                findAndQueueUnresolvedSubitemsJson(responseDoc["os_list"].toArray(), _networkManager.get());
+                emit osListPrepared();
+            } else {
+                qDebug() << "Incorrectly formatted OS list at: " << data->url();
+            }
+        } else if (httpStatusCode >= 300 && httpStatusCode < 400) {
+            auto request = QNetworkRequest(data->url());
+
+            request.setAttribute(QNetworkRequest::RedirectionTargetAttribute, QNetworkRequest::NoLessSafeRedirectPolicy);
+
+            data->manager()->get(request);
+
+            // maintain manager
+            return;
+        } else if (httpStatusCode >= 400 && httpStatusCode < 600) {
+            // HTTP Error
+            qDebug() << "Failed to fetch URL [" << data->url() << "], got: " << httpStatusCode;
+        } else {
+            // Completely unknown error, worth logging separately
+            qDebug() << "Failed to fetch URL [" << data->url() << "], got unknown response code: " << httpStatusCode;
+        }
+    } else {
+        // QT Error.
+        qDebug() << "Unrecognised QT error: " << data->error() << ", explainer: " << data->errorString();
+    }
+}
+
+/** You are expected to have called setDeviceFilter from the UI before making this call.
+ *  If you have not, you will effectively get the full list. With some extras.
+*/
+QByteArray ImageWriter::getFilteredOSlist() {
+    QJsonArray referenceArray = {};
+    QJsonObject referenceImagerMeta = {};
+    if (!_completeOsList.isEmpty()) {
+        std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
+        referenceArray = _completeOsList.object()["os_list"].toArray();
+        referenceImagerMeta = _completeOsList.object()["imager"].toObject();
+    }
+
+    referenceArray.append(QJsonObject({
+            {"name",  tr("Erase")},
+            {"description", tr("Format card as FAT32")},
+            {"icon", "icons/erase.png"},
+            {"url", "internal://format"},
+        }));
+
+    referenceArray.append(QJsonObject({
+            {"name",  tr("Use custom")},
+            {"description", tr("Select a custom .img from your computer")},
+            {"icon", "icons/use_custom.png"},
+            {"url", ""},
+        }));
+
+    return QJsonDocument(
+        QJsonObject({
+            {"imager", referenceImagerMeta},
+            {"os_list", referenceArray},
+        }
+    )).toJson();
+}
+
+void ImageWriter::beginOSListFetch() {
+    QNetworkRequest request = QNetworkRequest(constantOsListUrl());
+    request.setAttribute(QNetworkRequest::RedirectPolicyAttribute,
+                         QNetworkRequest::NoLessSafeRedirectPolicy);
+    
+    // This will set up a chain of requests that culiminate in the eventual fetch and assembly of
+    // a complete cached OS list.
+   _networkManager->get(QNetworkRequest(request));
+}
+
 void ImageWriter::setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256)
 {
     _cacheFileName = cacheFile;

src/imagewriter.h

@@ -6,6 +6,11 @@
  * Copyright (C) 2020 Raspberry Pi Ltd
  */
 
+#include <memory>
+
+#include <QJsonArray>
+#include <QJsonDocument>
+#include <QNetworkAccessManager>
 #include <QObject>
 #include <QTimer>
 #include <QUrl>
@@ -74,6 +79,15 @@ public:
     /* Set custom repository */
     Q_INVOKABLE void setCustomOsListUrl(const QUrl &url);
 
+    /* Get the cached OS list. This may be empty if network connectivity is not available. */
+    Q_INVOKABLE QByteArray getFilteredOSlist();
+
+    /** Begin the asynchronous fetch of the OS lists, and associated sublists. */
+    Q_INVOKABLE void beginOSListFetch();
+
+    /** Set the HW filter, for a filtered view of the OS list */
+    Q_INVOKABLE void setHWFilterList(const QByteArray &json);
+
     /* Set custom cache file */
     void setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256);
 
@@ -144,6 +158,7 @@ signals:
     void finalizing();
     void networkOnline();
     void preparationStatusUpdate(QVariant msg);
+    void osListPrepared();
 
 protected slots:
 
@@ -160,6 +175,16 @@ protected slots:
     void onFinalizing();
     void onTimeSyncReply(QNetworkReply *reply);
     void onPreparationStatusUpdate(QString msg);
+    void handleNetworkRequestFinished(QNetworkReply *data);
+
+private:
+    // Recursively walk all the entries with subitems and, for any which
+    // refer to an external JSON list, fetch the list and put it in place.
+    void fillSubLists(QJsonArray &topLevel);
+    std::unique_ptr<QNetworkAccessManager> _networkManager;
+    QJsonDocument _completeOsList;
+    QJsonArray _deviceFilter;
+    std::mutex _deviceListMutationMutex;
 
 protected:
     QUrl _src, _repo;

src/main.cpp

@@ -346,6 +346,7 @@ int main(int argc, char *argv[])
     qmlwindow->connect(&imageWriter, SIGNAL(cancelled()), qmlwindow, SLOT(onCancelled()));
     qmlwindow->connect(&imageWriter, SIGNAL(finalizing()), qmlwindow, SLOT(onFinalizing()));
     qmlwindow->connect(&imageWriter, SIGNAL(networkOnline()), qmlwindow, SLOT(fetchOSlist()));
+    qmlwindow->connect(&imageWriter, SIGNAL(osListPrepared()), qmlwindow, SLOT(onOsListPrepared()));
 
 #ifndef QT_NO_WIDGETS
     /* Set window position */
@@ -374,6 +375,8 @@ int main(int argc, char *argv[])
     qmlwindow->setProperty("y", y);
 #endif
 
+    imageWriter.beginOSListFetch();
+
     int rc = app.exec();
 
 #ifndef QT_NO_WIDGETS

src/main.qml

@@ -1221,29 +1221,6 @@ ApplicationWindow {
         }
     }
 
-    /* Utility functions */
-    function httpRequest(url, callback) {
-        var xhr = new XMLHttpRequest();
-        xhr.timeout = 5000
-        xhr.onreadystatechange = (function(x) {
-            return function() {
-                if (x.readyState === x.DONE)
-                {
-                    if (x.status === 200)
-                    {
-                        callback(x)
-                    }
-                    else
-                    {
-                        onError(qsTr("Error downloading OS list from Internet"))
-                    }
-                }
-            }
-        })(xhr)
-        xhr.open("GET", url)
-        xhr.send()
-    }
-
     /* Slots for signals imagewrite emits */
     function onDownloadProgress(now,total) {
         var newPos
@@ -1289,6 +1266,11 @@ ApplicationWindow {
         progressText.text = qsTr("Preparing to write... (%1)").arg(msg)
     }
 
+    function onOsListPrepared() {
+        console.log("OS list updated.");
+        fetchOSlist()
+    }
+
     function resetWriteButton() {
         progressText.visible = false
         progressBar.visible = false
@@ -1454,42 +1436,42 @@ ApplicationWindow {
     }
 
     function oslistFromJson(o) {
-        var oslist = false
+        var oslist_parsed = false
         var lang_country = Qt.locale().name
         if ("os_list_"+lang_country in o) {
-            oslist = o["os_list_"+lang_country]
+            oslist_parsed = o["os_list_"+lang_country]
         }
         else if (lang_country.includes("_")) {
             var lang = lang_country.substr(0, lang_country.indexOf("_"))
             if ("os_list_"+lang in o) {
-                oslist = o["os_list_"+lang]
+                oslist_parsed = o["os_list_"+lang]
             }
         }
 
-        if (!oslist) {
+        if (!oslist_parsed) {
             if (!"os_list" in o) {
                 onError(qsTr("Error parsing os_list.json"))
                 return false
             }
 
-            oslist = o["os_list"]
+            oslist_parsed = o["os_list"]
         }
 
         if (hwTags != "") {
-            filterItems(oslist, JSON.parse(hwTags), hwTagMatchingType)
+            filterItems(oslist_parsed, JSON.parse(hwTags), hwTagMatchingType)
         }
-        checkForRandom(oslist)
+        checkForRandom(oslist_parsed)
 
         /* Flatten subitems to subitems_json */
-        for (var i in oslist) {
+        for (var i in oslist_parsed) {
-            var entry = oslist[i];
+            var entry = oslist_parsed[i];
             if ("subitems" in entry) {
                 entry["subitems_json"] = JSON.stringify(entry["subitems"])
                 delete entry["subitems"]
             }
         }
 
-        return oslist
+        return oslist_parsed
     }
 
     function selectNamedOS(name, collection)
@@ -1508,80 +1490,54 @@ ApplicationWindow {
     }
 
     function fetchOSlist() {
-        httpRequest(imageWriter.constantOsListUrl(), function (x) {
+        var oslist_json = imageWriter.getFilteredOSlist();
-            var o = JSON.parse(x.responseText)
+        var o = JSON.parse(oslist_json)
-            var oslist = oslistFromJson(o)
+        var oslist_parsed = oslistFromJson(o)
-            if (oslist === false)
+        if (oslist_parsed === false)
-                return
+            return
-            osmodel.clear()
+        osmodel.clear()
-            for (var i in oslist) {
+        for (var i in oslist_parsed) {
-                osmodel.append(oslist[i])
+            osmodel.append(oslist_parsed[i])
-            }
+        }
 
-            if ("imager" in o) {
+        if ("imager" in o) {
-                var imager = o["imager"]
+            var imager = o["imager"]
 
-                if ("devices" in imager)
+            if ("devices" in imager)
+            {
+                deviceModel.clear()
+                var devices = imager["devices"]
+                for (var j in devices)
                 {
-                    deviceModel.clear()
+                    devices[j]["tags"] = JSON.stringify(devices[j]["tags"])
-                    var devices = imager["devices"]
+                    deviceModel.append(devices[j])
-                    for (var j in devices)
+                    if ("default" in devices[j] && devices[j]["default"])
                     {
-                        devices[j]["tags"] = JSON.stringify(devices[j]["tags"])
+                        hwlist.currentIndex = deviceModel.count-1
-                        deviceModel.append(devices[j])
-                        if ("default" in devices[j] && devices[j]["default"])
-                        {
-                            hwlist.currentIndex = deviceModel.count-1
-                        }
-                    }
-                }
-
-                if (imageWriter.getBoolSetting("check_version") && "latest_version" in imager && "url" in imager) {
-                    if (!imageWriter.isEmbeddedMode() && imageWriter.isVersionNewer(imager["latest_version"])) {
-                        updatepopup.url = imager["url"]
-                        updatepopup.openPopup()
-                    }
-                }
-                if ("default_os" in imager) {
-                    selectNamedOS(imager["default_os"], osmodel)
-                }
-                if (imageWriter.isEmbeddedMode()) {
-                    if ("embedded_default_os" in imager) {
-                        selectNamedOS(imager["embedded_default_os"], osmodel)
-                    }
-                    if ("embedded_default_destination" in imager) {
-                        imageWriter.startDriveListPolling()
-                        setDefaultDest.drive = imager["embedded_default_destination"]
-                        setDefaultDest.start()
                     }
                 }
             }
 
-            /* Add in our 'special' items. */
+            if (imageWriter.getBoolSetting("check_version") && "latest_version" in imager && "url" in imager) {
-            osmodel.append({
+                if (!imageWriter.isEmbeddedMode() && imageWriter.isVersionNewer(imager["latest_version"])) {
-                url: "internal://format",
+                    updatepopup.url = imager["url"]
-                icon: "icons/erase.png",
+                    updatepopup.openPopup()
-                extract_size: 0,
+                }
-                image_download_size: 0,
+            }
-                extract_sha256: "",
+            if ("default_os" in imager) {
-                contains_multiple_files: false,
+                selectNamedOS(imager["default_os"], osmodel)
-                release_date: "",
+            }
-                subitems_url: "",
+            if (imageWriter.isEmbeddedMode()) {
-                subitems_json: "",
+                if ("embedded_default_os" in imager) {
-                name: qsTr("Erase"),
+                    selectNamedOS(imager["embedded_default_os"], osmodel)
-                description: qsTr("Format card as FAT32"),
+                }
-                tooltip: "",
+                if ("embedded_default_destination" in imager) {
-                website: "",
+                    imageWriter.startDriveListPolling()
-                init_format: ""
+                    setDefaultDest.drive = imager["embedded_default_destination"]
-            })
+                    setDefaultDest.start()
-
+                }
-            osmodel.append({
+            }
-                url: "",
+        }
-                icon: "icons/use_custom.png",
-                name: qsTr("Use custom"),
-                description: qsTr("Select a custom .img from your computer")
-            })
-        })
     }
 
     Timer {
@@ -1648,30 +1604,29 @@ ApplicationWindow {
         }
 
         /* Reload list */
-        httpRequest(imageWriter.constantOsListUrl(), function (x) {
+        var oslist_json = imageWriter.getFilteredOSlist();
-            var o = JSON.parse(x.responseText)
+        var o = JSON.parse(oslist_json)
-            var oslist = oslistFromJson(o)
+        var oslist_parsed = oslistFromJson(o)
-            if (oslist === false)
+        if (oslist_parsed === false)
-                return
+            return
 
-            /* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS.
+        /* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS.
-             * To do this, we exploit a convention of how we build the OS list. By convention,
+            * To do this, we exploit a convention of how we build the OS list. By convention,
-             * the preferred OS for a device is listed at the top level of the list, and is at the
+            * the preferred OS for a device is listed at the top level of the list, and is at the
-             * lowest index. So..
+            * lowest index. So..
-             */
+            */
-            if (oslist.length != 0) {
+        if (oslist_parsed.length != 0) {
-                var candidate = oslist[0]
+            var candidate = oslist_parsed[0]
 
-                if ("description" in candidate && !("subitems" in candidate)) {
+            if ("description" in candidate && !("subitems" in candidate)) {
-                    candidate["description"] += " (Recommended)"
+                candidate["description"] += " (Recommended)"
-                }
             }
+        }
 
-            osmodel.remove(0, osmodel.count-2)
+        osmodel.clear()
-            for (var i in oslist) {
+        for (var i in oslist_parsed) {
-                osmodel.insert(osmodel.count-2, oslist[i])
+            osmodel.append(oslist_parsed[i])
-            }
+        }
-        })
 
         // When the HW device is changed, reset the OS selection otherwise
         // you get a weird effect with the selection moving around in the list
@@ -1732,19 +1687,7 @@ ApplicationWindow {
             }
             else
             {
-                ospopup.categorySelected = d.name
+                console.log("Failure: Backend should have pre-flattened the JSON!");
-                var suburl = d.subitems_url
-                var m = newSublist()
-
-                httpRequest(suburl, function (x) {
-                    var o = JSON.parse(x.responseText)
-                    var oslist = oslistFromJson(o)
-                    if (oslist === false)
-                        return
-                    for (var i in oslist) {
-                        m.append(oslist[i])
-                    }
-                })
 
                 osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1
                 osswipeview.incrementCurrentIndex()
@@ -1757,9 +1700,9 @@ ApplicationWindow {
                 if (imageWriter.mountUsbSourceMedia()) {
                     var m = newSublist()
 
-                    var oslist = JSON.parse(imageWriter.getUsbSourceOSlist())
+                    var usboslist = JSON.parse(imageWriter.getUsbSourceOSlist())
-                    for (var i in oslist) {
+                    for (var i in usboslist) {
-                        m.append(oslist[i])
+                        m.append(usboslist[i])
                     }
                     osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1
                     osswipeview.incrementCurrentIndex()