diff --git a/src/imagewriter.cpp b/src/imagewriter.cpp index 2a86281..de3d185 100644 --- a/src/imagewriter.cpp +++ b/src/imagewriter.cpp @@ -53,6 +53,10 @@ #include #endif +namespace { + constexpr uint MAX_SUBITEMS_DEPTH = 16; +} // namespace anonymous + ImageWriter::ImageWriter(QObject *parent) : QObject(parent), _repo(QUrl(QString(OSLIST_URL))), _dlnow(0), _verifynow(0), _engine(nullptr), _thread(nullptr), _verifyEnabled(false), _cachingEnabled(false), @@ -167,6 +171,10 @@ ImageWriter::ImageWriter(QObject *parent) } } //_currentKeyboard = "us"; + + // Centralised network manager, for fetching OS lists + _networkManager = std::make_unique(this); + connect(_networkManager.get(), SIGNAL(finished(QNetworkReply *)), this, SLOT(handleNetworkRequestFinished(QNetworkReply *))); } ImageWriter::~ImageWriter() @@ -416,6 +424,166 @@ void ImageWriter::setCustomOsListUrl(const QUrl &url) _repo = url; } +namespace { + QJsonArray findAndInsertJsonResult(QJsonArray parent_list, QJsonArray incomingBody, QUrl referenceUrl, uint8_t count = 0) { + if (count > MAX_SUBITEMS_DEPTH) { + qDebug() << "Aborting insertion of subitems, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels."; + return {}; + } + + QJsonArray returnArray = {}; + + for (auto ositem : parent_list) { + auto ositemObject = ositem.toObject(); + + if (ositemObject.contains("subitems")) { + // Recurse! + ositemObject["subitems"] = findAndInsertJsonResult(ositemObject["subitems"].toArray(), incomingBody, referenceUrl, count++); + } else if (ositemObject.contains("subitems_url")) { + if ( !ositemObject["subitems_url"].toString().compare(referenceUrl.toString())) { + // qDebug() << "Replacing URL [" << ositemObject["subitems_url"] << "] with body"; + ositemObject.insert("subitems", incomingBody); + ositemObject.remove("subitems_url"); + } else { + //qDebug() << "Moving to next match for " << referenceUrl << " as we didn't match with " << ositemObject["subitems_url"].toString(); + } + } + + returnArray += ositemObject; + } + + return returnArray; + } + + void findAndQueueUnresolvedSubitemsJson(QJsonArray incoming, QNetworkAccessManager *manager, uint8_t count = 0) { + // Step 2: Queue the other downloads. + if (count > MAX_SUBITEMS_DEPTH) { + qDebug() << "Aborting fetch of subitems JSON, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels."; + return; + } + + for (auto entry : incoming) { + auto entryObject = entry.toObject(); + if (entryObject.contains("subitems")) { + // No need to handle a return - this isn't processing a list, it's searching and queuing downloads. + findAndQueueUnresolvedSubitemsJson(entryObject["subitems"].toArray(), manager, count++); + } else if (entryObject.contains("subitems_url")) { + auto url = entryObject["subitems_url"].toString(); + manager->get(QNetworkRequest(url)); + } + } + } +} // namespace anonymous + + +void ImageWriter::setHWFilterList(const QByteArray &json) { + QJsonDocument json_document = QJsonDocument::fromJson(json); + + _deviceFilter = json_document.array(); +} + +void ImageWriter::handleNetworkRequestFinished(QNetworkReply *data) { + // Defer deletion + //data->deleteLater(); + + if (data->error() == QNetworkReply::NoError) { + auto httpStatusCode = data->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); + + if (httpStatusCode >= 200 && httpStatusCode < 300) { + // TODO: Indicate download complete, hand back for re-assembly. Userdata to point to qJsonDocument? + + auto responseDoc = QJsonDocument::fromJson(data->readAll()).object(); + + if (responseDoc.contains("os_list")) { + // Step 1: Insert the items into the canonical JSON document. + // It doesn't matter that these may still contain subitems_url items + // As these will be fixed up as the subitems_url instances are blinked in + if (_completeOsList.isEmpty()) { + std::lock_guard lock(_deviceListMutationMutex); + _completeOsList = QJsonDocument(responseDoc); + } else { + // TODO: Insert into current graph + std::lock_guard lock(_deviceListMutationMutex); + auto new_list = findAndInsertJsonResult(_completeOsList["os_list"].toArray(), responseDoc["os_list"].toArray(), data->url()); + auto imager_meta = _completeOsList["imager"].toObject(); + _completeOsList = QJsonDocument(QJsonObject({ + {"imager", imager_meta}, + {"os_list", new_list} + })); + } + + // TODO: Find and queue subitem downloads. Recursively? + findAndQueueUnresolvedSubitemsJson(responseDoc["os_list"].toArray(), _networkManager.get()); + emit osListPrepared(); + } else { + qDebug() << "Incorrectly formatted OS list at: " << data->url(); + } + } else if (httpStatusCode >= 300 && httpStatusCode < 400) { + auto request = QNetworkRequest(data->url()); + + request.setAttribute(QNetworkRequest::RedirectionTargetAttribute, QNetworkRequest::NoLessSafeRedirectPolicy); + + data->manager()->get(request); + + // maintain manager + return; + } else if (httpStatusCode >= 400 && httpStatusCode < 600) { + // HTTP Error + qDebug() << "Failed to fetch URL [" << data->url() << "], got: " << httpStatusCode; + } else { + // Completely unknown error, worth logging separately + qDebug() << "Failed to fetch URL [" << data->url() << "], got unknown response code: " << httpStatusCode; + } + } else { + // QT Error. + qDebug() << "Unrecognised QT error: " << data->error() << ", explainer: " << data->errorString(); + } +} + +/** You are expected to have called setDeviceFilter from the UI before making this call. + * If you have not, you will effectively get the full list. With some extras. +*/ +QByteArray ImageWriter::getFilteredOSlist() { + QJsonArray referenceArray = {}; + QJsonObject referenceImagerMeta = {}; + if (!_completeOsList.isEmpty()) { + std::lock_guard lock(_deviceListMutationMutex); + referenceArray = _completeOsList.object()["os_list"].toArray(); + referenceImagerMeta = _completeOsList.object()["imager"].toObject(); + } + + referenceArray.append(QJsonObject({ + {"name", tr("Erase")}, + {"description", tr("Format card as FAT32")}, + {"icon", "icons/erase.png"}, + {"url", "internal://format"}, + })); + + referenceArray.append(QJsonObject({ + {"name", tr("Use custom")}, + {"description", tr("Select a custom .img from your computer")}, + {"icon", "icons/use_custom.png"}, + {"url", ""}, + })); + + return QJsonDocument( + QJsonObject({ + {"imager", referenceImagerMeta}, + {"os_list", referenceArray}, + } + )).toJson(); +} + +void ImageWriter::beginOSListFetch() { + QNetworkRequest request = QNetworkRequest(constantOsListUrl()); + request.setAttribute(QNetworkRequest::RedirectPolicyAttribute, + QNetworkRequest::NoLessSafeRedirectPolicy); + + // This will set up a chain of requests that culiminate in the eventual fetch and assembly of + // a complete cached OS list. + _networkManager->get(QNetworkRequest(request)); +} + void ImageWriter::setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256) { _cacheFileName = cacheFile; diff --git a/src/imagewriter.h b/src/imagewriter.h index 2fd63da..9694030 100644 --- a/src/imagewriter.h +++ b/src/imagewriter.h @@ -6,6 +6,11 @@ * Copyright (C) 2020 Raspberry Pi Ltd */ +#include + +#include +#include +#include #include #include #include @@ -74,6 +79,15 @@ public: /* Set custom repository */ Q_INVOKABLE void setCustomOsListUrl(const QUrl &url); + /* Get the cached OS list. This may be empty if network connectivity is not available. */ + Q_INVOKABLE QByteArray getFilteredOSlist(); + + /** Begin the asynchronous fetch of the OS lists, and associated sublists. */ + Q_INVOKABLE void beginOSListFetch(); + + /** Set the HW filter, for a filtered view of the OS list */ + Q_INVOKABLE void setHWFilterList(const QByteArray &json); + /* Set custom cache file */ void setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256); @@ -144,6 +158,7 @@ signals: void finalizing(); void networkOnline(); void preparationStatusUpdate(QVariant msg); + void osListPrepared(); protected slots: @@ -160,6 +175,16 @@ protected slots: void onFinalizing(); void onTimeSyncReply(QNetworkReply *reply); void onPreparationStatusUpdate(QString msg); + void handleNetworkRequestFinished(QNetworkReply *data); + +private: + // Recursively walk all the entries with subitems and, for any which + // refer to an external JSON list, fetch the list and put it in place. + void fillSubLists(QJsonArray &topLevel); + std::unique_ptr _networkManager; + QJsonDocument _completeOsList; + QJsonArray _deviceFilter; + std::mutex _deviceListMutationMutex; protected: QUrl _src, _repo; diff --git a/src/main.cpp b/src/main.cpp index 3469109..9ebe85c 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -346,6 +346,7 @@ int main(int argc, char *argv[]) qmlwindow->connect(&imageWriter, SIGNAL(cancelled()), qmlwindow, SLOT(onCancelled())); qmlwindow->connect(&imageWriter, SIGNAL(finalizing()), qmlwindow, SLOT(onFinalizing())); qmlwindow->connect(&imageWriter, SIGNAL(networkOnline()), qmlwindow, SLOT(fetchOSlist())); + qmlwindow->connect(&imageWriter, SIGNAL(osListPrepared()), qmlwindow, SLOT(onOsListPrepared())); #ifndef QT_NO_WIDGETS /* Set window position */ @@ -374,6 +375,8 @@ int main(int argc, char *argv[]) qmlwindow->setProperty("y", y); #endif + imageWriter.beginOSListFetch(); + int rc = app.exec(); #ifndef QT_NO_WIDGETS diff --git a/src/main.qml b/src/main.qml index d511a9d..c10bb09 100644 --- a/src/main.qml +++ b/src/main.qml @@ -1221,29 +1221,6 @@ ApplicationWindow { } } - /* Utility functions */ - function httpRequest(url, callback) { - var xhr = new XMLHttpRequest(); - xhr.timeout = 5000 - xhr.onreadystatechange = (function(x) { - return function() { - if (x.readyState === x.DONE) - { - if (x.status === 200) - { - callback(x) - } - else - { - onError(qsTr("Error downloading OS list from Internet")) - } - } - } - })(xhr) - xhr.open("GET", url) - xhr.send() - } - /* Slots for signals imagewrite emits */ function onDownloadProgress(now,total) { var newPos @@ -1289,6 +1266,11 @@ ApplicationWindow { progressText.text = qsTr("Preparing to write... (%1)").arg(msg) } + function onOsListPrepared() { + console.log("OS list updated."); + fetchOSlist() + } + function resetWriteButton() { progressText.visible = false progressBar.visible = false @@ -1454,42 +1436,42 @@ ApplicationWindow { } function oslistFromJson(o) { - var oslist = false + var oslist_parsed = false var lang_country = Qt.locale().name if ("os_list_"+lang_country in o) { - oslist = o["os_list_"+lang_country] + oslist_parsed = o["os_list_"+lang_country] } else if (lang_country.includes("_")) { var lang = lang_country.substr(0, lang_country.indexOf("_")) if ("os_list_"+lang in o) { - oslist = o["os_list_"+lang] + oslist_parsed = o["os_list_"+lang] } } - if (!oslist) { + if (!oslist_parsed) { if (!"os_list" in o) { onError(qsTr("Error parsing os_list.json")) return false } - oslist = o["os_list"] + oslist_parsed = o["os_list"] } if (hwTags != "") { - filterItems(oslist, JSON.parse(hwTags), hwTagMatchingType) + filterItems(oslist_parsed, JSON.parse(hwTags), hwTagMatchingType) } - checkForRandom(oslist) + checkForRandom(oslist_parsed) /* Flatten subitems to subitems_json */ - for (var i in oslist) { - var entry = oslist[i]; + for (var i in oslist_parsed) { + var entry = oslist_parsed[i]; if ("subitems" in entry) { entry["subitems_json"] = JSON.stringify(entry["subitems"]) delete entry["subitems"] } } - return oslist + return oslist_parsed } function selectNamedOS(name, collection) @@ -1508,80 +1490,54 @@ ApplicationWindow { } function fetchOSlist() { - httpRequest(imageWriter.constantOsListUrl(), function (x) { - var o = JSON.parse(x.responseText) - var oslist = oslistFromJson(o) - if (oslist === false) - return - osmodel.clear() - for (var i in oslist) { - osmodel.append(oslist[i]) - } + var oslist_json = imageWriter.getFilteredOSlist(); + var o = JSON.parse(oslist_json) + var oslist_parsed = oslistFromJson(o) + if (oslist_parsed === false) + return + osmodel.clear() + for (var i in oslist_parsed) { + osmodel.append(oslist_parsed[i]) + } - if ("imager" in o) { - var imager = o["imager"] + if ("imager" in o) { + var imager = o["imager"] - if ("devices" in imager) + if ("devices" in imager) + { + deviceModel.clear() + var devices = imager["devices"] + for (var j in devices) { - deviceModel.clear() - var devices = imager["devices"] - for (var j in devices) + devices[j]["tags"] = JSON.stringify(devices[j]["tags"]) + deviceModel.append(devices[j]) + if ("default" in devices[j] && devices[j]["default"]) { - devices[j]["tags"] = JSON.stringify(devices[j]["tags"]) - deviceModel.append(devices[j]) - if ("default" in devices[j] && devices[j]["default"]) - { - hwlist.currentIndex = deviceModel.count-1 - } - } - } - - if (imageWriter.getBoolSetting("check_version") && "latest_version" in imager && "url" in imager) { - if (!imageWriter.isEmbeddedMode() && imageWriter.isVersionNewer(imager["latest_version"])) { - updatepopup.url = imager["url"] - updatepopup.openPopup() - } - } - if ("default_os" in imager) { - selectNamedOS(imager["default_os"], osmodel) - } - if (imageWriter.isEmbeddedMode()) { - if ("embedded_default_os" in imager) { - selectNamedOS(imager["embedded_default_os"], osmodel) - } - if ("embedded_default_destination" in imager) { - imageWriter.startDriveListPolling() - setDefaultDest.drive = imager["embedded_default_destination"] - setDefaultDest.start() + hwlist.currentIndex = deviceModel.count-1 } } } - /* Add in our 'special' items. */ - osmodel.append({ - url: "internal://format", - icon: "icons/erase.png", - extract_size: 0, - image_download_size: 0, - extract_sha256: "", - contains_multiple_files: false, - release_date: "", - subitems_url: "", - subitems_json: "", - name: qsTr("Erase"), - description: qsTr("Format card as FAT32"), - tooltip: "", - website: "", - init_format: "" - }) - - osmodel.append({ - url: "", - icon: "icons/use_custom.png", - name: qsTr("Use custom"), - description: qsTr("Select a custom .img from your computer") - }) - }) + if (imageWriter.getBoolSetting("check_version") && "latest_version" in imager && "url" in imager) { + if (!imageWriter.isEmbeddedMode() && imageWriter.isVersionNewer(imager["latest_version"])) { + updatepopup.url = imager["url"] + updatepopup.openPopup() + } + } + if ("default_os" in imager) { + selectNamedOS(imager["default_os"], osmodel) + } + if (imageWriter.isEmbeddedMode()) { + if ("embedded_default_os" in imager) { + selectNamedOS(imager["embedded_default_os"], osmodel) + } + if ("embedded_default_destination" in imager) { + imageWriter.startDriveListPolling() + setDefaultDest.drive = imager["embedded_default_destination"] + setDefaultDest.start() + } + } + } } Timer { @@ -1648,30 +1604,29 @@ ApplicationWindow { } /* Reload list */ - httpRequest(imageWriter.constantOsListUrl(), function (x) { - var o = JSON.parse(x.responseText) - var oslist = oslistFromJson(o) - if (oslist === false) - return + var oslist_json = imageWriter.getFilteredOSlist(); + var o = JSON.parse(oslist_json) + var oslist_parsed = oslistFromJson(o) + if (oslist_parsed === false) + return - /* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS. - * To do this, we exploit a convention of how we build the OS list. By convention, - * the preferred OS for a device is listed at the top level of the list, and is at the - * lowest index. So.. - */ - if (oslist.length != 0) { - var candidate = oslist[0] + /* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS. + * To do this, we exploit a convention of how we build the OS list. By convention, + * the preferred OS for a device is listed at the top level of the list, and is at the + * lowest index. So.. + */ + if (oslist_parsed.length != 0) { + var candidate = oslist_parsed[0] - if ("description" in candidate && !("subitems" in candidate)) { - candidate["description"] += " (Recommended)" - } + if ("description" in candidate && !("subitems" in candidate)) { + candidate["description"] += " (Recommended)" } + } - osmodel.remove(0, osmodel.count-2) - for (var i in oslist) { - osmodel.insert(osmodel.count-2, oslist[i]) - } - }) + osmodel.clear() + for (var i in oslist_parsed) { + osmodel.append(oslist_parsed[i]) + } // When the HW device is changed, reset the OS selection otherwise // you get a weird effect with the selection moving around in the list @@ -1732,19 +1687,7 @@ ApplicationWindow { } else { - ospopup.categorySelected = d.name - var suburl = d.subitems_url - var m = newSublist() - - httpRequest(suburl, function (x) { - var o = JSON.parse(x.responseText) - var oslist = oslistFromJson(o) - if (oslist === false) - return - for (var i in oslist) { - m.append(oslist[i]) - } - }) + console.log("Failure: Backend should have pre-flattened the JSON!"); osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1 osswipeview.incrementCurrentIndex() @@ -1757,9 +1700,9 @@ ApplicationWindow { if (imageWriter.mountUsbSourceMedia()) { var m = newSublist() - var oslist = JSON.parse(imageWriter.getUsbSourceOSlist()) - for (var i in oslist) { - m.append(oslist[i]) + var usboslist = JSON.parse(imageWriter.getUsbSourceOSlist()) + for (var i in usboslist) { + m.append(usboslist[i]) } osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1 osswipeview.incrementCurrentIndex()