qml: move OS list fetch to backend

- Simple implementation of OS list fetching in backend
- Replace frontend OS list fetching by calls to backend
- OS list updates are brought in asynchronously, avoiding excessive UI
  blockage.
- "Erase" and "Custom" OS list options are always present, even in a
  no-internet scenario

Based-On: cillian64/rpi-imager/oslist_backend
This commit is contained in:
Tom Dewey 2023-11-09 14:38:34 +00:00 committed by Tom Dewey
parent 54ae0f889c
commit 673b3c7a33
4 changed files with 274 additions and 135 deletions

View file

@ -53,6 +53,10 @@
#include <QtPlatformHeaders/QEglFSFunctions>
#endif
namespace {
constexpr uint MAX_SUBITEMS_DEPTH = 16;
} // namespace anonymous
ImageWriter::ImageWriter(QObject *parent)
: QObject(parent), _repo(QUrl(QString(OSLIST_URL))), _dlnow(0), _verifynow(0),
_engine(nullptr), _thread(nullptr), _verifyEnabled(false), _cachingEnabled(false),
@ -167,6 +171,10 @@ ImageWriter::ImageWriter(QObject *parent)
}
}
//_currentKeyboard = "us";
// Centralised network manager, for fetching OS lists
_networkManager = std::make_unique<QNetworkAccessManager>(this);
connect(_networkManager.get(), SIGNAL(finished(QNetworkReply *)), this, SLOT(handleNetworkRequestFinished(QNetworkReply *)));
}
ImageWriter::~ImageWriter()
@ -416,6 +424,166 @@ void ImageWriter::setCustomOsListUrl(const QUrl &url)
_repo = url;
}
namespace {
QJsonArray findAndInsertJsonResult(QJsonArray parent_list, QJsonArray incomingBody, QUrl referenceUrl, uint8_t count = 0) {
if (count > MAX_SUBITEMS_DEPTH) {
qDebug() << "Aborting insertion of subitems, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels.";
return {};
}
QJsonArray returnArray = {};
for (auto ositem : parent_list) {
auto ositemObject = ositem.toObject();
if (ositemObject.contains("subitems")) {
// Recurse!
ositemObject["subitems"] = findAndInsertJsonResult(ositemObject["subitems"].toArray(), incomingBody, referenceUrl, count++);
} else if (ositemObject.contains("subitems_url")) {
if ( !ositemObject["subitems_url"].toString().compare(referenceUrl.toString())) {
// qDebug() << "Replacing URL [" << ositemObject["subitems_url"] << "] with body";
ositemObject.insert("subitems", incomingBody);
ositemObject.remove("subitems_url");
} else {
//qDebug() << "Moving to next match for " << referenceUrl << " as we didn't match with " << ositemObject["subitems_url"].toString();
}
}
returnArray += ositemObject;
}
return returnArray;
}
void findAndQueueUnresolvedSubitemsJson(QJsonArray incoming, QNetworkAccessManager *manager, uint8_t count = 0) {
// Step 2: Queue the other downloads.
if (count > MAX_SUBITEMS_DEPTH) {
qDebug() << "Aborting fetch of subitems JSON, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels.";
return;
}
for (auto entry : incoming) {
auto entryObject = entry.toObject();
if (entryObject.contains("subitems")) {
// No need to handle a return - this isn't processing a list, it's searching and queuing downloads.
findAndQueueUnresolvedSubitemsJson(entryObject["subitems"].toArray(), manager, count++);
} else if (entryObject.contains("subitems_url")) {
auto url = entryObject["subitems_url"].toString();
manager->get(QNetworkRequest(url));
}
}
}
} // namespace anonymous
void ImageWriter::setHWFilterList(const QByteArray &json) {
QJsonDocument json_document = QJsonDocument::fromJson(json);
_deviceFilter = json_document.array();
}
void ImageWriter::handleNetworkRequestFinished(QNetworkReply *data) {
// Defer deletion
//data->deleteLater();
if (data->error() == QNetworkReply::NoError) {
auto httpStatusCode = data->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
if (httpStatusCode >= 200 && httpStatusCode < 300) {
// TODO: Indicate download complete, hand back for re-assembly. Userdata to point to qJsonDocument?
auto responseDoc = QJsonDocument::fromJson(data->readAll()).object();
if (responseDoc.contains("os_list")) {
// Step 1: Insert the items into the canonical JSON document.
// It doesn't matter that these may still contain subitems_url items
// As these will be fixed up as the subitems_url instances are blinked in
if (_completeOsList.isEmpty()) {
std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
_completeOsList = QJsonDocument(responseDoc);
} else {
// TODO: Insert into current graph
std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
auto new_list = findAndInsertJsonResult(_completeOsList["os_list"].toArray(), responseDoc["os_list"].toArray(), data->url());
auto imager_meta = _completeOsList["imager"].toObject();
_completeOsList = QJsonDocument(QJsonObject({
{"imager", imager_meta},
{"os_list", new_list}
}));
}
// TODO: Find and queue subitem downloads. Recursively?
findAndQueueUnresolvedSubitemsJson(responseDoc["os_list"].toArray(), _networkManager.get());
emit osListPrepared();
} else {
qDebug() << "Incorrectly formatted OS list at: " << data->url();
}
} else if (httpStatusCode >= 300 && httpStatusCode < 400) {
auto request = QNetworkRequest(data->url());
request.setAttribute(QNetworkRequest::RedirectionTargetAttribute, QNetworkRequest::NoLessSafeRedirectPolicy);
data->manager()->get(request);
// maintain manager
return;
} else if (httpStatusCode >= 400 && httpStatusCode < 600) {
// HTTP Error
qDebug() << "Failed to fetch URL [" << data->url() << "], got: " << httpStatusCode;
} else {
// Completely unknown error, worth logging separately
qDebug() << "Failed to fetch URL [" << data->url() << "], got unknown response code: " << httpStatusCode;
}
} else {
// QT Error.
qDebug() << "Unrecognised QT error: " << data->error() << ", explainer: " << data->errorString();
}
}
/** You are expected to have called setDeviceFilter from the UI before making this call.
* If you have not, you will effectively get the full list. With some extras.
*/
QByteArray ImageWriter::getFilteredOSlist() {
QJsonArray referenceArray = {};
QJsonObject referenceImagerMeta = {};
if (!_completeOsList.isEmpty()) {
std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
referenceArray = _completeOsList.object()["os_list"].toArray();
referenceImagerMeta = _completeOsList.object()["imager"].toObject();
}
referenceArray.append(QJsonObject({
{"name", tr("Erase")},
{"description", tr("Format card as FAT32")},
{"icon", "icons/erase.png"},
{"url", "internal://format"},
}));
referenceArray.append(QJsonObject({
{"name", tr("Use custom")},
{"description", tr("Select a custom .img from your computer")},
{"icon", "icons/use_custom.png"},
{"url", ""},
}));
return QJsonDocument(
QJsonObject({
{"imager", referenceImagerMeta},
{"os_list", referenceArray},
}
)).toJson();
}
void ImageWriter::beginOSListFetch() {
QNetworkRequest request = QNetworkRequest(constantOsListUrl());
request.setAttribute(QNetworkRequest::RedirectPolicyAttribute,
QNetworkRequest::NoLessSafeRedirectPolicy);
// This will set up a chain of requests that culiminate in the eventual fetch and assembly of
// a complete cached OS list.
_networkManager->get(QNetworkRequest(request));
}
void ImageWriter::setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256)
{
_cacheFileName = cacheFile;

View file

@ -6,6 +6,11 @@
* Copyright (C) 2020 Raspberry Pi Ltd
*/
#include <memory>
#include <QJsonArray>
#include <QJsonDocument>
#include <QNetworkAccessManager>
#include <QObject>
#include <QTimer>
#include <QUrl>
@ -74,6 +79,15 @@ public:
/* Set custom repository */
Q_INVOKABLE void setCustomOsListUrl(const QUrl &url);
/* Get the cached OS list. This may be empty if network connectivity is not available. */
Q_INVOKABLE QByteArray getFilteredOSlist();
/** Begin the asynchronous fetch of the OS lists, and associated sublists. */
Q_INVOKABLE void beginOSListFetch();
/** Set the HW filter, for a filtered view of the OS list */
Q_INVOKABLE void setHWFilterList(const QByteArray &json);
/* Set custom cache file */
void setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256);
@ -144,6 +158,7 @@ signals:
void finalizing();
void networkOnline();
void preparationStatusUpdate(QVariant msg);
void osListPrepared();
protected slots:
@ -160,6 +175,16 @@ protected slots:
void onFinalizing();
void onTimeSyncReply(QNetworkReply *reply);
void onPreparationStatusUpdate(QString msg);
void handleNetworkRequestFinished(QNetworkReply *data);
private:
// Recursively walk all the entries with subitems and, for any which
// refer to an external JSON list, fetch the list and put it in place.
void fillSubLists(QJsonArray &topLevel);
std::unique_ptr<QNetworkAccessManager> _networkManager;
QJsonDocument _completeOsList;
QJsonArray _deviceFilter;
std::mutex _deviceListMutationMutex;
protected:
QUrl _src, _repo;

View file

@ -346,6 +346,7 @@ int main(int argc, char *argv[])
qmlwindow->connect(&imageWriter, SIGNAL(cancelled()), qmlwindow, SLOT(onCancelled()));
qmlwindow->connect(&imageWriter, SIGNAL(finalizing()), qmlwindow, SLOT(onFinalizing()));
qmlwindow->connect(&imageWriter, SIGNAL(networkOnline()), qmlwindow, SLOT(fetchOSlist()));
qmlwindow->connect(&imageWriter, SIGNAL(osListPrepared()), qmlwindow, SLOT(onOsListPrepared()));
#ifndef QT_NO_WIDGETS
/* Set window position */
@ -374,6 +375,8 @@ int main(int argc, char *argv[])
qmlwindow->setProperty("y", y);
#endif
imageWriter.beginOSListFetch();
int rc = app.exec();
#ifndef QT_NO_WIDGETS

View file

@ -1221,29 +1221,6 @@ ApplicationWindow {
}
}
/* Utility functions */
function httpRequest(url, callback) {
var xhr = new XMLHttpRequest();
xhr.timeout = 5000
xhr.onreadystatechange = (function(x) {
return function() {
if (x.readyState === x.DONE)
{
if (x.status === 200)
{
callback(x)
}
else
{
onError(qsTr("Error downloading OS list from Internet"))
}
}
}
})(xhr)
xhr.open("GET", url)
xhr.send()
}
/* Slots for signals imagewrite emits */
function onDownloadProgress(now,total) {
var newPos
@ -1289,6 +1266,11 @@ ApplicationWindow {
progressText.text = qsTr("Preparing to write... (%1)").arg(msg)
}
function onOsListPrepared() {
console.log("OS list updated.");
fetchOSlist()
}
function resetWriteButton() {
progressText.visible = false
progressBar.visible = false
@ -1454,42 +1436,42 @@ ApplicationWindow {
}
function oslistFromJson(o) {
var oslist = false
var oslist_parsed = false
var lang_country = Qt.locale().name
if ("os_list_"+lang_country in o) {
oslist = o["os_list_"+lang_country]
oslist_parsed = o["os_list_"+lang_country]
}
else if (lang_country.includes("_")) {
var lang = lang_country.substr(0, lang_country.indexOf("_"))
if ("os_list_"+lang in o) {
oslist = o["os_list_"+lang]
oslist_parsed = o["os_list_"+lang]
}
}
if (!oslist) {
if (!oslist_parsed) {
if (!"os_list" in o) {
onError(qsTr("Error parsing os_list.json"))
return false
}
oslist = o["os_list"]
oslist_parsed = o["os_list"]
}
if (hwTags != "") {
filterItems(oslist, JSON.parse(hwTags), hwTagMatchingType)
filterItems(oslist_parsed, JSON.parse(hwTags), hwTagMatchingType)
}
checkForRandom(oslist)
checkForRandom(oslist_parsed)
/* Flatten subitems to subitems_json */
for (var i in oslist) {
var entry = oslist[i];
for (var i in oslist_parsed) {
var entry = oslist_parsed[i];
if ("subitems" in entry) {
entry["subitems_json"] = JSON.stringify(entry["subitems"])
delete entry["subitems"]
}
}
return oslist
return oslist_parsed
}
function selectNamedOS(name, collection)
@ -1508,80 +1490,54 @@ ApplicationWindow {
}
function fetchOSlist() {
httpRequest(imageWriter.constantOsListUrl(), function (x) {
var o = JSON.parse(x.responseText)
var oslist = oslistFromJson(o)
if (oslist === false)
return
osmodel.clear()
for (var i in oslist) {
osmodel.append(oslist[i])
}
var oslist_json = imageWriter.getFilteredOSlist();
var o = JSON.parse(oslist_json)
var oslist_parsed = oslistFromJson(o)
if (oslist_parsed === false)
return
osmodel.clear()
for (var i in oslist_parsed) {
osmodel.append(oslist_parsed[i])
}
if ("imager" in o) {
var imager = o["imager"]
if ("imager" in o) {
var imager = o["imager"]
if ("devices" in imager)
if ("devices" in imager)
{
deviceModel.clear()
var devices = imager["devices"]
for (var j in devices)
{
deviceModel.clear()
var devices = imager["devices"]
for (var j in devices)
devices[j]["tags"] = JSON.stringify(devices[j]["tags"])
deviceModel.append(devices[j])
if ("default" in devices[j] && devices[j]["default"])
{
devices[j]["tags"] = JSON.stringify(devices[j]["tags"])
deviceModel.append(devices[j])
if ("default" in devices[j] && devices[j]["default"])
{
hwlist.currentIndex = deviceModel.count-1
}
}
}
if (imageWriter.getBoolSetting("check_version") && "latest_version" in imager && "url" in imager) {
if (!imageWriter.isEmbeddedMode() && imageWriter.isVersionNewer(imager["latest_version"])) {
updatepopup.url = imager["url"]
updatepopup.openPopup()
}
}
if ("default_os" in imager) {
selectNamedOS(imager["default_os"], osmodel)
}
if (imageWriter.isEmbeddedMode()) {
if ("embedded_default_os" in imager) {
selectNamedOS(imager["embedded_default_os"], osmodel)
}
if ("embedded_default_destination" in imager) {
imageWriter.startDriveListPolling()
setDefaultDest.drive = imager["embedded_default_destination"]
setDefaultDest.start()
hwlist.currentIndex = deviceModel.count-1
}
}
}
/* Add in our 'special' items. */
osmodel.append({
url: "internal://format",
icon: "icons/erase.png",
extract_size: 0,
image_download_size: 0,
extract_sha256: "",
contains_multiple_files: false,
release_date: "",
subitems_url: "",
subitems_json: "",
name: qsTr("Erase"),
description: qsTr("Format card as FAT32"),
tooltip: "",
website: "",
init_format: ""
})
osmodel.append({
url: "",
icon: "icons/use_custom.png",
name: qsTr("Use custom"),
description: qsTr("Select a custom .img from your computer")
})
})
if (imageWriter.getBoolSetting("check_version") && "latest_version" in imager && "url" in imager) {
if (!imageWriter.isEmbeddedMode() && imageWriter.isVersionNewer(imager["latest_version"])) {
updatepopup.url = imager["url"]
updatepopup.openPopup()
}
}
if ("default_os" in imager) {
selectNamedOS(imager["default_os"], osmodel)
}
if (imageWriter.isEmbeddedMode()) {
if ("embedded_default_os" in imager) {
selectNamedOS(imager["embedded_default_os"], osmodel)
}
if ("embedded_default_destination" in imager) {
imageWriter.startDriveListPolling()
setDefaultDest.drive = imager["embedded_default_destination"]
setDefaultDest.start()
}
}
}
}
Timer {
@ -1648,30 +1604,29 @@ ApplicationWindow {
}
/* Reload list */
httpRequest(imageWriter.constantOsListUrl(), function (x) {
var o = JSON.parse(x.responseText)
var oslist = oslistFromJson(o)
if (oslist === false)
return
var oslist_json = imageWriter.getFilteredOSlist();
var o = JSON.parse(oslist_json)
var oslist_parsed = oslistFromJson(o)
if (oslist_parsed === false)
return
/* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS.
* To do this, we exploit a convention of how we build the OS list. By convention,
* the preferred OS for a device is listed at the top level of the list, and is at the
* lowest index. So..
*/
if (oslist.length != 0) {
var candidate = oslist[0]
/* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS.
* To do this, we exploit a convention of how we build the OS list. By convention,
* the preferred OS for a device is listed at the top level of the list, and is at the
* lowest index. So..
*/
if (oslist_parsed.length != 0) {
var candidate = oslist_parsed[0]
if ("description" in candidate && !("subitems" in candidate)) {
candidate["description"] += " (Recommended)"
}
if ("description" in candidate && !("subitems" in candidate)) {
candidate["description"] += " (Recommended)"
}
}
osmodel.remove(0, osmodel.count-2)
for (var i in oslist) {
osmodel.insert(osmodel.count-2, oslist[i])
}
})
osmodel.clear()
for (var i in oslist_parsed) {
osmodel.append(oslist_parsed[i])
}
// When the HW device is changed, reset the OS selection otherwise
// you get a weird effect with the selection moving around in the list
@ -1732,19 +1687,7 @@ ApplicationWindow {
}
else
{
ospopup.categorySelected = d.name
var suburl = d.subitems_url
var m = newSublist()
httpRequest(suburl, function (x) {
var o = JSON.parse(x.responseText)
var oslist = oslistFromJson(o)
if (oslist === false)
return
for (var i in oslist) {
m.append(oslist[i])
}
})
console.log("Failure: Backend should have pre-flattened the JSON!");
osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1
osswipeview.incrementCurrentIndex()
@ -1757,9 +1700,9 @@ ApplicationWindow {
if (imageWriter.mountUsbSourceMedia()) {
var m = newSublist()
var oslist = JSON.parse(imageWriter.getUsbSourceOSlist())
for (var i in oslist) {
m.append(oslist[i])
var usboslist = JSON.parse(imageWriter.getUsbSourceOSlist())
for (var i in usboslist) {
m.append(usboslist[i])
}
osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1
osswipeview.incrementCurrentIndex()