mirror of
https://github.com/cmclark00/retro-imager.git
synced 2025-05-18 16:05:21 +01:00
qml: move OS list fetch to backend
- Simple implementation of OS list fetching in backend - Replace frontend OS list fetching by calls to backend - OS list updates are brought in asynchronously, avoiding excessive UI blockage. - "Erase" and "Custom" OS list options are always present, even in a no-internet scenario Based-On: cillian64/rpi-imager/oslist_backend
This commit is contained in:
Tom Dewey
Tom Dewey
parent
54ae0f889c
commit
673b3c7a33
4 changed files with 274 additions and 135 deletions
|
|
@ -53,6 +53,10 @@
|
|||
#include <QtPlatformHeaders/QEglFSFunctions>
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
constexpr uint MAX_SUBITEMS_DEPTH = 16;
|
||||
} // namespace anonymous
|
||||
|
||||
ImageWriter::ImageWriter(QObject *parent)
|
||||
: QObject(parent), _repo(QUrl(QString(OSLIST_URL))), _dlnow(0), _verifynow(0),
|
||||
_engine(nullptr), _thread(nullptr), _verifyEnabled(false), _cachingEnabled(false),
|
||||
|
|
@ -167,6 +171,10 @@ ImageWriter::ImageWriter(QObject *parent)
|
|||
}
|
||||
}
|
||||
//_currentKeyboard = "us";
|
||||
|
||||
// Centralised network manager, for fetching OS lists
|
||||
_networkManager = std::make_unique<QNetworkAccessManager>(this);
|
||||
connect(_networkManager.get(), SIGNAL(finished(QNetworkReply *)), this, SLOT(handleNetworkRequestFinished(QNetworkReply *)));
|
||||
}
|
||||
|
||||
ImageWriter::~ImageWriter()
|
||||
|
|
@ -416,6 +424,166 @@ void ImageWriter::setCustomOsListUrl(const QUrl &url)
|
|||
_repo = url;
|
||||
}
|
||||
|
||||
namespace {
|
||||
QJsonArray findAndInsertJsonResult(QJsonArray parent_list, QJsonArray incomingBody, QUrl referenceUrl, uint8_t count = 0) {
|
||||
if (count > MAX_SUBITEMS_DEPTH) {
|
||||
qDebug() << "Aborting insertion of subitems, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels.";
|
||||
return {};
|
||||
}
|
||||
|
||||
QJsonArray returnArray = {};
|
||||
|
||||
for (auto ositem : parent_list) {
|
||||
auto ositemObject = ositem.toObject();
|
||||
|
||||
if (ositemObject.contains("subitems")) {
|
||||
// Recurse!
|
||||
ositemObject["subitems"] = findAndInsertJsonResult(ositemObject["subitems"].toArray(), incomingBody, referenceUrl, count++);
|
||||
} else if (ositemObject.contains("subitems_url")) {
|
||||
if ( !ositemObject["subitems_url"].toString().compare(referenceUrl.toString())) {
|
||||
// qDebug() << "Replacing URL [" << ositemObject["subitems_url"] << "] with body";
|
||||
ositemObject.insert("subitems", incomingBody);
|
||||
ositemObject.remove("subitems_url");
|
||||
} else {
|
||||
//qDebug() << "Moving to next match for " << referenceUrl << " as we didn't match with " << ositemObject["subitems_url"].toString();
|
||||
}
|
||||
}
|
||||
|
||||
returnArray += ositemObject;
|
||||
}
|
||||
|
||||
return returnArray;
|
||||
}
|
||||
|
||||
void findAndQueueUnresolvedSubitemsJson(QJsonArray incoming, QNetworkAccessManager *manager, uint8_t count = 0) {
|
||||
// Step 2: Queue the other downloads.
|
||||
if (count > MAX_SUBITEMS_DEPTH) {
|
||||
qDebug() << "Aborting fetch of subitems JSON, exceeded maximum configured limit of " << MAX_SUBITEMS_DEPTH << " levels.";
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto entry : incoming) {
|
||||
auto entryObject = entry.toObject();
|
||||
if (entryObject.contains("subitems")) {
|
||||
// No need to handle a return - this isn't processing a list, it's searching and queuing downloads.
|
||||
findAndQueueUnresolvedSubitemsJson(entryObject["subitems"].toArray(), manager, count++);
|
||||
} else if (entryObject.contains("subitems_url")) {
|
||||
auto url = entryObject["subitems_url"].toString();
|
||||
manager->get(QNetworkRequest(url));
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace anonymous
|
||||
|
||||
|
||||
void ImageWriter::setHWFilterList(const QByteArray &json) {
|
||||
QJsonDocument json_document = QJsonDocument::fromJson(json);
|
||||
|
||||
_deviceFilter = json_document.array();
|
||||
}
|
||||
|
||||
void ImageWriter::handleNetworkRequestFinished(QNetworkReply *data) {
|
||||
// Defer deletion
|
||||
//data->deleteLater();
|
||||
|
||||
if (data->error() == QNetworkReply::NoError) {
|
||||
auto httpStatusCode = data->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt();
|
||||
|
||||
if (httpStatusCode >= 200 && httpStatusCode < 300) {
|
||||
// TODO: Indicate download complete, hand back for re-assembly. Userdata to point to qJsonDocument?
|
||||
|
||||
auto responseDoc = QJsonDocument::fromJson(data->readAll()).object();
|
||||
|
||||
if (responseDoc.contains("os_list")) {
|
||||
// Step 1: Insert the items into the canonical JSON document.
|
||||
// It doesn't matter that these may still contain subitems_url items
|
||||
// As these will be fixed up as the subitems_url instances are blinked in
|
||||
if (_completeOsList.isEmpty()) {
|
||||
std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
|
||||
_completeOsList = QJsonDocument(responseDoc);
|
||||
} else {
|
||||
// TODO: Insert into current graph
|
||||
std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
|
||||
auto new_list = findAndInsertJsonResult(_completeOsList["os_list"].toArray(), responseDoc["os_list"].toArray(), data->url());
|
||||
auto imager_meta = _completeOsList["imager"].toObject();
|
||||
_completeOsList = QJsonDocument(QJsonObject({
|
||||
{"imager", imager_meta},
|
||||
{"os_list", new_list}
|
||||
}));
|
||||
}
|
||||
|
||||
// TODO: Find and queue subitem downloads. Recursively?
|
||||
findAndQueueUnresolvedSubitemsJson(responseDoc["os_list"].toArray(), _networkManager.get());
|
||||
emit osListPrepared();
|
||||
} else {
|
||||
qDebug() << "Incorrectly formatted OS list at: " << data->url();
|
||||
}
|
||||
} else if (httpStatusCode >= 300 && httpStatusCode < 400) {
|
||||
auto request = QNetworkRequest(data->url());
|
||||
|
||||
request.setAttribute(QNetworkRequest::RedirectionTargetAttribute, QNetworkRequest::NoLessSafeRedirectPolicy);
|
||||
|
||||
data->manager()->get(request);
|
||||
|
||||
// maintain manager
|
||||
return;
|
||||
} else if (httpStatusCode >= 400 && httpStatusCode < 600) {
|
||||
// HTTP Error
|
||||
qDebug() << "Failed to fetch URL [" << data->url() << "], got: " << httpStatusCode;
|
||||
} else {
|
||||
// Completely unknown error, worth logging separately
|
||||
qDebug() << "Failed to fetch URL [" << data->url() << "], got unknown response code: " << httpStatusCode;
|
||||
}
|
||||
} else {
|
||||
// QT Error.
|
||||
qDebug() << "Unrecognised QT error: " << data->error() << ", explainer: " << data->errorString();
|
||||
}
|
||||
}
|
||||
|
||||
/** You are expected to have called setDeviceFilter from the UI before making this call.
|
||||
* If you have not, you will effectively get the full list. With some extras.
|
||||
*/
|
||||
QByteArray ImageWriter::getFilteredOSlist() {
|
||||
QJsonArray referenceArray = {};
|
||||
QJsonObject referenceImagerMeta = {};
|
||||
if (!_completeOsList.isEmpty()) {
|
||||
std::lock_guard<std::mutex> lock(_deviceListMutationMutex);
|
||||
referenceArray = _completeOsList.object()["os_list"].toArray();
|
||||
referenceImagerMeta = _completeOsList.object()["imager"].toObject();
|
||||
}
|
||||
|
||||
referenceArray.append(QJsonObject({
|
||||
{"name", tr("Erase")},
|
||||
{"description", tr("Format card as FAT32")},
|
||||
{"icon", "icons/erase.png"},
|
||||
{"url", "internal://format"},
|
||||
}));
|
||||
|
||||
referenceArray.append(QJsonObject({
|
||||
{"name", tr("Use custom")},
|
||||
{"description", tr("Select a custom .img from your computer")},
|
||||
{"icon", "icons/use_custom.png"},
|
||||
{"url", ""},
|
||||
}));
|
||||
|
||||
return QJsonDocument(
|
||||
QJsonObject({
|
||||
{"imager", referenceImagerMeta},
|
||||
{"os_list", referenceArray},
|
||||
}
|
||||
)).toJson();
|
||||
}
|
||||
|
||||
void ImageWriter::beginOSListFetch() {
|
||||
QNetworkRequest request = QNetworkRequest(constantOsListUrl());
|
||||
request.setAttribute(QNetworkRequest::RedirectPolicyAttribute,
|
||||
QNetworkRequest::NoLessSafeRedirectPolicy);
|
||||
|
||||
// This will set up a chain of requests that culiminate in the eventual fetch and assembly of
|
||||
// a complete cached OS list.
|
||||
_networkManager->get(QNetworkRequest(request));
|
||||
}
|
||||
|
||||
void ImageWriter::setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256)
|
||||
{
|
||||
_cacheFileName = cacheFile;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,11 @@
|
|||
* Copyright (C) 2020 Raspberry Pi Ltd
|
||||
*/
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include <QJsonArray>
|
||||
#include <QJsonDocument>
|
||||
#include <QNetworkAccessManager>
|
||||
#include <QObject>
|
||||
#include <QTimer>
|
||||
#include <QUrl>
|
||||
|
|
@ -74,6 +79,15 @@ public:
|
|||
/* Set custom repository */
|
||||
Q_INVOKABLE void setCustomOsListUrl(const QUrl &url);
|
||||
|
||||
/* Get the cached OS list. This may be empty if network connectivity is not available. */
|
||||
Q_INVOKABLE QByteArray getFilteredOSlist();
|
||||
|
||||
/** Begin the asynchronous fetch of the OS lists, and associated sublists. */
|
||||
Q_INVOKABLE void beginOSListFetch();
|
||||
|
||||
/** Set the HW filter, for a filtered view of the OS list */
|
||||
Q_INVOKABLE void setHWFilterList(const QByteArray &json);
|
||||
|
||||
/* Set custom cache file */
|
||||
void setCustomCacheFile(const QString &cacheFile, const QByteArray &sha256);
|
||||
|
||||
|
|
@ -144,6 +158,7 @@ signals:
|
|||
void finalizing();
|
||||
void networkOnline();
|
||||
void preparationStatusUpdate(QVariant msg);
|
||||
void osListPrepared();
|
||||
|
||||
protected slots:
|
||||
|
||||
|
|
@ -160,6 +175,16 @@ protected slots:
|
|||
void onFinalizing();
|
||||
void onTimeSyncReply(QNetworkReply *reply);
|
||||
void onPreparationStatusUpdate(QString msg);
|
||||
void handleNetworkRequestFinished(QNetworkReply *data);
|
||||
|
||||
private:
|
||||
// Recursively walk all the entries with subitems and, for any which
|
||||
// refer to an external JSON list, fetch the list and put it in place.
|
||||
void fillSubLists(QJsonArray &topLevel);
|
||||
std::unique_ptr<QNetworkAccessManager> _networkManager;
|
||||
QJsonDocument _completeOsList;
|
||||
QJsonArray _deviceFilter;
|
||||
std::mutex _deviceListMutationMutex;
|
||||
|
||||
protected:
|
||||
QUrl _src, _repo;
|
||||
|
|
|
|||
|
|
@ -346,6 +346,7 @@ int main(int argc, char *argv[])
|
|||
qmlwindow->connect(&imageWriter, SIGNAL(cancelled()), qmlwindow, SLOT(onCancelled()));
|
||||
qmlwindow->connect(&imageWriter, SIGNAL(finalizing()), qmlwindow, SLOT(onFinalizing()));
|
||||
qmlwindow->connect(&imageWriter, SIGNAL(networkOnline()), qmlwindow, SLOT(fetchOSlist()));
|
||||
qmlwindow->connect(&imageWriter, SIGNAL(osListPrepared()), qmlwindow, SLOT(onOsListPrepared()));
|
||||
|
||||
#ifndef QT_NO_WIDGETS
|
||||
/* Set window position */
|
||||
|
|
@ -374,6 +375,8 @@ int main(int argc, char *argv[])
|
|||
qmlwindow->setProperty("y", y);
|
||||
#endif
|
||||
|
||||
imageWriter.beginOSListFetch();
|
||||
|
||||
int rc = app.exec();
|
||||
|
||||
#ifndef QT_NO_WIDGETS
|
||||
|
|
|
|||
125
src/main.qml
125
src/main.qml
|
|
@ -1221,29 +1221,6 @@ ApplicationWindow {
|
|||
}
|
||||
}
|
||||
|
||||
/* Utility functions */
|
||||
function httpRequest(url, callback) {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.timeout = 5000
|
||||
xhr.onreadystatechange = (function(x) {
|
||||
return function() {
|
||||
if (x.readyState === x.DONE)
|
||||
{
|
||||
if (x.status === 200)
|
||||
{
|
||||
callback(x)
|
||||
}
|
||||
else
|
||||
{
|
||||
onError(qsTr("Error downloading OS list from Internet"))
|
||||
}
|
||||
}
|
||||
}
|
||||
})(xhr)
|
||||
xhr.open("GET", url)
|
||||
xhr.send()
|
||||
}
|
||||
|
||||
/* Slots for signals imagewrite emits */
|
||||
function onDownloadProgress(now,total) {
|
||||
var newPos
|
||||
|
|
@ -1289,6 +1266,11 @@ ApplicationWindow {
|
|||
progressText.text = qsTr("Preparing to write... (%1)").arg(msg)
|
||||
}
|
||||
|
||||
function onOsListPrepared() {
|
||||
console.log("OS list updated.");
|
||||
fetchOSlist()
|
||||
}
|
||||
|
||||
function resetWriteButton() {
|
||||
progressText.visible = false
|
||||
progressBar.visible = false
|
||||
|
|
@ -1454,42 +1436,42 @@ ApplicationWindow {
|
|||
}
|
||||
|
||||
function oslistFromJson(o) {
|
||||
var oslist = false
|
||||
var oslist_parsed = false
|
||||
var lang_country = Qt.locale().name
|
||||
if ("os_list_"+lang_country in o) {
|
||||
oslist = o["os_list_"+lang_country]
|
||||
oslist_parsed = o["os_list_"+lang_country]
|
||||
}
|
||||
else if (lang_country.includes("_")) {
|
||||
var lang = lang_country.substr(0, lang_country.indexOf("_"))
|
||||
if ("os_list_"+lang in o) {
|
||||
oslist = o["os_list_"+lang]
|
||||
oslist_parsed = o["os_list_"+lang]
|
||||
}
|
||||
}
|
||||
|
||||
if (!oslist) {
|
||||
if (!oslist_parsed) {
|
||||
if (!"os_list" in o) {
|
||||
onError(qsTr("Error parsing os_list.json"))
|
||||
return false
|
||||
}
|
||||
|
||||
oslist = o["os_list"]
|
||||
oslist_parsed = o["os_list"]
|
||||
}
|
||||
|
||||
if (hwTags != "") {
|
||||
filterItems(oslist, JSON.parse(hwTags), hwTagMatchingType)
|
||||
filterItems(oslist_parsed, JSON.parse(hwTags), hwTagMatchingType)
|
||||
}
|
||||
checkForRandom(oslist)
|
||||
checkForRandom(oslist_parsed)
|
||||
|
||||
/* Flatten subitems to subitems_json */
|
||||
for (var i in oslist) {
|
||||
var entry = oslist[i];
|
||||
for (var i in oslist_parsed) {
|
||||
var entry = oslist_parsed[i];
|
||||
if ("subitems" in entry) {
|
||||
entry["subitems_json"] = JSON.stringify(entry["subitems"])
|
||||
delete entry["subitems"]
|
||||
}
|
||||
}
|
||||
|
||||
return oslist
|
||||
return oslist_parsed
|
||||
}
|
||||
|
||||
function selectNamedOS(name, collection)
|
||||
|
|
@ -1508,14 +1490,14 @@ ApplicationWindow {
|
|||
}
|
||||
|
||||
function fetchOSlist() {
|
||||
httpRequest(imageWriter.constantOsListUrl(), function (x) {
|
||||
var o = JSON.parse(x.responseText)
|
||||
var oslist = oslistFromJson(o)
|
||||
if (oslist === false)
|
||||
var oslist_json = imageWriter.getFilteredOSlist();
|
||||
var o = JSON.parse(oslist_json)
|
||||
var oslist_parsed = oslistFromJson(o)
|
||||
if (oslist_parsed === false)
|
||||
return
|
||||
osmodel.clear()
|
||||
for (var i in oslist) {
|
||||
osmodel.append(oslist[i])
|
||||
for (var i in oslist_parsed) {
|
||||
osmodel.append(oslist_parsed[i])
|
||||
}
|
||||
|
||||
if ("imager" in o) {
|
||||
|
|
@ -1556,32 +1538,6 @@ ApplicationWindow {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Add in our 'special' items. */
|
||||
osmodel.append({
|
||||
url: "internal://format",
|
||||
icon: "icons/erase.png",
|
||||
extract_size: 0,
|
||||
image_download_size: 0,
|
||||
extract_sha256: "",
|
||||
contains_multiple_files: false,
|
||||
release_date: "",
|
||||
subitems_url: "",
|
||||
subitems_json: "",
|
||||
name: qsTr("Erase"),
|
||||
description: qsTr("Format card as FAT32"),
|
||||
tooltip: "",
|
||||
website: "",
|
||||
init_format: ""
|
||||
})
|
||||
|
||||
osmodel.append({
|
||||
url: "",
|
||||
icon: "icons/use_custom.png",
|
||||
name: qsTr("Use custom"),
|
||||
description: qsTr("Select a custom .img from your computer")
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
Timer {
|
||||
|
|
@ -1648,10 +1604,10 @@ ApplicationWindow {
|
|||
}
|
||||
|
||||
/* Reload list */
|
||||
httpRequest(imageWriter.constantOsListUrl(), function (x) {
|
||||
var o = JSON.parse(x.responseText)
|
||||
var oslist = oslistFromJson(o)
|
||||
if (oslist === false)
|
||||
var oslist_json = imageWriter.getFilteredOSlist();
|
||||
var o = JSON.parse(oslist_json)
|
||||
var oslist_parsed = oslistFromJson(o)
|
||||
if (oslist_parsed === false)
|
||||
return
|
||||
|
||||
/* As we're filtering the OS list, we need to ensure we present a 'Recommended' OS.
|
||||
|
|
@ -1659,19 +1615,18 @@ ApplicationWindow {
|
|||
* the preferred OS for a device is listed at the top level of the list, and is at the
|
||||
* lowest index. So..
|
||||
*/
|
||||
if (oslist.length != 0) {
|
||||
var candidate = oslist[0]
|
||||
if (oslist_parsed.length != 0) {
|
||||
var candidate = oslist_parsed[0]
|
||||
|
||||
if ("description" in candidate && !("subitems" in candidate)) {
|
||||
candidate["description"] += " (Recommended)"
|
||||
}
|
||||
}
|
||||
|
||||
osmodel.remove(0, osmodel.count-2)
|
||||
for (var i in oslist) {
|
||||
osmodel.insert(osmodel.count-2, oslist[i])
|
||||
osmodel.clear()
|
||||
for (var i in oslist_parsed) {
|
||||
osmodel.append(oslist_parsed[i])
|
||||
}
|
||||
})
|
||||
|
||||
// When the HW device is changed, reset the OS selection otherwise
|
||||
// you get a weird effect with the selection moving around in the list
|
||||
|
|
@ -1732,19 +1687,7 @@ ApplicationWindow {
|
|||
}
|
||||
else
|
||||
{
|
||||
ospopup.categorySelected = d.name
|
||||
var suburl = d.subitems_url
|
||||
var m = newSublist()
|
||||
|
||||
httpRequest(suburl, function (x) {
|
||||
var o = JSON.parse(x.responseText)
|
||||
var oslist = oslistFromJson(o)
|
||||
if (oslist === false)
|
||||
return
|
||||
for (var i in oslist) {
|
||||
m.append(oslist[i])
|
||||
}
|
||||
})
|
||||
console.log("Failure: Backend should have pre-flattened the JSON!");
|
||||
|
||||
osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1
|
||||
osswipeview.incrementCurrentIndex()
|
||||
|
|
@ -1757,9 +1700,9 @@ ApplicationWindow {
|
|||
if (imageWriter.mountUsbSourceMedia()) {
|
||||
var m = newSublist()
|
||||
|
||||
var oslist = JSON.parse(imageWriter.getUsbSourceOSlist())
|
||||
for (var i in oslist) {
|
||||
m.append(oslist[i])
|
||||
var usboslist = JSON.parse(imageWriter.getUsbSourceOSlist())
|
||||
for (var i in usboslist) {
|
||||
m.append(usboslist[i])
|
||||
}
|
||||
osswipeview.itemAt(osswipeview.currentIndex+1).currentIndex = (selectFirstSubitem === true) ? 0 : -1
|
||||
osswipeview.incrementCurrentIndex()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue