First attempt at cloudinit support

Experimental

OptionsPopup.qml

@@ -23,6 +23,10 @@ Popup {
     property string config
     property string cmdline
     property string firstrun
+    property string cloudinit
+    property string cloudinitrun
+    property string cloudinitwrite
+    property string cloudinitnetwork
 
     // background of title
     Rectangle {
@@ -496,12 +500,29 @@ Popup {
     function escapeshellarg(arg) {
         return "'"+arg.replace(/'/g, "\\'")+"'"
     }
+    function addCloudInit(s) {
+        cloudinit += s+"\n"
+    }
+    function addCloudInitWriteFile(name, content, perms) {
+        cloudinitwrite += "- encoding: b64\n"
+        cloudinitwrite += "  content: "+Qt.btoa(content)+"\n"
+        cloudinitwrite += "  owner: root:root\n"
+        cloudinitwrite += "  path: "+name+"\n"
+        cloudinitwrite += "  permissions: '"+perms+"'\n"
+    }
+    function addCloudInitRun(cmd) {
+        cloudinitrun += "- "+cmd+"\n"
+    }
 
     function applySettings()
     {
         cmdline = ""
         config = ""
         firstrun = ""
+        cloudinit = ""
+        cloudinitrun = ""
+        cloudinitwrite = ""
+        cloudinitnetwork = ""
 
         if (chkOverscan.checked) {
             addConfig("disable_overscan=1")
@@ -510,15 +531,31 @@ Popup {
             addFirstRun("CURRENT_HOSTNAME=`cat /etc/hostname | tr -d \" \\t\\n\\r\"`")
             addFirstRun("echo "+fieldHostname.text+" >/etc/hostname")
             addFirstRun("sed -i \"s/127.0.1.1.*$CURRENT_HOSTNAME/127.0.1.1\\t"+fieldHostname.text+"/g\" /etc/hosts")
+
+            addCloudInit("hostname: "+fieldHostname.text)
+            addCloudInit("manage_etc_hosts: true")
+            addCloudInit("packages:")
+            addCloudInit("- avahi-daemon")
+            addCloudInit("")
         }
         if (chkSSH.checked) {
             // First user may not be called 'pi' on all distributions, so look username up
             addFirstRun("FIRSTUSER=`getent passwd 1000 | cut -d: -f1`");
             addFirstRun("FIRSTUSERHOME=`getent passwd 1000 | cut -d: -f6`")
 
+            addCloudInit("users:")
+            addCloudInit("- name: pi")
+            addCloudInit("  groups: users,adm,dialout,audio,netdev,video,plugdev,sudo")
+            addCloudInit("  shell: /bin/bash")
+
             if (radioPasswordAuthentication.checked) {
                 var cryptedPassword = fieldUserPassword.alreadyCrypted ? fieldUserPassword.text : imageWriter.crypt(fieldUserPassword.text)
                 addFirstRun("echo \"$FIRSTUSER:\""+escapeshellarg(cryptedPassword)+" | chpasswd -e")
+
+                addCloudInit("  lock_passwd: false")
+                addCloudInit("  passwd: "+cryptedPassword)
+                addCloudInit("")
+                addCloudInit("ssh_pwauth: true")
             }
             if (radioPubKeyAuthentication.checked) {
                 var pubkey = fieldPublicKey.text.replace(/\n/g, "")
@@ -527,8 +564,14 @@ Popup {
                     addFirstRun("install -o \"$FIRSTUSER\" -m 600 <(echo \""+pubkey+"\") \"$FIRSTUSERHOME/.ssh/authorized_keys\"")
                 }
                 addFirstRun("echo 'PasswordAuthentication no' >>/etc/ssh/sshd_config")
+
+                addCloudInit("  lock_passwd: true")
+                addCloudInit("  ssh_authorized_keys:")
+                addCloudInit("    - "+pubkey)
+                addCloudInit("  sudo: ALL=(ALL) NOPASSWD:ALL")
             }
             addFirstRun("systemctl enable ssh")
+            addCloudInit("")
         }
         if (chkWifi.checked) {
             var wpaconfig = "country="+fieldWifiCountry.editText+"\n"
@@ -549,22 +592,39 @@ Popup {
             addFirstRun("for filename in /var/lib/systemd/rfkill/*:wlan ; do")
             addFirstRun("  echo 0 > $filename")
             addFirstRun("done")
+
+            cloudinitnetwork  = "version: 2\n"
+            cloudinitnetwork += "wifis:\n"
+            cloudinitnetwork += "  renderer: networkd\n"
+            cloudinitnetwork += "  wlan0:\n"
+            cloudinitnetwork += "    dhcp4: true\n"
+            cloudinitnetwork += "    optional: true\n"
+            cloudinitnetwork += "    access-points:\n"
+            cloudinitnetwork += "      "+fieldWifiSSID.text+":\n"
+            cloudinitnetwork += "        password: \""+cryptedPsk+"\"\n"
         }
         if (chkLocale.checked) {
             if (chkSkipFirstUse) {
                 addFirstRun("rm -f /etc/xdg/autostart/piwiz.desktop")
+                addCloudInitRun("rm -f /etc/xdg/autostart/piwiz.desktop")
             }
 
+            var kbdconfig = "XKBMODEL=\"pc105\"\n"
+            kbdconfig += "XKBLAYOUT=\""+fieldKeyboardLayout.text+"\"\n"
+            kbdconfig += "XKBVARIANT=\"\"\n"
+            kbdconfig += "XKBOPTIONS=\"\"\n"
+
             addFirstRun("rm -f /etc/localtime")
             addFirstRun("echo \""+fieldTimezone.editText+"\" >/etc/timezone")
             addFirstRun("dpkg-reconfigure -f noninteractive tzdata")
             addFirstRun("cat >/etc/default/keyboard <<'KBEOF'")
-            addFirstRun("XKBMODEL=\"pc105\"")
+            addFirstRun(kbdconfig)
-            addFirstRun("XKBLAYOUT=\""+fieldKeyboardLayout.text+"\"")
-            addFirstRun("XKBVARIANT=\"\"")
-            addFirstRun("XKBOPTIONS=\"\"")
             addFirstRun("KBEOF")
             addFirstRun("dpkg-reconfigure -f noninteractive keyboard-configuration")
+
+            addCloudInit("timezone: "+fieldTimezone.editText)
+            addCloudInitWriteFile("/etc/default/keyboard", kbdconfig, '0644')
+            addCloudInitRun("dpkg-reconfigure -f noninteractive keyboard-configuration || true")
         }
 
         if (firstrun.length) {
@@ -574,10 +634,19 @@ Popup {
             addFirstRun("exit 0")
             /* using systemd.run_success_action=none does not seem to have desired effect
                systemd then stays at "reached target kernel command line", so use reboot instead */
-            addCmdline("systemd.run=/boot/firstrun.sh systemd.run_success_action=reboot systemd.unit=kernel-command-line.target")
+            //addCmdline("systemd.run=/boot/firstrun.sh systemd.run_success_action=reboot systemd.unit=kernel-command-line.target")
+            // cmdline changing moved to DownloadThread::_customizeImage()
         }
 
-        imageWriter.setImageCustomization(config, cmdline, firstrun)
+        if (cloudinitwrite !== "") {
+            addCloudInit("write_files:\n"+cloudinitwrite+"\n")
+        }
+
+        if (cloudinitrun !== "") {
+            addCloudInit("runcmd:\n"+cloudinitrun+"\n")
+        }
+
+        imageWriter.setImageCustomization(config, cmdline, firstrun, cloudinit, cloudinitnetwork)
     }
 
     function saveSettings()

downloadthread.cpp

@@ -829,11 +829,14 @@ qint64 DownloadThread::_sectorsWritten()
     return -1;
 }
 
-void DownloadThread::setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun)
+void DownloadThread::setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun, const QByteArray &cloudinit, const QByteArray &cloudInitNetwork, const QByteArray &initFormat)
 {
     _config = config;
     _cmdline = cmdline;
     _firstrun = firstrun;
+    _cloudinit = cloudinit;
+    _cloudinitNetwork = cloudInitNetwork;
+    _initFormat = initFormat;
 }
 
 bool DownloadThread::_customizeImage()
@@ -987,20 +990,6 @@ bool DownloadThread::_customizeImage()
 
     emit preparationStatusUpdate(tr("Customizing image"));
 
-    if (!_firstrun.isEmpty())
-    {
-        QFile f(folder+"/firstrun.sh");
-        if (f.open(f.WriteOnly) && f.write(_firstrun) == _firstrun.length())
-        {
-            f.close();
-        }
-        else
-        {
-            emit error(tr("Error creating firstrun.sh on FAT partition"));
-            return false;
-        }
-    }
-
     if (!_config.isEmpty())
     {
         auto configItems = _config.split('\n');
@@ -1041,6 +1030,85 @@ bool DownloadThread::_customizeImage()
         }
     }
 
+    if (_initFormat == "auto")
+    {
+        /* Do an attempt at auto-detecting what customization format a custom
+           image provided by the user supports */
+        QByteArray issue;
+        QFile fi(folder+"/issue.txt");
+        if (fi.exists() && fi.open(fi.ReadOnly))
+        {
+            issue = fi.readAll();
+            fi.close();
+        }
+
+        if (QFile::exists(folder+"/user-data"))
+        {
+            /* If we have user-data file on FAT partition, then it must be cloudinit */
+            _initFormat = "cloudinit";
+            qDebug() << "user-data found on FAT partition. Assuming cloudinit support";
+        }
+        else if (issue.contains("pi-gen"))
+        {
+            /* If issue.txt mentions pi-gen, and there is no user-data file assume
+             * it is a RPI OS flavor, and use the old systemd unit firstrun script stuff */
+            _initFormat = "systemd";
+            qDebug() << "using firstrun script invoked by systemd customization method";
+        }
+        else
+        {
+            /* Fallback to writing cloudinit file, as it does not hurt having one
+             * Will just have no customization if OS does not support it */
+            _initFormat = "cloudinit";
+            qDebug() << "Unknown what customization method image supports. Falling back to cloudinit";
+        }
+    }
+
+    if (!_firstrun.isEmpty() && _initFormat == "systemd")
+    {
+        QFile f(folder+"/firstrun.sh");
+        if (f.open(f.WriteOnly) && f.write(_firstrun) == _firstrun.length())
+        {
+            f.close();
+        }
+        else
+        {
+            emit error(tr("Error creating firstrun.sh on FAT partition"));
+            return false;
+        }
+
+        _cmdline += " systemd.run=/boot/firstrun.sh systemd.run_success_action=reboot systemd.unit=kernel-command-line.target";
+    }
+
+    if (!_cloudinit.isEmpty() && _initFormat == "cloudinit")
+    {
+        _cloudinit = "#cloud-config\n"+_cloudinit;
+        QFile f(folder+"/user-data");
+        if (f.open(f.WriteOnly) && f.write(_cloudinit) == _cloudinit.length())
+        {
+            f.close();
+        }
+        else
+        {
+            emit error(tr("Error creating user-data cloudinit file on FAT partition"));
+            return false;
+        }
+    }
+
+    if (!_cloudinitNetwork.isEmpty() && _initFormat == "cloudinit")
+    {
+        QFile f(folder+"/network-config");
+        if (f.open(f.WriteOnly) && f.write(_cloudinitNetwork) == _cloudinitNetwork.length())
+        {
+            f.close();
+        }
+        else
+        {
+            emit error(tr("Error creating network-config cloudinit file on FAT partition"));
+            return false;
+        }
+    }
+
     if (!_cmdline.isEmpty())
     {
         QByteArray cmdline;

downloadthread.h

@@ -112,7 +112,7 @@ public:
     /*
      * Enable image customization
      */
-    void setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun);
+    void setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun, const QByteArray &cloudinit, const QByteArray &cloudinitNetwork, const QByteArray &initFormat);
 
     /*
      * Thread safe download progress query functions
@@ -164,7 +164,7 @@ protected:
     curl_off_t _startOffset;
     std::atomic<std::uint64_t> _lastDlTotal, _lastDlNow, _verifyTotal, _lastVerifyNow, _bytesWritten;
     qint64 _sectorsStart;
-    QByteArray _url, _useragent, _buf, _filename, _lastError, _expectedHash, _config, _cmdline, _firstrun;
+    QByteArray _url, _useragent, _buf, _filename, _lastError, _expectedHash, _config, _cmdline, _firstrun, _cloudinit, _cloudinitNetwork, _initFormat;
     char *_firstBlock;
     size_t _firstBlockSize;
     static QByteArray _proxy;

icons/ic_cog_40px.svg

@@ -0,0 +1 @@
+<svg id="Layer_1" data-name="Layer 1" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 40 40"><defs><style>.cls-1{fill:#f2c5c5;}.cls-2{fill:none;stroke:#f2c5c5;stroke-miterlimit:10;stroke-width:3px;}</style></defs><path class="cls-1" d="M20,16a4,4,0,1,1-4,4,4.00455,4.00455,0,0,1,4-4m0-2a6,6,0,1,0,6,6,6,6,0,0,0-6-6Z"/><path class="cls-2" d="M18.41429,7.42824,15.25814,3.11A17.506,17.506,0,0,0,7.742,7.45608l2.17348,4.88276a12.31422,12.31422,0,0,0-1.59722,2.75283l-5.29836.57179a17.50582,17.50582,0,0,0,.00577,8.68221l5.31534.5591a12.34163,12.34163,0,0,0,1.58374,2.75854L7.76287,32.55559a17.50588,17.50588,0,0,0,7.52189,4.33611L18.42662,32.568a12.34585,12.34585,0,0,0,3.18078.00781l3.15589,4.31881a17.506,17.506,0,0,0,7.51614-4.3461L30.106,27.66578a12.34511,12.34511,0,0,0,1.59709-2.7506l5.31778-.57412a17.50593,17.50593,0,0,0-.00576-8.68221l-5.31534-.5591a12.32179,12.32179,0,0,0-1.59627-2.77415l2.15518-4.87708a17.50589,17.50589,0,0,0-7.5219-4.33612L21.59487,7.43607A12.36027,12.36027,0,0,0,18.41429,7.42824Z"/></svg>

imagewriter.cpp

@@ -138,7 +138,7 @@ void ImageWriter::setEngine(QQmlApplicationEngine *engine)
 }
 
 /* Set URL to download from */
-void ImageWriter::setSrc(const QUrl &url, quint64 downloadLen, quint64 extrLen, QByteArray expectedHash, bool multifilesinzip, QString parentcategory, QString osname)
+void ImageWriter::setSrc(const QUrl &url, quint64 downloadLen, quint64 extrLen, QByteArray expectedHash, bool multifilesinzip, QString parentcategory, QString osname, QByteArray initFormat)
 {
     _src = url;
     _downloadLen = downloadLen;
@@ -147,11 +147,13 @@ void ImageWriter::setSrc(const QUrl &url, quint64 downloadLen, quint64 extrLen,
     _multipleFilesInZip = multifilesinzip;
     _parentCategory = parentcategory;
     _osName = osname;
+    _initFormat = (initFormat == "none") ? "" : initFormat;
 
     if (!_downloadLen && url.isLocalFile())
     {
         QFileInfo fi(url.toLocalFile());
         _downloadLen = fi.size();
+        _initFormat = "auto";
     }
 }
 
@@ -238,7 +240,7 @@ void ImageWriter::startWrite()
     connect(_thread, SIGNAL(preparationStatusUpdate(QString)), SLOT(onPreparationStatusUpdate(QString)));
     _thread->setVerifyEnabled(_verifyEnabled);
     _thread->setUserAgent(QString("Mozilla/5.0 rpi-imager/%1").arg(constantVersion()).toUtf8());
-    _thread->setImageCustomization(_config, _cmdline, _firstrun);
+    _thread->setImageCustomization(_config, _cmdline, _firstrun, _cloudinit, _cloudinitNetwork, _initFormat);
 
     if (!_expectedHash.isEmpty() && _cachedFileHash != _expectedHash && _cachingEnabled)
     {
@@ -941,15 +943,18 @@ void ImageWriter::setSetting(const QString &key, const QVariant &value)
     _settings.sync();
 }
 
-void ImageWriter::setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun)
+void ImageWriter::setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun, const QByteArray &cloudinit, const QByteArray &cloudinitNetwork)
 {
     _config = config;
     _cmdline = cmdline;
     _firstrun = firstrun;
+    _cloudinit = cloudinit;
+    _cloudinitNetwork = cloudinitNetwork;
 
     qDebug() << "Custom config.txt entries:" << config;
     qDebug() << "Custom cmdline.txt entries:" << cmdline;
     qDebug() << "Custom firstuse.sh:" << firstrun;
+    qDebug() << "Cloudinit:" << cloudinit;
 }
 
 QString ImageWriter::crypt(const QByteArray &password)
@@ -1022,6 +1027,11 @@ bool ImageWriter::hasSavedCustomizationSettings()
     return result;
 }
 
+bool ImageWriter::imageSupportsCustomization()
+{
+    return !_initFormat.isEmpty();
+}
+
 void MountUtilsLog(std::string msg) {
     Q_UNUSED(msg)
     //qDebug() << "mountutils:" << msg.c_str();

imagewriter.h

@@ -29,7 +29,7 @@ public:
     void setEngine(QQmlApplicationEngine *engine);
 
     /* Set URL to download from, and if known download length and uncompressed length */
-    Q_INVOKABLE void setSrc(const QUrl &url, quint64 downloadLen = 0, quint64 extrLen = 0, QByteArray expectedHash = "", bool multifilesinzip = false, QString parentcategory = "", QString osname = "");
+    Q_INVOKABLE void setSrc(const QUrl &url, quint64 downloadLen = 0, quint64 extrLen = 0, QByteArray expectedHash = "", bool multifilesinzip = false, QString parentcategory = "", QString osname = "", QByteArray initFormat = "");
 
     /* Set device to write to */
     Q_INVOKABLE void setDst(const QString &device, quint64 deviceSize = 0);
@@ -102,11 +102,12 @@ public:
 
     Q_INVOKABLE bool getBoolSetting(const QString &key);
     Q_INVOKABLE void setSetting(const QString &key, const QVariant &value);
-    Q_INVOKABLE void setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun);
+    Q_INVOKABLE void setImageCustomization(const QByteArray &config, const QByteArray &cmdline, const QByteArray &firstrun, const QByteArray &cloudinit, const QByteArray &cloudinitNetwork);
     Q_INVOKABLE void setSavedCustomizationSettings(const QVariantMap &map);
     Q_INVOKABLE QVariantMap getSavedCustomizationSettings();
     Q_INVOKABLE void clearSavedCustomizationSettings();
     Q_INVOKABLE bool hasSavedCustomizationSettings();
+    Q_INVOKABLE bool imageSupportsCustomization();
 
     Q_INVOKABLE QString crypt(const QByteArray &password);
     Q_INVOKABLE QString pbkdf2(const QByteArray &psk, const QByteArray &ssid);
@@ -143,7 +144,7 @@ protected slots:
 protected:
     QUrl _src, _repo;
     QString _dst, _cacheFileName, _parentCategory, _osName;
-    QByteArray _expectedHash, _cachedFileHash, _cmdline, _config, _firstrun;
+    QByteArray _expectedHash, _cachedFileHash, _cmdline, _config, _firstrun, _cloudinit, _cloudinitNetwork, _initFormat;
     quint64 _downloadLen, _extrLen, _devLen, _dlnow, _verifynow;
     DriveListModel _drivelist;
     QQmlApplicationEngine *_engine;

main.qml

@@ -192,7 +192,7 @@ ApplicationWindow {
                                 return
                             }
 
-                            if (!optionspopup.initialized && imageWriter.hasSavedCustomizationSettings()) {
+                            if (!optionspopup.initialized && imageWriter.imageSupportsCustomization() && imageWriter.hasSavedCustomizationSettings()) {
                                 usesavedsettingspopup.openPopup()
                             } else {
                                 confirmwritepopup.askForConfirmation()
@@ -205,7 +205,7 @@ ApplicationWindow {
                 ColumnLayout {
                     id: columnLayout3
                     Layout.columnSpan: 3
-                    Layout.alignment: Qt.AlignHCenter | Qt.AlignVCenter
+                    Layout.alignment: Qt.AlignRight | Qt.AlignVCenter
 
                     Text {
                         id: progressText
@@ -255,6 +255,17 @@ ApplicationWindow {
                         font.family: roboto.name
                         Accessible.onPressAction: clicked()
                     }
+                    Image {
+                        id: customizebutton
+                        source: "icons/ic_cog_40px.svg"
+                        visible: false
+                        MouseArea {
+                            anchors.fill: parent
+                            onClicked: {
+                                optionspopup.openPopup()
+                            }
+                        }
+                    }
                 }
             }
         }
@@ -375,6 +386,7 @@ ApplicationWindow {
                     description: qsTr("Go back to main menu")
                     tooltip: ""
                     website: ""
+                    init_format: ""
                 }
             }
 
@@ -416,6 +428,7 @@ ApplicationWindow {
             description: qsTr("Format card as FAT32")
             tooltip: ""
             website: ""
+            init_format: ""
         }
 
         ListElement {
@@ -767,6 +780,7 @@ ApplicationWindow {
         title: qsTr("Warning")
         onYes: {
             writebutton.enabled = false
+            customizebutton.visible = false
             cancelwritebutton.enabled = true
             cancelwritebutton.visible = true
             cancelverifybutton.enabled = true
@@ -892,6 +906,7 @@ ApplicationWindow {
     function resetWriteButton() {
         progressText.visible = false
         progressBar.visible = false
+        customizebutton.visible = imageWriter.imageSupportsCustomization()
         osbutton.enabled = true
         dstbutton.enabled = true
         writebutton.visible = true
@@ -931,6 +946,7 @@ ApplicationWindow {
         if (imageWriter.readyToWrite()) {
             writebutton.enabled = true
         }
+        customizebutton.visible = imageWriter.imageSupportsCustomization()
     }
 
     function onCancelled() {
@@ -1059,12 +1075,13 @@ ApplicationWindow {
                 }
             }
         } else {
-            imageWriter.setSrc(d.url, d.image_download_size, d.extract_size, typeof(d.extract_sha256) != "undefined" ? d.extract_sha256 : "", typeof(d.contains_multiple_files) != "undefined" ? d.contains_multiple_files : false, ospopup.categorySelected, d.name)
+            imageWriter.setSrc(d.url, d.image_download_size, d.extract_size, typeof(d.extract_sha256) != "undefined" ? d.extract_sha256 : "", typeof(d.contains_multiple_files) != "undefined" ? d.contains_multiple_files : false, ospopup.categorySelected, d.name, typeof(d.init_format) != "undefined" ? d.init_format : "")
             osbutton.text = d.name
             ospopup.close()
             if (imageWriter.readyToWrite()) {
                 writebutton.enabled = true
             }
+            customizebutton.visible = imageWriter.imageSupportsCustomization()
         }
     }
 

qml.qrc

@@ -28,5 +28,6 @@
         <file>UseSavedSettingsPopup.qml</file>
         <file>icons/ic_info_16px.png</file>
         <file>icons/ic_info_12px.png</file>
+        <file>icons/ic_cog_40px.svg</file>
     </qresource>
 </RCC>