Add integration tests

Tests if repository json files conform to the json schema.
If all resources (images/icons/website URLs) they mention actually
exists.

And can also test writing images and the FAT modification code.

tests/README.md

@@ -0,0 +1,30 @@
+Integration tests
+===
+
+Test if all json files in the public repository are correct (validate against json schema, and test all image files, icons and websites mentioned in the json do exist by performing HEAD requests)
+
+```
+$ cd tests
+$ pytest
+
+```
+
+Test if a specific json file validates correctly
+
+```
+$ cd tests
+$ pytest --repo=http://my-repo/os_list.json
+```
+
+Test image writes for all images in a repository
+
+```
+$ cd tests
+$ truncate -s 16G loopfile
+$ udisksctl loop-setup --file loopfile
+Mapped file loopfile as /dev/loop24
+$ sudo -g disk pytest test_write_images.py --repo=http://my-repo/os_list.json --device=/dev/loop24
+```
+
+Note: make sure automatic mounting of removable media is disabled in your Linux distribution during write tests.
+You can also use real drives instead of loop files as device. But be very careful not to enter the wrong device. Writes are done for real, it is not a mock test...

tests/cache/.gitignore

@@ -0,0 +1 @@
+*

tests/conftest.py

@@ -0,0 +1,90 @@
+import pytest
+import json
+import urllib.request
+
+os_list_files = []
+icon_urls = set()
+website_urls = set()
+item_json = []
+already_processed_urls = set()
+total_download_size = 0
+largest_extract_size = 0
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--repo",
+        action="store",
+        default="https://downloads.raspberrypi.com/os_list_imagingutility_v3.json",
+        help="Repository URL to test"
+    )
+    parser.addoption(
+        "--device",
+        action="store",
+        default="",
+        help="(Loop) device if you want to perform actual image write tests"
+    )
+
+def parse_json_entries(j):
+    global total_download_size, largest_extract_size
+
+    for item in j:
+        if "subitems" in item:
+            parse_json_entries(item["subitems"])
+        elif "subitems_url" in item:
+            parse_os_list(item["subitems_url"])
+        else:
+            if "icon" in item and not item["icon"].startswith("data:"):
+                icon_urls.add(item["icon"])
+            if "website" in item:
+                website_urls.add(item["website"])
+            if "url" in item:
+                item_json.append(item)
+            if "image_download_size" in item:
+                total_download_size += int(item["image_download_size"])
+            if "extract_size" in item:
+                largest_extract_size = max(largest_extract_size, int(item["extract_size"]))
+
+def parse_os_list(url):
+    if url in already_processed_urls:
+        print("Circular reference! Already processed URL: {}".format(url))
+        return
+
+    already_processed_urls.add(url)
+
+    try:
+        print("Fetching OS list file {}".format(url))
+        req = urllib.request.Request(
+            url, 
+            data=None, 
+            headers={
+                'User-Agent': 'rpi-imager automated tests'
+            }
+        )
+        response = urllib.request.urlopen(req)
+        body = response.read()
+        j = json.loads(body)
+        os_list_files.append( (url,body) )
+
+        if "os_list" in j:
+            parse_json_entries(j["os_list"])
+
+    except Exception as err:
+        print("Error processing '{}': {}".format(url, repr(err) ))
+
+def pytest_configure(config):
+    parse_os_list(config.getoption("--repo"))
+    print("Found {} os_list.json files {} OS images {} icons {} website URLs".format( 
+        len(os_list_files), len(item_json), len(icon_urls), len(website_urls) ) )
+    print("Total compressed image download size: {} GB".format(round(total_download_size / 1024 ** 3) ))
+    print("Largest uncompressed image size: {} GB".format(round(largest_extract_size / 1024 ** 3) ))
+
+
+def pytest_generate_tests(metafunc):
+    if "oslisttuple" in metafunc.fixturenames:
+        metafunc.parametrize("oslisttuple", os_list_files)
+    if "iconurl" in metafunc.fixturenames:
+        metafunc.parametrize("iconurl", icon_urls)
+    if "websiteurl" in metafunc.fixturenames:
+        metafunc.parametrize("websiteurl", website_urls)
+    if "imageitem" in metafunc.fixturenames:
+        metafunc.parametrize("imageitem", item_json)

tests/test_firstrun.txt

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+echo "Bogus firstrun.sh file, used for automated testing of FAT16/FAT32 modification code only"
+
+exit 0

tests/test_schema.py

@@ -0,0 +1,28 @@
+import os
+import json
+import pytest
+from jsonschema import validate
+from jsonschema.exceptions import ValidationError
+
+
+def test_os_list_json_against_schema(oslisttuple, schema):
+    oslisturl = oslisttuple[0]
+    oslistdata = oslisttuple[1]
+    errorMsg = ""
+
+    j = json.loads(oslistdata)
+
+    try:
+        validate(instance=j, schema=schema)
+    except ValidationError as err:
+        errorMsg = err.message
+
+    if errorMsg != "":
+        pytest.fail(oslisturl+" failed schema validation: "+errorMsg, False)
+
+
+@pytest.fixture
+def schema():
+    f = open(os.path.dirname(__file__)+"/../doc/json-schema/os-list-schema.json","r")
+    data = f.read()
+    return json.loads(data)

tests/test_urls.py

@@ -0,0 +1,25 @@
+import urllib.request
+
+def _head_request(url):
+    req = urllib.request.Request(
+        url, 
+        data=None, 
+        headers={
+            'User-Agent': 'rpi-imager automated tests'
+        },
+        method="HEAD"
+    )
+    return urllib.request.urlopen(req)    
+
+
+def test_icon_url_exists(iconurl):
+    assert _head_request(iconurl).status == 200
+
+def test_website_url_exists(websiteurl):
+    assert _head_request(websiteurl).status == 200
+
+def test_image_url_exists_and_has_correct_image_download_size(imageitem):
+    response = _head_request(imageitem["url"])
+
+    assert response.status == 200
+    assert str(imageitem["image_download_size"]) == response.headers["Content-Length"]

tests/test_write_images.py

@@ -0,0 +1,40 @@
+import pytest
+import re
+import subprocess
+import os
+import time
+from shlex import quote
+
+
+def shell(cmd, url):
+    msg = ''
+    try:
+        subprocess.run(cmd, shell=True, check=True, capture_output=True)
+    except subprocess.CalledProcessError as err:
+        msg = "{} Error running '{}' exit code {} stderr: '{}'".format(url, err.cmd, err.returncode, err.output)
+
+    if msg != '':
+        pytest.fail(msg, False)
+
+
+def test_write_image_and_modify_fat(imageitem, device):
+    if not device:
+        pytest.skip("--device=<device> not specified. Skipping write tests")
+        return
+
+    assert "extract_sha256" in imageitem, "{}: missing extract_sha256. Cannot perform write test.".format(imageitem["url"])
+    assert "image_download_size" in imageitem, "{}: missing image_download_size. Cannot perform write test.".format(imageitem["url"])
+    assert re.search("^[a-z0-9]{64}$", imageitem["extract_sha256"]) != None
+
+    cacheFile = "cache/"+imageitem["extract_sha256"]
+    if os.path.exists(cacheFile) and os.path.getsize(cacheFile) != imageitem["image_download_size"]:
+        os.remove(cacheFile)
+
+    shell("rpi-imager --cli --quiet --enable-writing-system-drives --sha256 {} --cache-file {} --first-run-script test_firstrun.txt {} {}".format(
+        quote(imageitem["extract_sha256"]), quote(cacheFile), quote(imageitem["url"]), quote(device) ), imageitem["url"])
+    time.sleep(0.5)
+    shell("fsck.vfat -n "+quote(device+"p1"), imageitem["url"])
+
+@pytest.fixture
+def device(request):
+    return request.config.getoption("--device")