Address comments

.gitlab/ci/build_releng.sh

@@ -14,6 +14,7 @@
 # * zsync
 # * python
 # * python-jinja
+# * python-orjson
 
 set -eu
 shopt -s extglob
@@ -306,13 +307,15 @@ run_mkarchiso() {
 }
 
 generate_archlinux_ipxe() {
-  # generate the archlinux.ipxe intermediate artifact that is downloaded by the ipxe image
+  # generate the archlinux.ipxe target script that is downloaded by the ipxe image
   print_section_start "generate_archlinux_ipxe" "Generating archlinux.ipxe image"
 
   local _ipxe_dir="${orig_pwd}/ipxe"
-  local _ipxe_output_dir="${output}/ipxe"
+  local _ipxe_output="${output}/ipxe/ipxe-${version}"
 
-  python "${_ipxe_dir}/generate_archlinux_ipxe.py" > "${_ipxe_output_dir}/archlinux.ipxe"
+  python "${_ipxe_dir}/generate_archlinux_ipxe.py" > "${_ipxe_output}/archlinux.ipxe"
+
+  create_checksums "${_ipxe_output}/archlinux.ipxe"
 
   print_section_end "generate_archlinux_ipxe"
 }
@@ -322,7 +325,7 @@ sign_archlinux_ipxe() {
   print_section_start "sign_archlinux_ipxe" "Signing archlinux.ipxe image"
 
   local _ipxe_dir="${orig_pwd}/ipxe"
-  local _ipxe_output_dir="${output}/ipxe"
+  local _ipxe_output="${output}/ipxe/ipxe-${version}"
 
   openssl cms \
       -sign \
@@ -332,7 +335,7 @@ sign_archlinux_ipxe() {
       -signer "${codesigning_cert}" \
       -inkey "${codesigning_key}" \
       -outform DER \
-      -out "${_ipxe_output_dir}/archlinux.ipxe.sig"
+      -out "${_ipxe_output}/archlinux.ipxe.sig"
 
   print_section_end "sign_archlinux_ipxe"
 }

ipxe/generate_archlinux_ipxe.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import os
-import urllib.request, json
+import urllib.request
+import orjson
 import jinja2
 from collections import namedtuple
 from operator import itemgetter
@@ -8,35 +9,34 @@ from operator import itemgetter
 mirrors_url = "https://archlinux.org/mirrors/status/json"
 releases_url = "https://archlinux.org/releng/releases/json/"
 
-dir_path = os.path.dirname(os.path.realpath(__file__))
+def retrieve_releases():
-archlinux_ipxe_template = "archlinux.ipxe.jinja"
+    releases = []
+    with urllib.request.urlopen(releases_url) as url:
+        data = orjson.loads(url.read())
+        releases = sorted(data["releases"], key=itemgetter('release_date'), reverse=True)
+        releases = [ release["version"] for release in releases if release["available"]]
 
-templateLoader = jinja2.FileSystemLoader(dir_path)
+    return releases
-templateEnv = jinja2.Environment(loader=templateLoader)
-template = templateEnv.get_template(archlinux_ipxe_template)
-
-releases = []
-with urllib.request.urlopen(releases_url) as url:
-    data = json.loads(url.read())
-    releases = sorted(data["releases"], key=itemgetter('release_date'), reverse=True)
-    releases = [ release["version"] for release in releases if release["available"]]
     
     
-mirrors_by_country = []
+def retrieve_mirrors():
-with urllib.request.urlopen(mirrors_url) as url:
-    data = json.loads(url.read())
-
     mirrorurls = []
-    for mirror in data["urls"]: 
+    with urllib.request.urlopen(mirrors_url) as url:
-        if mirror["protocol"] == "http" and mirror["active"] and mirror["isos"]:
+        data = orjson.loads(url.read())
-            keys = ["url", "name", "country_code", "country"]
-            mirrorObj = namedtuple("Mirror", keys)
-            mirror = mirrorObj(mirror["url"], mirror["details"].rsplit('/',3)[1], mirror["country_code"], mirror["country"])
-            mirrorurls.append(mirror)
 
-    mirrorurls = sorted(mirrorurls, key=lambda x: x.name)
+        for mirror in data["urls"]: 
-    mirrorurls = sorted(mirrorurls, key=lambda x: x.country)
+            if mirror["protocol"] == "http" and mirror["active"] and mirror["isos"]:
+                keys = ["url", "name", "country_code", "country"]
+                mirrorObj = namedtuple("Mirror", keys)
+                mirror = mirrorObj(mirror["url"], mirror["details"].rsplit('/',3)[1], mirror["country_code"], mirror["country"])
+                mirrorurls.append(mirror)
 
+        mirrorurls = sorted(mirrorurls, key=lambda x: x.name)
+        mirrorurls = sorted(mirrorurls, key=lambda x: x.country)
+    
+    return mirrorurls
+
+def sort_mirrors_by_country(mirrorurls):
     mirrors_by_country = {}
     for mirror in mirrorurls:
         if mirror.country_code not in mirrors_by_country.keys():
@@ -45,5 +45,27 @@ with urllib.request.urlopen(mirrors_url) as url:
         mirrors_by_country[mirror.country_code]["list"].append({"url": mirror.url, "name": mirror.name})
 
     mirrors_by_country = mirrors_by_country.values()
+
+    return mirrors_by_country
+
+def render_from_jinja_template(mirrors_by_country, releases):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    archlinux_ipxe_template = "archlinux.ipxe.jinja"
+
+    templateLoader = jinja2.FileSystemLoader(dir_path)
+    templateEnv = jinja2.Environment(loader=templateLoader)
+    template = templateEnv.get_template(archlinux_ipxe_template)
+
+    print((template.render(mirrors_by_country = mirrors_by_country, releases = releases)))
    
-print((template.render(mirrors_by_country = mirrors_by_country, releases = releases)))
+
+def main():
+    releases = retrieve_releases()
+    mirrorurls = retrieve_mirrors()
+    mirrors_by_country = sort_mirrors_by_country(mirrorurls)
+
+    render_from_jinja_template(mirrors_by_country, releases)
+
+
+if __name__ == "__main__":
+    main()

ipxe/test.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-python generate_archlinux_ipxe.py
+python generate_archlinux_ipxe.py > archlinux.ipxe
 curl https://ipxe.archlinux.org/releng/netboot/archlinux.ipxe > archweb-archlinux.ipxe
 
 diff archweb-archlinux.ipxe archlinux.ipxe