Merge remote-tracking branch 'sespiros/issues/9'

* sespiros/issues/9:
  Fix build script for gitlab CI
  Address comments
  Add ipxe generation and signing to the build script
  Add script for offline generation of the ipxe intermediate artifact

.gitlab-ci.yml

@@ -8,7 +8,7 @@ include:
 
 variables:
   BUILD_SCRIPT: ./.gitlab/ci/build_releng.sh
-  PACKAGE_LIST: archiso erofs-utils ipxe openssl zsync
+  PACKAGE_LIST: archiso erofs-utils ipxe openssl zsync python python-jinja python-orjson
   QEMU_BUILD_TIMEOUT: 2400
   QEMU_COPY_ARTIFACTS_TIMEOUT: 120
   QEMU_VM_MEMORY: 3072

.gitlab/ci/build_releng.sh

@@ -12,6 +12,9 @@
 # * gnupg
 # * openssl
 # * zsync
+# * python
+# * python-jinja
+# * python-orjson
 
 set -eu
 shopt -s extglob
@@ -249,7 +252,6 @@ copy_ipxe_binaries() {
 
   print_section_start "copy_ipxe" "Copy iPXE binaries"
 
-  install -vdm 755 -- "${_ipxe_output}"
   cp -av -- "${_ipxe_base}/"{ipxe-arch.{lkrn,pxe},x86_64/ipxe-arch.efi} "${_ipxe_output}"
 
   print_section_end "copy_ipxe"
@@ -303,6 +305,41 @@ run_mkarchiso() {
   move_build_artifacts
 }
 
+generate_archlinux_ipxe() {
+  # generate the archlinux.ipxe target script that is downloaded by the ipxe image
+  print_section_start "generate_archlinux_ipxe" "Generating archlinux.ipxe image"
+
+  local _ipxe_dir="${orig_pwd}/ipxe"
+  local _ipxe_output="${output}/ipxe/ipxe-${version}"
+
+  install -vdm 755 -- "${_ipxe_output}"
+  python "${_ipxe_dir}/generate_archlinux_ipxe.py" > "${_ipxe_output}/archlinux.ipxe"
+
+  create_checksums "${_ipxe_output}/archlinux.ipxe"
+
+  print_section_end "generate_archlinux_ipxe"
+}
+
+sign_archlinux_ipxe() {
+  # sign the archlinux.ipxe intermediate artifact
+  print_section_start "sign_archlinux_ipxe" "Signing archlinux.ipxe image"
+
+  local _ipxe_dir="${orig_pwd}/ipxe"
+  local _ipxe_output="${output}/ipxe/ipxe-${version}"
+
+  openssl cms \
+      -sign \
+      -binary \
+      -noattr \
+      -in "${_ipxe_output}/archlinux.ipxe" \
+      -signer "${codesigning_cert}" \
+      -inkey "${codesigning_key}" \
+      -outform DER \
+      -out "${_ipxe_output}/archlinux.ipxe.sig"
+
+  print_section_end "sign_archlinux_ipxe"
+}
+
 trap cleanup EXIT
 
 if (( EUID != 0 )); then
@@ -313,6 +350,8 @@ fi
 create_ephemeral_pgp_key
 select_codesigning_key
 check_codesigning_cert_validity
+generate_archlinux_ipxe
+sign_archlinux_ipxe
 run_mkarchiso
 copy_ipxe_binaries
 set_ownership

ipxe/archlinux.ipxe.jinja

@@ -0,0 +1,157 @@
+#!ipxe
+
+
+# Figure out if client is 64-bit capable
+cpuid --ext 29 && set cpuarch x86_64 || set cpuarch i686
+
+# allow only trusted images
+imgtrust
+
+# initial options
+set release {{ releases.0 }}
+set extrabootoptions ip=dhcp net.ifnames=0 BOOTIF=01-${netX/mac}
+set countrycode
+
+:main
+iseq ${cpuarch} x86_64 && goto main_x86_64 || goto main_i686
+
+:main_x86_64
+menu Arch Linux Netboot
+item --gap Settings
+item --gap Architecture: x86_64
+item set_release Release: ${release}
+isset ${mirrorurl} && item set_mirror Mirror: ${mirrorurl} || item set_mirror Choose a mirror
+item set_options Boot options: ${extrabootoptions}
+item
+isset ${mirrorurl} && item boot Boot Arch Linux || item --gap Boot Arch Linux
+item shell Drop to iPXE shell
+item reboot Reboot
+item exit Exit iPXE
+isset ${mirrorurl} && choose --default set_options selected || choose --default set_mirror selected || goto shell
+goto ${selected} || goto main
+
+:main_i686
+menu Arch Linux Netboot
+item --gap ERROR: i686 is no longer supported!
+item shell Drop to iPXE shell
+item reboot Reboot
+item exit Exit iPXE
+choose --default shell selected || goto shell
+goto ${selected} || goto main
+
+:shell
+echo Type 'exit' to get the back to the menu
+shell
+goto main
+
+:reboot
+reboot
+
+:exit
+exit
+
+:set_release
+menu Arch Linux Netboot: Select Release
+item back back
+item
+item --gap Available releases:
+{% for release in releases %}item {{ release }} {{ release }}
+{% endfor %}
+choose selected || goto main
+iseq ${selected} back && goto main ||
+set release ${selected}
+goto main
+
+:set_mirror
+goto select_mirror_country
+
+:select_mirror_country
+menu Arch Linux Netboot: Select Mirror
+item back back
+item
+item --gap Custom Configuration:
+item custom Enter mirror URL
+item
+item --gap Select Mirror By Country:
+{% for mirrorgroup in mirrors_by_country %}
+{% if mirrorgroup.grouper == '' %}
+item 00 Global
+{% else %}
+item {{ mirrorgroup.grouper }} {{ mirrorgroup.grouper_name }}
+{% endif %}
+{% endfor %}
+isset ${countrycode} && choose --default ${countrycode} selected || choose selected || goto main
+iseq ${selected} back && goto main ||
+iseq ${selected} custom && goto enter_mirror_url ||
+set countrycode ${selected}
+goto select_mirror_url
+
+:enter_mirror_url
+set countrycode
+echo Enter the mirror URL including the trailing slash.
+echo Example: http://some.host/archlinux/
+echo
+echo -n Mirror URL: ${} && read mirrorurl || goto select_mirror_country
+goto main
+
+:select_mirror_url
+goto select_mirror_url_${countrycode} || goto select_mirror_country
+
+{% for mirrorgroup in mirrors_by_country %}
+{% if mirrorgroup.grouper == '' %}
+:select_mirror_url_00
+set countryname Global
+{% else %}
+:select_mirror_url_{{ mirrorgroup.grouper }}
+set countryname {{ mirrorgroup.grouper_name }}
+{% endif %}
+menu Arch Linux Netboot: Select Mirror
+item back back
+item
+item --gap Available mirrors in ${countryname}
+{% for mirror in mirrorgroup.list %}item {{ mirror.url }} {{ mirror.name }}
+{% endfor %}
+choose selected || goto select_mirror_country
+iseq ${selected} back && goto select_mirror_country ||
+set mirrorurl ${selected}
+goto main
+{% endfor %}
+
+:set_options
+echo -n Boot options: ${} && read extrabootoptions ||
+goto main
+
+:boot
+echo Booting Arch Linux x86_64 ${release} from ${mirrorurl}
+echo
+kernel ${mirrorurl}iso/${release}/arch/boot/x86_64/vmlinuz-linux || goto failed_download
+imgverify vmlinuz-linux ${mirrorurl}iso/${release}/arch/boot/x86_64/vmlinuz-linux.ipxe.sig || goto failed_verify
+initrd ${mirrorurl}iso/${release}/arch/boot/amd-ucode.img || goto failed_download
+imgverify amd-ucode.img ${mirrorurl}iso/${release}/arch/boot/amd-ucode.img.ipxe.sig || goto failed_verify
+initrd ${mirrorurl}iso/${release}/arch/boot/intel-ucode.img || goto failed_download
+imgverify intel-ucode.img ${mirrorurl}iso/${release}/arch/boot/intel-ucode.img.ipxe.sig || goto failed_verify
+initrd ${mirrorurl}iso/${release}/arch/boot/x86_64/initramfs-linux.img || goto failed_download
+imgverify initramfs-linux.img ${mirrorurl}iso/${release}/arch/boot/x86_64/initramfs-linux.img.ipxe.sig || goto failed_verify
+imgargs vmlinuz-linux initrd=amd-ucode.img initrd=intel-ucode.img initrd=initramfs-linux.img archiso_http_srv=${mirrorurl}iso/${release}/ archisobasedir=arch verify=y ${extrabootoptions}
+boot || goto failed_boot
+
+:failed_download
+echo
+echo Failed to download a file.
+goto failed
+
+:failed_verify
+echo
+echo Failed to verify a file.
+goto failed
+
+:failed_boot
+echo
+echo Boot failed.
+goto failed
+
+:failed
+echo Press a key to return to the menu.
+prompt
+imgfree
+goto main

ipxe/generate_archlinux_ipxe.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python
+import os
+import urllib.request
+import orjson
+import jinja2
+from collections import namedtuple
+from operator import itemgetter
+
+mirrors_url = "https://archlinux.org/mirrors/status/json"
+releases_url = "https://archlinux.org/releng/releases/json/"
+
+def retrieve_releases():
+    releases = []
+    with urllib.request.urlopen(releases_url) as url:
+        data = orjson.loads(url.read())
+        releases = sorted(data["releases"], key=itemgetter('release_date'), reverse=True)
+        releases = [ release["version"] for release in releases if release["available"]]
+
+    return releases
+    
+    
+def retrieve_mirrors():
+    mirrorurls = []
+    with urllib.request.urlopen(mirrors_url) as url:
+        data = orjson.loads(url.read())
+
+        for mirror in data["urls"]: 
+            if mirror["protocol"] == "http" and mirror["active"] and mirror["isos"]:
+                keys = ["url", "name", "country_code", "country"]
+                mirrorObj = namedtuple("Mirror", keys)
+                mirror = mirrorObj(mirror["url"], mirror["details"].rsplit('/',3)[1], mirror["country_code"], mirror["country"])
+                mirrorurls.append(mirror)
+
+        mirrorurls = sorted(mirrorurls, key=lambda x: x.name)
+        mirrorurls = sorted(mirrorurls, key=lambda x: x.country)
+    
+    return mirrorurls
+
+def sort_mirrors_by_country(mirrorurls):
+    mirrors_by_country = {}
+    for mirror in mirrorurls:
+        if mirror.country_code not in mirrors_by_country.keys():
+            mirrors_by_country[mirror.country_code] = {"grouper": mirror.country_code, "grouper_name": mirror.country, "list": []} 
+        
+        mirrors_by_country[mirror.country_code]["list"].append({"url": mirror.url, "name": mirror.name})
+
+    mirrors_by_country = mirrors_by_country.values()
+
+    return mirrors_by_country
+
+def render_from_jinja_template(mirrors_by_country, releases):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    archlinux_ipxe_template = "archlinux.ipxe.jinja"
+
+    templateLoader = jinja2.FileSystemLoader(dir_path)
+    templateEnv = jinja2.Environment(loader=templateLoader)
+    template = templateEnv.get_template(archlinux_ipxe_template)
+
+    print((template.render(mirrors_by_country = mirrors_by_country, releases = releases)))
+   
+
+def main():
+    releases = retrieve_releases()
+    mirrorurls = retrieve_mirrors()
+    mirrors_by_country = sort_mirrors_by_country(mirrorurls)
+
+    render_from_jinja_template(mirrors_by_country, releases)
+
+
+if __name__ == "__main__":
+    main()

ipxe/test.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+python generate_archlinux_ipxe.py > archlinux.ipxe
+curl https://ipxe.archlinux.org/releng/netboot/archlinux.ipxe > archweb-archlinux.ipxe
+
+diff archweb-archlinux.ipxe archlinux.ipxe