diff --git a/.gitlab/ci/build_releng.sh b/.gitlab/ci/build_releng.sh index 979cba1..8eb7672 100755 --- a/.gitlab/ci/build_releng.sh +++ b/.gitlab/ci/build_releng.sh @@ -14,6 +14,7 @@ # * zsync # * python # * python-jinja +# * python-orjson set -eu shopt -s extglob @@ -306,13 +307,15 @@ run_mkarchiso() { } generate_archlinux_ipxe() { - # generate the archlinux.ipxe intermediate artifact that is downloaded by the ipxe image + # generate the archlinux.ipxe target script that is downloaded by the ipxe image print_section_start "generate_archlinux_ipxe" "Generating archlinux.ipxe image" local _ipxe_dir="${orig_pwd}/ipxe" - local _ipxe_output_dir="${output}/ipxe" + local _ipxe_output="${output}/ipxe/ipxe-${version}" - python "${_ipxe_dir}/generate_archlinux_ipxe.py" > "${_ipxe_output_dir}/archlinux.ipxe" + python "${_ipxe_dir}/generate_archlinux_ipxe.py" > "${_ipxe_output}/archlinux.ipxe" + + create_checksums "${_ipxe_output}/archlinux.ipxe" print_section_end "generate_archlinux_ipxe" } @@ -322,7 +325,7 @@ sign_archlinux_ipxe() { print_section_start "sign_archlinux_ipxe" "Signing archlinux.ipxe image" local _ipxe_dir="${orig_pwd}/ipxe" - local _ipxe_output_dir="${output}/ipxe" + local _ipxe_output="${output}/ipxe/ipxe-${version}" openssl cms \ -sign \ @@ -332,7 +335,7 @@ sign_archlinux_ipxe() { -signer "${codesigning_cert}" \ -inkey "${codesigning_key}" \ -outform DER \ - -out "${_ipxe_output_dir}/archlinux.ipxe.sig" + -out "${_ipxe_output}/archlinux.ipxe.sig" print_section_end "sign_archlinux_ipxe" } diff --git a/ipxe/generate_archlinux_ipxe.py b/ipxe/generate_archlinux_ipxe.py index e0decf7..511ca4c 100755 --- a/ipxe/generate_archlinux_ipxe.py +++ b/ipxe/generate_archlinux_ipxe.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os -import urllib.request, json +import urllib.request +import orjson import jinja2 from collections import namedtuple from operator import itemgetter @@ -8,35 +9,34 @@ from operator import itemgetter mirrors_url = "https://archlinux.org/mirrors/status/json" releases_url = "https://archlinux.org/releng/releases/json/" -dir_path = os.path.dirname(os.path.realpath(__file__)) -archlinux_ipxe_template = "archlinux.ipxe.jinja" +def retrieve_releases(): + releases = [] + with urllib.request.urlopen(releases_url) as url: + data = orjson.loads(url.read()) + releases = sorted(data["releases"], key=itemgetter('release_date'), reverse=True) + releases = [ release["version"] for release in releases if release["available"]] -templateLoader = jinja2.FileSystemLoader(dir_path) -templateEnv = jinja2.Environment(loader=templateLoader) -template = templateEnv.get_template(archlinux_ipxe_template) - -releases = [] -with urllib.request.urlopen(releases_url) as url: - data = json.loads(url.read()) - releases = sorted(data["releases"], key=itemgetter('release_date'), reverse=True) - releases = [ release["version"] for release in releases if release["available"]] + return releases -mirrors_by_country = [] -with urllib.request.urlopen(mirrors_url) as url: - data = json.loads(url.read()) - +def retrieve_mirrors(): mirrorurls = [] - for mirror in data["urls"]: - if mirror["protocol"] == "http" and mirror["active"] and mirror["isos"]: - keys = ["url", "name", "country_code", "country"] - mirrorObj = namedtuple("Mirror", keys) - mirror = mirrorObj(mirror["url"], mirror["details"].rsplit('/',3)[1], mirror["country_code"], mirror["country"]) - mirrorurls.append(mirror) + with urllib.request.urlopen(mirrors_url) as url: + data = orjson.loads(url.read()) - mirrorurls = sorted(mirrorurls, key=lambda x: x.name) - mirrorurls = sorted(mirrorurls, key=lambda x: x.country) + for mirror in data["urls"]: + if mirror["protocol"] == "http" and mirror["active"] and mirror["isos"]: + keys = ["url", "name", "country_code", "country"] + mirrorObj = namedtuple("Mirror", keys) + mirror = mirrorObj(mirror["url"], mirror["details"].rsplit('/',3)[1], mirror["country_code"], mirror["country"]) + mirrorurls.append(mirror) + mirrorurls = sorted(mirrorurls, key=lambda x: x.name) + mirrorurls = sorted(mirrorurls, key=lambda x: x.country) + + return mirrorurls + +def sort_mirrors_by_country(mirrorurls): mirrors_by_country = {} for mirror in mirrorurls: if mirror.country_code not in mirrors_by_country.keys(): @@ -45,5 +45,27 @@ with urllib.request.urlopen(mirrors_url) as url: mirrors_by_country[mirror.country_code]["list"].append({"url": mirror.url, "name": mirror.name}) mirrors_by_country = mirrors_by_country.values() + + return mirrors_by_country + +def render_from_jinja_template(mirrors_by_country, releases): + dir_path = os.path.dirname(os.path.realpath(__file__)) + archlinux_ipxe_template = "archlinux.ipxe.jinja" + + templateLoader = jinja2.FileSystemLoader(dir_path) + templateEnv = jinja2.Environment(loader=templateLoader) + template = templateEnv.get_template(archlinux_ipxe_template) + + print((template.render(mirrors_by_country = mirrors_by_country, releases = releases))) -print((template.render(mirrors_by_country = mirrors_by_country, releases = releases))) + +def main(): + releases = retrieve_releases() + mirrorurls = retrieve_mirrors() + mirrors_by_country = sort_mirrors_by_country(mirrorurls) + + render_from_jinja_template(mirrors_by_country, releases) + + +if __name__ == "__main__": + main() diff --git a/ipxe/test.sh b/ipxe/test.sh index 778d811..9b19825 100755 --- a/ipxe/test.sh +++ b/ipxe/test.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -python generate_archlinux_ipxe.py +python generate_archlinux_ipxe.py > archlinux.ipxe curl https://ipxe.archlinux.org/releng/netboot/archlinux.ipxe > archweb-archlinux.ipxe diff archweb-archlinux.ipxe archlinux.ipxe