Add base-devel tag

2020-08-29 04:51:05 +02:00 · 2020-08-29 04:51:05 +02:00 · d51a887efb
parent a0d65ca9a9
commit d51a887efb
10 changed files with 305 additions and 54 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,3 +0,0 @@
-*
-!archlinux.tar
-!archlinux.tar.xz
--- a/.gitignore
+++ b/.gitignore
@ -1,6 +1,6 @@
 *~
 *.orig
 /.idea
-/archlinux.tar
-/archlinux.tar.xz
+/base.tar*
+/base-devel.tar*
 rootfs/etc/pacman.conf
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,44 +1,91 @@
 stages:
+  - lint
  - rootfs
  - docker
  - test
+  - release
+  - publish

-roofs:
+lint:
+  stage: lint
+  image: hadolint/hadolint:latest
+  script: hadolint --ignore DL3020 Dockerfile.template
+
+rootfs:base:
  stage: rootfs
  image: archlinux:latest
+  needs:
+    - job: "lint"
+  before_script:
+    - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env
  script:
    - pacman -Syu --noconfirm make devtools fakechroot fakeroot
-    - make compress-rootfs
+    - make base.tar.xz
  artifacts:
    paths:
-      - archlinux.tar.xz
+      - base.tar.xz
    expire_in: 10m
+    reports:
+      dotenv: build.env

-docker:
+rootfs:base-devel:
+  stage: rootfs
+  image: archlinux:latest
+  needs:
+    - job: "lint"
+  before_script:
+    - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env
+  script:
+    - pacman -Syu --noconfirm make devtools fakechroot fakeroot
+    - make base-devel.tar.xz
+  artifacts:
+    paths:
+      - base-devel.tar.xz
+    expire_in: 10m
+    reports:
+      dotenv: build.env
+
+docker:base:
  stage: docker
  image:
    name: gcr.io/kaniko-project/executor:debug
    entrypoint: [""]
-  script:
-    - test -f archlinux.tar.xz
-    # kaniko can't process .tar.xz archives
-    # https://github.com/GoogleContainerTools/kaniko/issues/1107
-    - unxz archlinux.tar.xz
-    - test -f archlinux.tar
-    - sed -i 's/archlinux\.tar\.xz/archlinux\.tar/g' Dockerfile
-    - echo "Building ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}"
+  needs:
+    - job: "rootfs:base"
+  before_script:
    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+    - sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base
+    - unxz base.tar.xz
+  script:
    - /kaniko/executor
      --whitelist-var-run="false"
      --context $CI_PROJECT_DIR
-      --dockerfile $CI_PROJECT_DIR/Dockerfile
-      --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}
+      --dockerfile $CI_PROJECT_DIR/Dockerfile.base
+      --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG

-test:
-  stage: test
-  image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}
+docker:base-devel:
+  stage: docker
+  image:
+    name: gcr.io/kaniko-project/executor:debug
+    entrypoint: [""]
  needs:
-    - job: docker
+    - job: "rootfs:base-devel"
+  before_script:
+    - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json
+    - sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel
+    - unxz base-devel.tar.xz
+  script:
+    - /kaniko/executor
+      --whitelist-var-run="false"
+      --context $CI_PROJECT_DIR
+      --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel
+      --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG
+
+test:base:
+  stage: test
+  image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG
+  needs:
+    - job: "docker:base"
      artifacts: false
  script:
    - pacman -Sy
@ -48,3 +95,89 @@ test:
    - id -u http
    - locale | grep -q UTF-8

+test:base-devel:
+  stage: test
+  image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG
+  needs:
+    - job: "docker:base-devel"
+      artifacts: false
+  script:
+    - pacman -Sy
+    - pacman -Qqk
+    - pacman -Syu --noconfirm docker grep
+    - docker -v
+    - id -u http
+    - locale | grep -q UTF-8
+    - gcc -v
+    - g++ -v
+    - make -v
+
+release:
+  stage: release
+  image: archlinux:latest
+  only:
+    refs:
+      - master
+      - add-base-devel-tags
+    variables:
+      - $SCHEDULED_PUBLISH == "TRUE"
+  needs:
+    - job: "test:base"
+    - job: "test:base-devel"
+  before_script:
+    - pacman -Syu python-gitlab
+  script:
+    - python ci/release.py
+  tags:
+    - secure
+
+# Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base
+# publish:org:base:
+#   stage: publish
+#   image:
+#     name: gcr.io/go-containerregistry/crane:debug
+#     entrypoint: [""]
+#   needs:
+#     - job: "test:base"
+#       artifacts: true
+#   tags:
+#     - secure
+#   variables:
+#     GIT_STRATEGY: none
+#   before_script:
+#     - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+#     - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux
+#   script:
+#     - crane cp $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG archlinux/archlinux:base
+#     - crane tag archlinux/archlinux:base latest
+#     - crane tag archlinux/archlinux:base base-$BUILD_DATE
+#   only:
+#     variables:
+#       - $SCHEDULED_PUBLISH == "TRUE"
+
+# Publish base-devel to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base-devel
+# publish:org:base-devel:
+#   stage: publish
+#   image:
+#     name: gcr.io/go-containerregistry/crane:debug
+#     entrypoint: [""]
+#   needs:
+#     - job: "test:base-devel"
+#       artifacts: true
+#   tags:
+#     - secure
+#   variables:
+#     GIT_STRATEGY: none
+#   before_script:
+#     - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+#     - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux
+#   script:
+#     - crane cp $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG archlinux/archlinux:base-devel
+#     - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_DATE
+#   only:
+#     variables:
+#       - $SCHEDULED_PUBLISH == "TRUE"
+
+# Publish to the official Docker namespace: https://hub.docker.com/_/archlinux
+# publish:official:
+# TODO No idea right now how we're going to automatically do the official Docker Hub pull request
--- a/Dockerfile.template
+++ b/Dockerfile.template
@ -1,15 +1,15 @@
-FROM scratch
-ADD archlinux.tar.xz /
+FROM scratch AS base
+ADD TEMPLATE_LOCATION_HERE /

 # manually run all alpm hooks that can't be run inside the fakechroot
-RUN ldconfig && update-ca-trust && locale-gen 
+RUN ldconfig && update-ca-trust && locale-gen
 RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers '

 # update /etc/os-release
 RUN ln -s /usr/lib/os-release /etc/os-release

 # initialize the archlinux keyring, but discard any private key that may be shipped.
-RUN pacman-key --init && pacman-key --populate archlinux && rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*
+RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*"

 ENV LANG=en_US.UTF-8
 CMD ["/usr/bin/bash"]
--- a/70
+++ b/70
@ -1,16 +1,14 @@
 DOCKER_USER:=pierres
-DOCKER_ORGANIZATION=archlinux
-DOCKER_IMAGE:=base
 BUILDDIR=build
 PWD=$(shell pwd)

-XZ_THREADS ?= 0
-
+.PHONY: hooks
 hooks:
 	mkdir -p alpm-hooks/usr/share/libalpm/hooks
 	find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(PWD)/alpm-hooks{} \;

-rootfs: hooks
+.PHONY: rootfs-base
+rootfs-base: hooks
 	mkdir -vp $(BUILDDIR)/var/lib/pacman/
 	cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf
 	cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf
@ -18,35 +16,61 @@ rootfs: hooks
 		--noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \
 		--config rootfs/etc/pacman.conf \
 		--noscriptlet \
-		--hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages)
+		--hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base
 	cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/
-	
+
 	# remove passwordless login for root (see CVE-2019-5021 for reference)
 	sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow"

 	# fakeroot to map the gid/uid of the builder process to root
 	# fixes #22
-	fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar
+	fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base.tar
 	rm -rf $(BUILDDIR) alpm-hooks

-archlinux.tar: rootfs
+.PHONY: rootfs-base-devel
+rootfs-base-devel: hooks
+	mkdir -vp $(BUILDDIR)/var/lib/pacman/
+	cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf
+	cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf
+	fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \
+		--noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \
+		--config rootfs/etc/pacman.conf \
+		--noscriptlet \
+		--hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base base-devel
+	cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/

-compress-rootfs: archlinux.tar
-	xz -9 -T"$(XZ_THREADS)" -f archlinux.tar
+	# remove passwordless login for root (see CVE-2019-5021 for reference)
+	sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow"

-docker-image: compress-rootfs
-	docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) .
+	# fakeroot to map the gid/uid of the builder process to root
+	# fixes #22
+	fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base-devel.tar
+	rm -rf $(BUILDDIR) alpm-hooks

-docker-image-test: docker-image
-	# FIXME: /etc/mtab is hidden by docker so the stricter -Qkk fails
-	docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Sy && /usr/bin/pacman -Qqk"
-	docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm docker && docker -v" # Ensure that the image does not include a private key
-	! docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) pacman-key --lsign-key pierre@archlinux.de
-	docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/id -u http"
-	docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm grep && locale | grep -q UTF-8"
+base.tar.xz: rootfs-base
+	xz -9 -T0 -f base.tar

-docker-push:
+base-devel.tar.xz: rootfs-base-devel
+	xz -9 -T0 -f base-devel.tar
+
+.PHONY: docker-image-base
+docker-image-base: base.tar.xz
+	unxz base.tar.xz
+	sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base
+	docker build -f Dockerfile.base -t archlinux/archlinux:base .
+
+.PHONY: docker-image-base-devel
+docker-image-base-devel: base-devel.tar.xz
+	unxz base-devel.tar.xz
+	sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel
+	docker build -f Dockerfile.base-devel -t archlinux/archlinux:base-devel .
+
+.PHONY: docker-push-base
+docker-push-base:
 	docker login -u $(DOCKER_USER)
-	docker push $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE)
+	docker push archlinux/archlinux:base

-.PHONY: rootfs docker-image docker-image-test docker-push
+.PHONY: docker-push-base-devel
+docker-push-base-devel:
+	docker login -u $(DOCKER_USER)
+	docker push archlinux/archlinux:base-devel
--- a/README.md
+++ b/README.md
@ -1,16 +1,26 @@
-# Docker Base Image for Arch Linux [![Build Status](https://travis-ci.org/archlinux/archlinux-docker.svg?branch=master)](https://travis-ci.org/archlinux/archlinux-docker)
-This repository contains all scripts and files needed to create a Docker base image for the Arch Linux distribution.
+# Docker Base Image for Arch Linux
+[![pipeline status](https://gitlab.archlinux.org/archlinux/archlinux-docker/badges/master/pipeline.svg)](https://gitlab.archlinux.org/archlinux/archlinux-docker/-/commits/master)
+
+This repository contains all scripts and files needed to create a Docker base image for Arch Linux.
+
 ## Dependencies
 Install the following Arch Linux packages:
+
 * make
 * devtools
 * docker
 * fakechroot
 * fakeroot
+
+Make sure your user can directly interact with Docker (ie. `docker info` works).
+
 ## Usage
-Run `make docker-image` as root to build the base image.
+Run `make docker-image-base` to build the image `archlinux:base` with the
+`base` group installed. You can also run `make docker-image-base-devel` to
+build the image `archlinux:base-devel` with the `base-devel` group installed.
+
 ## Purpose
-* Provide the Arch experience in a Docker Image
+* Provide the Arch experience in a Docker image
 * Provide the most simple but complete image to base every other upon
 * `pacman` needs to work out of the box
 * All installed packages have to be kept unmodified
--- a/base-devel/Dockerfile
+++ b/base-devel/Dockerfile
--- a/base/Dockerfile
+++ b/base/Dockerfile
--- a/ci/release.py
+++ b/ci/release.py
@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+"""
+Should only be called from GitLab CI!
+
+Required env vars:
+ - GITLAB_PROJECT_TOKEN
+ - BUILD_DATE
+ - CI_PROJECT_ID
+ - CI_PROJECT_URL
+"""
+
+import os
+from pathlib import Path
+import gitlab
+
+token = os.environ['GITLAB_PROJECT_TOKEN']
+build_date = os.environ['BUILD_DATE']
+project_id = os.environ['CI_PROJECT_ID']
+project_url = os.environ['CI_PROJECT_URL']
+
+if __name__ == "__main__":
+    gl = gitlab.Gitlab("https://gitlab.archlinux.org", token)
+    project = gl.projects.get(project_id)
+
+    print("Uploading base.tar.xz")
+    base_uploaded_url = project.upload(
+        f"base-{build_date}.tar.xz", filepath="base.tar.xz"
+    )["url"]
+    base_template = Path("Dockerfile.template").read_text()
+    base_full_url = f"{project_url}{base_uploaded_url}"
+    base_replaced = base_template.replace("TEMPLATE_LOCATION_HERE", base_full_url)
+
+    print("Uploading base-devel.tar.xz")
+    base_devel_uploaded_url = project.upload(
+        f"base-devel-{build_date}.tar.xz", filepath="base-devel.tar.xz"
+    )["url"]
+    base_devel_template = Path("Dockerfile.template").read_text()
+    base_devel_full_url = f"{project_url}{base_devel_uploaded_url}"
+    base_devel_replaced = base_devel_template.replace(
+        "TEMPLATE_LOCATION_HERE", base_devel_full_url
+    )
+
+    print("Templating Dockerfiles")
+    data = {
+        "branch": "add-base-devel-tags",
+        "commit_message": f"Release {build_date}",
+        "actions": [
+            {
+                "action": "update",
+                "file_path": "base/Dockerfile",
+                "content": base_replaced,
+            },
+            {
+                "action": "update",
+                "file_path": "base-devel/Dockerfile",
+                "content": base_devel_replaced,
+            },
+        ],
+    }
+    project.commits.create(data)
+
+    print("Creating release")
+    release = project.releases.create(
+        {
+            "name": f"Release {build_date}",
+            "tag_name": build_date,
+            "description": f"Release {build_date}",
+            "ref": "add-base-devel-tags",
+            "assets": {
+                "links": [
+                    {
+                        "name": "base.tar.xz",
+                        "url": base_full_url,
+                        "link_type": "package",
+                    },
+                    {
+                        "name": "base-devel.tar.xz",
+                        "url": base_devel_full_url,
+                        "link_type": "package",
+                    }
+                ]
+            },
+        }
+    )
+    print("Created release", release.get_id())
--- a/rootfs/etc/pacman.d/mirrorlist
+++ b/rootfs/etc/pacman.d/mirrorlist
@ -1,2 +1,3 @@
+Server = https://mirror.pkgbuild.com/$repo/os/$arch
 Server = https://mirror.rackspace.com/archlinux/$repo/os/$arch
 Server = https://mirror.leaseweb.net/archlinux/$repo/os/$arch