ci/cd - init work

.github/workflows/reusables/staging-container-build.yml

@@ -0,0 +1,65 @@
+name: Build staging container (REUSABLE)
+
+  workflow_call:
+    inputs:
+      IMAGE:
+        required: true
+        type: string
+      DOCKERFILE:
+        required: true
+        type: string
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_TOKEN:
+        required: true
+      PRIVATE_REGISTRY:
+        required: true
+      PRIVATE_REGISTRY_TOKEN:
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Setup Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to private repository
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.PRIVATE_REGISTRY }}
+          username: registry
+          password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+      # Build image
+      - name: Build container for amd64
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ${{ inputs.DOCKERFILE }}
+          platforms: linux/amd64
+          load: true
+          tags: local/${{ inputs.IMAGE }}
+          cache-from: type=registry,ref=bunkerity/cache:${{ inputs.IMAGE }}-staging
+          cache-to: type=registry,ref=bunkerity/cache:${{ inputs.IMAGE }}-staging,mode=min
+      # Check OS vulnerabilities
+      - name: Check OS vulnerabilities
+        uses: aquasecurity/trivy-action@master
+        with:
+          vuln-type: os
+          image-ref: local/${{ inputs.IMAGE }}
+          format: table
+          exit-code: 1
+          ignore-unfixed: false
+          severity: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
+          trivyignores: .trivyignore
+      # Push image
+      - name: Push image
+        run: docker tag local/${{ inputs.IMAGE }} ${{ secrets.PRIVATE_REGISTRY }}/infra/${{ inputs.IMAGE }}-tests:staging && docker push ${{ secrets.PRIVATE_REGISTRY }}/infra/${{ inputs.IMAGE }}-tests:staging

.github/workflows/reusables/staging-create-infra.yml

@@ -0,0 +1,49 @@
+name: Create staging infra (REUSABLE)
+
+  workflow_call:
+    inputs:
+      TYPE:
+        required: true
+        type: string
+    secrets:
+      CICD_SECRETS:
+        required: true
+
+jobs:
+  create:
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Generate SSH keypair
+        run: ssh-keygen -b 2048 -t rsa -f ~/.ssh/id_rsa -q -N "" && ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub && echo -e "Host *\n  StrictHostKeyChecking no" > ~/.ssh/ssh_config
+        if: inputs.TYPE != 'k8s'
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v2
+      - name: Install kubectl
+        uses: azure/setup-kubectl@v3
+        if: inputs.TYPE == 'k8s'
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        if: inputs.TYPE != 'k8s'
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Install ansible
+        run: pip install ansible
+        if: inputs.TYPE != 'k8s'
+      - name: Install ansible libs
+        run: ansible-galaxy install --timeout 120 monolithprojects.github_actions_runner && ansible-galaxy collection install --timeout 120 community.general
+        if: inputs.TYPE != 'k8s'
+      # Create infra
+      - run: ./tests/create.sh ${{ inputs.TYPE }}
+        env:
+          CICD_SECRETS: ${{ secrets.CICD_SECRETS }}
+      - run: tar -cvf terraform.tar /tmp/${{ inputs.TYPE }}
+        if: always()
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: tf-${{ inputs.TYPE }}
+          path: terraform.tar

.github/workflows/reusables/staging-delete-infra.yml

@@ -0,0 +1,38 @@
+name: Delete staging infra (REUSABLE)
+
+on:
+  workflow_call:
+    inputs:
+      TYPE:
+        required: true
+        type: string
+    secrets:
+      CICD_SECRETS:
+        required: true
+
+jobs:
+  delete:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Install terraform
+        uses: hashicorp/setup-terraform@v2
+      - uses: actions/download-artifact@v3
+        with:
+          name: tf-${{ inputs.TYPE }}
+          path: /tmp
+      - run: tar xvf /tmp/terraform.tar -C / && mkdir ~/.ssh && touch ~/.ssh/id_rsa.pub
+      - uses: azure/setup-kubectl@v3
+        if: inputs.TYPE == "k8s"
+      # Remove infra
+      - run: kubectl delete daemonsets,replicasets,services,deployments,pods,rc,ingress,statefulsets --all --all-namespaces --timeout=60s ; kubectl delete pvc --all --timeout=60s ; kubectl delete pv --all --timeout=60s
+        if: inputs.TYPE == "k8s"
+        continue-on-error: true
+        env:
+          KUBECONFIG: /tmp/k8s/kubeconfig
+      - run: ./tests/rm.sh ${{ inputs.TYPE }}
+        env:
+          CICD_SECRETS: ${{ secrets.CICD_SECRETS }}

.github/workflows/reusables/staging-linux-build.yml

@@ -0,0 +1,67 @@
+name: Build staging Linux package (REUSABLE)
+
+  workflow_call:
+    inputs:
+      LINUX:
+        required: true
+        type: string
+      PACKAGE:
+        required: true
+        type: string
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_TOKEN:
+        required: true
+      PRIVATE_REGISTRY:
+        required: true
+      PRIVATE_REGISTRY_TOKEN:
+        required: true
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Setup Buildx
+        uses: docker/setup-buildx-action@v2
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to private repository
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.PRIVATE_REGISTRY }}
+          username: registry
+          password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+      # Build package image
+      - name: Build package image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          load: true
+          file: src/linux/Dockerfile-${{ inputs.LINUX }}
+          platforms: linux/amd64
+          tags: local/bunkerweb-${{ inputs.LINUX }}:latest
+          cache-from: type=registry,ref=bunkerity/cache:${{ inputs.LINUX }}-staging
+          cache-to: type=registry,ref=bunkerity/cache:${{ inputs.LINUX }}-staging,mode=min
+      # Generate package
+      - name: Generate package
+        run: ./src/linux/package.sh ${{ inputs.LINUX }}
+      - uses: actions/upload-artifact@v3
+        with:
+          name: package-${{ inputs.LINUX }}
+          path: package-${{ inputs.LINUX }}/*.${{ inputs.PACKAGE }}
+      # Build test image
+      - name: Build test image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: tests/linux/Dockerfile-${{ inputs.LINUX }}
+          platforms: linux/amd64
+          push: true
+          tags: ${{ secrets.PRIVATE_REGISTRY }}/infra/${{ inputs.LINUX }}-tests:staging

.github/workflows/reusables/staging-push-docker.yml

@@ -0,0 +1,40 @@
+name: Push staging container (REUSABLE)
+
+on:
+  workflow_call:
+    inputs:
+      PRIVATE_IMAGE:
+        required: true
+        type: string
+      PUBLIC_IMAGE:
+        required: true
+        type: string
+    secrets:
+      DOCKER_USERNAME:
+        required: true
+      DOCKER_TOKEN:
+        required: true
+      PRIVATE_REGISTRY:
+        required: true
+      PRIVATE_REGISTRY_TOKEN:
+        required: true
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to private repository
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.PRIVATE_REGISTRY }}
+          username: registry
+          password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+      # Push
+      - name: Push bunkerweb
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/${{ inputs.PRIVATE_IMAGE }} && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/${{ inputs.PRIVATE_IMAGE }} bunkerity/${{ inputs.PUBLIC_IMAGE }} && docker push bunkerity/${{ inputs.PUBLIC_IMAGE }}

.github/workflows/reusables/staging-push-packagecloud.yml

@@ -0,0 +1,89 @@
+name: Push staging packagecloud (REUSABLE)
+
+on:
+  workflow_call:
+    inputs:
+      SEPARATOR:
+        required: true
+        type: string
+      SUFFIX:
+        required: true
+        type: string
+      REPO:
+        required: true
+        type: string
+      LINUX:
+        required: true
+        type: string
+      VERSION:
+        required: true
+        type: string
+      PACKAGE:
+        required: true
+        type: string
+    secrets:
+      PACKAGECLOUD_TOKEN:
+        required: true
+
+jobs:
+  push:
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      - name: Set variables
+        run: |
+          VER=$(cat src/VERSION | tr -d '\n')
+          echo "VERSION=$VER" >> $GITHUB_ENV
+      - name: Install ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.0'
+      - name: Install packagecloud
+        run: gem install package_cloud
+      # Download packages
+      - uses: actions/download-artifact@v3
+        with:
+          name: package-${{ inputs.LINUX }}
+          path: /tmp/${{ inputs.LINUX }}
+      # Remove existing packages
+      - name: Remove existing package
+        run: package_cloud yank bunkerity/${{ inputs.REPO }}/${{ inputs.LINUX }}/${{ inputs.VERSION }} bunkerweb${{ inputs.SEPARATOR }}${{ env.VERSION }}${{ inputs.SEPARATOR }}${{ inputs.SUFFIX }}.${{ inputs.PACKAGE }}
+        continue-on-error: true
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      # TODO : push
+      # Push packages
+      - name: Push Ubuntu DEB to packagecloud
+        uses: danielmundi/upload-packagecloud@v1
+        with:
+          PACKAGE-NAME: /tmp/ubuntu/bunkerweb_${{ env.VERSION }}-1_amd64.deb
+          PACKAGECLOUD-USERNAME: bunkerity
+          PACKAGECLOUD-REPO: bunkerweb-dev
+          PACKAGECLOUD-DISTRIB: ubuntu/jammy
+          PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - name: Push Debian DEB to packagecloud
+        uses: danielmundi/upload-packagecloud@v1
+        with:
+          PACKAGE-NAME: /tmp/debian/bunkerweb_${{ env.VERSION }}-1_amd64.deb
+          PACKAGECLOUD-USERNAME: bunkerity
+          PACKAGECLOUD-REPO: bunkerweb-dev
+          PACKAGECLOUD-DISTRIB: debian/bullseye
+          PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - name: Push CentOS RPM to packagecloud
+        uses: danielmundi/upload-packagecloud@v1
+        with:
+          PACKAGE-NAME: /tmp/centos/bunkerweb-${{ env.VERSION }}-1.x86_64.rpm
+          PACKAGECLOUD-USERNAME: bunkerity
+          PACKAGECLOUD-REPO: bunkerweb-dev
+          PACKAGECLOUD-DISTRIB: el/8
+          PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - name: Push Fedora RPM to packagecloud
+        uses: danielmundi/upload-packagecloud@v1
+        with:
+          PACKAGE-NAME: /tmp/fedora/bunkerweb-${{ env.VERSION }}-1.x86_64.rpm
+          PACKAGECLOUD-USERNAME: bunkerity
+          PACKAGECLOUD-REPO: bunkerweb-dev
+          PACKAGECLOUD-DISTRIB: fedora/36
+          PACKAGECLOUD-TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}

.github/workflows/reusables/staging-tests.yml

@@ -0,0 +1,111 @@
+name: Perform staging tests (REUSABLE)
+
+on:
+  workflow_call:
+    inputs:
+      TYPE:
+        required: true
+        type: string
+    secrets:
+      PRIVATE_REGISTRY:
+        required: true
+      PRIVATE_REGISTRY_TOKEN:
+        required: true
+      TEST_DOMAINS:
+        required: true
+      ROOT_DOMAIN:
+        required: true
+
+jobs:
+  tests:
+    runs-on: [self-hosted, bw-${{ inputs.TYPE }}]
+    steps:
+      # Prepare
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Login to private repository
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.PRIVATE_REGISTRY }}
+          username: registry
+          password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+      - name: Pull BW image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests:staging local/bunkerweb-tests:latest
+        if: ! contains(fromJSON('["linux", "k8s"]'), inputs.TYPE)
+      - name: Pull Scheduler image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/scheduler-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/scheduler-tests:staging local/scheduler-tests:latest
+        if: ! contains(fromJSON('["linux", "k8s"]'), inputs.TYPE)
+      - name: Pull Autoconf image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/autoconf-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/autoconf-tests:staging local/scheduler-tests:latest
+        if: contains(fromJSON('["autoconf", "swarm"]'), inputs.TYPE)
+      - name: Push images to local repo
+        run: docker tag local/bunkerweb-tests:latest 192.168.42.100:5000/bunkerweb-tests:latest && docker push 192.168.42.100:5000/bunkerweb-tests:latest && docker tag local/scheduler-tests:latest 192.168.42.100:5000/scheduler-tests:latest && docker push 192.168.42.100:5000/scheduler-tests:latest && docker tag local/autoconf-tests:latest 192.168.42.100:5000/autoconf-tests:latest && docker push 192.168.42.100:5000/autoconf-tests:latest
+        if: inputs.TYPE == 'swarm'
+      - name: Install test dependencies
+        run: pip3 install -r tests/requirements.txt
+      - uses: actions/download-artifact@v3
+        with:
+          name: tf-k8s
+          path: /tmp
+        if: inputs.TYPE == 'k8s'
+      - run: tar xvf /tmp/terraform.tar -C /
+        if: inputs.TYPE == 'k8s'
+      - uses: azure/setup-kubectl@v3
+        if: inputs.TYPE == 'k8s'
+      - uses: azure/setup-helm@v3
+        if: inputs.TYPE == 'k8s'
+      - name: Pull BW linux ubuntu test image
+        if: inputs.TYPE == "linux"
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/ubuntu-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/ubuntu-tests:staging local/ubuntu:latest
+      - name: Pull BW linux debian test image
+        if: inputs.TYPE == "linux"
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/debian-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/debian-tests:staging local/debian:latest
+      - name: Pull BW linux centos test image
+        if: inputs.TYPE == "linux"
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/centos-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/centos-tests:staging local/centos:latest
+      - name: Pull BW linux fedora test image
+        if: inputs.TYPE == "linux"
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/fedora-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/fedora-tests:staging local/fedora:latest
+      - name: Pull BW linux redhat test image
+        if: inputs.TYPE == "linux"
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/redhat-tests:staging && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/redhat-tests:staging local/redhat:latest
+      # Do tests
+      - name: Run tests
+        if: contains(fromJSON('["docker", "autoconf", "swarm"]'), inputs.TYPE)
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "${{ inputs.TYPE }}"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      - name: Run tests
+        if: inputs.TYPE == "k8s"
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "kubernetes"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+          KUBECONFIG: "/tmp/k8s/kubeconfig"
+          PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
+          IMAGE_TAG: "latest"
+      - name: Run Linux ubuntu tests
+        if: inputs.TYPE == "linux"
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "ubuntu"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      - name: Run Linux debian tests
+        if: inputs.TYPE == "linux"
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "debian"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      - name: Run Linux centos tests
+        if: inputs.TYPE == "linux"
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "centos"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      - name: Run Linux fedora tests
+        if: inputs.TYPE == "linux"
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "fedora"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}