ci/cd start working on linux core tests

2023-09-18 12:12:03 +02:00 · 2023-09-18 12:12:03 +02:00 · 80d56fcca6
parent 69307fba6f
commit 80d56fcca6
5 changed files with 266 additions and 55 deletions
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@ -27,14 +27,42 @@ jobs:
    uses: ./.github/workflows/container-build.yml
    with:
      RELEASE: dev
-      CACHE: true
      ARCH: linux/amd64
+      CACHE: true
      IMAGE: ${{ matrix.image }}
      DOCKERFILE: ${{ matrix.dockerfile }}
    secrets:
      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}

+  # Build Linux packages
+  build-packages:
+    permissions:
+      contents: read
+      packages: write
+    strategy:
+      matrix:
+        linux: [ubuntu, debian, fedora, rhel]
+        include:
+          - linux: ubuntu
+            package: deb
+          - linux: debian
+            package: deb
+          - linux: fedora
+            package: rpm
+          - linux: rhel
+            package: rpm
+    uses: ./.github/workflows/linux-build.yml
+    with:
+      RELEASE: dev
+      LINUX: ${{ matrix.linux }}
+      PACKAGE: ${{ matrix.package }}
+      TEST: true
+      PLATFORMS: linux/amd64
+    secrets:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+
  # Python code security
  code-security:
    runs-on: ubuntu-latest
@ -68,7 +96,7 @@ jobs:

  # Core tests
  prepare-tests-core:
-    needs: [code-security, build-containers]
+    needs: [code-security, build-containers, build-packages]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
@ -89,6 +117,16 @@ jobs:
    with:
      TEST: ${{ matrix.test }}
      RELEASE: dev
+  tests-core-linux:
+    needs: prepare-tests-core
+    strategy:
+      fail-fast: false
+      matrix:
+        test: ${{ fromJson(needs.prepare-tests-core.outputs.tests) }}
+    uses: ./.github/workflows/test-core-linux.yml
+    with:
+      TEST: ${{ matrix.test }}
+      RELEASE: dev

  # Push with dev tag
  push-dev:
@ -129,3 +167,59 @@ jobs:
        env:
          FROM: "autoconf"
          TO: "bunkerweb-autoconf"
+
+  # Push Linux packages
+  push-packages:
+    needs: [staging-tests, tests-ui, tests-core]
+    strategy:
+      matrix:
+        linux: [ubuntu, debian, fedora, el]
+        arch: [amd64]
+        include:
+          - release: dev
+            repo: bunkerweb
+          - linux: ubuntu
+            separator: _
+            suffix: ""
+            version: jammy
+            package: deb
+          - linux: debian
+            separator: _
+            suffix: ""
+            version: bullseye
+            package: deb
+          - linux: fedora
+            separator: "-"
+            suffix: "1."
+            version: 38
+            package: rpm
+          - linux: el
+            separator: "-"
+            suffix: "1."
+            version: 8
+            package: rpm
+          - linux: ubuntu
+            arch: amd64
+            package_arch: amd64
+          - linux: debian
+            arch: amd64
+            package_arch: amd64
+          - linux: fedora
+            arch: amd64
+            package_arch: x86_64
+          - linux: el
+            arch: amd64
+            package_arch: x86_64
+    uses: ./.github/workflows/push-packagecloud.yml
+    with:
+      SEPARATOR: ${{ matrix.separator }}
+      SUFFIX: ${{ matrix.suffix }}
+      REPO: ${{ matrix.repo }}
+      LINUX: ${{ matrix.linux }}
+      VERSION: ${{ matrix.version }}
+      PACKAGE: ${{ matrix.package }}
+      BW_VERSION: ${{ matrix.release }}
+      PACKAGE_ARCH: ${{ matrix.package_arch }}
+      ARCH: ${{ matrix.arch }}
+    secrets:
+      PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
--- a/.github/workflows/linux-build.yml
+++ b/.github/workflows/linux-build.yml
@ -39,8 +39,8 @@ jobs:
      - name: Checkout source code
        uses: actions/checkout@v4
      - name: Replace VERSION
-        if: inputs.RELEASE == 'testing'
-        run: ./misc/update-version.sh testing
+        if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev'
+        run: ./misc/update-version.sh ${{ inputs.RELEASE }}
      - name: Extract arch
        run: |
          echo "ARCH=${{ env.PLATFORMS }}" | sed 's/linux//g' | sed 's@/@@g' >> "$GITHUB_ENV"
@ -91,7 +91,7 @@ jobs:
          password: ${{ secrets.GITHUB_TOKEN }}
      # Build testing package image
      - name: Build package image
-        if: inputs.RELEASE == 'testing'
+        if: inputs.RELEASE == 'testing' || inputs.RELEASE == 'dev'
        uses: docker/build-push-action@v5
        with:
          context: .
@ -99,11 +99,11 @@ jobs:
          file: src/linux/Dockerfile-${{ inputs.LINUX }}
          platforms: ${{ inputs.PLATFORMS }}
          tags: local/bunkerweb-${{ inputs.LINUX }}:latest
-          cache-from: type=gha,scope=${{ inputs.LINUX }}-testing
-          cache-to: type=gha,scope=${{ inputs.LINUX }}-testing,mode=min
+          cache-from: type=gha,scope=${{ inputs.LINUX }}-${{ inputs.RELEASE }}
+          cache-to: type=gha,scope=${{ inputs.LINUX }}-${{ inputs.RELEASE }},mode=min
      # Build non-testing package image
      - name: Build package image
-        if: inputs.RELEASE != 'testing'
+        if: inputs.RELEASE != 'testing' && inputs.RELEASE != 'dev'
        uses: docker/build-push-action@v5
        with:
          context: .
--- a/.github/workflows/test-core-linux.yml
+++ b/.github/workflows/test-core-linux.yml
@ -0,0 +1,52 @@
+name: Core test Linux (REUSABLE)
+
+on:
+  workflow_call:
+    inputs:
+      TEST:
+        required: true
+        type: string
+      RELEASE:
+        required: true
+        type: string
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      # Prepare
+      - name: Checkout source code
+        uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: "pip"
+      - name: Login to ghcr
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Pull BW linux ubuntu test image
+        run: docker pull ghcr.io/bunkerity/ubuntu-tests:${{ inputs.RELEASE }} && docker tag ghcr.io/bunkerity/ubuntu-tests:${{ inputs.RELEASE }} local/ubuntu:latest
+      - name: Run BW linux ubuntu test image
+        run: docker run --rm -v /tmp/bunkerweb_deb:/data local/ubuntu:latest
+      - name: Install BunkerWeb
+        run: |
+          # Install NGINX
+          apt update
+          apt install -y curl gnupg2 ca-certificates lsb-release ubuntu-keyring
+          curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor | tee /usr/share/keyrings/nginx-archive-keyring.gpg >/dev/null
+          echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] http://nginx.org/packages/ubuntu `lsb_release -cs` nginx" | tee /etc/apt/sources.list.d/nginx.list
+          apt update
+          apt install -y nginx=1.24.0-1~jammy
+          # Install BunkerWeb
+          apt install -y /tmp/bunkerweb_deb/bunkerweb.deb
+      - name: Edit configuration files
+        run: |
+          echo "www.example.com 127.0.0.1" >> /etc/hosts
+          echo "SERVER_NAME=www.example.com" > /etc/bunkerweb/variables.env
+          { echo "HTTP_PORT=80"; echo "HTTPS_PORT=443"; echo "DNS_RESOLVERS=9.9.9.9 8.8.8.8 8.8.4.4"; } >> /etc/bunkerweb/variables.env
+      - name: Run tests
+        run: ./test.sh "linux"
--- a/.github/workflows/test-core.yml
+++ b/.github/workflows/test-core.yml
@ -9,6 +9,7 @@ on:
      RELEASE:
        required: true
        type: string
+
 jobs:
  test:
    runs-on: ubuntu-latest
@ -26,15 +27,10 @@ jobs:
        run: docker pull ghcr.io/bunkerity/bunkerweb-tests:${{ inputs.RELEASE }} && docker tag ghcr.io/bunkerity/bunkerweb-tests:${{ inputs.RELEASE }} bunkerweb-tests
      - name: Pull Scheduler image
        run: docker pull ghcr.io/bunkerity/scheduler-tests:${{ inputs.RELEASE }} && docker tag ghcr.io/bunkerity/scheduler-tests:${{ inputs.RELEASE }} scheduler-tests
-      # # Temp fix "is not connected to the network" until compose v2.19.1 is available
-      # - name: Downgrade compose
-      #   run: |
-      #     sudo apt update
-      #     sudo apt install -y --allow-downgrades moby-compose=2.18.1+azure-ubuntu22.04u2
      # Run test
      - name: Run test
        run: |
          cd ./tests/core/${{ inputs.TEST }}
          find . -type f -name 'docker-compose.*' -exec sed -i "s@bunkerity/bunkerweb:.*@bunkerweb-tests@" {} \;
          find . -type f -name 'docker-compose.*' -exec sed -i "s@bunkerity/bunkerweb-scheduler:.*@scheduler-tests@" {} \;
-          ./test.sh
+          ./test.sh "docker"
--- a/tests/core/antibot/test.sh
+++ b/tests/core/antibot/test.sh
@ -1,17 +1,32 @@
 #!/bin/bash

-echo "🤖 Building antibot stack ..."
+integration=$1

-# Starting stack
-docker compose pull bw-docker app1
-if [ $? -ne 0 ] ; then
-    echo "🤖 Pull failed ❌"
+if [ -z "$integration" ] ; then
+    echo "🤖 Please provide an integration name as argument ❌"
+    exit 1
+elif [ "$integration" != "docker" ] && [ "$integration" != "linux" ] ; then
+    echo "🤖 Integration \"$integration\" is not supported ❌"
    exit 1
 fi
-docker compose -f docker-compose.test.yml build
-if [ $? -ne 0 ] ; then
-    echo "🤖 Build failed ❌"
-    exit 1
+
+echo "🤖 Building antibot stack for integration \"$integration\" ..."
+
+# Starting stack
+if [ "$integration" = "docker" ] ; then
+    docker compose pull bw-docker app1
+    if [ $? -ne 0 ] ; then
+        echo "🤖 Pull failed ❌"
+        exit 1
+    fi
+    docker compose -f docker-compose.test.yml build
+    if [ $? -ne 0 ] ; then
+        echo "🤖 Build failed ❌"
+        exit 1
+    fi
+else
+    systemctl stop bunkerweb
+    { echo "USE_ANTIBOT=no"; echo "ANTIBOT_URI=/challenge"; } >> /etc/bunkerweb/variables.env
 fi

 manual=0
@ -19,8 +34,13 @@ end=0
 cleanup_stack () {
    exit_code=$?
    if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then
-        find . -type f -name 'docker-compose.*' -exec sed -i 's@ANTIBOT_URI: "/custom"@ANTIBOT_URI: "/challenge"@' {} \;
-        find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "no"@' {} \;
+        if [ "$integration" == "docker" ] ; then
+            find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "no"@' {} \;
+            find . -type f -name 'docker-compose.*' -exec sed -i 's@ANTIBOT_URI: "/custom"@ANTIBOT_URI: "/challenge"@' {} \;
+        else
+            sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "no"@' /etc/bunkerweb/variables.env
+            sed -i 's@ANTIBOT_URI: "/custom"@ANTIBOT_URI: "/challenge"@' /etc/bunkerweb/variables.env
+        fi
        if [[ $end -eq 1 && $exit_code = 0 ]] ; then
            return
        fi
@ -28,10 +48,14 @@ cleanup_stack () {

    echo "🤖 Cleaning up current stack ..."

-    docker compose down -v --remove-orphans
+    if [ "$integration" == "docker" ] ; then
+        docker compose down -v --remove-orphans
+    else
+        systemctl stop bunkerweb
+    fi

    if [ $? -ne 0 ] ; then
-        echo "🤖 Down failed ❌"
+        echo "🤖 cleanup failed ❌"
        exit 1
    fi

@ -47,20 +71,36 @@ do
        echo "🤖 Running tests without antibot ..."
    elif [ "$test" = "endpoint" ] ; then
        echo "🤖 Running tests where antibot is on a different endpoint ..."
-        find . -type f -name 'docker-compose.*' -exec sed -i 's@ANTIBOT_URI: "/challenge"@ANTIBOT_URI: "/custom"@' {} \;
+        if [ "$integration" == "docker" ] ; then
+            find . -type f -name 'docker-compose.*' -exec sed -i 's@ANTIBOT_URI: "/challenge"@ANTIBOT_URI: "/custom"@' {} \;
+        else
+            sed -i 's@ANTIBOT_URI: "/challenge"@ANTIBOT_URI: "/custom"@' /etc/bunkerweb/variables.env
+        fi
    elif [ "$test" != "deactivated" ] ; then
        echo "🤖 Running tests with antibot \"$test\" ..."
-        find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "'"${test}"'"@' {} \;
+        if [ "$integration" == "docker" ] ; then
+            find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "'"${test}"'"@' {} \;
+        else
+            sed -i 's@USE_ANTIBOT: ".*"$@USE_ANTIBOT: "'"${test}"'"@' /etc/bunkerweb/variables.env
+        fi
    fi

    echo "🤖 Starting stack ..."
-    docker compose up -d
-    if [ $? -ne 0 ] ; then
-        echo "🤖 Up failed, retrying ... ⚠️"
-        manual=1
-        cleanup_stack
-        manual=0
+    if [ "$integration" == "docker" ] ; then
        docker compose up -d
+        if [ $? -ne 0 ] ; then
+            echo "🤖 Up failed, retrying ... ⚠️"
+            manual=1
+            cleanup_stack
+            manual=0
+            docker compose up -d
+            if [ $? -ne 0 ] ; then
+                echo "🤖 Up failed ❌"
+                exit 1
+            fi
+        fi
+    else
+        systemctl start bunkerweb
        if [ $? -ne 0 ] ; then
            echo "🤖 Up failed ❌"
            exit 1
@ -69,38 +109,67 @@ do

    # Check if stack is healthy
    echo "🤖 Waiting for stack to be healthy ..."
-    i=0
-    while [ $i -lt 120 ] ; do
-        containers=("antibot-bw-1" "antibot-bw-scheduler-1")
-        healthy="true"
-        for container in "${containers[@]}" ; do
-            check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")"
-            if [ "$check" = "" ] ; then
-                healthy="false"
+    if [ "$integration" == "docker" ] ; then
+        i=0
+        while [ $i -lt 120 ] ; do
+            containers=("antibot-bw-1" "antibot-bw-scheduler-1")
+            healthy="true"
+            for container in "${containers[@]}" ; do
+                check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")"
+                if [ "$check" = "" ] ; then
+                    healthy="false"
+                    break
+                fi
+            done
+            if [ "$healthy" = "true" ] ; then
+                echo "🤖 Docker stack is healthy ✅"
                break
            fi
+            sleep 1
+            i=$((i+1))
        done
-        if [ "$healthy" = "true" ] ; then
-            echo "🤖 Docker stack is healthy ✅"
-            break
+        if [ $i -ge 120 ] ; then
+            docker compose logs
+            echo "🤖 Docker stack is not healthy ❌"
+            exit 1
+        fi
+    else
+        i=0
+        while [ $i -lt 120 ] ; do
+            check="$(cat /var/log/bunkerweb/error.log | grep "BunkerWeb is ready")"
+            if ! [ -z "$check" ] ; then
+                echo "🤖 Linux stack is healthy ✅"
+                break
+            fi
+            sleep 1
+            i=$((i+1))
+        done
+        if [ $i -ge 120 ] ; then
+            journalctl -u bunkerweb --no-pager
+            echo "🤖 Linux stack is not healthy ❌"
+            exit 1
        fi
-        sleep 1
-        i=$((i+1))
-    done
-    if [ $i -ge 120 ] ; then
-        docker compose logs
-        echo "🤖 Docker stack is not healthy ❌"
-        exit 1
    fi

    # Start tests

-    docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests
+    if [ "$integration" == "docker" ] ; then
+        docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests
+    else
+        source /etc/bunkerweb/variables.env
+        python3 main.py
+    fi

    if [ $? -ne 0 ] ; then
        echo "🤖 Test \"$test\" failed ❌"
        echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..."
-        docker compose logs bw bw-scheduler
+        if [ "$integration" == "docker" ] ; then
+            docker compose logs bw bw-scheduler
+        else
+            journalctl -u bunkerweb --no-pager
+            cat /var/log/bunkerweb/error.log
+            cat /var/log/bunkerweb/access.log
+        fi
        exit 1
    else
        echo "🤖 Test \"$test\" succeeded ✅"