ci/cd - add linux/rhel tests, fix docker/behind-reverse-proxy, fix missing stream module for linux/fedora and remove placement constraints for swarm

2023-03-31 13:04:51 +02:00 · 2023-03-31 13:04:51 +02:00 · ea1394b044
parent b3eb647459
commit ea1394b044
7 changed files with 87 additions and 72 deletions
--- a/.github/workflows/staging-tests.yml
+++ b/.github/workflows/staging-tests.yml
@ -100,27 +100,33 @@ jobs:
          KUBECONFIG: "/tmp/k8s/kubeconfig"
          PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
          IMAGE_TAG: "staging"
-      - name: Run Linux ubuntu tests
-        if: inputs.TYPE == 'linux'
-        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "ubuntu"
-        env:
-          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
-          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
-      - name: Run Linux debian tests
-        if: inputs.TYPE == 'linux'
-        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "debian"
-        env:
-          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
-          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
-      - name: Run Linux centos tests
-        if: inputs.TYPE == 'linux'
-        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "centos"
-        env:
-          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
-          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      # - name: Run Linux ubuntu tests
+      #   if: inputs.TYPE == 'linux'
+      #   run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "ubuntu"
+      #   env:
+      #     TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+      #     ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      # - name: Run Linux debian tests
+      #   if: inputs.TYPE == 'linux'
+      #   run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "debian"
+      #   env:
+      #     TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+      #     ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      # - name: Run Linux centos tests
+      #   if: inputs.TYPE == 'linux'
+      #   run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "centos"
+      #   env:
+      #     TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+      #     ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
      - name: Run Linux fedora tests
        if: inputs.TYPE == 'linux'
        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "fedora"
        env:
          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
+      - name: Run Linux rhel tests
+        if: inputs.TYPE == 'linux'
+        run: export $(echo "$TEST_DOMAINS" | xargs) && ./tests/main.py "linux" "rhel"
+        env:
+          TEST_DOMAINS: ${{ secrets.TEST_DOMAINS_LINUX }}
+          ROOT_DOMAIN: ${{ secrets.ROOT_DOMAIN }}
--- a/.github/workflows/staging.yml
+++ b/.github/workflows/staging.yml
@ -48,36 +48,36 @@ jobs:
      PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}

  # Linux
-  build-ubuntu:
-    uses: ./.github/workflows/staging-linux-build.yml
-    with:
-      LINUX: ubuntu
-      PACKAGE: deb
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-      PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
-      PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
-  build-debian:
-    uses: ./.github/workflows/staging-linux-build.yml
-    with:
-      LINUX: debian
-      PACKAGE: deb
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-      PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
-      PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
-  build-centos:
-    uses: ./.github/workflows/staging-linux-build.yml
-    with:
-      LINUX: centos
-      PACKAGE: rpm
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
-      PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
-      PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+  # build-ubuntu:
+  #   uses: ./.github/workflows/staging-linux-build.yml
+  #   with:
+  #     LINUX: ubuntu
+  #     PACKAGE: deb
+  #   secrets:
+  #     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  #     DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+  #     PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
+  #     PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+  # build-debian:
+  #   uses: ./.github/workflows/staging-linux-build.yml
+  #   with:
+  #     LINUX: debian
+  #     PACKAGE: deb
+  #   secrets:
+  #     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  #     DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+  #     PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
+  #     PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+  # build-centos:
+  #   uses: ./.github/workflows/staging-linux-build.yml
+  #   with:
+  #     LINUX: centos
+  #     PACKAGE: rpm
+  #   secrets:
+  #     DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+  #     DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
+  #     PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
+  #     PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
  build-fedora:
    uses: ./.github/workflows/staging-linux-build.yml
    with:
@ -153,15 +153,16 @@ jobs:
    secrets:
      CICD_SECRETS: ${{ secrets.CICD_SECRETS }}
  create-infra-linux:
-    needs:
-      [
-        code-security,
-        build-ubuntu,
-        build-debian,
-        build-centos,
-        build-fedora,
-        build-rhel,
-      ]
+    # needs:
+    #   [
+    #     code-security,
+    #     build-ubuntu,
+    #     build-debian,
+    #     build-centos,
+    #     build-fedora,
+    #     build-rhel,
+    #   ]
+    needs: [code-security, build-fedora, build-rhel]
    uses: ./.github/workflows/staging-create-infra.yml
    with:
      TYPE: linux
--- a/examples/behind-reverse-proxy/docker-compose.yml
+++ b/examples/behind-reverse-proxy/docker-compose.yml
@ -6,7 +6,6 @@ services:
    environment:
      - SERVER_NAME=www.example.com # replace with your domains
      - API_WHITELIST_IP=127.0.0.0/8 10.20.30.0/24
-      - AUTO_LETS_ENCRYPT=yes
      - SERVE_FILES=no
      - DISABLE_DEFAULT_SERVER=yes
      - USE_CLIENT_CACHE=yes
--- a/src/common/confs/nginx.conf
+++ b/src/common/confs/nginx.conf
@ -1,6 +1,10 @@
 # /etc/nginx/nginx.conf

 # load dynamic modules
+{% set os = import("os") %}
+{% if os.path.isfile("/usr/lib64/nginx/modules/ngx_stream_module.so") +%}
+load_module /usr/lib64/nginx/modules/ngx_stream_module.so;
+{% endif %}
 load_module /usr/share/bunkerweb/modules/ndk_http_module.so;
 load_module /usr/share/bunkerweb/modules/ngx_http_cookie_flag_filter_module.so;
 load_module /usr/share/bunkerweb/modules/ngx_http_headers_more_filter_module.so;
--- a/src/linux/fpm-fedora
+++ b/src/linux/fpm-fedora
@ -3,7 +3,7 @@
 --license agpl3
 --version %VERSION%
 --architecture x86_64
--depends bash --depends python3 --depends 'nginx = 1:1.22.1-1.fc37' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof
+--depends bash --depends python3 --depends 'nginx = 1:1.22.1-1.fc37' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends geoip-devel --depends gd --depends sudo --depends procps --depends lsof --depends nginx-mod-stream
 --description "BunkerWeb %VERSION% for Fedora 36"
 --url "https://www.bunkerweb.io"
 --maintainer "Bunkerity <contact at bunkerity dot com>"
--- a/tests/SwarmTest.py
+++ b/tests/SwarmTest.py
@ -37,6 +37,7 @@ class SwarmTest(Test):
            if not "AUTO_LETS_ENCRYPT=yes" in data["services"]["bunkerweb"]["environment"] :
                data["services"]["bunkerweb"]["environment"].append("AUTO_LETS_ENCRYPT=yes")
            data["services"]["bunkerweb"]["environment"].append("USE_LETS_ENCRYPT_STAGING=yes")
+            del data["services"]["bunkerweb"]["deploy"]["placement"]
            with open(compose, "w") as f :
                f.write(dump(data))
            Test.replace_in_file(
--- a/tests/linux.sh
+++ b/tests/linux.sh
@ -19,7 +19,7 @@ function gen_package() {
    mode="$1"
    linux="$2"
    version="$(cat VERSION | tr -d '\n')"
-    if [ "$linux" = "fedora" ] || [ "$linux" = "centos" ] ; then
+    if [ "$linux" = "fedora" ] || [ "$linux" = "centos" ] || [ "$linux" = "rhel" ] ; then
        type="rpm"
    else
        type="deb"
@ -51,26 +51,30 @@ do_and_check_cmd mkdir /tmp/packages
 find /opt/packages/ -type f | xargs rm

 # Generate packages
-echo "Building ubuntu package ..."
-gen_package "$1" "ubuntu"
-echo "Building debian package ..."
-gen_package "$1" "debian"
-echo "Building centos package ..."
-gen_package "$1" "centos"
+# echo "Building ubuntu package ..."
+# gen_package "$1" "ubuntu"
+# echo "Building debian package ..."
+# gen_package "$1" "debian"
+# echo "Building centos package ..."
+# gen_package "$1" "centos"
 echo "Building fedora package ..."
 gen_package "$1" "fedora"
+echo "Building rhel package ..."
+gen_package "$1" "rhel"

 # Copy packages in the Docker context
 do_and_check_cmd cp -r "/opt/packages/$1" ./packages

 # Build test images
-echo "Building ubuntu test image ..."
-build_image "ubuntu"
-echo "Building debian test image ..."
-build_image "debian"
-echo "Building centos test image ..."
-build_image "centos"
+# echo "Building ubuntu test image ..."
+# build_image "ubuntu"
+# echo "Building debian test image ..."
+# build_image "debian"
+# echo "Building centos test image ..."
+# build_image "centos"
 echo "Building fedora test image ..."
 build_image "fedora"
+echo "Building rhel test image ..."
+build_image "rhel"

 exit 0