Update core db tests to add the settings.json file and optimizations

tests/core/db/Dockerfile

@@ -14,9 +14,10 @@ RUN addgroup -g 101 nginx && \
 
 COPY --chown=nginx:nginx main.py .
 ADD ./init/plugins external
+ADD ./init/bunkerweb bunkerweb
 
-RUN chown -R nginx:nginx external && \
-  chmod -R 777 external
+RUN chown -R nginx:nginx external bunkerweb && \
+  chmod -R 777 external bunkerweb
 
 USER nginx:nginx
 

tests/core/db/docker-compose.init.yml

@@ -6,4 +6,10 @@ services:
     environment:
       PYTHONUNBUFFERED: "1"
     volumes:
+      - bw-volume:/opt/init/bunkerweb
       - ./init/plugins:/plugins
+      - ./init/bunkerweb:/bunkerweb
+
+volumes:
+  bw-volume:
+    external: true

tests/core/db/docker-compose.test.yml

@@ -5,8 +5,6 @@ services:
     build: .
     volumes:
       - bw-data:/data/lib
-      - bw-db:/opt/tests/db
-      - bw-core-plugins:/opt/tests/core
     environment:
       PYTHONUNBUFFERED: "1"
       DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3"
@@ -30,10 +28,6 @@ services:
 volumes:
   bw-data:
     external: true
-  bw-db:
-    external: true
-  bw-core-plugins:
-    external: true
 
 networks:
   bw-services:

tests/core/db/docker-compose.yml

@@ -31,8 +31,7 @@ services:
       - bw-docker
     volumes:
       - bw-data:/data/lib
-      - bw-db:/usr/share/bunkerweb/db
-      - bw-core-plugins:/usr/share/bunkerweb/core
+      - bw-volume:/usr/share/bunkerweb
       - ./init/plugins:/data/plugins
     environment:
       DOCKER_HOST: "tcp://bw-docker:2375"
@@ -90,10 +89,8 @@ services:
 volumes:
   bw-data:
     name: bw-data
-  bw-db:
-    name: bw-db
-  bw-core-plugins:
-    name: bw-core-plugins
+  bw-volume:
+    name: bw-volume
 
 networks:
   bw-universe:

tests/core/db/init/entrypoint.sh

@@ -13,5 +13,13 @@ echo "ℹ️ Extracting ClamAV plugin ..."
 
 cp -r clamav /plugins/
 
-chown -R root:101 /plugins
-chmod -R 777 /plugins
+cd ..
+
+echo "ℹ️ Extracting settings.json file, db and core directory ..."
+
+cp bunkerweb/settings.json /bunkerweb/
+cp -r bunkerweb/core /bunkerweb/
+cp -r bunkerweb/db /bunkerweb/
+
+chown -R root:101 /plugins /bunkerweb
+chmod -R 777 /plugins /bunkerweb

tests/core/db/main.py

@@ -17,7 +17,7 @@ from sqlalchemy.orm import scoped_session, sessionmaker
 from traceback import format_exc
 from time import sleep
 
-from db.model import (
+from bunkerweb.db.model import (
     Custom_configs,
     Global_values,
     Jobs,
@@ -292,6 +292,9 @@ try:
     print("   ", flush=True)
     print("ℹ️ Checking if the plugins are correct ...", flush=True)
 
+    with open(join("bunkerweb", "settings.json"), "r") as f:
+        global_settings = load(f)
+
     core_plugins = {
         "general": {
             "order": 999,
@@ -302,292 +305,10 @@ try:
             "external": False,
             "checked": False,
             "page_checked": True,
-            "settings": {
-                "IS_LOADING": {
-                    "context": "global",
-                    "default": "no",
-                    "help": "Internal use : set to yes when BW is loading.",
-                    "id": "internal-use",
-                    "label": "internal use",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "NGINX_PREFIX": {
-                    "context": "global",
-                    "default": "/etc/nginx/",
-                    "help": "Where nginx will search for configurations.",
-                    "id": "nginx-prefix",
-                    "label": "nginx prefix",
-                    "regex": "^(/[\\w. -]+)*/$",
-                    "type": "text",
-                },
-                "HTTP_PORT": {
-                    "context": "global",
-                    "default": "8080",
-                    "help": "HTTP port number which bunkerweb binds to.",
-                    "id": "http-port",
-                    "label": "HTTP port",
-                    "regex": "^\\d+$",
-                    "type": "text",
-                },
-                "HTTPS_PORT": {
-                    "context": "global",
-                    "default": "8443",
-                    "help": "HTTPS port number which bunkerweb binds to.",
-                    "id": "https-port",
-                    "label": "HTTPS port",
-                    "regex": "^\\d+$",
-                    "type": "text",
-                },
-                "MULTISITE": {
-                    "context": "global",
-                    "default": "no",
-                    "help": "Multi site activation.",
-                    "id": "multisite",
-                    "label": "Multisite",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "SERVER_NAME": {
-                    "context": "multisite",
-                    "default": "www.example.com",
-                    "help": "List of the virtual hosts served by bunkerweb.",
-                    "id": "server-name",
-                    "label": "Server name",
-                    "regex": "^(?! )( ?((?=.{1,255}$)[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?(?:\\.[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\\.?)(?!.* \\2))*$",
-                    "type": "text",
-                },
-                "WORKER_PROCESSES": {
-                    "context": "global",
-                    "default": "auto",
-                    "help": "Number of worker processes.",
-                    "id": "worker-processes",
-                    "label": "Worker processes",
-                    "regex": "^(auto|\\d+)$",
-                    "type": "text",
-                },
-                "WORKER_RLIMIT_NOFILE": {
-                    "context": "global",
-                    "default": "2048",
-                    "help": "Maximum number of open files for worker processes.",
-                    "id": "worker-rlimit-nofile",
-                    "label": "Open files per worker",
-                    "regex": "^\\d+$",
-                    "type": "text",
-                },
-                "WORKER_CONNECTIONS": {
-                    "context": "global",
-                    "default": "1024",
-                    "help": "Maximum number of connections per worker.",
-                    "id": "worker-connections",
-                    "label": "Connections per worker",
-                    "regex": "^\\d+$",
-                    "type": "text",
-                },
-                "LOG_FORMAT": {
-                    "context": "global",
-                    "default": '$host $remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"',
-                    "help": "The format to use for access logs.",
-                    "id": "log-format",
-                    "label": "Log format",
-                    "regex": "^.*$",
-                    "type": "text",
-                },
-                "LOG_LEVEL": {
-                    "context": "global",
-                    "default": "notice",
-                    "help": "The level to use for error logs.",
-                    "id": "log-level",
-                    "label": "Log level",
-                    "regex": "^(debug|info|notice|warn|error|crit|alert|emerg)$",
-                    "type": "select",
-                    "select": [
-                        "debug",
-                        "info",
-                        "notice",
-                        "warn",
-                        "error",
-                        "crit",
-                        "alert",
-                        "emerg",
-                    ],
-                },
-                "DNS_RESOLVERS": {
-                    "context": "global",
-                    "default": "127.0.0.11",
-                    "help": "DNS addresses of resolvers to use.",
-                    "id": "dns-resolvers",
-                    "label": "DNS resolvers",
-                    "regex": "^(?! )( *(((\\b25[0-5]|\\b2[0-4]\\d|\\b[01]?\\d\\d?)(\\.(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)){3})(\\/([1-2][0-9]?|3[0-2]?|[04-9]))?|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]Z{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d)|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d))(\\/(12[0-8]|1[01][0-9]|[0-9][0-9]?))?)(?!.*\\D\\2([^\\d\\/]|$)) *)*$",
-                    "type": "text",
-                },
-                "DATASTORE_MEMORY_SIZE": {
-                    "context": "global",
-                    "default": "64m",
-                    "help": "Size of the internal datastore.",
-                    "id": "datastore-memory-size",
-                    "label": "Datastore memory size",
-                    "regex": "^\\d+[kKmMgG]?$",
-                    "type": "text",
-                },
-                "CACHESTORE_MEMORY_SIZE": {
-                    "context": "global",
-                    "default": "64m",
-                    "help": "Size of the internal cachestore.",
-                    "id": "cachestore-memory-size",
-                    "label": "Cachestore memory size",
-                    "regex": "^\\d+[kKmMgG]?$",
-                    "type": "text",
-                },
-                "CACHESTORE_IPC_MEMORY_SIZE": {
-                    "context": "global",
-                    "default": "16m",
-                    "help": "Size of the internal cachestore (ipc).",
-                    "id": "cachestore-ipc-memory-size",
-                    "label": "Cachestore ipc memory size",
-                    "regex": "^\\d+[kKmMgG]?$",
-                    "type": "text",
-                },
-                "CACHESTORE_MISS_MEMORY_SIZE": {
-                    "context": "global",
-                    "default": "16m",
-                    "help": "Size of the internal cachestore (miss).",
-                    "id": "cachestore-miss-memory-size",
-                    "label": "Cachestore miss memory size",
-                    "regex": "^\\d+[kKmMgG]?$",
-                    "type": "text",
-                },
-                "CACHESTORE_LOCKS_MEMORY_SIZE": {
-                    "context": "global",
-                    "default": "16m",
-                    "help": "Size of the internal cachestore (locks).",
-                    "id": "cachestore-locks-memory-size",
-                    "label": "Cachestore locks memory size",
-                    "regex": "^\\d+[kKmMgG]?$",
-                    "type": "text",
-                },
-                "USE_API": {
-                    "context": "global",
-                    "default": "yes",
-                    "help": "Activate the API to control BunkerWeb.",
-                    "id": "use-api",
-                    "label": "Activate API",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "API_HTTP_PORT": {
-                    "context": "global",
-                    "default": "5000",
-                    "help": "Listen port number for the API.",
-                    "id": "api-http-listen",
-                    "label": "API port number",
-                    "regex": "^\\d+$",
-                    "type": "text",
-                },
-                "API_LISTEN_IP": {
-                    "context": "global",
-                    "default": "0.0.0.0",
-                    "help": "Listen IP address for the API.",
-                    "id": "api-ip-listen",
-                    "label": "API listen IP",
-                    "regex": "^.*$",
-                    "type": "text",
-                },
-                "API_SERVER_NAME": {
-                    "context": "global",
-                    "default": "bwapi",
-                    "help": "Server name (virtual host) for the API.",
-                    "id": "api-server-name",
-                    "label": "API server name",
-                    "regex": "^(?=.{1,255}$)[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?(?:\\.[0-9A-Za-z](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\\.?$",
-                    "type": "text",
-                },
-                "API_WHITELIST_IP": {
-                    "context": "global",
-                    "default": "127.0.0.0/8",
-                    "help": "List of IP/network allowed to contact the API.",
-                    "id": "api-whitelist-ip",
-                    "label": "API whitelist IP",
-                    "regex": "^(?! )( *(((\\b25[0-5]|\\b2[0-4]\\d|\\b[01]?\\d\\d?)(\\.(25[0-5]|2[0-4]\\d|[01]?\\d\\d?)){3})(\\/([1-2][0-9]?|3[0-2]?|[04-9]))?|(([0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]Z{0,4}){0,4}%[0-9a-zA-Z]+|::(ffff(:0{1,4})?:)?((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d)|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1?\\d)?\\d)\\.){3}(25[0-5]|(2[0-4]|1?\\d)?\\d))(\\/(12[0-8]|1[01][0-9]|[0-9][0-9]?))?)(?!.*\\D\\2([^\\d\\/]|$)) *)*$",
-                    "type": "text",
-                },
-                "AUTOCONF_MODE": {
-                    "context": "global",
-                    "default": "no",
-                    "help": "Enable Autoconf Docker integration.",
-                    "id": "autoconf-mode",
-                    "label": "Autoconf mode",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "SWARM_MODE": {
-                    "context": "global",
-                    "default": "no",
-                    "help": "Enable Docker Swarm integration.",
-                    "id": "swarm-mode",
-                    "label": "Swarm mode",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "KUBERNETES_MODE": {
-                    "context": "global",
-                    "default": "no",
-                    "help": "Enable Kubernetes integration.",
-                    "id": "kubernetes-mode",
-                    "label": "Kubernetes mode",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "SERVER_TYPE": {
-                    "context": "multisite",
-                    "default": "http",
-                    "help": "Server type : http or stream.",
-                    "id": "server-type",
-                    "label": "Server type",
-                    "regex": "^(http|stream)$",
-                    "type": "select",
-                    "select": ["http", "stream"],
-                },
-                "LISTEN_STREAM": {
-                    "context": "multisite",
-                    "default": "yes",
-                    "help": "Enable listening for non-ssl (passthrough).",
-                    "id": "listen-stream",
-                    "label": "Listen stream",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-                "LISTEN_STREAM_PORT": {
-                    "context": "multisite",
-                    "default": "1337",
-                    "help": "Listening port for non-ssl (passthrough).",
-                    "id": "listen-stream-port",
-                    "label": "Listen stream port",
-                    "regex": "^[0-9]+$",
-                    "type": "text",
-                },
-                "LISTEN_STREAM_PORT_SSL": {
-                    "context": "multisite",
-                    "default": "4242",
-                    "help": "Listening port for ssl (passthrough).",
-                    "id": "listen-stream-port-ssl",
-                    "label": "Listen stream port ssl",
-                    "regex": "^[0-9]+$",
-                    "type": "text",
-                },
-                "USE_UDP": {
-                    "context": "multisite",
-                    "default": "no",
-                    "help": "UDP listen instead of TCP (stream).",
-                    "id": "use-udp",
-                    "label": "Listen UDP",
-                    "regex": "^(yes|no)$",
-                    "type": "check",
-                },
-            },
-        }
+            "settings": global_settings,
+        },
     }
-    for filename in iglob(join("core", "*", "plugin.json")):
+    for filename in iglob(join("bunkerweb", "core", "*", "plugin.json")):
         with open(filename, "r") as f:
             data = load(f)
             data["checked"] = False
@@ -802,8 +523,8 @@ try:
                 exit(1)
 
             path_ui = (
-                Path(join("core", plugin_page.plugin_id, "ui"))
-                if Path(join("core", plugin_page.plugin_id, "ui")).exists()
+                Path(join("bunkerweb", "core", plugin_page.plugin_id, "ui"))
+                if Path(join("bunkerweb", "core", plugin_page.plugin_id, "ui")).exists()
                 else Path(join("external", plugin_page.plugin_id, "ui"))
             )
 

tests/core/db/test.sh

@@ -15,6 +15,7 @@ cleanup_stack () {
     exit_code=$?
     if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then
         rm -rf init/plugins
+        rm -rf init/bunkerweb
         find . -type f -name 'docker-compose.*' -exec sed -i 's@DATABASE_URI: ".*"$@DATABASE_URI: "sqlite:////var/lib/bunkerweb/db.sqlite3"@' {} \;
         find . -type f -name 'docker-compose.*' -exec sed -i 's@MULTISITE: "yes"$@MULTISITE: "no"@' {} \;
         sed -i 's@bwadm.example.com_USE_REVERSE_PROXY@USE_REVERSE_PROXY@' docker-compose.yml
@@ -60,9 +61,16 @@ cleanup_stack () {
 # Cleanup stack on exit
 trap cleanup_stack EXIT
 
+echo "💾 Starting stack ..."
+docker compose up -d 2>/dev/null
+if [ $? -ne 0 ] ; then
+    echo "💾 Up failed ❌"
+    exit 1
+fi
+
 echo "💾 Initializing workspace ..."
-rm -rf init/plugins
-mkdir -p init/plugins
+rm -rf init/plugins init/bunkerweb
+mkdir -p init/plugins init/bunkerweb
 docker compose -f docker-compose.init.yml up --build
 if [ $? -ne 0 ] ; then
     echo "💾 Build failed ❌"
@@ -70,8 +78,21 @@ if [ $? -ne 0 ] ; then
 elif ! [[ -d "init/plugins/clamav" ]]; then
     echo "💾 ClamAV plugin not found ❌"
     exit 1
+elif ! [[ -d "init/bunkerweb/core" ]]; then
+    echo "💾 BunkerWeb's core plugins directory not found ❌"
+    exit 1
+elif ! [[ -d "init/bunkerweb/db" ]]; then
+    echo "💾 BunkerWeb's database directory not found ❌"
+    exit 1
+elif ! [[ -f "init/bunkerweb/settings.json" ]]; then
+    echo "💾 BunkerWeb's settings file not found ❌"
+    exit 1
 fi
 
+manual=1
+cleanup_stack
+manual=0
+
 docker compose -f docker-compose.test.yml build
 if [ $? -ne 0 ] ; then
     echo "💾 Build failed ❌"