add missing init-stream-lua.conf and various fixes for ci/cd

src/common/confs/init-stream-lua.conf

@@ -0,0 +1,118 @@
+init_by_lua_block {
+
+local logger	= require "logger"
+local datastore	= require "datastore"
+local plugins	= require "plugins"
+local utils		= require "utils"
+local cjson		= require "cjson"
+
+logger.log(ngx.NOTICE, "INIT-STREAM", "Init phase started")
+
+-- Remove previous data from the datastore
+local data_keys = {"^plugin_", "^variable_", "^plugins$", "^api_", "^misc_"}
+for i, key in pairs(data_keys) do
+	local ok, err = datastore:delete_all(key)
+	if not ok then
+		logger.log(ngx.ERR, "INIT-STREAM", "Can't delete " .. key .. " from datastore : " .. err)
+		return false
+	end
+	logger.log(ngx.INFO, "INIT-STREAM", "Deleted " .. key .. " from datastore")
+end
+
+-- Load variables into the datastore
+local file = io.open("/etc/nginx/variables.env")
+if not file then
+	logger.log(ngx.ERR, "INIT-STREAM", "Can't open /etc/nginx/variables.env file")
+	return false
+end
+file:close()
+for line in io.lines("/etc/nginx/variables.env") do
+	local variable, value = line:match("(.+)=(.*)")
+	ok, err = datastore:set("variable_" .. variable, value)
+	if not ok then
+		logger.log(ngx.ERR, "INIT-STREAM", "Can't save variable " .. variable .. " into datastore")
+		return false
+	end
+end
+
+-- Set default values into the datastore
+ok, err = datastore:set("plugins", cjson.encode({}))
+if not ok then
+	logger.log(ngx.ERR, "INIT-STREAM", "Can't set default value for plugins into the datastore : " .. err)
+	return false
+end
+ok, err = utils.set_values()
+if not ok then
+	logger.log(ngx.ERR, "INIT-STREAM", "Error while setting default values : " .. err)
+	return false
+end
+
+-- API setup
+local value, err = datastore:get("variable_USE_API")
+if not value then
+	logger.log(ngx.ERR, "INIT-STREAM", "Can't get variable USE_API from the datastore")
+	return false
+end
+if value == "yes" then
+	value, err = datastore:get("variable_API_WHITELIST_IP")
+	if not value then
+		logger.log(ngx.ERR, "INIT-STREAM", "Can't get variable API_WHITELIST_IP from the datastore")
+		return false
+	end
+	local whitelists = { data = {}}
+	for whitelist in value:gmatch("%S+") do
+		table.insert(whitelists.data, whitelist)
+	end
+	ok, err = datastore:set("api_whitelist_ip", cjson.encode(whitelists))
+	if not ok then
+		logger.log(ngx.ERR, "INIT-STREAM", "Can't save api_whitelist_ip to datastore : " .. err)
+		return false
+	end
+end
+
+-- Load plugins into the datastore
+local plugin_paths = {"/usr/share/bunkerweb/core", "/etc/bunkerweb/plugins"}
+for i, plugin_path in ipairs(plugin_paths) do
+	local paths = io.popen("find -L " .. plugin_path .. " -maxdepth 1 -type d ! -path " .. plugin_path)
+	for path in paths:lines() do
+		plugin, err = plugins:load(path)
+		if not plugin then
+			logger.log(ngx.ERR, "INIT-STREAM", "Error while loading plugin from " .. path .. " : " .. err)
+			return false
+		end
+		logger.log(ngx.NOTICE, "INIT-STREAM", "Loaded plugin " .. plugin.id .. " v" .. plugin.version)
+	end
+end
+
+-- Call init method of plugins
+local list, err = plugins:list()
+if not list then
+	logger.log(ngx.ERR, "INIT-STREAM", "Can't list loaded plugins : " .. err)
+	list = {}
+end
+for i, plugin in ipairs(list) do
+	local ret, plugin_lua = pcall(require, plugin.id .. "/" .. plugin.id)
+	if ret then
+		local plugin_obj = plugin_lua.new()
+		if plugin_obj.init ~= nil then
+			ok, err = plugin_obj:init()
+			if not ok then
+				logger.log(ngx.ERR, "INIT-STREAM", "Plugin " .. plugin.id .. " failed on init() : " .. err)
+			else
+				logger.log(ngx.INFO, "INIT-STREAM", "Successfull init() call for plugin " .. plugin.id .. " : " .. err)
+			end
+		else
+			logger.log(ngx.INFO, "INIT-STREAM", "init() method not found in " .. plugin.id .. ", skipped execution")
+		end
+	else
+		if plugin_lua:match("not found") then
+			logger.log(ngx.INFO, "INIT-STREAM", "can't require " .. plugin.id .. " : not found")
+		else
+			logger.log(ngx.ERR, "INIT-STREAM", "can't require " .. plugin.id .. " : " .. plugin_lua)
+		end
+	end
+end
+
+logger.log(ngx.NOTICE, "INIT-STREAM", "Init phase ended")
+
+}

src/common/confs/stream.conf

@@ -29,7 +29,6 @@ lua_ssl_trusted_certificate "/usr/share/bunkerweb/misc/root-ca.pem";
 lua_ssl_verify_depth 2;
 {% if has_variable(all, "SERVER_TYPE", "stream") +%}
 lua_shared_dict datastore_stream {{ DATASTORE_MEMORY_SIZE }};
-{% endif %}
 
 # LUA init block
 include /etc/nginx/init-stream-lua.conf;
@@ -63,3 +62,5 @@ include /etc/nginx/{{ first_server }}/server-stream.conf;
 {% elif MULTISITE == "no" and SERVER_NAME != "" and SERVER_TYPE == "stream" +%}
 include /etc/nginx/server-stream.conf;
 {% endif %}
+
+{% endif %}

tests/AutoconfTest.py

@@ -45,7 +45,7 @@ class AutoconfTest(Test) :
             i = 0
             healthy = False
             while i < 30 :
-                proc = run('docker inspect --format "{{json .State.Health }}" autoconf-mybunker-1', cwd="/tmp/autoconf", shell=True, capture_output=True)
+                proc = run('docker inspect --format "{{json .State.Health }}" autoconf-bunkerweb-1', cwd="/tmp/autoconf", shell=True, capture_output=True)
                 if proc.returncode != 0 :
                     raise(Exception("docker inspect failed (autoconf stack)"))
                 if "healthy" in proc.stdout.decode() :

tests/KubernetesTest.py

@@ -61,8 +61,11 @@ class KubernetesTest(Test) :
                 sleep(1)
                 i += 1
             if not healthy :
+                run("kubectl describe daemonset/bunkerweb", cwd="/tmp/kubernetes", shell=True)
                 run("kubectl logs daemonset/bunkerweb", cwd="/tmp/kubernetes", shell=True)
+                run("kubectl describe deployment/bunkerweb-controller", cwd="/tmp/kubernetes", shell=True)
                 run("kubectl logs deployment/bunkerweb-controller", cwd="/tmp/kubernetes", shell=True)
+                run("kubectl describe deployment/bunkerweb-scheduler", cwd="/tmp/kubernetes", shell=True)
                 run("kubectl logs deployment/bunkerweb-scheduler", cwd="/tmp/kubernetes", shell=True)
                 run("kubectl logs deployment/bunkerweb-db", cwd="/tmp/kubernetes", shell=True)
                 run("kubectl logs deployment/bunkerweb-redis", cwd="/tmp/kubernetes", shell=True)

tests/ui/docker-compose.yml

@@ -33,7 +33,7 @@ services:
   bw-scheduler:
     image: bunkerity/bunkerweb-scheduler:1.5.0-beta
     depends_on:
-      - mybunker
+      - bw
     environment:
       DOCKER_HOST: "tcp://docker-proxy:2375"
     volumes:
@@ -45,7 +45,7 @@ services:
   bw-ui:
     image: bunkerity/bunkerweb-ui:1.5.0-beta
     depends_on:
-      - mybunker
+      - bw
       - docker-proxy
     environment:
       ABSOLUTE_URI: "http://www.example.com:8080/admin/"