mirror of
https://github.com/bunkerity/bunkerized-nginx
synced 2023-12-13 21:30:18 +01:00
Merge pull request #324 from TheophileDiot/1.5
Fix gen for Docker integration
This commit is contained in:
commit
cfaeb10133
|
@ -216,20 +216,20 @@ class Config(ApiCaller, ConfigCaller):
|
||||||
# run(cmd.split(" "), stdin=DEVNULL, stdout=DEVNULL, stderr=STDOUT)
|
# run(cmd.split(" "), stdin=DEVNULL, stdout=DEVNULL, stderr=STDOUT)
|
||||||
|
|
||||||
# send nginx configs
|
# send nginx configs
|
||||||
# send data folder
|
|
||||||
# reload nginx
|
|
||||||
ret = self._send_files("/etc/nginx", "/confs")
|
ret = self._send_files("/etc/nginx", "/confs")
|
||||||
if not ret:
|
if not ret:
|
||||||
success = False
|
success = False
|
||||||
self.__logger.error(
|
self.__logger.error(
|
||||||
"sending nginx configs failed, configuration will not work as expected...",
|
"sending nginx configs failed, configuration will not work as expected...",
|
||||||
)
|
)
|
||||||
|
# send data/configs folder
|
||||||
ret = self._send_files("/data/configs", "/custom_configs")
|
ret = self._send_files("/data/configs", "/custom_configs")
|
||||||
if not ret:
|
if not ret:
|
||||||
success = False
|
success = False
|
||||||
self.__logger.error(
|
self.__logger.error(
|
||||||
"sending custom configs failed, configuration will not work as expected...",
|
"sending custom configs failed, configuration will not work as expected...",
|
||||||
)
|
)
|
||||||
|
# reload nginx
|
||||||
ret = self._send_to_apis("POST", "/reload")
|
ret = self._send_to_apis("POST", "/reload")
|
||||||
if not ret:
|
if not ret:
|
||||||
success = False
|
success = False
|
||||||
|
|
|
@ -13,6 +13,9 @@ sys_path.append("/opt/bunkerweb/deps/python")
|
||||||
sys_path.append("/opt/bunkerweb/utils")
|
sys_path.append("/opt/bunkerweb/utils")
|
||||||
sys_path.append("/opt/bunkerweb/api")
|
sys_path.append("/opt/bunkerweb/api")
|
||||||
|
|
||||||
|
from docker import DockerClient
|
||||||
|
from docker.errors import DockerException
|
||||||
|
|
||||||
from logger import setup_logger
|
from logger import setup_logger
|
||||||
from API import API
|
from API import API
|
||||||
|
|
||||||
|
@ -85,7 +88,34 @@ try:
|
||||||
|
|
||||||
# Docker or Linux case
|
# Docker or Linux case
|
||||||
elif bw_integration == "Docker":
|
elif bw_integration == "Docker":
|
||||||
api = API(f"{getenv('BW_API_URL')}:5000")
|
try:
|
||||||
|
docker_client = DockerClient(base_url="tcp://docker-proxy:2375")
|
||||||
|
except DockerException:
|
||||||
|
docker_client = DockerClient(
|
||||||
|
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
|
||||||
|
)
|
||||||
|
|
||||||
|
apis = []
|
||||||
|
for instance in docker_client.containers.list(
|
||||||
|
filters={"label": "bunkerweb.INSTANCE"}
|
||||||
|
):
|
||||||
|
api = None
|
||||||
|
|
||||||
|
for var in instance.attrs["Config"]["Env"]:
|
||||||
|
if var.startswith("API_HTTP_PORT="):
|
||||||
|
api = API(
|
||||||
|
f"http://{instance.name}:{var.replace('API_HTTP_PORT=', '', 1)}"
|
||||||
|
)
|
||||||
|
break
|
||||||
|
|
||||||
|
if api:
|
||||||
|
apis.append(api)
|
||||||
|
else:
|
||||||
|
apis.append(
|
||||||
|
API(f"http://{instance.name}:{getenv('API_HTTP_PORT', '5000')}")
|
||||||
|
)
|
||||||
|
|
||||||
|
for api in apis:
|
||||||
sent, err, status, resp = api.request("POST", "/reload")
|
sent, err, status, resp = api.request("POST", "/reload")
|
||||||
if not sent:
|
if not sent:
|
||||||
status = 1
|
status = 1
|
||||||
|
|
|
@ -4,20 +4,31 @@ from logging import Logger
|
||||||
from re import search as re_search
|
from re import search as re_search
|
||||||
from sys import path as sys_path
|
from sys import path as sys_path
|
||||||
from traceback import format_exc
|
from traceback import format_exc
|
||||||
|
from typing import Union
|
||||||
|
|
||||||
sys_path.append("/opt/bunkerweb/utils")
|
sys_path.append("/opt/bunkerweb/utils")
|
||||||
|
|
||||||
|
|
||||||
class Configurator:
|
class Configurator:
|
||||||
def __init__(
|
def __init__(
|
||||||
self, settings: str, core: str, plugins: str, variables: str, logger: Logger
|
self,
|
||||||
|
settings: str,
|
||||||
|
core: str,
|
||||||
|
plugins: str,
|
||||||
|
variables: Union[str, dict],
|
||||||
|
logger: Logger,
|
||||||
):
|
):
|
||||||
self.__logger = logger
|
self.__logger = logger
|
||||||
self.__settings = self.__load_settings(settings)
|
self.__settings = self.__load_settings(settings)
|
||||||
self.__core = core
|
self.__core = core
|
||||||
self.__plugins_settings = []
|
self.__plugins_settings = []
|
||||||
self.__plugins = self.__load_plugins(plugins, "plugins")
|
self.__plugins = self.__load_plugins(plugins, "plugins")
|
||||||
|
|
||||||
|
if isinstance(variables, str):
|
||||||
self.__variables = self.__load_variables(variables)
|
self.__variables = self.__load_variables(variables)
|
||||||
|
else:
|
||||||
|
self.__variables = variables
|
||||||
|
|
||||||
self.__multisite = (
|
self.__multisite = (
|
||||||
"MULTISITE" in self.__variables and self.__variables["MULTISITE"] == "yes"
|
"MULTISITE" in self.__variables and self.__variables["MULTISITE"] == "yes"
|
||||||
)
|
)
|
||||||
|
@ -89,23 +100,26 @@ class Configurator:
|
||||||
variables[var] = value
|
variables[var] = value
|
||||||
return variables
|
return variables
|
||||||
|
|
||||||
def get_config(self, default: bool = False):
|
def get_config(self):
|
||||||
config = {}
|
config = {}
|
||||||
# Extract default settings
|
# Extract default settings
|
||||||
default_settings = [self.__settings, self.__core, self.__plugins]
|
default_settings = [self.__settings, self.__core, self.__plugins]
|
||||||
|
for settings in default_settings:
|
||||||
if default:
|
for setting, data in settings.items():
|
||||||
return default_settings
|
config[setting] = data["default"]
|
||||||
|
|
||||||
# Override with variables
|
# Override with variables
|
||||||
for variable, value in self.__variables.items():
|
for variable, value in self.__variables.items():
|
||||||
ret, err = self.__check_var(variable)
|
ret, err = self.__check_var(variable)
|
||||||
if ret:
|
if ret:
|
||||||
config[variable] = value
|
config[variable] = value
|
||||||
elif (
|
elif not variable.startswith("PYTHON") and variable not in (
|
||||||
not variable.startswith("PYTHON")
|
"GPG_KEY",
|
||||||
and variable != "GPG_KEY"
|
"LANG",
|
||||||
and variable != "LANG"
|
"PATH",
|
||||||
|
"NGINX_VERSION",
|
||||||
|
"NJS_VERSION",
|
||||||
|
"PKG_RELEASE",
|
||||||
):
|
):
|
||||||
self.__logger.warning(f"Ignoring variable {variable} : {err}")
|
self.__logger.warning(f"Ignoring variable {variable} : {err}")
|
||||||
# Expand variables to each sites if MULTISITE=yes and if not present
|
# Expand variables to each sites if MULTISITE=yes and if not present
|
||||||
|
|
126
bw/gen/main.py
126
bw/gen/main.py
|
@ -103,10 +103,13 @@ if __name__ == "__main__":
|
||||||
logger.info(f"Method : {args.method}")
|
logger.info(f"Method : {args.method}")
|
||||||
logger.info(f"Init : {args.init}")
|
logger.info(f"Init : {args.init}")
|
||||||
|
|
||||||
if args.variables or args.init:
|
custom_confs_rx = re_compile(
|
||||||
|
r"^([0-9a-z\.\-]*)_?CUSTOM_CONF_(HTTP|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC|MODSEC_CRS)_(.+)$"
|
||||||
|
)
|
||||||
|
|
||||||
# Check existences and permissions
|
# Check existences and permissions
|
||||||
logger.info("Checking arguments ...")
|
logger.info("Checking arguments ...")
|
||||||
files = [args.settings, args.variables]
|
files = [args.settings] + ([args.variables] if args.variables else [])
|
||||||
paths_rx = [args.core, args.plugins, args.templates]
|
paths_rx = [args.core, args.plugins, args.templates]
|
||||||
paths_rwx = [args.output]
|
paths_rwx = [args.output]
|
||||||
for file in files:
|
for file in files:
|
||||||
|
@ -160,6 +163,7 @@ if __name__ == "__main__":
|
||||||
for plugin in core_plugins[order]:
|
for plugin in core_plugins[order]:
|
||||||
core_settings.update(plugin["settings"])
|
core_settings.update(plugin["settings"])
|
||||||
|
|
||||||
|
if args.variables or args.init:
|
||||||
# Compute the config
|
# Compute the config
|
||||||
logger.info("Computing config ...")
|
logger.info("Computing config ...")
|
||||||
config = Configurator(
|
config = Configurator(
|
||||||
|
@ -167,10 +171,8 @@ if __name__ == "__main__":
|
||||||
)
|
)
|
||||||
config_files = config.get_config()
|
config_files = config.get_config()
|
||||||
|
|
||||||
if config_files.get("LOG_LEVEL", "INFO") != logger.level:
|
if config_files.get("LOG_LEVEL", logger.level) != logger.level:
|
||||||
logger = setup_logger(
|
logger = setup_logger("Generator", config_files["LOG_LEVEL"])
|
||||||
"Generator", config_files.get("LOG_LEVEL", "INFO")
|
|
||||||
)
|
|
||||||
|
|
||||||
bw_integration = None
|
bw_integration = None
|
||||||
if config_files.get("SWARM_MODE", "no") == "yes":
|
if config_files.get("SWARM_MODE", "no") == "yes":
|
||||||
|
@ -263,9 +265,6 @@ if __name__ == "__main__":
|
||||||
"Database not initialized, initializing ...",
|
"Database not initialized, initializing ...",
|
||||||
)
|
)
|
||||||
|
|
||||||
custom_confs_rx = re_compile(
|
|
||||||
r"^([0-9a-z\.\-]*)_?CUSTOM_CONF_(HTTP|DEFAULT_SERVER_HTTP|SERVER_HTTP|MODSEC|MODSEC_CRS)_(.+)$"
|
|
||||||
)
|
|
||||||
custom_confs = [
|
custom_confs = [
|
||||||
{"value": v, "exploded": custom_confs_rx.search(k).groups()}
|
{"value": v, "exploded": custom_confs_rx.search(k).groups()}
|
||||||
for k, v in environ.items()
|
for k, v in environ.items()
|
||||||
|
@ -297,7 +296,7 @@ if __name__ == "__main__":
|
||||||
with open("/opt/bunkerweb/INTEGRATION", "r") as f:
|
with open("/opt/bunkerweb/INTEGRATION", "r") as f:
|
||||||
bw_integration = f.read().strip()
|
bw_integration = f.read().strip()
|
||||||
|
|
||||||
if bw_integration in ("Docker", "Linux"):
|
if bw_integration == "Linux":
|
||||||
err = db.save_config(config_files, args.method)
|
err = db.save_config(config_files, args.method)
|
||||||
|
|
||||||
if not err:
|
if not err:
|
||||||
|
@ -324,6 +323,88 @@ if __name__ == "__main__":
|
||||||
|
|
||||||
sys_exit(0)
|
sys_exit(0)
|
||||||
|
|
||||||
|
config = config_files
|
||||||
|
elif args.method != "autoconf":
|
||||||
|
bw_integration = "Docker"
|
||||||
|
|
||||||
|
try:
|
||||||
|
docker_client = DockerClient(base_url="tcp://docker-proxy:2375")
|
||||||
|
except DockerException:
|
||||||
|
docker_client = DockerClient(
|
||||||
|
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
|
||||||
|
)
|
||||||
|
|
||||||
|
tmp_config = {}
|
||||||
|
custom_confs = []
|
||||||
|
apis = []
|
||||||
|
db = None
|
||||||
|
for instance in docker_client.containers.list(
|
||||||
|
filters={"label": "bunkerweb.INSTANCE"}
|
||||||
|
):
|
||||||
|
api = None
|
||||||
|
|
||||||
|
for var in instance.attrs["Config"]["Env"]:
|
||||||
|
if custom_confs_rx.match(var.split("=", 1)[0]):
|
||||||
|
splitted = var.split("=", 1)
|
||||||
|
custom_confs.append(
|
||||||
|
{
|
||||||
|
"value": var.pop(0),
|
||||||
|
"exploded": custom_confs_rx.search(
|
||||||
|
"=".join(var)
|
||||||
|
).groups(),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
tmp_config[var.split("=", 1)[0]] = var.split("=", 1)[1]
|
||||||
|
|
||||||
|
if var.startswith("DATABASE_URI="):
|
||||||
|
db = Database(logger, var.replace("DATABASE_URI=", "", 1))
|
||||||
|
elif var.startswith("API_HTTP_PORT="):
|
||||||
|
api = API(
|
||||||
|
f"http://{instance.name}:{var.replace('API_HTTP_PORT=', '', 1)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
if api:
|
||||||
|
apis.append(api)
|
||||||
|
else:
|
||||||
|
apis.append(
|
||||||
|
API(f"http://{instance.name}:{getenv('API_HTTP_PORT', '5000')}")
|
||||||
|
)
|
||||||
|
|
||||||
|
if db is None:
|
||||||
|
db = Database(logger)
|
||||||
|
|
||||||
|
api_caller = ApiCaller(apis=apis)
|
||||||
|
|
||||||
|
# Compute the config
|
||||||
|
logger.info("Computing config ...")
|
||||||
|
config = Configurator(
|
||||||
|
args.settings, core_settings, args.plugins, tmp_config, logger
|
||||||
|
)
|
||||||
|
config_files = config.get_config()
|
||||||
|
|
||||||
|
if config_files.get("LOG_LEVEL", logger.level) != logger.level:
|
||||||
|
logger = setup_logger("Generator", config_files["LOG_LEVEL"])
|
||||||
|
|
||||||
|
err = db.save_config(config_files, args.method)
|
||||||
|
|
||||||
|
if not err:
|
||||||
|
err1 = db.save_custom_configs(custom_confs, args.method)
|
||||||
|
else:
|
||||||
|
err = None
|
||||||
|
err1 = None
|
||||||
|
|
||||||
|
with open("/opt/bunkerweb/VERSION", "r") as f:
|
||||||
|
bw_version = f.read().strip()
|
||||||
|
|
||||||
|
if err or err1:
|
||||||
|
logger.error(
|
||||||
|
f"Can't save config to database : {err or err1}",
|
||||||
|
)
|
||||||
|
sys_exit(1)
|
||||||
|
else:
|
||||||
|
logger.info("Config successfully saved to database")
|
||||||
|
|
||||||
config = config_files
|
config = config_files
|
||||||
else:
|
else:
|
||||||
db = None
|
db = None
|
||||||
|
@ -347,33 +428,16 @@ if __name__ == "__main__":
|
||||||
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
|
base_url=getenv("DOCKER_HOST", "unix:///var/run/docker.sock")
|
||||||
)
|
)
|
||||||
|
|
||||||
apis = []
|
|
||||||
for instance in docker_client.containers.list(
|
for instance in docker_client.containers.list(
|
||||||
filters={"label": "bunkerweb.INSTANCE"}
|
filters={"label": "bunkerweb.INSTANCE"}
|
||||||
):
|
):
|
||||||
api = None
|
|
||||||
|
|
||||||
if db is None:
|
|
||||||
for var in instance.attrs["Config"]["Env"]:
|
for var in instance.attrs["Config"]["Env"]:
|
||||||
if var.startswith("DATABASE_URI="):
|
if var.startswith("DATABASE_URI="):
|
||||||
db = Database(
|
db = Database(logger, var.replace("DATABASE_URI=", "", 1))
|
||||||
logger, var.replace("DATABASE_URI=", "", 1)
|
break
|
||||||
)
|
|
||||||
elif var.startswith("API_HTTP_PORT="):
|
|
||||||
api = API(
|
|
||||||
f"http://{instance.name}:{var.replace('API_HTTP_PORT=', '', 1)}"
|
|
||||||
)
|
|
||||||
|
|
||||||
if api:
|
if db:
|
||||||
apis.append(api)
|
break
|
||||||
else:
|
|
||||||
apis.append(
|
|
||||||
API(
|
|
||||||
f"http://{instance.name}:{getenv('API_HTTP_PORT', '5000')}"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
api_caller = ApiCaller(apis=apis)
|
|
||||||
|
|
||||||
if db is None:
|
if db is None:
|
||||||
db = Database(logger)
|
db = Database(logger)
|
||||||
|
|
|
@ -5,7 +5,7 @@
|
||||||
log "$1" "ℹ️" "Setup and check /data folder ..."
|
log "$1" "ℹ️" "Setup and check /data folder ..."
|
||||||
|
|
||||||
# Create folders if missing and check permissions
|
# Create folders if missing and check permissions
|
||||||
rwx_folders=("cache" "letsencrypt")
|
rwx_folders=("cache" "cache/letsencrypt")
|
||||||
rx_folders=("configs" "configs/http" "configs/stream" "configs/server-http" "configs/server-stream" "configs/default-server-http" "configs/default-server-stream" "configs/modsec" "configs/modsec-crs" "plugins" "www")
|
rx_folders=("configs" "configs/http" "configs/stream" "configs/server-http" "configs/server-stream" "configs/default-server-http" "configs/default-server-stream" "configs/modsec" "configs/modsec-crs" "plugins" "www")
|
||||||
for folder in "${rwx_folders[@]}" ; do
|
for folder in "${rwx_folders[@]}" ; do
|
||||||
if [ ! -d "/data/${folder}" ] ; then
|
if [ ! -d "/data/${folder}" ] ; then
|
||||||
|
|
|
@ -18,7 +18,6 @@ API_WHITELIST_IP=127.0.0.0/8 10.20.30.0/24
|
||||||
AUTOCONF_MODE=no
|
AUTOCONF_MODE=no
|
||||||
SWARM_MODE=no
|
SWARM_MODE=no
|
||||||
KUBERNETES_MODE=no
|
KUBERNETES_MODE=no
|
||||||
BW_API_URL=http://mybunker
|
|
||||||
USE_BROTLI=no
|
USE_BROTLI=no
|
||||||
BROTLI_TYPES=application/atom+xml application/javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/x-javascript application/xhtml+xml application/xml font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml
|
BROTLI_TYPES=application/atom+xml application/javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-opentype application/x-font-truetype application/x-font-ttf application/x-javascript application/xhtml+xml application/xml font/eot font/opentype font/otf font/truetype image/svg+xml image/vnd.microsoft.icon image/x-icon image/x-win-bitmap text/css text/javascript text/plain text/xml
|
||||||
BROTLI_MIN_LENGTH=1000
|
BROTLI_MIN_LENGTH=1000
|
||||||
|
|
|
@ -51,6 +51,6 @@ VOLUME /data /etc/nginx
|
||||||
|
|
||||||
WORKDIR /opt/bunkerweb/scheduler
|
WORKDIR /opt/bunkerweb/scheduler
|
||||||
|
|
||||||
USER scheduler:scheduler
|
USER root:scheduler
|
||||||
|
|
||||||
ENTRYPOINT ["/opt/bunkerweb/scheduler/entrypoint.sh"]
|
ENTRYPOINT ["/opt/bunkerweb/scheduler/entrypoint.sh"]
|
||||||
|
|
|
@ -44,10 +44,21 @@ if [ "$?" -ne 0 ] ; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ -v VARIABLES_PATH ] && [ -f "/etc/nginx/variables.env" ] && grep -q "^TEMP_NGINX=no$" /etc/nginx/variables.env ; then
|
||||||
|
log "ENTRYPOINT" "⚠️ " "Looks like BunkerWeb configuration is already generated, will not generate it again"
|
||||||
|
elif [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] && [ "$AUTOCONF_MODE" != "yes" ] ; then
|
||||||
|
# Generate configuration and send config to bunkerweb
|
||||||
|
/opt/bunkerweb/gen/main.py --method scheduler
|
||||||
|
if [ "$?" -ne 0 ] ; then
|
||||||
|
log "ENTRYPOINT" "❌" "Scheduler generator failed"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
# execute jobs
|
# execute jobs
|
||||||
log "ENTRYPOINT" "ℹ️ " "Executing jobs ..."
|
log "ENTRYPOINT" "ℹ️ " "Executing jobs ..."
|
||||||
if [ -v VARIABLES_PATH ] ; then
|
if [ -v VARIABLES_PATH ] ; then
|
||||||
/opt/bunkerweb/scheduler/main.py --variables /tmp/variables.env --run
|
/opt/bunkerweb/scheduler/main.py --variables $VARIABLES_PATH --run
|
||||||
else
|
else
|
||||||
/opt/bunkerweb/scheduler/main.py --run
|
/opt/bunkerweb/scheduler/main.py --run
|
||||||
fi
|
fi
|
||||||
|
@ -56,16 +67,6 @@ if [ "$?" -ne 0 ] ; then
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ -v VARIABLES_PATH ] && [ -f "/etc/nginx/variables.env" ] && grep -q "^TEMP_NGINX=no$" /etc/nginx/variables.env ; then
|
|
||||||
log "ENTRYPOINT" "⚠️" "Looks like BunkerWeb configuration is already generated, will not generate it again"
|
|
||||||
elif [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] && [ "$AUTOCONF_MODE" != "yes" ] ; then
|
|
||||||
# Generate configuration and send config to bunkerweb
|
|
||||||
/opt/bunkerweb/gen/main.py --variables /tmp/variables.env --method scheduler
|
|
||||||
if [ "$?" -ne 0 ] ; then
|
|
||||||
log "ENTRYPOINT" "❌" "Scheduler generator failed"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "ENTRYPOINT" "ℹ️ " "Executing job scheduler ..."
|
log "ENTRYPOINT" "ℹ️ " "Executing job scheduler ..."
|
||||||
if [ -v VARIABLES_PATH ] ; then
|
if [ -v VARIABLES_PATH ] ; then
|
||||||
|
|
Loading…
Reference in a new issue