librewolf
/
bunkerized-nginx
mirror of https://github.com/bunkerity/bunkerized-nginx
4
0
Fork 1
Code Issues Projects Releases Wiki Activity

Edit scheduler and change DB

This commit is contained in:
TheophileDiot 2022-11-09 15:53:49 +01:00
parent 858f6e00f4
commit 0811aad7f5
8 changed files with 215 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

233
bw/gen/save_config.py
View File

@ -110,7 +110,7 @@ if __name__ == "__main__":
)
args = parser.parse_args()
logger.info("First gen started ...")
logger.info("Save config started ...")
logger.info(f"Settings : {args.settings}")
logger.info(f"Core : {args.core}")
logger.info(f"Plugins : {args.plugins}")
@ -128,138 +128,76 @@ if __name__ == "__main__":
integration = f.read().strip()
logger.info(f"Detected {integration} integration")
config_files = None
db = None
# Check existences and permissions
logger.info("Checking arguments ...")
files = [args.settings] + ([args.variables] if args.variables else [])
paths_rx = [args.core, args.plugins]
for file in files:
if not path.exists(file):
logger.error(f"Missing file : {file}")
sys_exit(1)
if not access(file, R_OK):
logger.error(f"Can't read file : {file}")
sys_exit(1)
for _path in paths_rx:
if not path.isdir(_path):
logger.error(f"Missing directory : {_path}")
sys_exit(1)
if not access(_path, R_OK | X_OK):
logger.error(
f"Missing RX rights on directory : {_path}",
)
sys_exit(1)
# Check core plugins orders
logger.info("Checking core plugins orders ...")
core_plugins = {}
files = glob(f"{args.core}/*/plugin.json")
for file in files:
try:
with open(file) as f:
core_plugin = loads(f.read())
if core_plugin["order"] not in core_plugins:
core_plugins[core_plugin["order"]] = []
core_plugins[core_plugin["order"]].append(core_plugin)
except:
logger.error(
f"Exception while loading JSON from {file} : {format_exc()}",
)
core_settings = {}
for order in core_plugins:
if len(core_plugins[order]) > 1 and order != 999:
logger.warning(
f"Multiple plugins have the same order ({order}) : {', '.join(plugin['id'] for plugin in core_plugins[order])}. Therefor, the execution order will be random.",
)
for plugin in core_plugins[order]:
core_settings.update(plugin["settings"])
if args.variables:
logger.info(f"Variables : {args.variables}")
# Check existences and permissions
logger.info("Checking arguments ...")
files = [args.settings, args.variables]
paths_rx = [args.core, args.plugins]
for file in files:
if not path.exists(file):
logger.error(f"Missing file : {file}")
sys_exit(1)
if not access(file, R_OK):
logger.error(f"Can't read file : {file}")
sys_exit(1)
for _path in paths_rx:
if not path.isdir(_path):
logger.error(f"Missing directory : {_path}")
sys_exit(1)
if not access(_path, R_OK | X_OK):
logger.error(
f"Missing RX rights on directory : {_path}",
)
sys_exit(1)
# Check core plugins orders
logger.info("Checking core plugins orders ...")
core_plugins = {}
files = glob(f"{args.core}/*/plugin.json")
for file in files:
try:
with open(file) as f:
core_plugin = loads(f.read())
if core_plugin["order"] not in core_plugins:
core_plugins[core_plugin["order"]] = []
core_plugins[core_plugin["order"]].append(core_plugin)
except:
logger.error(
f"Exception while loading JSON from {file} : {format_exc()}",
)
core_settings = {}
for order in core_plugins:
if len(core_plugins[order]) > 1 and order != 999:
logger.warning(
f"Multiple plugins have the same order ({order}) : {', '.join(plugin['id'] for plugin in core_plugins[order])}. Therefor, the execution order will be random.",
)
for plugin in core_plugins[order]:
core_settings.update(plugin["settings"])
# Compute the config
logger.info("Computing config ...")
config = Configurator(
args.settings, core_settings, args.plugins, args.variables, logger
)
config_files = config.get_config()
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
)
is_initialized = db.is_initialized()
if not is_initialized:
ret, err = db.init_tables(
[
config.get_settings(),
list(chain.from_iterable(core_plugins.values())),
config.get_plugins_settings(),
]
)
# Initialize database tables
if err:
logger.error(
f"Exception while initializing database : {err}",
)
sys_exit(1)
elif ret is False:
logger.info(
"Database tables are already initialized, skipping creation ...",
)
else:
logger.info("Database tables initialized")
logger.info(
"Database not initialized, initializing ...",
)
custom_confs = [
{"value": v, "exploded": custom_confs_rx.search(k).groups()}
for k, v in environ.items()
if custom_confs_rx.match(k)
]
with open("/opt/bunkerweb/VERSION", "r") as f:
bw_version = f.read().strip()
err = db.save_config(config_files, "scheduler")
if not err:
err1 = db.save_custom_configs(custom_confs, "scheduler")
if not err1:
err2 = db.initialize_db(
version=bw_version, integration=integration
)
if err or err1 or err2:
logger.error(
f"Can't Initialize database : {err or err1 or err2}",
)
sys_exit(1)
else:
logger.info("Database initialized")
if args.init:
sys_exit(0)
elif is_initialized:
logger.info(
"Database is already initialized, skipping ...",
)
sys_exit(0)
custom_confs = [
{"value": v, "exploded": custom_confs_rx.search(k).groups()}
for k, v in environ.items()
if custom_confs_rx.match(k)
]
elif integration == "Kubernetes":
corev1 = kube_client.CoreV1Api()
tmp_config = {}
apis = []
db = None
for pod in corev1.list_pod_for_all_namespaces(watch=False).items:
if (
@ -343,7 +281,6 @@ if __name__ == "__main__":
tmp_config = {}
custom_confs = []
apis = []
db = None
for instance in (
docker_client.containers.list(filters={"label": "bunkerweb.INSTANCE"})
@ -365,11 +302,57 @@ if __name__ == "__main__":
db = Database(logger)
# Compute the config
logger.info("Computing config ...")
config = Configurator(
args.settings, args.core, args.plugins, tmp_config, logger
)
config_files = config.get_config()
if config_files is None:
logger.info("Computing config ...")
config = Configurator(
args.settings, core_settings, args.plugins, tmp_config, logger
)
config_files = config.get_config()
if not db.is_initialized():
logger.info(
"Database not initialized, initializing ...",
)
ret, err = db.init_tables(
[
config.get_settings(),
list(chain.from_iterable(core_plugins.values())),
config.get_plugins_settings(),
]
)
# Initialize database tables
if err:
logger.error(
f"Exception while initializing database : {err}",
)
sys_exit(1)
elif ret is False:
logger.info(
"Database tables are already initialized, skipping creation ...",
)
else:
logger.info("Database tables initialized")
with open("/opt/bunkerweb/VERSION", "r") as f:
version = f.read().strip()
err = db.initialize_db(version=version, integration=integration)
if err:
logger.error(
f"Can't Initialize database : {err}",
)
sys_exit(1)
else:
logger.info("Database initialized")
else:
logger.info(
"Database is already initialized, skipping ...",
)
if args.init:
sys_exit(0)
err = db.save_config(config_files, "scheduler")

10
bw/helpers/entrypoint.sh
View File

@ -40,9 +40,13 @@ elif [ "$AUTOCONF_MODE" == "yes" ] ; then
echo "Autoconf" > /opt/bunkerweb/INTEGRATION
fi
# generate "temp" config
echo -e "IS_LOADING=yes\nSERVER_NAME=\nAPI_HTTP_PORT=${API_HTTP_PORT:-5000}\nAPI_SERVER_NAME=${API_SERVER_NAME:-bwapi}\nAPI_WHITELIST_IP=${API_WHITELIST_IP:-127.0.0.0/8}" > /tmp/variables.env
python3 /opt/bunkerweb/gen/main.py --variables /tmp/variables.env
if [ -f "/etc/nginx/variables.env" ] ; then
log "ENTRYPOINT" "⚠️ " "Looks like BunkerWeb has already been loaded, will not generate temp config"
else
# generate "temp" config
echo -e "IS_LOADING=yes\nSERVER_NAME=\nAPI_HTTP_PORT=${API_HTTP_PORT:-5000}\nAPI_SERVER_NAME=${API_SERVER_NAME:-bwapi}\nAPI_WHITELIST_IP=${API_WHITELIST_IP:-127.0.0.0/8}" > /tmp/variables.env
python3 /opt/bunkerweb/gen/main.py --variables /tmp/variables.env
fi
# start nginx
log "ENTRYPOINT" "ℹ️" "Starting nginx ..."

2
db/Database.py
View File

@ -8,7 +8,7 @@ from os.path import exists
from re import search
from sys import path as sys_path
from typing import Any, Dict, List, Optional, Tuple
from sqlalchemy import create_engine, inspect, text
from sqlalchemy import create_engine, inspect
from sqlalchemy.exc import OperationalError, ProgrammingError, SQLAlchemyError
from sqlalchemy.orm import scoped_session, sessionmaker
from time import sleep

2
db/model.py
View File

@ -150,7 +150,7 @@ class Jobs(Base):
file = Column(String(255), nullable=False)
every = Column(SCHEDULES_ENUM, nullable=False)
reload = Column(Boolean, nullable=False)
success = Column(Boolean, nullable=False)
success = Column(Boolean, nullable=True)
last_run = Column(DateTime, nullable=True)
plugin = relationship("Plugins", back_populates="jobs")

22
scheduler/entrypoint.sh
View File

@ -37,23 +37,19 @@ elif [ "$AUTOCONF_MODE" == "yes" ] ; then
echo "Autoconf" > /opt/bunkerweb/INTEGRATION
fi
# Init database
get_env > "/tmp/variables.env"
/opt/bunkerweb/gen/save_config.py --variables /tmp/variables.env --init
if [ "$?" -ne 0 ] ; then
log "ENTRYPOINT" "❌" "Scheduler generator failed"
exit 1
fi
generate=yes
if [ -f "/etc/nginx/variables.env" ] && grep -q "^TEMP_NGINX=no$" /etc/nginx/variables.env ; then
log "ENTRYPOINT" "⚠️ " "Looks like BunkerWeb configuration is already generated, will not generate it again"
generate=no
if ! grep -q "Docker" /opt/bunkerweb/INTEGRATION ; then
# Init database
get_env > "/tmp/variables.env"
/opt/bunkerweb/gen/save_config.py --variables /tmp/variables.env --init
if [ "$?" -ne 0 ] ; then
log "ENTRYPOINT" "❌" "Scheduler generator failed"
exit 1
fi
fi
# execute jobs
log "ENTRYPOINT" "ℹ️ " "Executing scheduler ..."
/opt/bunkerweb/scheduler/main.py --generate $generate
/opt/bunkerweb/scheduler/main.py
log "ENTRYPOINT" "ℹ️ " "Scheduler stopped"
exit 0

80
scheduler/main.py
View File

@ -16,7 +16,7 @@ from os import (
walk,
)
from os.path import dirname, exists, isdir, isfile, islink, join
from shutil import chown, rmtree
from shutil import chown, copy, rmtree
from signal import SIGINT, SIGTERM, SIGUSR1, SIGUSR2, signal
from subprocess import PIPE, run as subprocess_run, DEVNULL, STDOUT
from sys import path as sys_path
@ -110,14 +110,8 @@ if __name__ == "__main__":
type=str,
help="path to the file containing environment variables",
)
parser.add_argument(
"--generate",
default="no",
type=str,
help="Precise if the configuration needs to be generated directly or not",
)
args = parser.parse_args()
generate = args.generate == "yes"
generate = False
integration = "Linux"
api_caller = ApiCaller()
@ -136,27 +130,11 @@ if __name__ == "__main__":
integration = f.read().strip()
api_caller.auto_setup(bw_integration=integration)
if integration == "Docker" and generate is True:
# run the config saver
cmd = f"python /opt/bunkerweb/gen/save_config.py --settings /opt/bunkerweb/settings.json"
proc = subprocess_run(cmd.split(" "), stdin=DEVNULL, stderr=STDOUT)
if proc.returncode != 0:
logger.error(
"Config saver failed, configuration will not work as expected...",
)
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
)
while not db.is_initialized():
logger.warning(
"Database is not initialized, retrying in 5s ...",
)
sleep(5)
if integration in (
"Swarm",
"Kubernetes",
@ -174,6 +152,23 @@ if __name__ == "__main__":
"Autoconf is not loaded yet in the database, retrying in 5s ...",
)
sleep(5)
elif integration == "Docker" and (
not exists("/opt/bunkerweb/tmp/variables.env")
or db.get_config() != dotenv_values("/opt/bunkerweb/tmp/variables.env")
):
# run the config saver
cmd = f"python /opt/bunkerweb/gen/save_config.py --settings /opt/bunkerweb/settings.json"
proc = subprocess_run(cmd.split(" "), stdin=DEVNULL, stderr=STDOUT)
if proc.returncode != 0:
logger.error(
"Config saver failed, configuration will not work as expected...",
)
while not db.is_initialized():
logger.warning(
"Database is not initialized, retrying in 5s ...",
)
sleep(5)
env = db.get_config()
while not db.is_first_config_saved() or not env:
@ -208,7 +203,10 @@ if __name__ == "__main__":
}
)
old_configs = None
if custom_confs:
old_configs = db.get_custom_configs()
ret = db.save_custom_configs(custom_confs, "manual")
if ret:
logger.error(
@ -217,16 +215,19 @@ if __name__ == "__main__":
custom_configs = db.get_custom_configs()
original_path = "/data/configs"
makedirs(original_path, exist_ok=True)
for custom_config in custom_configs:
tmp_path = f"{original_path}/{custom_config['type'].replace('_', '-')}"
if custom_config["service_id"]:
tmp_path += f"/{custom_config['service_id']}"
tmp_path += f"/{custom_config['name']}.conf"
makedirs(dirname(tmp_path), exist_ok=True)
with open(tmp_path, "wb") as f:
f.write(custom_config["data"])
if old_configs != custom_configs:
original_path = "/data/configs"
makedirs(original_path, exist_ok=True)
for custom_config in custom_configs:
tmp_path = (
f"{original_path}/{custom_config['type'].replace('_', '-')}"
)
if custom_config["service_id"]:
tmp_path += f"/{custom_config['service_id']}"
tmp_path += f"/{custom_config['name']}.conf"
makedirs(dirname(tmp_path), exist_ok=True)
with open(tmp_path, "wb") as f:
f.write(custom_config["data"])
# Fix permissions for the custom configs folder
for root, dirs, files in walk("/data/configs", topdown=False):
@ -248,6 +249,15 @@ if __name__ == "__main__":
)
logger.info("Executing scheduler ...")
generate = not exists(
"/opt/bunkerweb/tmp/variables.env"
) or env != dotenv_values("/opt/bunkerweb/tmp/variables.env")
if generate is False:
logger.warning(
"Looks like BunkerWeb configuration is already generated, will not generate it again ..."
)
while True:
# Instantiate scheduler
scheduler = JobScheduler(
@ -278,6 +288,8 @@ if __name__ == "__main__":
chown(join(root, name), "scheduler", "scheduler")
chmod(join(root, name), 0o770)
copy("/etc/nginx/variables.env", "/opt/bunkerweb/tmp/variables.env")
if len(api_caller._get_apis()) > 0:
# send nginx configs
logger.info("Sending /etc/nginx folder ...")

64
ui/main.py
View File

@ -627,7 +627,11 @@ def configs():
db_configs = db.get_custom_configs()
return render_template(
"configs.html",
folders=[path_to_dict("/opt/bunkerweb/configs", db_configs=db_configs)],
folders=[
path_to_dict(
"/opt/bunkerweb/configs", db_configs=db_configs, integration=integration
)
],
)
@ -1147,7 +1151,11 @@ def logs():
first_instance = instances[0] if instances else None
return render_template(
"logs.html", first_instance=first_instance, instances=instances
"logs.html",
first_instance=first_instance,
instances=instances,
is_swarm=getenv("SWARM_MODE", "no") == "yes",
is_kubernetes=getenv("KUBERNETES_MODE", "no") == "yes",
)
@ -1204,7 +1212,11 @@ def logs_linux():
for line in raw_logs_error:
line_lower = line.lower()
if "[info]" in line.lower() and line.endswith(":") or "[error]" in line.lower():
if (
("[info]" in line_lower or "ℹ️" in line_lower)
and line.endswith(":")
or ("[error]" in line_lower or "❌" in line_lower)
):
if temp_multiple_lines:
logs_error.append("\n".join(temp_multiple_lines))
@ -1244,37 +1256,19 @@ def logs_linux():
or "❌" in log_lower
else (
"warn"
if "[warn]" in log_lower
else ("info" if "[info]" in log_lower else "message")
if "[warn]" in log_lower or "⚠️" in log_lower
else (
"info" if "[info]" in log_lower or "ℹ️" in log_lower else "message"
)
)
)
if "\n" in log:
splitted_one_line = log.split("\n")
logs.append(
{
"content": " ".join(
splitted_one_line.pop(0).strip().split(" ")[1:]
),
"type": error_type,
"separator": True,
}
)
for splitted_log in splitted_one_line:
logs.append(
{
"content": splitted_log,
"type": error_type,
}
)
else:
logs.append(
{
"content": " ".join(log.strip().split(" ")[1:]),
"type": error_type,
}
)
logs.append(
{
"content": " ".join(log.strip().split(" ")[1:]),
"type": error_type,
}
)
count_error_logs = 0
for log in logs_error:
@ -1336,8 +1330,12 @@ def logs_container(container_id):
or "❌" in log_lower
else (
"warn"
if "[warn]" in log_lower
else ("info" if "[info]" in log_lower else "message")
if "[warn]" in log_lower or "⚠️" in log_lower
else (
"info"
if "[info]" in log_lower or "ℹ️" in log_lower
else "message"
)
),
}
)

35
ui/utils.py
View File

@ -423,50 +423,39 @@ def path_to_dict(
}
for conf in db_configs:
type_lower = conf["type"].replace("_", "-")
file_info = {
"name": conf["name"],
"name": f"{conf['name']}.conf",
"type": "file",
"path": f"{path}/{conf['type'].replace('_', '-')}{'/' + conf['service_id'] if conf['service_id'] else ''}/{conf['name']}.conf",
"path": f"{path}/{type_lower}{'/' + conf['service_id'] if conf['service_id'] else ''}/{conf['name']}.conf",
"can_edit": conf["method"] == "ui",
"can_download": is_cache,
"content": b64encode(conf["data"]).decode("utf-8"),
}
if (
d["children"][config_types.index(conf["type"].replace("_", "-"))][
"children"
]
d["children"][config_types.index(type_lower)]["children"]
and conf["service_id"]
and conf["service_id"]
in [
x["name"]
for x in d["children"][
config_types.index(conf["type"].replace("_", "-"))
]["children"]
for x in d["children"][config_types.index(type_lower)]["children"]
]
):
d["children"][config_types.index(conf["type"].replace("_", "-"))][
"children"
][
d["children"][config_types.index(type_lower)]["children"][
[
x["name"]
for x in d["children"][
config_types.index(conf["type"].replace("_", "-"))
]["children"]
for x in d["children"][config_types.index(type_lower)][
"children"
]
].index(conf["service_id"])
][
"children"
].append(
file_info
)
]["children"].append(file_info)
else:
d["children"][config_types.index(conf["type"].replace("_", "-"))][
"children"
].append(
d["children"][config_types.index(type_lower)]["children"].append(
{
"name": conf["service_id"],
"type": "folder",
"path": f"{path}/{conf['type']}/{conf['service_id']}",
"path": f"{path}/{type_lower}/{conf['service_id']}",
"can_create_files": True,
"can_create_folders": False,
"can_edit": True,