From 0356250d9dc43d7354f79e6176772647b9294d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Th=C3=A9ophile=20Diot?= Date: Fri, 10 Mar 2023 13:16:00 +0100 Subject: [PATCH] Fix problem with the bunkerweb container and plugins --- src/bw/lua/api.lua | 48 ++-- src/common/core/jobs/jobs/download-plugins.py | 123 ++++++--- src/common/db/Database.py | 251 ++++++++++++------ src/common/db/model.py | 5 +- src/common/gen/Configurator.py | 30 ++- src/common/gen/save_config.py | 2 - src/scheduler/main.py | 83 +++++- src/ui/main.py | 202 +++++++++----- src/ui/src/Config.py | 46 +++- 9 files changed, 553 insertions(+), 237 deletions(-) diff --git a/src/bw/lua/api.lua b/src/bw/lua/api.lua index dfe9da3c..ee8d6261 100644 --- a/src/bw/lua/api.lua +++ b/src/bw/lua/api.lua @@ -1,24 +1,24 @@ -local datastore = require "datastore" -local utils = require "utils" -local cjson = require "cjson" -local plugins = require "plugins" -local upload = require "resty.upload" -local logger = require "logger" +local datastore = require "datastore" +local utils = require "utils" +local cjson = require "cjson" +local plugins = require "plugins" +local upload = require "resty.upload" +local logger = require "logger" -local api = { global = { GET = {}, POST = {}, PUT = {}, DELETE = {} } } +local api = { global = { GET = {}, POST = {}, PUT = {}, DELETE = {} } } -api.response = function(self, http_status, api_status, msg) +api.response = function(self, http_status, api_status, msg) local resp = {} resp["status"] = api_status resp["msg"] = msg return http_status, resp end -api.global.GET["^/ping$"] = function(api) +api.global.GET["^/ping$"] = function(api) return api:response(ngx.HTTP_OK, "success", "pong") end -api.global.POST["^/reload$"] = function(api) +api.global.POST["^/reload$"] = function(api) local status = os.execute("nginx -s reload") if status == 0 then return api:response(ngx.HTTP_OK, "success", "reload successful") @@ -26,7 +26,7 @@ api.global.POST["^/reload$"] = function(api) return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "exit status = " .. tostring(status)) end -api.global.POST["^/stop$"] = function(api) +api.global.POST["^/stop$"] = function(api) local status = os.execute("nginx -s quit") if status == 0 then return api:response(ngx.HTTP_OK, "success", "stop successful") @@ -34,7 +34,7 @@ api.global.POST["^/stop$"] = function(api) return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "exit status = " .. tostring(status)) end -api.global.POST["^/confs$"] = function(api) +api.global.POST["^/confs$"] = function(api) local tmp = "/var/tmp/bunkerweb/api_" .. ngx.var.uri:sub(2) .. ".tar.gz" local destination = "/usr/share/bunkerweb/" .. ngx.var.uri:sub(2) if ngx.var.uri == "/confs" then @@ -45,6 +45,8 @@ api.global.POST["^/confs$"] = function(api) destination = "/data/cache" elseif ngx.var.uri == "/custom_configs" then destination = "/data/configs" + elseif ngx.var.uri == "/plugins" then + destination = "/data/plugins" end local form, err = upload:new(4096) if not form then @@ -78,13 +80,15 @@ api.global.POST["^/confs$"] = function(api) return api:response(ngx.HTTP_OK, "success", "saved data at " .. destination) end -api.global.POST["^/data$"] = api.global.POST["^/confs$"] +api.global.POST["^/data$"] = api.global.POST["^/confs$"] -api.global.POST["^/cache$"] = api.global.POST["^/confs$"] +api.global.POST["^/cache$"] = api.global.POST["^/confs$"] api.global.POST["^/custom_configs$"] = api.global.POST["^/confs$"] -api.global.POST["^/unban$"] = function(api) +api.global.POST["^/plugins$"] = api.global.POST["^/confs$"] + +api.global.POST["^/unban$"] = function(api) ngx.req.read_body() local data = ngx.req.get_body_data() if not data then @@ -103,7 +107,7 @@ api.global.POST["^/unban$"] = function(api) return api:response(ngx.HTTP_OK, "success", "ip " .. ip["ip"] .. " unbanned") end -api.global.POST["^/ban$"] = function(api) +api.global.POST["^/ban$"] = function(api) ngx.req.read_body() local data = ngx.req.get_body_data() if not data then @@ -122,17 +126,19 @@ api.global.POST["^/ban$"] = function(api) return api:response(ngx.HTTP_OK, "success", "ip " .. ip["ip"] .. " banned") end -api.global.GET["^/bans$"] = function(api) +api.global.GET["^/bans$"] = function(api) local data = {} for i, k in ipairs(datastore:keys()) do if k:find("^bans_ip_") then local ret, reason = datastore:get(k) if not ret then - return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't access " .. k .. " from datastore : " + reason) + return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", + "can't access " .. k .. " from datastore : " + reason) end local ret, exp = datastore:exp(k) if not ret then - return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't access exp " .. k .. " from datastore : " + exp) + return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", + "can't access exp " .. k .. " from datastore : " + exp) end local ban = { ip = k:sub(9, #k), reason = reason, exp = exp } table.insert(data, ban) @@ -141,7 +147,7 @@ api.global.GET["^/bans$"] = function(api) return api:response(ngx.HTTP_OK, "success", data) end -api.is_allowed_ip = function(self) +api.is_allowed_ip = function(self) local data, err = datastore:get("api_whitelist_ip") if not data then return false, "can't access api_allowed_ips in datastore" @@ -152,7 +158,7 @@ api.is_allowed_ip = function(self) return false, "IP is not in API_WHITELIST_IP" end -api.do_api_call = function(self) +api.do_api_call = function(self) if self.global[ngx.var.request_method] ~= nil then for uri, api_fun in pairs(self.global[ngx.var.request_method]) do if string.match(ngx.var.uri, uri) then diff --git a/src/common/core/jobs/jobs/download-plugins.py b/src/common/core/jobs/jobs/download-plugins.py index 4bab3399..fb8cbbce 100644 --- a/src/common/core/jobs/jobs/download-plugins.py +++ b/src/common/core/jobs/jobs/download-plugins.py @@ -1,16 +1,18 @@ #!/usr/bin/python3 +from hashlib import sha256 from io import BytesIO from os import getenv, listdir, makedirs, chmod, stat, _exit, walk -from os.path import dirname, join +from os.path import basename, dirname, join from pathlib import Path from stat import S_IEXEC from sys import exit as sys_exit, path as sys_path from threading import Lock from uuid import uuid4 from glob import glob -from json import load, loads +from json import loads from shutil import chown, copytree, rmtree +from tarfile import open as tar_open from traceback import format_exc from zipfile import ZipFile @@ -18,6 +20,7 @@ sys_path.extend( ( "/usr/share/bunkerweb/deps/python", "/usr/share/bunkerweb/utils", + "/usr/share/bunkerweb/api", "/usr/share/bunkerweb/db", ) ) @@ -28,31 +31,29 @@ from Database import Database from logger import setup_logger -logger = setup_logger("Jobs", getenv("LOG_LEVEL", "INFO")) -db = Database( - logger, - sqlalchemy_string=getenv("DATABASE_URI", None), -) +logger = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO")) lock = Lock() status = 0 -def install_plugin(plugin_dir): +def install_plugin(plugin_dir) -> bool: # Load plugin.json - with open(f"{plugin_dir}plugin.json", "rb") as f: + with open(f"{plugin_dir}/plugin.json", "rb") as f: metadata = loads(f.read()) # Don't go further if plugin is already installed if Path(f"/data/plugins/{metadata['id']}/plugin.json").is_file(): - logger.info( + logger.warning( f"Skipping installation of plugin {metadata['id']} (already installed)", ) - return + return False # Copy the plugin copytree(plugin_dir, f"/data/plugins/{metadata['id']}") # Add u+x permissions to jobs files for job_file in glob(f"{plugin_dir}jobs/*"): st = stat(job_file) chmod(job_file, st.st_mode | S_IEXEC) + logger.info(f"Plugin {metadata['id']} installed") + return True try: @@ -62,7 +63,15 @@ try: logger.info("No external plugins to download") _exit(0) + db = Database( + logger, + sqlalchemy_string=getenv("DATABASE_URI"), + ) + + plugin_nbr = 0 + # Loop on URLs + logger.info(f"Downloading external plugins from {plugin_urls}...") for plugin_url in plugin_urls.split(" "): # Download ZIP file try: @@ -75,9 +84,9 @@ try: continue # Extract it to tmp folder - temp_dir = f"/var/tmp/bunkerweb/plugins-{uuid4()}/" + temp_dir = f"/var/tmp/bunkerweb/plugins-{uuid4()}" try: - makedirs(temp_dir, exist_ok=True) + Path(temp_dir).mkdir(parents=True, exist_ok=True) with ZipFile(BytesIO(req.content)) as zf: zf.extractall(path=temp_dir) except: @@ -89,48 +98,76 @@ try: # Install plugins try: - for plugin_dir in glob(f"{temp_dir}**/plugin.json", recursive=True): - install_plugin(f"{dirname(plugin_dir)}/") + for plugin_dir in glob(f"{temp_dir}/**/plugin.json", recursive=True): + try: + if install_plugin(dirname(plugin_dir)): + plugin_nbr += 1 + except FileExistsError: + logger.warning( + f"Skipping installation of plugin {basename(dirname(plugin_dir))} (already installed)", + ) except: logger.error( f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}", ) status = 2 - continue - external_plugins = [] - external_plugins_ids = [] - for plugin in listdir("/etc/bunkerweb/plugins"): - with open( - f"/etc/bunkerweb/plugins/{plugin}/plugin.json", - "r", - ) as f: - plugin_file = load(f) - - external_plugins.append(plugin_file) - external_plugins_ids.append(plugin_file["id"]) - - with lock: - db_plugins = db.get_plugins() - - for plugin in db_plugins: - if plugin["external"] is True and plugin["id"] not in external_plugins_ids: - external_plugins.append(plugin) - - # Fix permissions for the certificates + # Fix permissions on plugins for root, dirs, files in walk("/data/plugins", topdown=False): for name in files + dirs: chown(join(root, name), "root", 101) chmod(join(root, name), 0o770) - if external_plugins: - with lock: - err = db.update_external_plugins(external_plugins) + if not plugin_nbr: + logger.info("No external plugins to update to database") + _exit(0) - if err: - logger.error( - f"Couldn't update external plugins to database: {err}", - ) + external_plugins = [] + external_plugins_ids = [] + for plugin in listdir("/data/plugins"): + path = f"/data/plugins/{plugin}" + if not Path(f"{path}/plugin.json").is_file(): + logger.warning(f"Plugin {plugin} is not valid, deleting it...") + rmtree(path) + continue + + plugin_file = loads(Path(f"{path}/plugin.json").read_text()) + + plugin_content = BytesIO() + with tar_open(fileobj=plugin_content, mode="w:gz") as tar: + tar.add(path, arcname=basename(path)) + plugin_content.seek(0) + value = plugin_content.getvalue() + + plugin_file.update( + { + "external": True, + "page": False, + "method": "scheduler", + "data": value, + "checksum": sha256(value).hexdigest(), + } + ) + + if "ui" in listdir(path): + plugin_file["ui"] = True + + external_plugins.append(plugin_file) + external_plugins_ids.append(plugin_file["id"]) + + for plugin in db.get_plugins(external=True, with_data=True): + if plugin["method"] != "scheduler" and plugin["id"] not in external_plugins_ids: + external_plugins.append(plugin) + + with lock: + err = db.update_external_plugins(external_plugins) + + if err: + logger.error( + f"Couldn't update external plugins to database: {err}", + ) + + logger.info("External plugins downloaded and installed") except: status = 2 diff --git a/src/common/db/Database.py b/src/common/db/Database.py index 539a6505..6b1d8b17 100644 --- a/src/common/db/Database.py +++ b/src/common/db/Database.py @@ -5,8 +5,9 @@ from hashlib import sha256 from logging import ( Logger, ) -from os import _exit, getenv, listdir, makedirs -from os.path import dirname, exists +from os import _exit, getenv +from os.path import dirname +from pathlib import Path from pymysql import install_as_MySQLdb from re import compile as re_compile from sys import path as sys_path @@ -61,7 +62,9 @@ class Database: if sqlalchemy_string.startswith("sqlite"): with suppress(FileExistsError): - makedirs(dirname(sqlalchemy_string.split("///")[1]), exist_ok=True) + Path(dirname(sqlalchemy_string.split("///")[1])).mkdir( + parents=True, exist_ok=True + ) elif "+" in sqlalchemy_string and "+pymysql" not in sqlalchemy_string: splitted = sqlalchemy_string.split("+") sqlalchemy_string = f"{splitted[0]}:{':'.join(splitted[1].split(':')[1:])}" @@ -117,6 +120,9 @@ class Database: sqlalchemy_string, future=True, ) + if "Unknown table" in str(e): + not_connected = False + continue else: self.__logger.warning( "Can't connect to database, retrying in 5 seconds ...", @@ -270,6 +276,7 @@ class Database: for plugin in plugins: settings = {} jobs = [] + page = False if "id" not in plugin: settings = plugin plugin = { @@ -283,6 +290,7 @@ class Database: else: settings = plugin.pop("settings", {}) jobs = plugin.pop("jobs", []) + page = plugin.pop("page", False) to_put.append(Plugins(**plugin)) @@ -298,40 +306,37 @@ class Database: for select in value.pop("select", []): to_put.append(Selects(setting_id=value["id"], value=select)) - to_put.append( - Settings( - **value, - ) - ) + to_put.append(Settings(**value)) for job in jobs: job["file_name"] = job.pop("file") to_put.append(Jobs(plugin_id=plugin["id"], **job)) - if exists(f"/usr/share/bunkerweb/core/{plugin['id']}/ui"): - if {"template.html", "actions.py"}.issubset( - listdir(f"/usr/share/bunkerweb/core/{plugin['id']}/ui") - ): - with open( - f"/usr/share/bunkerweb/core/{plugin['id']}/ui/template.html", - "r", - ) as file: - template = file.read().encode("utf-8") - with open( - f"/usr/share/bunkerweb/core/{plugin['id']}/ui/actions.py", - "r", - ) as file: - actions = file.read().encode("utf-8") + if page: + path_ui = ( + Path(f"/usr/share/bunkerweb/core/{plugin['id']}/ui") + if Path( + f"/usr/share/bunkerweb/core/{plugin['id']}/ui" + ).exists() + else Path(f"/etc/bunkerweb/plugins/{plugin['id']}/ui") + ) - to_put.append( - Plugin_pages( - plugin_id=plugin["id"], - template_file=template, - template_checksum=sha256(template).hexdigest(), - actions_file=actions, - actions_checksum=sha256(actions).hexdigest(), + if path_ui.exists(): + if {"template.html", "actions.py"}.issubset( + path_ui.iterdir() + ): + template = Path(f"{path_ui}/template.html").read_bytes() + actions = Path(f"{path_ui}/actions.py").read_bytes() + + to_put.append( + Plugin_pages( + plugin_id=plugin["id"], + template_file=template, + template_checksum=sha256(template).hexdigest(), + actions_file=actions, + actions_checksum=sha256(actions).hexdigest(), + ) ) - ) try: session.add_all(to_put) @@ -907,7 +912,9 @@ class Database: return "" - def update_external_plugins(self, plugins: List[Dict[str, Any]]) -> str: + def update_external_plugins( + self, plugins: List[Dict[str, Any]], *, delete_missing: bool = True + ) -> str: """Update external plugins from the database""" to_put = [] with self.__db_session() as session: @@ -919,7 +926,7 @@ class Database: ) db_ids = [] - if db_plugins: + if delete_missing and db_plugins: db_ids = [plugin.id for plugin in db_plugins] ids = [plugin["id"] for plugin in plugins] missing_ids = [plugin for plugin in db_ids if plugin not in ids] @@ -931,7 +938,7 @@ class Database: for plugin in plugins: settings = plugin.pop("settings", {}) jobs = plugin.pop("jobs", []) - pages = plugin.pop("pages", []) + page = plugin.pop("page", False) plugin["external"] = True db_plugin = ( session.query(Plugins) @@ -967,6 +974,15 @@ class Database: if plugin["version"] != db_plugin.version: updates[Plugins.version] = plugin["version"] + if plugin["method"] != db_plugin.method: + updates[Plugins.method] = plugin["method"] + + if plugin.get("data") != db_plugin.data: + updates[Plugins.data] = plugin["data"] + + if plugin.get("checksum") != db_plugin.checksum: + updates[Plugins.checksum] = plugin["checksum"] + if updates: session.query(Plugins).filter( Plugins.id == plugin["id"] @@ -1140,13 +1156,13 @@ class Database: ).update(updates) path_ui = ( - f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui" - if exists(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui") - else f"/etc/bunkerweb/plugins/{plugin['id']}/ui" + Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui") + if Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui").exists() + else Path(f"/etc/bunkerweb/plugins/{plugin['id']}/ui") ) - if exists(path_ui): - if {"template.html", "actions.py"}.issubset(listdir(path_ui)): + if path_ui.exists(): + if {"template.html", "actions.py"}.issubset(path_ui.iterdir()): db_plugin_page = ( session.query(Plugin_pages) .with_entities( @@ -1158,16 +1174,8 @@ class Database: ) if db_plugin_page is None: - with open( - f"{path_ui}/template.html", - "r", - ) as file: - template = file.read().encode("utf-8") - with open( - f"{path_ui}/actions.py", - "r", - ) as file: - actions = file.read().encode("utf-8") + template = Path(f"{path_ui}/template.html").read_bytes() + actions = Path(f"{path_ui}/actions.py").read_bytes() to_put.append( Plugin_pages( @@ -1178,7 +1186,7 @@ class Database: actions_checksum=sha256(actions).hexdigest(), ) ) - else: # TODO test this + else: updates = {} template_checksum = file_hash( f"{path_ui}/template.html" @@ -1189,32 +1197,24 @@ class Database: template_checksum != db_plugin_page.template_checksum ): - with open( - f"{path_ui}/template.html", - "r", - ) as file: - updates.update( - { - Plugin_pages.template_file: file.read().encode( - "utf-8" - ), - Plugin_pages.template_checksum: template_checksum, - } - ) + updates.update( + { + Plugin_pages.template_file: Path( + f"{path_ui}/template.html" + ).read_bytes(), + Plugin_pages.template_checksum: template_checksum, + } + ) if actions_checksum != db_plugin_page.actions_checksum: - with open( - f"{path_ui}/actions.py", - "r", - ) as file: - updates.update( - { - Plugin_pages.actions_file: file.read().encode( - "utf-8" - ), - Plugin_pages.actions_checksum: actions_checksum, - } - ) + updates.update( + { + Plugin_pages.actions_file: Path( + f"{path_ui}/actions.py" + ).read_bytes(), + Plugin_pages.actions_checksum: actions_checksum, + } + ) if updates: session.query(Plugin_pages).filter( @@ -1269,17 +1269,72 @@ class Database: job["reload"] = job.get("reload", False) to_put.append(Jobs(plugin_id=plugin["id"], **job)) - for page in pages: - to_put.append( - Plugin_pages( - plugin_id=plugin["id"], - template_file=page["template_file"], - template_checksum=sha256(page["template_file"]).hexdigest(), - actions_file=page["actions_file"], - actions_checksum=sha256(page["actions_file"]).hexdigest(), - ) + if page: + path_ui = ( + Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui") + if Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui").exists() + else Path(f"/etc/bunkerweb/plugins/{plugin['id']}/ui") ) + if path_ui.exists(): + if {"template.html", "actions.py"}.issubset(path_ui.iterdir()): + db_plugin_page = ( + session.query(Plugin_pages) + .with_entities( + Plugin_pages.template_checksum, + Plugin_pages.actions_checksum, + ) + .filter_by(plugin_id=plugin["id"]) + .first() + ) + + if db_plugin_page is None: + template = Path(f"{path_ui}/template.html").read_bytes() + actions = Path(f"{path_ui}/actions.py").read_bytes() + + to_put.append( + Plugin_pages( + plugin_id=plugin["id"], + template_file=template, + template_checksum=sha256(template).hexdigest(), + actions_file=actions, + actions_checksum=sha256(actions).hexdigest(), + ) + ) + else: + updates = {} + template_checksum = file_hash( + f"{path_ui}/template.html" + ) + actions_checksum = file_hash(f"{path_ui}/actions.py") + + if ( + template_checksum + != db_plugin_page.template_checksum + ): + updates.update( + { + Plugin_pages.template_file: Path( + f"{path_ui}/template.html" + ).read_bytes(), + Plugin_pages.template_checksum: template_checksum, + } + ) + + if actions_checksum != db_plugin_page.actions_checksum: + updates.update( + { + Plugin_pages.actions_file: Path( + f"{path_ui}/actions.py" + ).read_bytes(), + Plugin_pages.actions_checksum: actions_checksum, + } + ) + + if updates: + session.query(Plugin_pages).filter( + Plugin_pages.plugin_id == plugin["id"] + ).update(updates) try: session.add_all(to_put) session.commit() @@ -1288,8 +1343,10 @@ class Database: return "" - def get_plugins(self) -> List[Dict[str, Any]]: - """Get plugins.""" + def get_plugins( + self, *, external: bool = False, with_data: bool = False + ) -> List[Dict[str, Any]]: + """Get all plugins from the database.""" plugins = [] with self.__db_session() as session: for plugin in ( @@ -1301,10 +1358,29 @@ class Database: Plugins.description, Plugins.version, Plugins.external, + Plugins.method, + Plugins.data, + Plugins.checksum, + ) + .order_by(Plugins.order) + .all() + if with_data + else session.query(Plugins) + .with_entities( + Plugins.id, + Plugins.order, + Plugins.name, + Plugins.description, + Plugins.version, + Plugins.external, + Plugins.method, ) .order_by(Plugins.order) .all() ): + if external and not plugin.external: + continue + page = ( session.query(Plugin_pages) .with_entities(Plugin_pages.id) @@ -1318,9 +1394,14 @@ class Database: "description": plugin.description, "version": plugin.version, "external": plugin.external, + "method": plugin.method, "page": page is not None, "settings": {}, - } + } | ( + {"data": plugin.data, "checksum": plugin.checksum} + if with_data + else {} + ) for setting in ( session.query(Settings) diff --git a/src/common/db/model.py b/src/common/db/model.py index 07eede69..dd89b081 100644 --- a/src/common/db/model.py +++ b/src/common/db/model.py @@ -61,6 +61,9 @@ class Plugins(Base): description = Column(String(256), nullable=False) version = Column(String(32), nullable=False) external = Column(Boolean, default=False, nullable=False) + method = Column(METHODS_ENUM, default="manual", nullable=False) + data = Column(LargeBinary(length=(2**32) - 1), nullable=True) + checksum = Column(String(128), nullable=True) settings = relationship( "Settings", back_populates="plugin", cascade="all, delete-orphan" @@ -164,7 +167,7 @@ class Jobs(Base): ) file_name = Column(String(256), nullable=False) every = Column(SCHEDULES_ENUM, nullable=False) - reload = Column(Boolean, nullable=False) + reload = Column(Boolean, default=False, nullable=False) success = Column(Boolean, nullable=True) last_run = Column(DateTime, nullable=True) diff --git a/src/common/gen/Configurator.py b/src/common/gen/Configurator.py index b4e03554..3593853c 100644 --- a/src/common/gen/Configurator.py +++ b/src/common/gen/Configurator.py @@ -1,10 +1,15 @@ from glob import glob +from hashlib import sha256 +from io import BytesIO from json import loads from logging import Logger +from os import listdir +from os.path import basename, dirname from re import search as re_search from sys import path as sys_path +from tarfile import open as tar_open from traceback import format_exc -from typing import Union +from typing import Optional, Union sys_path.append("/usr/share/bunkerweb/utils") @@ -18,7 +23,7 @@ class Configurator: variables: Union[str, dict], logger: Logger, *, - plugins_settings: list = None, + plugins_settings: Optional[list] = None, ): self.__logger = logger self.__settings = self.__load_settings(settings) @@ -88,7 +93,26 @@ class Configurator: data = loads(f.read()) if type == "plugins": - self.__plugins_settings.append(data) + plugin_content = BytesIO() + with tar_open(fileobj=plugin_content, mode="w:gz") as tar: + tar.add( + dirname(file), + arcname=basename(dirname(file)), + recursive=True, + ) + plugin_content.seek(0) + value = plugin_content.getvalue() + + self.__plugins_settings.append( + data + | { + "external": path.startswith("/etc/bunkerweb/plugins"), + "page": "ui" in listdir(dirname(file)), + "method": "manual", + "data": value, + "checksum": sha256(value).hexdigest(), + } + ) plugins.update(data["settings"]) except: diff --git a/src/common/gen/save_config.py b/src/common/gen/save_config.py index 773c9225..d03c6245 100644 --- a/src/common/gen/save_config.py +++ b/src/common/gen/save_config.py @@ -151,8 +151,6 @@ if __name__ == "__main__": plugins = {} plugins_settings = [] for plugin in db.get_plugins(): - del plugin["page"] - del plugin["external"] plugins_settings.append(plugin) plugins.update(plugin["settings"]) diff --git a/src/scheduler/main.py b/src/scheduler/main.py index 7c0f6bde..ccef9932 100644 --- a/src/scheduler/main.py +++ b/src/scheduler/main.py @@ -17,6 +17,7 @@ from shutil import chown, copy, rmtree from signal import SIGINT, SIGTERM, signal, SIGHUP from subprocess import run as subprocess_run, DEVNULL, STDOUT from sys import path as sys_path +from tarfile import open as tar_open from time import sleep from traceback import format_exc from typing import Any, Dict, List @@ -117,6 +118,38 @@ def generate_custom_configs( ) +def generate_external_plugins( + plugins: List[Dict[str, Any]], + integration: str, + api_caller: ApiCaller, + *, + original_path: str = "/data/plugins", +): + Path(original_path).mkdir(parents=True, exist_ok=True) + for plugin in plugins: + tmp_path = f"{original_path}/{plugin['id']}/{plugin['name']}.tar.gz" + Path(dirname(tmp_path)).mkdir(parents=True, exist_ok=True) + Path(tmp_path).write_bytes(plugin["data"]) + with tar_open(tmp_path, "r:gz") as tar: + tar.extractall(original_path) + Path(tmp_path).unlink() + + # Fix permissions for the plugins folder + for root, dirs, files in walk("/data/plugins", topdown=False): + for name in files + dirs: + chown(join(root, name), "root", 101) + chmod(join(root, name), 0o770) + + if integration != "Linux": + logger.info("Sending plugins to BunkerWeb") + ret = api_caller._send_files("/data/plugins", "/plugins") + + if not ret: + logger.error( + "Sending plugins failed, configuration will not work as expected...", + ) + + if __name__ == "__main__": try: # Don't execute if pid file exists @@ -273,6 +306,14 @@ if __name__ == "__main__": if old_configs != custom_configs: generate_custom_configs(custom_configs, integration, api_caller) + external_plugins = db.get_plugins(external=True) + if external_plugins: + generate_external_plugins( + db.get_plugins(external=True, with_data=True), + integration, + api_caller, + ) + logger.info("Executing scheduler ...") generate = not Path( @@ -384,11 +425,13 @@ if __name__ == "__main__": f"Exception while reloading after running jobs once scheduling : {format_exc()}", ) - # infinite schedule for the jobs generate = True scheduler.setup() + need_reload = False + + # infinite schedule for the jobs logger.info("Executing job scheduler ...") - while run: + while run and not need_reload: scheduler.run_pending() sleep(1) @@ -397,15 +440,13 @@ if __name__ == "__main__": tmp_custom_configs = db.get_custom_configs() if custom_configs != tmp_custom_configs: logger.info("Custom configs changed, generating ...") - logger.debug(f"{tmp_custom_configs}") - logger.debug(f"{custom_configs}") - custom_configs = tmp_custom_configs - original_path = "/data/configs" + logger.debug(f"{tmp_custom_configs=}") + logger.debug(f"{custom_configs=}") + custom_configs = deepcopy(tmp_custom_configs) # Remove old custom configs files logger.info("Removing old custom configs files ...") - files = glob(f"{original_path}/*") - for file in files: + for file in glob("/data/configs/*"): if Path(file).is_symlink() or Path(file).is_file(): Path(file).unlink() elif Path(file).is_dir(): @@ -437,6 +478,30 @@ if __name__ == "__main__": else: logger.error("Error while reloading nginx") + # check if the plugins have changed since last time + tmp_external_plugins = db.get_plugins(external=True) + if external_plugins != tmp_external_plugins: + logger.info("External plugins changed, generating ...") + logger.debug(f"{tmp_external_plugins=}") + logger.debug(f"{external_plugins=}") + external_plugins = deepcopy(tmp_external_plugins) + + # Remove old external plugins files + logger.info("Removing old external plugins files ...") + for file in glob("/data/plugins/*"): + if Path(file).is_symlink() or Path(file).is_file(): + Path(file).unlink() + elif Path(file).is_dir(): + rmtree(file, ignore_errors=False) + + logger.info("Generating new external plugins ...") + generate_external_plugins( + db.get_plugins(external=True, with_data=True), + integration, + api_caller, + ) + need_reload = True + # check if the config have changed since last time tmp_env = db.get_config() if env != tmp_env: @@ -444,7 +509,7 @@ if __name__ == "__main__": logger.debug(f"{tmp_env=}") logger.debug(f"{env=}") env = deepcopy(tmp_env) - break + need_reload = True except: logger.error( f"Exception while executing scheduler : {format_exc()}", diff --git a/src/ui/main.py b/src/ui/main.py index 8e5b39ec..bf4585c9 100755 --- a/src/ui/main.py +++ b/src/ui/main.py @@ -1,3 +1,4 @@ +from hashlib import sha256 from bs4 import BeautifulSoup from contextlib import suppress from copy import deepcopy @@ -38,7 +39,7 @@ from os.path import join from pathlib import Path from re import match as re_match from requests import get -from shutil import rmtree, copytree, chown +from shutil import move, rmtree, copytree, chown from signal import SIGINT, signal, SIGTERM from subprocess import PIPE, Popen, call from sys import path as sys_path, modules as sys_modules @@ -754,6 +755,8 @@ def plugins(): errors = 0 files_count = 0 + new_plugins = [] + new_plugins_ids = [] for file in listdir("/var/tmp/bunkerweb/ui"): if not Path(f"/var/tmp/bunkerweb/ui/{file}").is_file(): @@ -797,22 +800,32 @@ def plugins(): raise Exception if not Path("/usr/sbin/nginx").is_file(): - plugins = app.config["CONFIG"].get_plugins() - for plugin in deepcopy(plugins): - if plugin["id"] == folder_name: - raise FileExistsError - elif plugin["external"] is False: - del plugins[plugins.index(plugin)] - - plugins.append(plugin_file) - err = db.update_external_plugins(plugins) - if err: - error = 1 - flash( - f"Couldn't update external plugins to database: {err}", - "error", + plugin_content = BytesIO() + with tar_open( + fileobj=plugin_content, mode="w:gz" + ) as tar: + tar.add( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}", + arcname=temp_folder_name, + recursive=True, ) - raise Exception + plugin_content.seek(0) + value = plugin_content.getvalue() + + new_plugins.append( + plugin_file + | { + "external": True, + "page": "ui" + in listdir( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}" + ), + "method": "ui", + "data": value, + "checksum": sha256(value).hexdigest(), + } + ) + new_plugins_ids.append(folder_name) else: if Path( f"/etc/bunkerweb/plugins/{folder_name}" @@ -871,22 +884,43 @@ def plugins(): raise Exception if not Path("/usr/sbin/nginx").is_file(): - plugins = app.config["CONFIG"].get_plugins() - for plugin in deepcopy(plugins): - if plugin["id"] == folder_name: - raise FileExistsError - elif plugin["external"] is False: - del plugins[plugins.index(plugin)] - - plugins.append(plugin_file) - err = db.update_external_plugins(plugins) - if err: - error = 1 - flash( - f"Couldn't update external plugins to database: {err}", - "error", + for file_name in listdir( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}" + ): + move( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}/{file_name}", + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{file_name}", ) - raise Exception + rmtree( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}" + ) + + plugin_content = BytesIO() + with tar_open( + fileobj=plugin_content, mode="w:gz" + ) as tar: + tar.add( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}", + arcname=temp_folder_name, + recursive=True, + ) + plugin_content.seek(0) + value = plugin_content.getvalue() + + new_plugins.append( + plugin_file + | { + "external": True, + "page": "ui" + in listdir( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}" + ), + "method": "ui", + "data": value, + "checksum": sha256(value).hexdigest(), + } + ) + new_plugins_ids.append(folder_name) else: if Path( f"/etc/bunkerweb/plugins/{folder_name}" @@ -940,22 +974,32 @@ def plugins(): raise Exception if not Path("/usr/sbin/nginx").is_file(): - plugins = app.config["CONFIG"].get_plugins() - for plugin in deepcopy(plugins): - if plugin["id"] == folder_name: - raise FileExistsError - elif plugin["external"] is False: - del plugins[plugins.index(plugin)] - - plugins.append(plugin_file) - err = db.update_external_plugins(plugins) - if err: - error = 1 - flash( - f"Couldn't update external plugins to database: {err}", - "error", + plugin_content = BytesIO() + with tar_open( + fileobj=plugin_content, mode="w:gz" + ) as tar: + tar.add( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}", + arcname=temp_folder_name, + recursive=True, ) - raise Exception + plugin_content.seek(0) + value = plugin_content.getvalue() + + new_plugins.append( + plugin_file + | { + "external": True, + "page": "ui" + in listdir( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}" + ), + "method": "ui", + "data": value, + "checksum": sha256(value).hexdigest(), + } + ) + new_plugins_ids.append(folder_name) else: if Path( f"/etc/bunkerweb/plugins/{folder_name}" @@ -1014,22 +1058,43 @@ def plugins(): raise Exception if not Path("/usr/sbin/nginx").is_file(): - plugins = app.config["CONFIG"].get_plugins() - for plugin in deepcopy(plugins): - if plugin["id"] == folder_name: - raise FileExistsError - elif plugin["external"] is False: - del plugins[plugins.index(plugin)] - - plugins.append(plugin_file) - err = db.update_external_plugins(plugins) - if err: - error = 1 - flash( - f"Couldn't update external plugins to database: {err}", - "error", + for file_name in listdir( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}" + ): + move( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}/{file_name}", + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{file_name}", ) - raise Exception + rmtree( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}" + ) + + plugin_content = BytesIO() + with tar_open( + fileobj=plugin_content, mode="w:gz" + ) as tar: + tar.add( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}", + arcname=temp_folder_name, + recursive=True, + ) + plugin_content.seek(0) + value = plugin_content.getvalue() + + new_plugins.append( + plugin_file + | { + "external": True, + "page": "ui" + in listdir( + f"/var/tmp/bunkerweb/ui/{temp_folder_name}" + ), + "method": "ui", + "data": value, + "checksum": sha256(value).hexdigest(), + } + ) + new_plugins_ids.append(folder_name) else: if Path( f"/etc/bunkerweb/plugins/{folder_name}" @@ -1105,7 +1170,7 @@ def plugins(): error = 0 - if errors < files_count: + if errors >= files_count: return redirect(url_for("loading", next=url_for("plugins"))) # Fix permissions for plugins folders @@ -1114,6 +1179,19 @@ def plugins(): chown(join(root, name), "root", 101) chmod(join(root, name), 0o770) + plugins = app.config["CONFIG"].get_plugins(external=True, with_data=True) + for plugin in deepcopy(plugins): + if plugin["id"] in new_plugins_ids: + flash(f"Plugin {plugin['id']} already exists", "error") + del new_plugins[new_plugins_ids.index(plugin["id"])] + + err = db.update_external_plugins(new_plugins, delete_missing=False) + if err: + flash( + f"Couldn't update external plugins to database: {err}", + "error", + ) + if operation: flash(operation) diff --git a/src/ui/src/Config.py b/src/ui/src/Config.py index 6cd7ba75..e55e970c 100644 --- a/src/ui/src/Config.py +++ b/src/ui/src/Config.py @@ -1,11 +1,15 @@ from copy import deepcopy +from hashlib import sha256 +from io import BytesIO from flask import flash from glob import iglob from json import load as json_load from os import listdir +from os.path import basename from pathlib import Path from re import search as re_search from subprocess import run, DEVNULL, STDOUT +from tarfile import open as tar_open from time import sleep from typing import List, Tuple from uuid import uuid4 @@ -134,25 +138,28 @@ class Config: **self.__settings, } - def get_plugins(self) -> List[dict]: + def get_plugins( + self, *, external: bool = False, with_data: bool = False + ) -> List[dict]: if not Path("/usr/sbin/nginx").exists(): - plugins = self.__db.get_plugins() + plugins = self.__db.get_plugins(external=external, with_data=with_data) plugins.sort(key=lambda x: x["name"]) - general_plugin = None - for x, plugin in enumerate(plugins): - if plugin["name"] == "General": - general_plugin = plugin - del plugins[x] - break - plugins.insert(0, general_plugin) + if not external: + general_plugin = None + for x, plugin in enumerate(plugins): + if plugin["name"] == "General": + general_plugin = plugin + del plugins[x] + break + plugins.insert(0, general_plugin) return plugins plugins = [] - for foldername in list(iglob("/etc/bunkerweb/plugins/*")) + list( - iglob("/usr/share/bunkerweb/core/*") + for foldername in list(iglob("/etc/bunkerweb/plugins/*")) + ( + list(iglob("/usr/share/bunkerweb/core/*") if not external else []) ): content = listdir(foldername) if "plugin.json" not in content: @@ -168,10 +175,26 @@ class Config: } ) + plugin["method"] = "ui" if plugin["external"] else "manual" + if "ui" in content: if "template.html" in listdir(f"{foldername}/ui"): plugin["page"] = True + if with_data: + plugin_content = BytesIO() + with tar_open(fileobj=plugin_content, mode="w:gz") as tar: + tar.add( + foldername, + arcname=basename(foldername), + recursive=True, + ) + plugin_content.seek(0) + value = plugin_content.getvalue() + + plugin["data"] = value + plugin["checksum"] = sha256(value).hexdigest() + plugins.append(plugin) plugins.sort(key=lambda x: x["name"]) @@ -186,6 +209,7 @@ class Config: "description": "The general settings for the server", "version": "0.1", "external": False, + "method": "manual", "page": False, "settings": json_load(f), },