librewolf
/
bunkerized-nginx
mirror of https://github.com/bunkerity/bunkerized-nginx
4
0
Fork 1
Code Issues Projects Releases Wiki Activity

Fix problem with the bunkerweb container and plugins

This commit is contained in:
Théophile Diot 2023-03-10 13:16:00 +01:00
parent 548d157fe3
commit 0356250d9d
No known key found for this signature in database
GPG Key ID: E752C80DB72BB014
9 changed files with 553 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/bw/lua/api.lua
View File

@ -1,24 +1,24 @@
local datastore = require "datastore"
local utils = require "utils"
local cjson = require "cjson"
local plugins = require "plugins"
local upload = require "resty.upload"
local logger = require "logger"
local datastore = require "datastore"
local utils = require "utils"
local cjson = require "cjson"
local plugins = require "plugins"
local upload = require "resty.upload"
local logger = require "logger"
local api = { global = { GET = {}, POST = {}, PUT = {}, DELETE = {} } }
local api = { global = { GET = {}, POST = {}, PUT = {}, DELETE = {} } }
api.response = function(self, http_status, api_status, msg)
api.response = function(self, http_status, api_status, msg)
local resp = {}
resp["status"] = api_status
resp["msg"] = msg
return http_status, resp
end
api.global.GET["^/ping$"] = function(api)
api.global.GET["^/ping$"] = function(api)
return api:response(ngx.HTTP_OK, "success", "pong")
end
api.global.POST["^/reload$"] = function(api)
api.global.POST["^/reload$"] = function(api)
local status = os.execute("nginx -s reload")
if status == 0 then
return api:response(ngx.HTTP_OK, "success", "reload successful")
@ -26,7 +26,7 @@ api.global.POST["^/reload$"] = function(api)
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "exit status = " .. tostring(status))
end
api.global.POST["^/stop$"] = function(api)
api.global.POST["^/stop$"] = function(api)
local status = os.execute("nginx -s quit")
if status == 0 then
return api:response(ngx.HTTP_OK, "success", "stop successful")
@ -34,7 +34,7 @@ api.global.POST["^/stop$"] = function(api)
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "exit status = " .. tostring(status))
end
api.global.POST["^/confs$"] = function(api)
api.global.POST["^/confs$"] = function(api)
local tmp = "/var/tmp/bunkerweb/api_" .. ngx.var.uri:sub(2) .. ".tar.gz"
local destination = "/usr/share/bunkerweb/" .. ngx.var.uri:sub(2)
if ngx.var.uri == "/confs" then
@ -45,6 +45,8 @@ api.global.POST["^/confs$"] = function(api)
destination = "/data/cache"
elseif ngx.var.uri == "/custom_configs" then
destination = "/data/configs"
elseif ngx.var.uri == "/plugins" then
destination = "/data/plugins"
end
local form, err = upload:new(4096)
if not form then
@ -78,13 +80,15 @@ api.global.POST["^/confs$"] = function(api)
return api:response(ngx.HTTP_OK, "success", "saved data at " .. destination)
end
api.global.POST["^/data$"] = api.global.POST["^/confs$"]
api.global.POST["^/data$"] = api.global.POST["^/confs$"]
api.global.POST["^/cache$"] = api.global.POST["^/confs$"]
api.global.POST["^/cache$"] = api.global.POST["^/confs$"]
api.global.POST["^/custom_configs$"] = api.global.POST["^/confs$"]
api.global.POST["^/unban$"] = function(api)
api.global.POST["^/plugins$"] = api.global.POST["^/confs$"]
api.global.POST["^/unban$"] = function(api)
ngx.req.read_body()
local data = ngx.req.get_body_data()
if not data then
@ -103,7 +107,7 @@ api.global.POST["^/unban$"] = function(api)
return api:response(ngx.HTTP_OK, "success", "ip " .. ip["ip"] .. " unbanned")
end
api.global.POST["^/ban$"] = function(api)
api.global.POST["^/ban$"] = function(api)
ngx.req.read_body()
local data = ngx.req.get_body_data()
if not data then
@ -122,17 +126,19 @@ api.global.POST["^/ban$"] = function(api)
return api:response(ngx.HTTP_OK, "success", "ip " .. ip["ip"] .. " banned")
end
api.global.GET["^/bans$"] = function(api)
api.global.GET["^/bans$"] = function(api)
local data = {}
for i, k in ipairs(datastore:keys()) do
if k:find("^bans_ip_") then
local ret, reason = datastore:get(k)
if not ret then
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't access " .. k .. " from datastore : " + reason)
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error",
"can't access " .. k .. " from datastore : " + reason)
end
local ret, exp = datastore:exp(k)
if not ret then
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't access exp " .. k .. " from datastore : " + exp)
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error",
"can't access exp " .. k .. " from datastore : " + exp)
end
local ban = { ip = k:sub(9, #k), reason = reason, exp = exp }
table.insert(data, ban)
@ -141,7 +147,7 @@ api.global.GET["^/bans$"] = function(api)
return api:response(ngx.HTTP_OK, "success", data)
end
api.is_allowed_ip = function(self)
api.is_allowed_ip = function(self)
local data, err = datastore:get("api_whitelist_ip")
if not data then
return false, "can't access api_allowed_ips in datastore"
@ -152,7 +158,7 @@ api.is_allowed_ip = function(self)
return false, "IP is not in API_WHITELIST_IP"
end
api.do_api_call = function(self)
api.do_api_call = function(self)
if self.global[ngx.var.request_method] ~= nil then
for uri, api_fun in pairs(self.global[ngx.var.request_method]) do
if string.match(ngx.var.uri, uri) then

123
src/common/core/jobs/jobs/download-plugins.py
View File

@ -1,16 +1,18 @@
#!/usr/bin/python3
from hashlib import sha256
from io import BytesIO
from os import getenv, listdir, makedirs, chmod, stat, _exit, walk
from os.path import dirname, join
from os.path import basename, dirname, join
from pathlib import Path
from stat import S_IEXEC
from sys import exit as sys_exit, path as sys_path
from threading import Lock
from uuid import uuid4
from glob import glob
from json import load, loads
from json import loads
from shutil import chown, copytree, rmtree
from tarfile import open as tar_open
from traceback import format_exc
from zipfile import ZipFile
@ -18,6 +20,7 @@ sys_path.extend(
(
"/usr/share/bunkerweb/deps/python",
"/usr/share/bunkerweb/utils",
"/usr/share/bunkerweb/api",
"/usr/share/bunkerweb/db",
)
)
@ -28,31 +31,29 @@ from Database import Database
from logger import setup_logger
logger = setup_logger("Jobs", getenv("LOG_LEVEL", "INFO"))
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI", None),
)
logger = setup_logger("Jobs.download-plugins", getenv("LOG_LEVEL", "INFO"))
lock = Lock()
status = 0
def install_plugin(plugin_dir):
def install_plugin(plugin_dir) -> bool:
# Load plugin.json
with open(f"{plugin_dir}plugin.json", "rb") as f:
with open(f"{plugin_dir}/plugin.json", "rb") as f:
metadata = loads(f.read())
# Don't go further if plugin is already installed
if Path(f"/data/plugins/{metadata['id']}/plugin.json").is_file():
logger.info(
logger.warning(
f"Skipping installation of plugin {metadata['id']} (already installed)",
)
return
return False
# Copy the plugin
copytree(plugin_dir, f"/data/plugins/{metadata['id']}")
# Add u+x permissions to jobs files
for job_file in glob(f"{plugin_dir}jobs/*"):
st = stat(job_file)
chmod(job_file, st.st_mode | S_IEXEC)
logger.info(f"Plugin {metadata['id']} installed")
return True
try:
@ -62,7 +63,15 @@ try:
logger.info("No external plugins to download")
_exit(0)
db = Database(
logger,
sqlalchemy_string=getenv("DATABASE_URI"),
)
plugin_nbr = 0
# Loop on URLs
logger.info(f"Downloading external plugins from {plugin_urls}...")
for plugin_url in plugin_urls.split(" "):
# Download ZIP file
try:
@ -75,9 +84,9 @@ try:
continue
# Extract it to tmp folder
temp_dir = f"/var/tmp/bunkerweb/plugins-{uuid4()}/"
temp_dir = f"/var/tmp/bunkerweb/plugins-{uuid4()}"
try:
makedirs(temp_dir, exist_ok=True)
Path(temp_dir).mkdir(parents=True, exist_ok=True)
with ZipFile(BytesIO(req.content)) as zf:
zf.extractall(path=temp_dir)
except:
@ -89,48 +98,76 @@ try:
# Install plugins
try:
for plugin_dir in glob(f"{temp_dir}**/plugin.json", recursive=True):
install_plugin(f"{dirname(plugin_dir)}/")
for plugin_dir in glob(f"{temp_dir}/**/plugin.json", recursive=True):
try:
if install_plugin(dirname(plugin_dir)):
plugin_nbr += 1
except FileExistsError:
logger.warning(
f"Skipping installation of plugin {basename(dirname(plugin_dir))} (already installed)",
)
except:
logger.error(
f"Exception while installing plugin(s) from {plugin_url} :\n{format_exc()}",
)
status = 2
continue
external_plugins = []
external_plugins_ids = []
for plugin in listdir("/etc/bunkerweb/plugins"):
with open(
f"/etc/bunkerweb/plugins/{plugin}/plugin.json",
"r",
) as f:
plugin_file = load(f)
external_plugins.append(plugin_file)
external_plugins_ids.append(plugin_file["id"])
with lock:
db_plugins = db.get_plugins()
for plugin in db_plugins:
if plugin["external"] is True and plugin["id"] not in external_plugins_ids:
external_plugins.append(plugin)
# Fix permissions for the certificates
# Fix permissions on plugins
for root, dirs, files in walk("/data/plugins", topdown=False):
for name in files + dirs:
chown(join(root, name), "root", 101)
chmod(join(root, name), 0o770)
if external_plugins:
with lock:
err = db.update_external_plugins(external_plugins)
if not plugin_nbr:
logger.info("No external plugins to update to database")
_exit(0)
if err:
logger.error(
f"Couldn't update external plugins to database: {err}",
)
external_plugins = []
external_plugins_ids = []
for plugin in listdir("/data/plugins"):
path = f"/data/plugins/{plugin}"
if not Path(f"{path}/plugin.json").is_file():
logger.warning(f"Plugin {plugin} is not valid, deleting it...")
rmtree(path)
continue
plugin_file = loads(Path(f"{path}/plugin.json").read_text())
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz") as tar:
tar.add(path, arcname=basename(path))
plugin_content.seek(0)
value = plugin_content.getvalue()
plugin_file.update(
{
"external": True,
"page": False,
"method": "scheduler",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
if "ui" in listdir(path):
plugin_file["ui"] = True
external_plugins.append(plugin_file)
external_plugins_ids.append(plugin_file["id"])
for plugin in db.get_plugins(external=True, with_data=True):
if plugin["method"] != "scheduler" and plugin["id"] not in external_plugins_ids:
external_plugins.append(plugin)
with lock:
err = db.update_external_plugins(external_plugins)
if err:
logger.error(
f"Couldn't update external plugins to database: {err}",
)
logger.info("External plugins downloaded and installed")
except:
status = 2

251
src/common/db/Database.py
View File

@ -5,8 +5,9 @@ from hashlib import sha256
from logging import (
Logger,
)
from os import _exit, getenv, listdir, makedirs
from os.path import dirname, exists
from os import _exit, getenv
from os.path import dirname
from pathlib import Path
from pymysql import install_as_MySQLdb
from re import compile as re_compile
from sys import path as sys_path
@ -61,7 +62,9 @@ class Database:
if sqlalchemy_string.startswith("sqlite"):
with suppress(FileExistsError):
makedirs(dirname(sqlalchemy_string.split("///")[1]), exist_ok=True)
Path(dirname(sqlalchemy_string.split("///")[1])).mkdir(
parents=True, exist_ok=True
)
elif "+" in sqlalchemy_string and "+pymysql" not in sqlalchemy_string:
splitted = sqlalchemy_string.split("+")
sqlalchemy_string = f"{splitted[0]}:{':'.join(splitted[1].split(':')[1:])}"
@ -117,6 +120,9 @@ class Database:
sqlalchemy_string,
future=True,
)
if "Unknown table" in str(e):
not_connected = False
continue
else:
self.__logger.warning(
"Can't connect to database, retrying in 5 seconds ...",
@ -270,6 +276,7 @@ class Database:
for plugin in plugins:
settings = {}
jobs = []
page = False
if "id" not in plugin:
settings = plugin
plugin = {
@ -283,6 +290,7 @@ class Database:
else:
settings = plugin.pop("settings", {})
jobs = plugin.pop("jobs", [])
page = plugin.pop("page", False)
to_put.append(Plugins(**plugin))
@ -298,40 +306,37 @@ class Database:
for select in value.pop("select", []):
to_put.append(Selects(setting_id=value["id"], value=select))
to_put.append(
Settings(
**value,
)
)
to_put.append(Settings(**value))
for job in jobs:
job["file_name"] = job.pop("file")
to_put.append(Jobs(plugin_id=plugin["id"], **job))
if exists(f"/usr/share/bunkerweb/core/{plugin['id']}/ui"):
if {"template.html", "actions.py"}.issubset(
listdir(f"/usr/share/bunkerweb/core/{plugin['id']}/ui")
):
with open(
f"/usr/share/bunkerweb/core/{plugin['id']}/ui/template.html",
"r",
) as file:
template = file.read().encode("utf-8")
with open(
f"/usr/share/bunkerweb/core/{plugin['id']}/ui/actions.py",
"r",
) as file:
actions = file.read().encode("utf-8")
if page:
path_ui = (
Path(f"/usr/share/bunkerweb/core/{plugin['id']}/ui")
if Path(
f"/usr/share/bunkerweb/core/{plugin['id']}/ui"
).exists()
else Path(f"/etc/bunkerweb/plugins/{plugin['id']}/ui")
)
to_put.append(
Plugin_pages(
plugin_id=plugin["id"],
template_file=template,
template_checksum=sha256(template).hexdigest(),
actions_file=actions,
actions_checksum=sha256(actions).hexdigest(),
if path_ui.exists():
if {"template.html", "actions.py"}.issubset(
path_ui.iterdir()
):
template = Path(f"{path_ui}/template.html").read_bytes()
actions = Path(f"{path_ui}/actions.py").read_bytes()
to_put.append(
Plugin_pages(
plugin_id=plugin["id"],
template_file=template,
template_checksum=sha256(template).hexdigest(),
actions_file=actions,
actions_checksum=sha256(actions).hexdigest(),
)
)
)
try:
session.add_all(to_put)
@ -907,7 +912,9 @@ class Database:
return ""
def update_external_plugins(self, plugins: List[Dict[str, Any]]) -> str:
def update_external_plugins(
self, plugins: List[Dict[str, Any]], *, delete_missing: bool = True
) -> str:
"""Update external plugins from the database"""
to_put = []
with self.__db_session() as session:
@ -919,7 +926,7 @@ class Database:
)
db_ids = []
if db_plugins:
if delete_missing and db_plugins:
db_ids = [plugin.id for plugin in db_plugins]
ids = [plugin["id"] for plugin in plugins]
missing_ids = [plugin for plugin in db_ids if plugin not in ids]
@ -931,7 +938,7 @@ class Database:
for plugin in plugins:
settings = plugin.pop("settings", {})
jobs = plugin.pop("jobs", [])
pages = plugin.pop("pages", [])
page = plugin.pop("page", False)
plugin["external"] = True
db_plugin = (
session.query(Plugins)
@ -967,6 +974,15 @@ class Database:
if plugin["version"] != db_plugin.version:
updates[Plugins.version] = plugin["version"]
if plugin["method"] != db_plugin.method:
updates[Plugins.method] = plugin["method"]
if plugin.get("data") != db_plugin.data:
updates[Plugins.data] = plugin["data"]
if plugin.get("checksum") != db_plugin.checksum:
updates[Plugins.checksum] = plugin["checksum"]
if updates:
session.query(Plugins).filter(
Plugins.id == plugin["id"]
@ -1140,13 +1156,13 @@ class Database:
).update(updates)
path_ui = (
f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui"
if exists(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui")
else f"/etc/bunkerweb/plugins/{plugin['id']}/ui"
Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui")
if Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui").exists()
else Path(f"/etc/bunkerweb/plugins/{plugin['id']}/ui")
)
if exists(path_ui):
if {"template.html", "actions.py"}.issubset(listdir(path_ui)):
if path_ui.exists():
if {"template.html", "actions.py"}.issubset(path_ui.iterdir()):
db_plugin_page = (
session.query(Plugin_pages)
.with_entities(
@ -1158,16 +1174,8 @@ class Database:
)
if db_plugin_page is None:
with open(
f"{path_ui}/template.html",
"r",
) as file:
template = file.read().encode("utf-8")
with open(
f"{path_ui}/actions.py",
"r",
) as file:
actions = file.read().encode("utf-8")
template = Path(f"{path_ui}/template.html").read_bytes()
actions = Path(f"{path_ui}/actions.py").read_bytes()
to_put.append(
Plugin_pages(
@ -1178,7 +1186,7 @@ class Database:
actions_checksum=sha256(actions).hexdigest(),
)
)
else: # TODO test this
else:
updates = {}
template_checksum = file_hash(
f"{path_ui}/template.html"
@ -1189,32 +1197,24 @@ class Database:
template_checksum
!= db_plugin_page.template_checksum
):
with open(
f"{path_ui}/template.html",
"r",
) as file:
updates.update(
{
Plugin_pages.template_file: file.read().encode(
"utf-8"
),
Plugin_pages.template_checksum: template_checksum,
}
)
updates.update(
{
Plugin_pages.template_file: Path(
f"{path_ui}/template.html"
).read_bytes(),
Plugin_pages.template_checksum: template_checksum,
}
)
if actions_checksum != db_plugin_page.actions_checksum:
with open(
f"{path_ui}/actions.py",
"r",
) as file:
updates.update(
{
Plugin_pages.actions_file: file.read().encode(
"utf-8"
),
Plugin_pages.actions_checksum: actions_checksum,
}
)
updates.update(
{
Plugin_pages.actions_file: Path(
f"{path_ui}/actions.py"
).read_bytes(),
Plugin_pages.actions_checksum: actions_checksum,
}
)
if updates:
session.query(Plugin_pages).filter(
@ -1269,17 +1269,72 @@ class Database:
job["reload"] = job.get("reload", False)
to_put.append(Jobs(plugin_id=plugin["id"], **job))
for page in pages:
to_put.append(
Plugin_pages(
plugin_id=plugin["id"],
template_file=page["template_file"],
template_checksum=sha256(page["template_file"]).hexdigest(),
actions_file=page["actions_file"],
actions_checksum=sha256(page["actions_file"]).hexdigest(),
)
if page:
path_ui = (
Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui")
if Path(f"/var/tmp/bunkerweb/ui/{plugin['id']}/ui").exists()
else Path(f"/etc/bunkerweb/plugins/{plugin['id']}/ui")
)
if path_ui.exists():
if {"template.html", "actions.py"}.issubset(path_ui.iterdir()):
db_plugin_page = (
session.query(Plugin_pages)
.with_entities(
Plugin_pages.template_checksum,
Plugin_pages.actions_checksum,
)
.filter_by(plugin_id=plugin["id"])
.first()
)
if db_plugin_page is None:
template = Path(f"{path_ui}/template.html").read_bytes()
actions = Path(f"{path_ui}/actions.py").read_bytes()
to_put.append(
Plugin_pages(
plugin_id=plugin["id"],
template_file=template,
template_checksum=sha256(template).hexdigest(),
actions_file=actions,
actions_checksum=sha256(actions).hexdigest(),
)
)
else:
updates = {}
template_checksum = file_hash(
f"{path_ui}/template.html"
)
actions_checksum = file_hash(f"{path_ui}/actions.py")
if (
template_checksum
!= db_plugin_page.template_checksum
):
updates.update(
{
Plugin_pages.template_file: Path(
f"{path_ui}/template.html"
).read_bytes(),
Plugin_pages.template_checksum: template_checksum,
}
)
if actions_checksum != db_plugin_page.actions_checksum:
updates.update(
{
Plugin_pages.actions_file: Path(
f"{path_ui}/actions.py"
).read_bytes(),
Plugin_pages.actions_checksum: actions_checksum,
}
)
if updates:
session.query(Plugin_pages).filter(
Plugin_pages.plugin_id == plugin["id"]
).update(updates)
try:
session.add_all(to_put)
session.commit()
@ -1288,8 +1343,10 @@ class Database:
return ""
def get_plugins(self) -> List[Dict[str, Any]]:
"""Get plugins."""
def get_plugins(
self, *, external: bool = False, with_data: bool = False
) -> List[Dict[str, Any]]:
"""Get all plugins from the database."""
plugins = []
with self.__db_session() as session:
for plugin in (
@ -1301,10 +1358,29 @@ class Database:
Plugins.description,
Plugins.version,
Plugins.external,
Plugins.method,
Plugins.data,
Plugins.checksum,
)
.order_by(Plugins.order)
.all()
if with_data
else session.query(Plugins)
.with_entities(
Plugins.id,
Plugins.order,
Plugins.name,
Plugins.description,
Plugins.version,
Plugins.external,
Plugins.method,
)
.order_by(Plugins.order)
.all()
):
if external and not plugin.external:
continue
page = (
session.query(Plugin_pages)
.with_entities(Plugin_pages.id)
@ -1318,9 +1394,14 @@ class Database:
"description": plugin.description,
"version": plugin.version,
"external": plugin.external,
"method": plugin.method,
"page": page is not None,
"settings": {},
}
} | (
{"data": plugin.data, "checksum": plugin.checksum}
if with_data
else {}
)
for setting in (
session.query(Settings)

5
src/common/db/model.py
View File

@ -61,6 +61,9 @@ class Plugins(Base):
description = Column(String(256), nullable=False)
version = Column(String(32), nullable=False)
external = Column(Boolean, default=False, nullable=False)
method = Column(METHODS_ENUM, default="manual", nullable=False)
data = Column(LargeBinary(length=(2**32) - 1), nullable=True)
checksum = Column(String(128), nullable=True)
settings = relationship(
"Settings", back_populates="plugin", cascade="all, delete-orphan"
@ -164,7 +167,7 @@ class Jobs(Base):
)
file_name = Column(String(256), nullable=False)
every = Column(SCHEDULES_ENUM, nullable=False)
reload = Column(Boolean, nullable=False)
reload = Column(Boolean, default=False, nullable=False)
success = Column(Boolean, nullable=True)
last_run = Column(DateTime, nullable=True)

30
src/common/gen/Configurator.py
View File

@ -1,10 +1,15 @@
from glob import glob
from hashlib import sha256
from io import BytesIO
from json import loads
from logging import Logger
from os import listdir
from os.path import basename, dirname
from re import search as re_search
from sys import path as sys_path
from tarfile import open as tar_open
from traceback import format_exc
from typing import Union
from typing import Optional, Union
sys_path.append("/usr/share/bunkerweb/utils")
@ -18,7 +23,7 @@ class Configurator:
variables: Union[str, dict],
logger: Logger,
*,
plugins_settings: list = None,
plugins_settings: Optional[list] = None,
):
self.__logger = logger
self.__settings = self.__load_settings(settings)
@ -88,7 +93,26 @@ class Configurator:
data = loads(f.read())
if type == "plugins":
self.__plugins_settings.append(data)
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz") as tar:
tar.add(
dirname(file),
arcname=basename(dirname(file)),
recursive=True,
)
plugin_content.seek(0)
value = plugin_content.getvalue()
self.__plugins_settings.append(
data
| {
"external": path.startswith("/etc/bunkerweb/plugins"),
"page": "ui" in listdir(dirname(file)),
"method": "manual",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
plugins.update(data["settings"])
except:

2
src/common/gen/save_config.py
View File

@ -151,8 +151,6 @@ if __name__ == "__main__":
plugins = {}
plugins_settings = []
for plugin in db.get_plugins():
del plugin["page"]
del plugin["external"]
plugins_settings.append(plugin)
plugins.update(plugin["settings"])

83
src/scheduler/main.py
View File

@ -17,6 +17,7 @@ from shutil import chown, copy, rmtree
from signal import SIGINT, SIGTERM, signal, SIGHUP
from subprocess import run as subprocess_run, DEVNULL, STDOUT
from sys import path as sys_path
from tarfile import open as tar_open
from time import sleep
from traceback import format_exc
from typing import Any, Dict, List
@ -117,6 +118,38 @@ def generate_custom_configs(
)
def generate_external_plugins(
plugins: List[Dict[str, Any]],
integration: str,
api_caller: ApiCaller,
*,
original_path: str = "/data/plugins",
):
Path(original_path).mkdir(parents=True, exist_ok=True)
for plugin in plugins:
tmp_path = f"{original_path}/{plugin['id']}/{plugin['name']}.tar.gz"
Path(dirname(tmp_path)).mkdir(parents=True, exist_ok=True)
Path(tmp_path).write_bytes(plugin["data"])
with tar_open(tmp_path, "r:gz") as tar:
tar.extractall(original_path)
Path(tmp_path).unlink()
# Fix permissions for the plugins folder
for root, dirs, files in walk("/data/plugins", topdown=False):
for name in files + dirs:
chown(join(root, name), "root", 101)
chmod(join(root, name), 0o770)
if integration != "Linux":
logger.info("Sending plugins to BunkerWeb")
ret = api_caller._send_files("/data/plugins", "/plugins")
if not ret:
logger.error(
"Sending plugins failed, configuration will not work as expected...",
)
if __name__ == "__main__":
try:
# Don't execute if pid file exists
@ -273,6 +306,14 @@ if __name__ == "__main__":
if old_configs != custom_configs:
generate_custom_configs(custom_configs, integration, api_caller)
external_plugins = db.get_plugins(external=True)
if external_plugins:
generate_external_plugins(
db.get_plugins(external=True, with_data=True),
integration,
api_caller,
)
logger.info("Executing scheduler ...")
generate = not Path(
@ -384,11 +425,13 @@ if __name__ == "__main__":
f"Exception while reloading after running jobs once scheduling : {format_exc()}",
)
# infinite schedule for the jobs
generate = True
scheduler.setup()
need_reload = False
# infinite schedule for the jobs
logger.info("Executing job scheduler ...")
while run:
while run and not need_reload:
scheduler.run_pending()
sleep(1)
@ -397,15 +440,13 @@ if __name__ == "__main__":
tmp_custom_configs = db.get_custom_configs()
if custom_configs != tmp_custom_configs:
logger.info("Custom configs changed, generating ...")
logger.debug(f"{tmp_custom_configs}")
logger.debug(f"{custom_configs}")
custom_configs = tmp_custom_configs
original_path = "/data/configs"
logger.debug(f"{tmp_custom_configs=}")
logger.debug(f"{custom_configs=}")
custom_configs = deepcopy(tmp_custom_configs)
# Remove old custom configs files
logger.info("Removing old custom configs files ...")
files = glob(f"{original_path}/*")
for file in files:
for file in glob("/data/configs/*"):
if Path(file).is_symlink() or Path(file).is_file():
Path(file).unlink()
elif Path(file).is_dir():
@ -437,6 +478,30 @@ if __name__ == "__main__":
else:
logger.error("Error while reloading nginx")
# check if the plugins have changed since last time
tmp_external_plugins = db.get_plugins(external=True)
if external_plugins != tmp_external_plugins:
logger.info("External plugins changed, generating ...")
logger.debug(f"{tmp_external_plugins=}")
logger.debug(f"{external_plugins=}")
external_plugins = deepcopy(tmp_external_plugins)
# Remove old external plugins files
logger.info("Removing old external plugins files ...")
for file in glob("/data/plugins/*"):
if Path(file).is_symlink() or Path(file).is_file():
Path(file).unlink()
elif Path(file).is_dir():
rmtree(file, ignore_errors=False)
logger.info("Generating new external plugins ...")
generate_external_plugins(
db.get_plugins(external=True, with_data=True),
integration,
api_caller,
)
need_reload = True
# check if the config have changed since last time
tmp_env = db.get_config()
if env != tmp_env:
@ -444,7 +509,7 @@ if __name__ == "__main__":
logger.debug(f"{tmp_env=}")
logger.debug(f"{env=}")
env = deepcopy(tmp_env)
break
need_reload = True
except:
logger.error(
f"Exception while executing scheduler : {format_exc()}",

202
src/ui/main.py
View File

@ -1,3 +1,4 @@
from hashlib import sha256
from bs4 import BeautifulSoup
from contextlib import suppress
from copy import deepcopy
@ -38,7 +39,7 @@ from os.path import join
from pathlib import Path
from re import match as re_match
from requests import get
from shutil import rmtree, copytree, chown
from shutil import move, rmtree, copytree, chown
from signal import SIGINT, signal, SIGTERM
from subprocess import PIPE, Popen, call
from sys import path as sys_path, modules as sys_modules
@ -754,6 +755,8 @@ def plugins():
errors = 0
files_count = 0
new_plugins = []
new_plugins_ids = []
for file in listdir("/var/tmp/bunkerweb/ui"):
if not Path(f"/var/tmp/bunkerweb/ui/{file}").is_file():
@ -797,22 +800,32 @@ def plugins():
raise Exception
if not Path("/usr/sbin/nginx").is_file():
plugins = app.config["CONFIG"].get_plugins()
for plugin in deepcopy(plugins):
if plugin["id"] == folder_name:
raise FileExistsError
elif plugin["external"] is False:
del plugins[plugins.index(plugin)]
plugins.append(plugin_file)
err = db.update_external_plugins(plugins)
if err:
error = 1
flash(
f"Couldn't update external plugins to database: {err}",
"error",
plugin_content = BytesIO()
with tar_open(
fileobj=plugin_content, mode="w:gz"
) as tar:
tar.add(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}",
arcname=temp_folder_name,
recursive=True,
)
raise Exception
plugin_content.seek(0)
value = plugin_content.getvalue()
new_plugins.append(
plugin_file
| {
"external": True,
"page": "ui"
in listdir(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}"
),
"method": "ui",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
new_plugins_ids.append(folder_name)
else:
if Path(
f"/etc/bunkerweb/plugins/{folder_name}"
@ -871,22 +884,43 @@ def plugins():
raise Exception
if not Path("/usr/sbin/nginx").is_file():
plugins = app.config["CONFIG"].get_plugins()
for plugin in deepcopy(plugins):
if plugin["id"] == folder_name:
raise FileExistsError
elif plugin["external"] is False:
del plugins[plugins.index(plugin)]
plugins.append(plugin_file)
err = db.update_external_plugins(plugins)
if err:
error = 1
flash(
f"Couldn't update external plugins to database: {err}",
"error",
for file_name in listdir(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}"
):
move(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}/{file_name}",
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{file_name}",
)
raise Exception
rmtree(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}"
)
plugin_content = BytesIO()
with tar_open(
fileobj=plugin_content, mode="w:gz"
) as tar:
tar.add(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}",
arcname=temp_folder_name,
recursive=True,
)
plugin_content.seek(0)
value = plugin_content.getvalue()
new_plugins.append(
plugin_file
| {
"external": True,
"page": "ui"
in listdir(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}"
),
"method": "ui",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
new_plugins_ids.append(folder_name)
else:
if Path(
f"/etc/bunkerweb/plugins/{folder_name}"
@ -940,22 +974,32 @@ def plugins():
raise Exception
if not Path("/usr/sbin/nginx").is_file():
plugins = app.config["CONFIG"].get_plugins()
for plugin in deepcopy(plugins):
if plugin["id"] == folder_name:
raise FileExistsError
elif plugin["external"] is False:
del plugins[plugins.index(plugin)]
plugins.append(plugin_file)
err = db.update_external_plugins(plugins)
if err:
error = 1
flash(
f"Couldn't update external plugins to database: {err}",
"error",
plugin_content = BytesIO()
with tar_open(
fileobj=plugin_content, mode="w:gz"
) as tar:
tar.add(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}",
arcname=temp_folder_name,
recursive=True,
)
raise Exception
plugin_content.seek(0)
value = plugin_content.getvalue()
new_plugins.append(
plugin_file
| {
"external": True,
"page": "ui"
in listdir(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}"
),
"method": "ui",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
new_plugins_ids.append(folder_name)
else:
if Path(
f"/etc/bunkerweb/plugins/{folder_name}"
@ -1014,22 +1058,43 @@ def plugins():
raise Exception
if not Path("/usr/sbin/nginx").is_file():
plugins = app.config["CONFIG"].get_plugins()
for plugin in deepcopy(plugins):
if plugin["id"] == folder_name:
raise FileExistsError
elif plugin["external"] is False:
del plugins[plugins.index(plugin)]
plugins.append(plugin_file)
err = db.update_external_plugins(plugins)
if err:
error = 1
flash(
f"Couldn't update external plugins to database: {err}",
"error",
for file_name in listdir(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}"
):
move(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}/{file_name}",
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{file_name}",
)
raise Exception
rmtree(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}/{dirs[0]}"
)
plugin_content = BytesIO()
with tar_open(
fileobj=plugin_content, mode="w:gz"
) as tar:
tar.add(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}",
arcname=temp_folder_name,
recursive=True,
)
plugin_content.seek(0)
value = plugin_content.getvalue()
new_plugins.append(
plugin_file
| {
"external": True,
"page": "ui"
in listdir(
f"/var/tmp/bunkerweb/ui/{temp_folder_name}"
),
"method": "ui",
"data": value,
"checksum": sha256(value).hexdigest(),
}
)
new_plugins_ids.append(folder_name)
else:
if Path(
f"/etc/bunkerweb/plugins/{folder_name}"
@ -1105,7 +1170,7 @@ def plugins():
error = 0
if errors < files_count:
if errors >= files_count:
return redirect(url_for("loading", next=url_for("plugins")))
# Fix permissions for plugins folders
@ -1114,6 +1179,19 @@ def plugins():
chown(join(root, name), "root", 101)
chmod(join(root, name), 0o770)
plugins = app.config["CONFIG"].get_plugins(external=True, with_data=True)
for plugin in deepcopy(plugins):
if plugin["id"] in new_plugins_ids:
flash(f"Plugin {plugin['id']} already exists", "error")
del new_plugins[new_plugins_ids.index(plugin["id"])]
err = db.update_external_plugins(new_plugins, delete_missing=False)
if err:
flash(
f"Couldn't update external plugins to database: {err}",
"error",
)
if operation:
flash(operation)

46
src/ui/src/Config.py
View File

@ -1,11 +1,15 @@
from copy import deepcopy
from hashlib import sha256
from io import BytesIO
from flask import flash
from glob import iglob
from json import load as json_load
from os import listdir
from os.path import basename
from pathlib import Path
from re import search as re_search
from subprocess import run, DEVNULL, STDOUT
from tarfile import open as tar_open
from time import sleep
from typing import List, Tuple
from uuid import uuid4
@ -134,25 +138,28 @@ class Config:
**self.__settings,
}
def get_plugins(self) -> List[dict]:
def get_plugins(
self, *, external: bool = False, with_data: bool = False
) -> List[dict]:
if not Path("/usr/sbin/nginx").exists():
plugins = self.__db.get_plugins()
plugins = self.__db.get_plugins(external=external, with_data=with_data)
plugins.sort(key=lambda x: x["name"])
general_plugin = None
for x, plugin in enumerate(plugins):
if plugin["name"] == "General":
general_plugin = plugin
del plugins[x]
break
plugins.insert(0, general_plugin)
if not external:
general_plugin = None
for x, plugin in enumerate(plugins):
if plugin["name"] == "General":
general_plugin = plugin
del plugins[x]
break
plugins.insert(0, general_plugin)
return plugins
plugins = []
for foldername in list(iglob("/etc/bunkerweb/plugins/*")) + list(
iglob("/usr/share/bunkerweb/core/*")
for foldername in list(iglob("/etc/bunkerweb/plugins/*")) + (
list(iglob("/usr/share/bunkerweb/core/*") if not external else [])
):
content = listdir(foldername)
if "plugin.json" not in content:
@ -168,10 +175,26 @@ class Config:
}
)
plugin["method"] = "ui" if plugin["external"] else "manual"
if "ui" in content:
if "template.html" in listdir(f"{foldername}/ui"):
plugin["page"] = True
if with_data:
plugin_content = BytesIO()
with tar_open(fileobj=plugin_content, mode="w:gz") as tar:
tar.add(
foldername,
arcname=basename(foldername),
recursive=True,
)
plugin_content.seek(0)
value = plugin_content.getvalue()
plugin["data"] = value
plugin["checksum"] = sha256(value).hexdigest()
plugins.append(plugin)
plugins.sort(key=lambda x: x["name"])
@ -186,6 +209,7 @@ class Config:
"description": "The general settings for the server",
"version": "0.1",
"external": False,
"method": "manual",
"page": False,
"settings": json_load(f),
},