Merge branch 'dev' (softly)

This commit is contained in:
TheophileDiot 2022-11-14 17:31:21 +01:00
parent 891757dab5
commit 5a8f812560
21 changed files with 244 additions and 71 deletions

View File

@ -111,23 +111,21 @@ class DockerController(Controller, ConfigCaller):
return ret
def process_events(self):
for event in self.__client.events(decode=True, filters={"type": "container"}):
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self._config.update_needed(
self._instances, self._services, configs=self._configs
):
continue
self.__logger.info(
"Catched docker event, deploying new configuration ...",
)
for _ in self.__client.events(decode=True, filters={"type": "container"}):
try:
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self._config.update_needed(
self._instances, self._services, configs=self._configs
):
continue
self.__logger.info(
"Catched Docker event, deploying new configuration ..."
)
ret = self.apply_config()
if not ret:
self.__logger.error(
"Error while deploying new configuration",
)
self.__logger.error("Error while deploying new configuration")
else:
self.__logger.info(
"Successfully deployed new configuration 🚀",
@ -136,10 +134,10 @@ class DockerController(Controller, ConfigCaller):
if not self._config._db.is_autoconf_loaded():
ret = self._config._db.set_autoconf_load(True)
if ret:
self.__logger.error(
self.__logger.warning(
f"Can't set autoconf loaded metadata to true in database: {ret}",
)
except:
self.__logger.error(
f"Exception while deploying new configuration :\n{format_exc()}",
f"Exception while processing events :\n{format_exc()}"
)

View File

@ -39,7 +39,7 @@ RUN apk add --no-cache bash && \
chown root:nginx /usr/bin/bwcli
# Fix CVEs
RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "libtirpc>=1.3.2-r1" "libtirpc-conf>=1.3.2-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1"
RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "libtirpc>=1.3.2-r1" "libtirpc-conf>=1.3.2-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0"
VOLUME /data /etc/nginx

View File

@ -1,4 +1,5 @@
from os import getenv
from time import sleep
from traceback import format_exc
from kubernetes import client, config, watch
from kubernetes.client.exceptions import ApiException
@ -224,8 +225,9 @@ class IngressController(Controller, ConfigCaller):
raise Exception(f"unsupported watch_type {watch_type}")
while True:
locked = False
error = False
try:
for event in w.stream(what):
for _ in w.stream(what):
self.__internal_lock.acquire()
locked = True
self._instances = self.get_instances()
@ -246,6 +248,7 @@ class IngressController(Controller, ConfigCaller):
self.__logger.error(
"Error while deploying new configuration ...",
)
error = True
else:
self.__logger.info(
"Successfully deployed new configuration 🚀",
@ -254,28 +257,33 @@ class IngressController(Controller, ConfigCaller):
if not self._config._db.is_autoconf_loaded():
ret = self._config._db.set_autoconf_load(True)
if ret:
self.__logger.error(
self.__logger.warning(
f"Can't set autoconf loaded metadata to true in database: {ret}",
)
except:
self.__logger.error(
f"Exception while deploying new configuration :\n{format_exc()}",
)
self.__internal_lock.release()
locked = False
error = True
except ApiException as e:
if e.status != 410:
self.__logger.error(
f"Exception while reading k8s event (type = {watch_type}) :\n{format_exc()}",
)
sys_exit(1)
if locked:
self.__internal_lock.release()
except:
self.__logger.error(
f"Unknown exception while reading k8s event (type = {watch_type}) :\n{format_exc()}",
)
sys_exit(2)
finally :
if locked:
self.__internal_lock.release()
locked = False
if error is True:
self.__logger.warning("Got exception, retrying in 10 seconds ...")
sleep(10)
def apply_config(self):
ret = self._config.apply(self._instances, self._services, configs=self._configs)

View File

@ -110,23 +110,20 @@ class SwarmController(Controller, ConfigCaller):
def __event(self, event_type):
for event in self.__client.events(decode=True, filters={"type": event_type}):
self.__internal_lock.acquire()
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self._config.update_needed(
self._instances, self._services, configs=self._configs
):
self.__internal_lock.release()
continue
self.__logger.info(
"Catched Swarm event, deploying new configuration ...",
)
try:
self._instances = self.get_instances()
self._services = self.get_services()
self._configs = self.get_configs()
if not self._config.update_needed(
self._instances, self._services, configs=self._configs
):
continue
self.__logger.info(
"Catched Swarm event, deploying new configuration ..."
)
ret = self.apply_config()
if not ret:
self.__logger.error(
"Error while deploying new configuration ...",
)
self.__logger.error("Error while deploying new configuration")
else:
self.__logger.info(
"Successfully deployed new configuration 🚀",
@ -135,12 +132,12 @@ class SwarmController(Controller, ConfigCaller):
if not self._config._db.is_autoconf_loaded():
ret = self._config._db.set_autoconf_load(True)
if ret:
self.__logger.error(
self.__logger.warning(
f"Can't set autoconf loaded metadata to true in database: {ret}",
)
except:
self.__logger.error(
f"Exception while deploying new configuration :\n{format_exc()}",
f"Exception while processing events :\n{format_exc()}"
)
self.__internal_lock.release()

View File

@ -75,7 +75,7 @@ RUN apk add --no-cache bash python3 && \
chmod 660 /usr/share/bunkerweb/INTEGRATION
# Fix CVEs
RUN apk add "freetype>=2.10.4-r3" "curl>=7.79.1-r2" "libcurl>=7.79.1-r2" "openssl>=1.1.1q-r0" "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1"
RUN apk add "freetype>=2.10.4-r3" "curl>=7.79.1-r2" "libcurl>=7.79.1-r2" "openssl>=1.1.1q-r0" "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0"
VOLUME /data /etc/nginx

View File

@ -103,6 +103,44 @@ api.global.POST["^/unban$"] = function(api)
return api:response(ngx.HTTP_OK, "success", "ip " .. ip["ip"] .. " unbanned")
end
api.global.POST["^/ban$"] = function(api)
ngx.req.read_body()
local data = ngx.req.get_body_data()
if not data then
local data_file = ngx.req.get_body_file()
if data_file then
local file = io.open(data_file)
data = file:read("*a")
file:close()
end
end
local ok, ip = pcall(cjson.decode, data)
if not ok then
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't decode JSON : " .. env)
end
datastore:set("bans_ip_" .. ip["ip"], "manual", ip["exp"])
return api:response(ngx.HTTP_OK, "success", "ip " .. ip["ip"] .. " banned")
end
api.global.GET["^/bans$"] = function(api)
local data = {}
for i, k in ipairs(datastore:keys()) do
if k:find("^bans_ip_") then
local ret, reason = datastore:get(k)
if not ret then
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't access " .. k .. " from datastore : " + reason)
end
local ret, exp = datastore:exp(k)
if not ret then
return api:response(ngx.HTTP_INTERNAL_SERVER_ERROR, "error", "can't access exp " .. k .. " from datastore : " + exp)
end
local ban = { ip = k:sub(9, #k), reason = reason, exp = exp }
table.insert(data, ban)
end
end
return api:response(ngx.HTTP_OK, "success", data)
end
api.is_allowed_ip = function(self)
local data, err = datastore:get("api_whitelist_ip")
if not data then

View File

@ -1,4 +1,4 @@
local datastore = {dict = ngx.shared.datastore }
local datastore = { dict = ngx.shared.datastore }
datastore.get = function(self, key)
local value, err = self.dict:get(key)
@ -22,6 +22,14 @@ datastore.delete = function(self, key)
return true, "success"
end
datastore.exp = function(self, key)
local ttl, err = self.dict:ttl(key)
if not ttl then
return false, err
end
return true, ttl
end
datastore.delete_all = function(self, pattern)
local keys = self.dict:get_keys(0)
for i, key in ipairs(keys) do

View File

@ -117,3 +117,18 @@ class CLI(ApiCaller):
if self._send_to_apis("POST", "/unban", data={"ip": ip}):
return True, f"IP {ip} has been unbanned"
return False, "error"
def ban(self, ip, exp):
if self._send_to_apis("POST", "/ban", data={"ip": ip, "exp": exp}):
return True, f"IP {ip} has been banned"
return False, "error"
def bans(self):
ret, resp = self._send_to_apis("GET", "/bans", response=True)
if ret:
bans = resp["bans"]
cli_str = "List of bans :\n"
for ban in bans:
cli_str += f"- {ban['ip']} for {ban['exp']}s : {ban['reason']}\n"
return True, cli_str
return False, "error"

View File

@ -27,6 +27,19 @@ if __name__ == "__main__":
)
parser_unban.add_argument("ip", type=str, help="IP address to unban")
# Ban subparser
parser_ban = subparsers.add_parser("ban", help="add a ban to the cache")
parser_ban.add_argument("ip", type=str, help="IP address to ban")
parser_ban.add_argument(
"exp",
type=int,
help="banning time in seconds (default : 86400)",
default=86400,
)
# Bans subparser
parser_bans = subparsers.add_parser("bans", help="list current bans")
# Parse args
args = parser.parse_args()
@ -37,6 +50,10 @@ if __name__ == "__main__":
ret, err = False, "unknown command"
if args.command == "unban":
ret, err = cli.unban(args.ip)
elif args.command == "ban":
ret, err = cli.ban(args.ip, args.exp)
elif args.command == "bans":
ret, err = cli.bans()
if not ret:
logger.error(f"CLI command status : ❌ (fail)\n{err}")
@ -50,5 +67,3 @@ if __name__ == "__main__":
except:
logger.error(f"Error while executing bwcli :\n{format_exc()}")
sys_exit(1)
sys_exit(0)

View File

@ -113,7 +113,7 @@ try:
checksum=new_hash,
)
if err:
logger.warning(f"Couldn't update db cache: {err}")
logger.warning(f"Couldn't update db ip.list cache: {err}")
logger.info("Successfully saved BunkerNet data")

View File

@ -117,15 +117,15 @@ try:
with open("/var/cache/bunkerweb/bunkernet/instance.id", "w") as f:
f.write(bunkernet_id)
# Update db
err = db.update_job_cache(
"bunkernet-register",
None,
"instance.id",
f"{bunkernet_id}".encode("utf-8"),
)
if err:
logger.warning(f"Couldn't update db cache: {err}")
# Update db
err = db.update_job_cache(
"bunkernet-register",
None,
"instance.id",
f"{bunkernet_id}".encode("utf-8"),
)
if err:
logger.warning(f"Couldn't update db cache: {err}")
else:
logger.error("Connectivity with BunkerWeb failed ...")
status = 2

View File

@ -1,10 +1,9 @@
#!/usr/bin/python3
from asyncio import run
from io import BytesIO
from os import environ, getenv
from os.path import exists
from subprocess import DEVNULL, STDOUT
from subprocess import run, DEVNULL, STDOUT
from sys import exit as sys_exit, path as sys_path
from tarfile import open as tar_open
from traceback import format_exc
@ -34,6 +33,8 @@ try:
with open("/usr/share/bunkerweb/INTEGRATION", "r") as f:
bw_integration = f.read().strip()
token = getenv("CERTBOT_TOKEN")
logger.info(f"Certificates renewal for {getenv('RENEWED_DOMAINS')} successful")
# Cluster case
if bw_integration in ("Swarm", "Kubernetes", "Autoconf"):

View File

@ -28,7 +28,6 @@ logger = setup_logger("LETS-ENCRYPT", getenv("LOG_LEVEL", "INFO"))
status = 0
try:
if getenv("MULTISITE") == "yes":
for first_server in getenv("SERVER_NAME").split(" "):
if first_server == "":
@ -46,11 +45,6 @@ try:
logger.error(
f"Certificates renewal for {first_server} failed",
)
else:
logger.info(
f"Certificates renewal for {first_server} successful",
)
elif getenv("AUTO_LETS_ENCRYPT") == "yes" and getenv("SERVER_NAME") != "":
first_server = getenv("SERVER_NAME").split(" ")[0]
if exists(f"/etc/letsencrypt/live/{first_server}/cert.pem"):
@ -60,10 +54,6 @@ try:
logger.error(
f"Certificates renewal for {first_server} failed",
)
else:
logger.info(
f"Certificates renewal for {first_server} successful",
)
except:
status = 2

View File

@ -100,7 +100,7 @@ class ApiCaller:
def _get_apis(self):
return self.__apis
def _send_to_apis(self, method, url, files=None, data=None):
def _send_to_apis(self, method, url, files=None, data=None, response=False):
ret = True
for api in self.__apis:
if files is not None:
@ -122,6 +122,9 @@ class ApiCaller:
self.__logger.info(
f"Successfully sent API request to {api.get_endpoint()}{url}",
)
if response:
return ret, resp.json()
return ret
def _send_files(self, path, url):

View File

@ -56,7 +56,6 @@ RUN cp /usr/share/bunkerweb/helpers/bwcli /usr/bin/ && \
mkdir /var/cache/bunkerweb/ && \
mkdir /etc/bunkerweb/plugins && \
mkdir /var/tmp/bunkerweb/ && \
#mkdir /var/www/html && \
echo "Linux" > /usr/share/bunkerweb/INTEGRATION && \
find /usr/share/bunkerweb -path /usr/share/bunkerweb/deps -prune -o -type f -exec chmod 0740 {} \; && \
find /usr/share/bunkerweb -path /usr/share/bunkerweb/deps -prune -o -type d -exec chmod 0750 {} \; && \

87
src/linux/Dockerfile-rhel Normal file
View File

@ -0,0 +1,87 @@
FROM redhat/ubi8:8.6
ENV OS=rhel
ENV NGINX_VERSION 1.20.2
# RHEL subscription
RUN subscription-manager register --username=username --password=password --auto-attach
# Install fpm
RUN dnf install -y ruby ruby-devel make gcc redhat-rpm-config rpm-build wget && \
wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
rpm -Uvh epel-release*rpm && \
dnf module -y reset ruby && dnf module -y enable ruby:2.6 && dnf module -y install ruby:2.6/common && \
gem install fpm
# Nginx
COPY linux/nginx.repo /etc/yum.repos.d/nginx.repo
RUN dnf install yum-utils -y && \
dnf install nginx-1.20.2 -y
# Copy dependencies sources folder
COPY src/common/deps /tmp/bunkerweb/deps
COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
RUN mkdir -p /usr/share/bunkerweb/deps && \
cat /tmp/req/requirements.txt /tmp/req/requirements.txt.1 /tmp/req/requirements.txt.2 /tmp/req/requirements.txt.3 > /tmp/bunkerweb/deps/requirements.txt && \
rm -rf /tmp/req
# Compile and install dependencies
RUN dnf install -y python39-pip brotli brotli-devel gperftools-devel perl libxslt-devel libxml2 libxslt bash gd gd-devel gcc-c++ kernel-devel curl znc-modtcl libmpc-devel gmp-devel gawk mpfr-devel libtool pcre-devel automake autoconf readline-devel gcc make openssl-devel git zlib-devel libxml2-devel pkgconf libcurl-devel geoip-devel lmdb-libs && \
mkdir -p /usr/share/bunkerweb/deps && \
chmod +x /tmp/bunkerweb/deps/install.sh && \
bash /tmp/bunkerweb/deps/install.sh && \
mkdir /usr/share/bunkerweb/deps/python && \
pip install --no-cache-dir --require-hashes --target /usr/share/bunkerweb/deps/python -r /tmp/bunkerweb/deps/requirements.txt
# Copy BW files
# can't exclude deps from . so we are copying everything by hand
COPY src/bw/loading /usr/share/bunkerweb/loading
COPY src/bw/lua /usr/share/bunkerweb/lua
COPY src/bw/misc /usr/share/bunkerweb/misc
COPY src/common/api /usr/share/bunkerweb/api
COPY src/common/cli /usr/share/bunkerweb/cli
COPY src/common/confs /usr/share/bunkerweb/confs
COPY src/common/core /usr/share/bunkerweb/core
COPY src/common/db /usr/share/bunkerweb/db
COPY src/common/gen /usr/share/bunkerweb/gen
COPY src/common/helpers /usr/share/bunkerweb/helpers
COPY src/common/settings.json /usr/share/bunkerweb/settings.json
COPY src/common/utils /usr/share/bunkerweb/utils
COPY src/scheduler /usr/share/bunkerweb/scheduler
COPY src/ui /usr/share/bunkerweb/ui
COPY src/VERSION /usr/share/bunkerweb/VERSION
# Setup BW
RUN cp /usr/share/bunkerweb/helpers/bwcli /usr/bin/ && \
chmod 755 /usr/bin/bwcli && \
mkdir /etc/bunkerweb/configs && \
mkdir /var/cache/bunkerweb/ && \
mkdir /etc/bunkerweb/plugins && \
mkdir /var/tmp/bunkerweb/ && \
echo "Linux" > /usr/share/bunkerweb/INTEGRATION && \
find /usr/share/bunkerweb -path /usr/share/bunkerweb/deps -prune -o -type f -exec chmod 0740 {} \; && \
find /usr/share/bunkerweb -path /usr/share/bunkerweb/deps -prune -o -type d -exec chmod 0750 {} \; && \
chmod 770 /var/cache/bunkerweb/ /var/tmp/bunkerweb/ && \
chmod 750 /usr/share/bunkerweb/gen/main.py /usr/share/bunkerweb/scheduler/main.py /usr/share/bunkerweb/cli/main.py /usr/share/bunkerweb/helpers/*.sh /usr/share/bunkerweb/ui/main.py && \
find /usr/share/bunkerweb/core/*/jobs/* -type f -exec chmod 750 {} \; && \
pip install --no-cache-dir --target /usr/share/bunkerweb/deps/python -r /usr/share/bunkerweb/ui/deps/requirements.txt && \
chmod 755 /usr/share/bunkerweb
# Copy Linux files
COPY src/linux/variables.env /etc/bunkerweb/variables.env
COPY src/linux/ui.env /etc/bunkerweb/ui.env
COPY src/linux/scripts /usr/share/bunkerweb/scripts
COPY src/linux/fpm.sh /usr/share/fpm.sh
RUN chmod +x /usr/share/bunkerweb/scripts/*.sh /usr/share/fpm.sh
COPY src/linux/fpm-rhel /usr/share/.fpm
COPY src/linux/bunkerweb.service /usr/share/bunkerweb.service
COPY src/linux/bunkerweb-ui.service /usr/share/bunkerweb-ui.service
# Generate RPM at startup
VOLUME /data
WORKDIR /usr/share/
ENTRYPOINT ["/usr/share/fpm.sh", "rpm"]

View File

@ -4,6 +4,7 @@ Documentation=https://docs.bunkerweb.io
After=network.target
[Service]
Restart=always
User=root
PIDFile=/var/tmp/bunkerweb/scheduler.pid
ExecStart=/usr/share/bunkerweb/scripts/start.sh start

13
src/linux/fpm-rhel Normal file
View File

@ -0,0 +1,13 @@
-s dir
--name bunkerweb
--license agpl3
--version %VERSION%
--architecture x86_64
--depends bash --depends epel-release --depends python39 --depends 'nginx = 1:1.20.2-1.el8.ngx' --depends libcurl-devel --depends libxml2 --depends lmdb-libs --depends GeoIP-devel --depends file-libs --depends net-tools --depends gd --depends sudo
--description "BunkerWeb %VERSION% for Rhel 8"
--url "https://www.bunkerweb.io"
--maintainer "Bunkerity <contact at bunkerity dot com>"
--after-install /usr/share/bunkerweb/scripts/postinstall.sh
--before-remove /usr/share/bunkerweb/scripts/beforeRemove.sh
--after-remove /usr/share/bunkerweb/scripts/afterRemove.sh
/usr/share/bunkerweb/=/usr/share/bunkerweb/ bunkerweb.service=/etc/systemd/system/bunkerweb.service bunkerweb-ui.service=/etc/systemd/system/bunkerweb-ui.service /usr/bin/bwcli=/usr/bin/bwcli

View File

@ -58,7 +58,7 @@ RUN apk add --no-cache bash libgcc libstdc++ openssl && \
chmod 660 /usr/share/bunkerweb/INTEGRATION
# Fix CVEs
RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "libtirpc>=1.3.2-r1" "libtirpc-conf>=1.3.2-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1"
RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "libtirpc>=1.3.2-r1" "libtirpc-conf>=1.3.2-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0"
VOLUME /data /etc/nginx

View File

@ -48,7 +48,7 @@ RUN apk add --no-cache bash file && \
chmod 750 /usr/share/bunkerweb/gen/main.py /usr/share/bunkerweb/deps/python/bin/*
# Fix CVEs
RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "libtirpc>=1.3.2-r1" "libtirpc-conf>=1.3.2-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1"
RUN apk add "libssl1.1>=1.1.1q-r0" "libcrypto1.1>=1.1.1q-r0" "git>=2.32.3-r0" "ncurses-libs>=6.2_p20210612-r1" "ncurses-terminfo-base>=6.2_p20210612-r1" "libtirpc>=1.3.2-r1" "libtirpc-conf>=1.3.2-r1" "zlib>=1.2.12-r2" "libxml2>=2.9.14-r1" "expat>=2.5.0-r0"
VOLUME /data /etc/nginx

View File

@ -24,14 +24,14 @@ class Config:
self.__logger.warning(
"Database is not initialized, retrying in 5s ...",
)
sleep(3)
sleep(5)
env = self.__db.get_config()
while not self.__db.is_first_config_saved() or not env:
self.__logger.warning(
"Database doesn't have any config saved yet, retrying in 5s ...",
)
sleep(3)
sleep(5)
env = self.__db.get_config()
self.reload_plugins()