librewolf/bunkerized-nginx
4
0
Fork 1
mirror of https://github.com/bunkerity/bunkerized-nginx synced 2023-12-13 21:30:18 +01:00
Code Issues Projects Releases Wiki Activity

Add refurb as a pre-commit-config hook and apply pre-commit-config

Browse source
This commit is contained in:
Théophile Diot 2023-11-07 11:09:18 +00:00
parent 966a78da9e
commit bcded8f7ce
No known key found for this signature in database
GPG key ID: 248FEA4BAE400D06
51 changed files with 77 additions and 122 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
.pre-commit-config.yaml
View file

@ -17,7 +17,7 @@ repos:
- id: check-case-conflict
- repo: https://github.com/ambv/black
rev: 9edba85f71d50d12996ef7bda576426362016171 # frozen: 23.10.0
rev: 744d23b34800c06e10272149b70752396e90eeb8 # frozen: 23.10.1
hooks:
- id: black
name: Black Python Formatter
@ -49,6 +49,13 @@ repos:
name: Flake8 Python Linter
args: ["--max-line-length=250", "--ignore=E266,E402,E722,W503"]
- repo: https://github.com/dosisod/refurb
rev: 229e7ea1c2ef7b4b0c0910f94bb46f44e2fd4dd1 # frozen: v1.22.1
hooks:
- id: refurb
name: Refurb Python Refactoring Tool
exclude: ^tests/
- repo: https://github.com/codespell-project/codespell
rev: 6e41aba91fb32e9feb741a6258eefeb9c6e4a482 # frozen: v2.2.6
hooks:

4
docs/json2md.py
View file

@ -3,6 +3,7 @@
from io import StringIO
from json import loads
from glob import glob
from pathlib import Path
from pytablewriter import MarkdownTableWriter
@ -85,5 +86,4 @@ content = doc.read()
doc = StringIO(content.replace("\\|", "|"))
doc.seek(0)
with open("docs/settings.md", "w") as f:
f.write(doc.read())
Path("docs", "settings.md").write_text(doc.read(), encoding="utf-8")

2
src/common/core/blacklist/jobs/blacklist-download.py
View file

@ -158,7 +158,7 @@ try:
for line in iterable:
line = line.strip()
if not line or line.startswith(b"#") or line.startswith(b";"):
if not line or line.startswith((b"#", b";")):
continue
elif kind != "USER_AGENT":
line = line.split(b" ")[0]

2
src/common/core/greylist/jobs/greylist-download.py
View file

@ -143,7 +143,7 @@ try:
for line in iterable:
line = line.strip()
if not line or line.startswith(b"#") or line.startswith(b";"):
if not line or line.startswith((b"#", b";")):
continue
elif kind != "USER_AGENT":
line = line.split(b" ")[0]

2
src/common/core/jobs/jobs/mmdb-asn.py
View file

@ -49,7 +49,7 @@ try:
if response and response.status_code == 200:
_sha1 = sha1()
with open(str(tmp_path), "rb") as f:
with tmp_path.open("rb") as f:
while True:
data = f.read(1024)
if not data:

2
src/common/core/jobs/jobs/mmdb-country.py
View file

@ -49,7 +49,7 @@ try:
if response and response.status_code == 200:
_sha1 = sha1()
with open(str(tmp_path), "rb") as f:
with tmp_path.open("rb") as f:
while True:
data = f.read(1024)
if not data:

2
src/common/core/realip/jobs/realip-download.py
View file

@ -107,7 +107,7 @@ try:
for line in iterable:
line = line.strip().split(b" ")[0]
if not line or line.startswith(b"#") or line.startswith(b";"):
if not line or line.startswith((b"#", b";")):
continue
ok, data = check_line(line)

2
src/common/core/whitelist/jobs/whitelist-download.py
View file

@ -142,7 +142,7 @@ try:
for line in iterable:
line = line.strip()
if not line or line.startswith(b"#") or line.startswith(b";"):
if not line or line.startswith((b"#", b";")):
continue
elif kind != "USER_AGENT":
line = line.split(b" ")[0]

10
src/common/db/Database.py
View file

@ -712,14 +712,14 @@ class Database:
.all()
):
default = setting.default or ""
config[setting.id] = default if methods is False else {"value": default, "global": True, "method": "default"}
config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"}
global_values = session.query(Global_values).with_entities(Global_values.value, Global_values.suffix, Global_values.method).filter_by(setting_id=setting.id).all()
for global_value in global_values:
config[setting.id + (f"_{global_value.suffix}" if setting.multiple and global_value.suffix > 0 else "")] = (
global_value.value
if methods is False
if not methods
else {
"value": global_value.value,
"global": True,
@ -764,7 +764,7 @@ class Database:
for service_setting in service_settings:
config[f"{service.id}_{key}" + (f"_{service_setting.suffix}" if service_setting.suffix > 0 else "")] = (
service_setting.value
if methods is False
if not methods
else {
"value": service_setting.value,
"global": False,
@ -774,7 +774,7 @@ class Database:
if is_multisite:
servers = " ".join(service.id for service in session.query(Services).all())
config["SERVER_NAME"] = servers if methods is False else {"value": servers, "global": True, "method": "default"}
config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
return config
@ -826,7 +826,7 @@ class Database:
"global": value["global"],
"method": value["method"],
}
if methods is True
if methods
else value
)

48
src/common/gen/Configurator.py
View file

@ -185,10 +185,7 @@ class Configurator:
config[variable] = value
elif (
"CUSTOM_CONF" not in variable
and not variable.startswith("PYTHON")
and not variable.startswith("KUBERNETES_SERVICE_")
and not variable.startswith("KUBERNETES_PORT_")
and not variable.startswith("SVC_")
and not variable.startswith(("PYTHON", "KUBERNETES_SERVICE_", "KUBERNETES_PORT_", "SVC_"))
and variable
not in (
"GPG_KEY",
@ -269,17 +266,7 @@ class Configurator:
return False, variable
def __validate_plugin(self, plugin: dict) -> Tuple[bool, str]:
if not all(
key in plugin.keys()
for key in [
"id",
"name",
"description",
"version",
"stream",
"settings",
]
):
if not all(key in plugin for key in ("id", "name", "description", "version", "stream", "settings")):
return (
False,
f"Missing mandatory keys for plugin {plugin.get('id', 'unknown')} (id, name, description, version, stream, settings)",
@ -305,25 +292,14 @@ class Configurator:
False,
f"Invalid version for plugin {plugin['id']} (Must be in format \\d+\\.\\d+(\\.\\d+)?)",
)
elif plugin["stream"] not in ["yes", "no", "partial"]:
elif plugin["stream"] not in ("yes", "no", "partial"):
return (
False,
f"Invalid stream for plugin {plugin['id']} (Must be yes, no or partial)",
)
for setting, data in plugin["settings"].items():
if not all(
key in data.keys()
for key in [
"context",
"default",
"help",
"id",
"label",
"regex",
"type",
]
):
if not all(key in data.keys() for key in ("context", "default", "help", "id", "label", "regex", "type")):
return (
False,
f"missing keys for setting {setting} in plugin {plugin['id']}, must have context, default, help, id, label, regex and type",
@ -334,7 +310,7 @@ class Configurator:
False,
f"Invalid setting name for setting {setting} in plugin {plugin['id']} (Can only contain capital letters and underscores (min 1 characters and max 256))",
)
elif data["context"] not in ["global", "multisite"]:
elif data["context"] not in ("global", "multisite"):
return (
False,
f"Invalid context for setting {setting} in plugin {plugin['id']} (Must be global or multisite)",
@ -359,7 +335,7 @@ class Configurator:
False,
f"Invalid regex for setting {setting} in plugin {plugin['id']} (Max 1024 characters)",
)
elif data["type"] not in ["password", "text", "check", "select"]:
elif data["type"] not in ("password", "text", "check", "select"):
return (
False,
f"Invalid type for setting {setting} in plugin {plugin['id']} (Must be password, text, check or select)",
@ -380,15 +356,7 @@ class Configurator:
)
for job in plugin.get("jobs", []):
if not all(
key in job.keys()
for key in [
"name",
"file",
"every",
"reload",
]
):
if not all(key in job.keys() for key in ("name", "file", "every", "reload")):
return (
False,
f"missing keys for job {job['name']} in plugin {plugin['id']}, must have name, file, every and reload",
@ -404,7 +372,7 @@ class Configurator:
False,
f"Invalid file for job {job['name']} in plugin {plugin['id']} (Can only contain numbers, letters, underscores, hyphens and slashes (min 1 characters and max 256))",
)
elif job["every"] not in ["once", "minute", "hour", "day", "week"]:
elif job["every"] not in ("once", "minute", "hour", "day", "week"):
return (
False,
f"Invalid every for job {job['name']} in plugin {plugin['id']} (Must be once, minute, hour, day or week)",

4
src/common/gen/save_config.py
View file

@ -156,9 +156,7 @@ if __name__ == "__main__":
external_plugins = args.plugins
if not Path(sep, "usr", "sbin", "nginx").exists() and args.method == "ui":
db = Database(logger, pool=False)
external_plugins = []
for plugin in db.get_plugins():
external_plugins.append(plugin)
external_plugins = db.get_plugins()
# Check existences and permissions
logger.info("Checking arguments ...")

5
src/common/gen/utils.py
View file

@ -1,6 +1,7 @@
#!/usr/bin/python3
from os import getegid, geteuid, stat
from os import getegid, geteuid
from pathlib import Path
from stat import (
S_IRGRP,
S_IROTH,
@ -18,7 +19,7 @@ from typing import List
def has_permissions(path: str, need_permissions: List[str]) -> bool:
uid = geteuid()
gid = getegid()
statinfo = stat(path)
statinfo = Path(path).stat()
permissions = {"R": False, "W": False, "X": False}
if statinfo.st_uid == uid:
if statinfo.st_mode & S_IRUSR:

15
src/scheduler/main.py
View file

@ -103,7 +103,7 @@ def generate_custom_configs(
if file.is_symlink() or file.is_file():
file.unlink()
elif file.is_dir():
rmtree(str(file), ignore_errors=True)
rmtree(file, ignore_errors=True)
if configs:
logger.info("Generating new custom configs ...")
@ -142,7 +142,7 @@ def generate_external_plugins(
if file.is_symlink() or file.is_file():
file.unlink()
elif file.is_dir():
rmtree(str(file), ignore_errors=True)
rmtree(file, ignore_errors=True)
if plugins:
logger.info("Generating new external plugins ...")
@ -526,19 +526,12 @@ if __name__ == "__main__":
else:
# Reload nginx
logger.info("Reloading nginx ...")
proc = subprocess_run(
[join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
stdin=DEVNULL,
stderr=STDOUT,
env=env.copy(),
check=False,
stdout=PIPE
)
proc = subprocess_run([join(sep, "usr", "sbin", "nginx"), "-s", "reload"], stdin=DEVNULL, stderr=STDOUT, env=env.copy(), check=False, stdout=PIPE)
if proc.returncode == 0:
logger.info("Successfully sent reload signal to nginx")
else:
logger.error(
f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stdout.decode('utf-8') if proc.stdout else 'no output'}",
f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stdout.decode('utf-8') if proc.stdout else 'no output'}",
)
# # Stop temp nginx
# logger.info("Stopping temp nginx ...")

4
src/ui/src/ConfigFiles.py
View file

@ -108,13 +108,13 @@ class ConfigFiles:
if path.is_file():
path.unlink()
elif path.is_dir():
rmtree(str(path), ignore_errors=False)
rmtree(path, ignore_errors=False)
else:
path = Path(f"{path}.conf")
if path.is_file():
path.unlink()
else:
rmtree(str(path), ignore_errors=False)
rmtree(path, ignore_errors=False)
except OSError:
return f"Could not delete {path}", 1

4
src/ui/utils.py
View file

@ -68,7 +68,7 @@ def path_to_dict(
"path": join(
path,
type_lower,
conf["service_id"] if conf["service_id"] else "",
conf["service_id"] or "",
f"{conf['name']}.conf",
),
"can_edit": conf["method"] == "ui",
@ -111,7 +111,7 @@ def path_to_dict(
"type": "file",
"path": join(
path,
conf["service_id"] if conf["service_id"] else "",
conf["service_id"] or "",
conf["file_name"],
),
"can_edit": False,

2
tests/core/antibot/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/authbasic/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/badbehavior/index.html
View file

@ -1 +1 @@
ready
ready

2
tests/core/badbehavior/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/blacklist/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/brotli/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/bunkernet/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/bwcli/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/clientcache/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/cors/main.py
View file

@ -14,7 +14,7 @@ try:
ready = False
retries = 0
while not ready:
with suppress(RequestException):
with suppress(RequestException):
resp = get(
f"http{'s' if ssl else ''}://www.example.com/ready",
headers={"Host": "www.example.com"},

2
tests/core/cors/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/country/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

10
tests/core/customcert/main.py
View file

@ -5,17 +5,11 @@ from traceback import format_exc
from time import sleep
try:
ready = False
retries = 0
while not ready:
with suppress(RequestException):
resp = get(
f"http://www.example.com/ready",
headers={"Host": "www.example.com"},
verify=False,
allow_redirects=True
)
with suppress(RequestException):
resp = get("http://www.example.com/ready", headers={"Host": "www.example.com"}, verify=False, allow_redirects=True)
status_code = resp.status_code
text = resp.text

2
tests/core/customcert/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/db/main.py
View file

@ -180,7 +180,7 @@ try:
for global_value in global_values:
if global_value.setting_id == "API_LISTEN_IP":
continue
continue
if global_value.setting_id in global_settings:
if global_value.value != global_settings[global_value.setting_id]["value"]:
print(

2
tests/core/db/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/dnsbl/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/errors/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/greylist/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/gzip/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/headers/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/inject/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

1
tests/core/limit/main.py
View file

@ -9,7 +9,6 @@ from time import sleep
from requests import get
try:
ready = False
retries = 0
while not ready:

2
tests/core/limit/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/misc/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/modsecurity/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/redirect/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

1
tests/core/redis/main.py
View file

@ -17,7 +17,6 @@ fastapi_proc = None
ip_to_check = "1.0.0.3" if getenv("TEST_TYPE", "docker") == "docker" else "127.0.0.1"
try:
ready = False
retries = 0
while not ready:

2
tests/core/redis/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/reversescan/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/selfsigned/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/sessions/main.py
View file

@ -1,6 +1,6 @@
from contextlib import suppress
from os import getenv
from subprocess import PIPE, run
from subprocess import run
from requests import get, post
from requests.exceptions import RequestException
from selenium import webdriver

2
tests/core/sessions/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

2
tests/core/whitelist/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}

8
tests/ui/main.py
View file

@ -4,7 +4,6 @@ from functools import partial
from os import getenv, listdir
from os.path import join
from pathlib import Path
from subprocess import PIPE, run
from time import sleep
from traceback import format_exc
from typing import List, Union
@ -76,6 +75,7 @@ def assert_button_click(driver, button: Union[str, WebElement]):
clicked = True
return clicked
def assert_alert_message(driver, message: str):
safe_get_element(driver, By.XPATH, "//button[@data-flash-sidebar-open='']")
@ -164,11 +164,7 @@ def access_page(
)
driver_func = partial(
webdriver.Firefox,
service=Service(log_output="./geckodriver.log"),
options=firefox_options
)
driver_func = partial(webdriver.Firefox, service=Service(log_output="./geckodriver.log"), options=firefox_options)
if TEST_TYPE == "dev":
driver_func = partial(
webdriver.Firefox,

2
tests/ui/ready.conf
View file

@ -5,4 +5,4 @@ location /ready {
ngx.flush(true)
ngx.exit(ngx.HTTP_OK)
}
}
}