Add refurb as a pre-commit-config hook and apply pre-commit-config

.pre-commit-config.yaml

@@ -17,7 +17,7 @@ repos:
       - id: check-case-conflict
 
   - repo: https://github.com/ambv/black
-    rev: 9edba85f71d50d12996ef7bda576426362016171 # frozen: 23.10.0
+    rev: 744d23b34800c06e10272149b70752396e90eeb8 # frozen: 23.10.1
     hooks:
       - id: black
         name: Black Python Formatter
@@ -49,6 +49,13 @@ repos:
         name: Flake8 Python Linter
         args: ["--max-line-length=250", "--ignore=E266,E402,E722,W503"]
 
+  - repo: https://github.com/dosisod/refurb
+    rev: 229e7ea1c2ef7b4b0c0910f94bb46f44e2fd4dd1 # frozen: v1.22.1
+    hooks:
+      - id: refurb
+        name: Refurb Python Refactoring Tool
+        exclude: ^tests/
+
   - repo: https://github.com/codespell-project/codespell
     rev: 6e41aba91fb32e9feb741a6258eefeb9c6e4a482 # frozen: v2.2.6
     hooks:

docs/json2md.py

@@ -3,6 +3,7 @@
 from io import StringIO
 from json import loads
 from glob import glob
+from pathlib import Path
 from pytablewriter import MarkdownTableWriter
 
 
@@ -85,5 +86,4 @@ content = doc.read()
 doc = StringIO(content.replace("\\|", "|"))
 doc.seek(0)
 
-with open("docs/settings.md", "w") as f:
-    f.write(doc.read())
+Path("docs", "settings.md").write_text(doc.read(), encoding="utf-8")

src/common/core/blacklist/jobs/blacklist-download.py

@@ -158,7 +158,7 @@ try:
                 for line in iterable:
                     line = line.strip()
 
-                    if not line or line.startswith(b"#") or line.startswith(b";"):
+                    if not line or line.startswith((b"#", b";")):
                         continue
                     elif kind != "USER_AGENT":
                         line = line.split(b" ")[0]

src/common/core/greylist/jobs/greylist-download.py

@@ -143,7 +143,7 @@ try:
                 for line in iterable:
                     line = line.strip()
 
-                    if not line or line.startswith(b"#") or line.startswith(b";"):
+                    if not line or line.startswith((b"#", b";")):
                         continue
                     elif kind != "USER_AGENT":
                         line = line.split(b" ")[0]

src/common/core/jobs/jobs/mmdb-asn.py

@@ -49,7 +49,7 @@ try:
 
         if response and response.status_code == 200:
             _sha1 = sha1()
-            with open(str(tmp_path), "rb") as f:
+            with tmp_path.open("rb") as f:
                 while True:
                     data = f.read(1024)
                     if not data:

src/common/core/jobs/jobs/mmdb-country.py

@@ -49,7 +49,7 @@ try:
 
         if response and response.status_code == 200:
             _sha1 = sha1()
-            with open(str(tmp_path), "rb") as f:
+            with tmp_path.open("rb") as f:
                 while True:
                     data = f.read(1024)
                     if not data:

src/common/core/realip/jobs/realip-download.py

@@ -107,7 +107,7 @@ try:
             for line in iterable:
                 line = line.strip().split(b" ")[0]
 
-                if not line or line.startswith(b"#") or line.startswith(b";"):
+                if not line or line.startswith((b"#", b";")):
                     continue
 
                 ok, data = check_line(line)

src/common/core/whitelist/jobs/whitelist-download.py

@@ -142,7 +142,7 @@ try:
                 for line in iterable:
                     line = line.strip()
 
-                    if not line or line.startswith(b"#") or line.startswith(b";"):
+                    if not line or line.startswith((b"#", b";")):
                         continue
                     elif kind != "USER_AGENT":
                         line = line.split(b" ")[0]

src/common/db/Database.py

@@ -712,14 +712,14 @@ class Database:
                 .all()
             ):
                 default = setting.default or ""
-                config[setting.id] = default if methods is False else {"value": default, "global": True, "method": "default"}
+                config[setting.id] = default if not methods else {"value": default, "global": True, "method": "default"}
 
                 global_values = session.query(Global_values).with_entities(Global_values.value, Global_values.suffix, Global_values.method).filter_by(setting_id=setting.id).all()
 
                 for global_value in global_values:
                     config[setting.id + (f"_{global_value.suffix}" if setting.multiple and global_value.suffix > 0 else "")] = (
                         global_value.value
-                        if methods is False
+                        if not methods
                         else {
                             "value": global_value.value,
                             "global": True,
@@ -764,7 +764,7 @@ class Database:
                         for service_setting in service_settings:
                             config[f"{service.id}_{key}" + (f"_{service_setting.suffix}" if service_setting.suffix > 0 else "")] = (
                                 service_setting.value
-                                if methods is False
+                                if not methods
                                 else {
                                     "value": service_setting.value,
                                     "global": False,
@@ -774,7 +774,7 @@ class Database:
 
             if is_multisite:
                 servers = " ".join(service.id for service in session.query(Services).all())
-                config["SERVER_NAME"] = servers if methods is False else {"value": servers, "global": True, "method": "default"}
+                config["SERVER_NAME"] = servers if not methods else {"value": servers, "global": True, "method": "default"}
 
             return config
 
@@ -826,7 +826,7 @@ class Database:
                                 "global": value["global"],
                                 "method": value["method"],
                             }
-                            if methods is True
+                            if methods
                             else value
                         )
 

src/common/gen/Configurator.py

@@ -185,10 +185,7 @@ class Configurator:
                 config[variable] = value
             elif (
                 "CUSTOM_CONF" not in variable
-                and not variable.startswith("PYTHON")
-                and not variable.startswith("KUBERNETES_SERVICE_")
-                and not variable.startswith("KUBERNETES_PORT_")
-                and not variable.startswith("SVC_")
+                and not variable.startswith(("PYTHON", "KUBERNETES_SERVICE_", "KUBERNETES_PORT_", "SVC_"))
                 and variable
                 not in (
                     "GPG_KEY",
@@ -269,17 +266,7 @@ class Configurator:
         return False, variable
 
     def __validate_plugin(self, plugin: dict) -> Tuple[bool, str]:
-        if not all(
-            key in plugin.keys()
-            for key in [
-                "id",
-                "name",
-                "description",
-                "version",
-                "stream",
-                "settings",
-            ]
-        ):
+        if not all(key in plugin for key in ("id", "name", "description", "version", "stream", "settings")):
             return (
                 False,
                 f"Missing mandatory keys for plugin {plugin.get('id', 'unknown')} (id, name, description, version, stream, settings)",
@@ -305,25 +292,14 @@ class Configurator:
                 False,
                 f"Invalid version for plugin {plugin['id']} (Must be in format \\d+\\.\\d+(\\.\\d+)?)",
             )
-        elif plugin["stream"] not in ["yes", "no", "partial"]:
+        elif plugin["stream"] not in ("yes", "no", "partial"):
             return (
                 False,
                 f"Invalid stream for plugin {plugin['id']} (Must be yes, no or partial)",
             )
 
         for setting, data in plugin["settings"].items():
-            if not all(
-                key in data.keys()
-                for key in [
-                    "context",
-                    "default",
-                    "help",
-                    "id",
-                    "label",
-                    "regex",
-                    "type",
-                ]
-            ):
+            if not all(key in data.keys() for key in ("context", "default", "help", "id", "label", "regex", "type")):
                 return (
                     False,
                     f"missing keys for setting {setting} in plugin {plugin['id']}, must have context, default, help, id, label, regex and type",
@@ -334,7 +310,7 @@ class Configurator:
                     False,
                     f"Invalid setting name for setting {setting} in plugin {plugin['id']} (Can only contain capital letters and underscores (min 1 characters and max 256))",
                 )
-            elif data["context"] not in ["global", "multisite"]:
+            elif data["context"] not in ("global", "multisite"):
                 return (
                     False,
                     f"Invalid context for setting {setting} in plugin {plugin['id']} (Must be global or multisite)",
@@ -359,7 +335,7 @@ class Configurator:
                     False,
                     f"Invalid regex for setting {setting} in plugin {plugin['id']} (Max 1024 characters)",
                 )
-            elif data["type"] not in ["password", "text", "check", "select"]:
+            elif data["type"] not in ("password", "text", "check", "select"):
                 return (
                     False,
                     f"Invalid type for setting {setting} in plugin {plugin['id']} (Must be password, text, check or select)",
@@ -380,15 +356,7 @@ class Configurator:
                     )
 
         for job in plugin.get("jobs", []):
-            if not all(
-                key in job.keys()
-                for key in [
-                    "name",
-                    "file",
-                    "every",
-                    "reload",
-                ]
-            ):
+            if not all(key in job.keys() for key in ("name", "file", "every", "reload")):
                 return (
                     False,
                     f"missing keys for job {job['name']} in plugin {plugin['id']}, must have name, file, every and reload",
@@ -404,7 +372,7 @@ class Configurator:
                     False,
                     f"Invalid file for job {job['name']} in plugin {plugin['id']} (Can only contain numbers, letters, underscores, hyphens and slashes (min 1 characters and max 256))",
                 )
-            elif job["every"] not in ["once", "minute", "hour", "day", "week"]:
+            elif job["every"] not in ("once", "minute", "hour", "day", "week"):
                 return (
                     False,
                     f"Invalid every for job {job['name']} in plugin {plugin['id']} (Must be once, minute, hour, day or week)",

src/common/gen/save_config.py

@@ -156,9 +156,7 @@ if __name__ == "__main__":
         external_plugins = args.plugins
         if not Path(sep, "usr", "sbin", "nginx").exists() and args.method == "ui":
             db = Database(logger, pool=False)
-            external_plugins = []
-            for plugin in db.get_plugins():
-                external_plugins.append(plugin)
+            external_plugins = db.get_plugins()
 
         # Check existences and permissions
         logger.info("Checking arguments ...")

src/common/gen/utils.py

@@ -1,6 +1,7 @@
 #!/usr/bin/python3
 
-from os import getegid, geteuid, stat
+from os import getegid, geteuid
+from pathlib import Path
 from stat import (
     S_IRGRP,
     S_IROTH,
@@ -18,7 +19,7 @@ from typing import List
 def has_permissions(path: str, need_permissions: List[str]) -> bool:
     uid = geteuid()
     gid = getegid()
-    statinfo = stat(path)
+    statinfo = Path(path).stat()
     permissions = {"R": False, "W": False, "X": False}
     if statinfo.st_uid == uid:
         if statinfo.st_mode & S_IRUSR:

src/scheduler/main.py

@@ -103,7 +103,7 @@ def generate_custom_configs(
         if file.is_symlink() or file.is_file():
             file.unlink()
         elif file.is_dir():
-            rmtree(str(file), ignore_errors=True)
+            rmtree(file, ignore_errors=True)
 
     if configs:
         logger.info("Generating new custom configs ...")
@@ -142,7 +142,7 @@ def generate_external_plugins(
         if file.is_symlink() or file.is_file():
             file.unlink()
         elif file.is_dir():
-            rmtree(str(file), ignore_errors=True)
+            rmtree(file, ignore_errors=True)
 
     if plugins:
         logger.info("Generating new external plugins ...")
@@ -526,19 +526,12 @@ if __name__ == "__main__":
                 else:
                     # Reload nginx
                     logger.info("Reloading nginx ...")
-                    proc = subprocess_run(
-                        [join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
-                        stdin=DEVNULL,
-                        stderr=STDOUT,
-                        env=env.copy(),
-                        check=False,
-                        stdout=PIPE
-                    )
+                    proc = subprocess_run([join(sep, "usr", "sbin", "nginx"), "-s", "reload"], stdin=DEVNULL, stderr=STDOUT, env=env.copy(), check=False, stdout=PIPE)
                     if proc.returncode == 0:
                         logger.info("Successfully sent reload signal to nginx")
                     else:
                         logger.error(
-                                f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stdout.decode('utf-8') if proc.stdout else 'no output'}",
+                            f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stdout.decode('utf-8') if proc.stdout else 'no output'}",
                         )
                     # # Stop temp nginx
                     # logger.info("Stopping temp nginx ...")

src/ui/src/ConfigFiles.py

@@ -108,13 +108,13 @@ class ConfigFiles:
             if path.is_file():
                 path.unlink()
             elif path.is_dir():
-                rmtree(str(path), ignore_errors=False)
+                rmtree(path, ignore_errors=False)
             else:
                 path = Path(f"{path}.conf")
                 if path.is_file():
                     path.unlink()
                 else:
-                    rmtree(str(path), ignore_errors=False)
+                    rmtree(path, ignore_errors=False)
         except OSError:
             return f"Could not delete {path}", 1
 

src/ui/utils.py

@@ -68,7 +68,7 @@ def path_to_dict(
                 "path": join(
                     path,
                     type_lower,
-                    conf["service_id"] if conf["service_id"] else "",
+                    conf["service_id"] or "",
                     f"{conf['name']}.conf",
                 ),
                 "can_edit": conf["method"] == "ui",
@@ -111,7 +111,7 @@ def path_to_dict(
                 "type": "file",
                 "path": join(
                     path,
-                    conf["service_id"] if conf["service_id"] else "",
+                    conf["service_id"] or "",
                     conf["file_name"],
                 ),
                 "can_edit": False,

tests/core/antibot/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/authbasic/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/badbehavior/index.html

@@ -1 +1 @@
-ready
+ready

tests/core/badbehavior/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/blacklist/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/brotli/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/bunkernet/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/bwcli/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/clientcache/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/cors/main.py

@@ -14,7 +14,7 @@ try:
     ready = False
     retries = 0
     while not ready:
-        with suppress(RequestException):      
+        with suppress(RequestException):
             resp = get(
                 f"http{'s' if ssl else ''}://www.example.com/ready",
                 headers={"Host": "www.example.com"},

tests/core/cors/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/country/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/customcert/main.py

@@ -5,17 +5,11 @@ from traceback import format_exc
 from time import sleep
 
 try:
-
     ready = False
     retries = 0
     while not ready:
-        with suppress(RequestException):      
-            resp = get(
-                f"http://www.example.com/ready",
-                headers={"Host": "www.example.com"},
-                verify=False,
-                allow_redirects=True
-            )
+        with suppress(RequestException):
+            resp = get("http://www.example.com/ready", headers={"Host": "www.example.com"}, verify=False, allow_redirects=True)
             status_code = resp.status_code
             text = resp.text
 

tests/core/customcert/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/db/main.py

@@ -180,7 +180,7 @@ try:
 
         for global_value in global_values:
             if global_value.setting_id == "API_LISTEN_IP":
-                    continue
+                continue
             if global_value.setting_id in global_settings:
                 if global_value.value != global_settings[global_value.setting_id]["value"]:
                     print(

tests/core/db/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/dnsbl/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/errors/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/greylist/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/gzip/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/headers/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/inject/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/limit/main.py

@@ -9,7 +9,6 @@ from time import sleep
 from requests import get
 
 try:
-
     ready = False
     retries = 0
     while not ready:

tests/core/limit/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/misc/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/modsecurity/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/redirect/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/redis/main.py

@@ -17,7 +17,6 @@ fastapi_proc = None
 ip_to_check = "1.0.0.3" if getenv("TEST_TYPE", "docker") == "docker" else "127.0.0.1"
 
 try:
-
     ready = False
     retries = 0
     while not ready:

tests/core/redis/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/reversescan/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/selfsigned/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/sessions/main.py

@@ -1,6 +1,6 @@
 from contextlib import suppress
 from os import getenv
-from subprocess import PIPE, run
+from subprocess import run
 from requests import get, post
 from requests.exceptions import RequestException
 from selenium import webdriver

tests/core/sessions/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/core/whitelist/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}

tests/ui/main.py

@@ -4,7 +4,6 @@ from functools import partial
 from os import getenv, listdir
 from os.path import join
 from pathlib import Path
-from subprocess import PIPE, run
 from time import sleep
 from traceback import format_exc
 from typing import List, Union
@@ -76,6 +75,7 @@ def assert_button_click(driver, button: Union[str, WebElement]):
             clicked = True
     return clicked
 
+
 def assert_alert_message(driver, message: str):
     safe_get_element(driver, By.XPATH, "//button[@data-flash-sidebar-open='']")
 
@@ -164,11 +164,7 @@ def access_page(
         )
 
 
-driver_func = partial(
-        webdriver.Firefox,
-        service=Service(log_output="./geckodriver.log"),
-        options=firefox_options
-    )
+driver_func = partial(webdriver.Firefox, service=Service(log_output="./geckodriver.log"), options=firefox_options)
 if TEST_TYPE == "dev":
     driver_func = partial(
         webdriver.Firefox,

tests/ui/ready.conf

@@ -5,4 +5,4 @@ location /ready {
         ngx.flush(true)
         ngx.exit(ngx.HTTP_OK)
     }
-}
+}