From 7b9722fac4ff74252c52f235f71ebec9a9bbe3e5 Mon Sep 17 00:00:00 2001 From: bunkerity Date: Wed, 6 Oct 2021 12:13:13 +0200 Subject: [PATCH] jobs - add remote API --- entrypoint/jobs.sh | 8 ++++++++ jobs/Job.py | 22 ++++++++++++++++------ jobs/RemoteApiDatabase.py | 15 +++++++++++++++ jobs/RemoteApiRegister.py | 16 ++++++++++++++++ jobs/main.py | 10 ++++++++++ misc/cron | 1 + misc/cron-autoconf | 1 + misc/cron-linux | 1 + 8 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 jobs/RemoteApiDatabase.py create mode 100644 jobs/RemoteApiRegister.py diff --git a/entrypoint/jobs.sh b/entrypoint/jobs.sh index 974b971c..dd5b15f5 100644 --- a/entrypoint/jobs.sh +++ b/entrypoint/jobs.sh @@ -87,3 +87,11 @@ fi if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then /opt/bunkerized-nginx/jobs/main.py --name abusers --cache fi + +# remote API +if [ "$(has_value USE_REMOTE_API yes)" != "" ] ; then + /opt/bunkerized-nginx/jobs/main.py --name remote-api-register --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" + if [ $? -eq 0 ] ; then + /opt/bunkerized-nginx/jobs/main.py --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" + fi +fi diff --git a/jobs/Job.py b/jobs/Job.py index 4f39982a..f24e3992 100644 --- a/jobs/Job.py +++ b/jobs/Job.py @@ -77,7 +77,7 @@ class JobManagement() : class Job(abc.ABC) : - def __init__(self, name, data, filename=None, redis_host=None, redis_ex=86400, type="line", regex=r"^.+$", copy_cache=False) : + def __init__(self, name, data, filename=None, redis_host=None, redis_ex=86400, type="line", regex=r"^.+$", copy_cache=False, json_data=None, method="GET") : self._name = name self._data = data self._filename = filename @@ -92,11 +92,13 @@ class Job(abc.ABC) : self._type = type self._regex = regex self._copy_cache = copy_cache + self._json_data = json_data + self._method = method def run(self) : ret = JobRet.KO try : - if self._type == "line" or self._type == "file" : + if self._type in ["line", "file", "json"] : if self._copy_cache : ret = self.__from_cache() if ret != JobRet.KO : @@ -123,18 +125,18 @@ class Job(abc.ABC) : for url in self._data : data = self.__download_data(url) for chunk in data : - if self._type == "line" : + if self._type == ["line", "json"] : if not re.match(self._regex, chunk.decode("utf-8")) : continue chunks = self._edit(chunk) if self._redis == None : - if self._type == "line" : + if self._type in ["line", "json"] : for chunk in chunks : file.write(chunk + b"\n") else : file.write(chunk) else : - if self._type == "line" : + if self._type in ["line", "json"] : for chunk in chunks : pipe.set(self._name + "_" + chunk, "1", ex=self._redis_ex) else : @@ -155,11 +157,16 @@ class Job(abc.ABC) : return JobRet.KO def __download_data(self, url) : - r = requests.get(url, stream=True) + r = requests.request(self._method, url, stream=True, json=self._json_data) if not r or r.status_code != 200 : raise Exception("can't download data at " + url) if self._type == "line" : return r.iter_lines() + if self._type == "json" : + try : + return self._json(r.json()) + except : + raise Exception("can't decode json from " + url) return r.iter_content(chunk_size=8192) def __exec(self) : @@ -177,6 +184,9 @@ class Job(abc.ABC) : self._callback(True) return JobRet.OK_RELOAD + def _json(self, data) : + return data + def _edit(self, chunk) : return [chunk] diff --git a/jobs/RemoteApiDatabase.py b/jobs/RemoteApiDatabase.py new file mode 100644 index 00000000..43814222 --- /dev/null +++ b/jobs/RemoteApiDatabase.py @@ -0,0 +1,15 @@ +from Job import Job + +class RemoteApiDatabase(Job) : + + def __init__(self, server="", version="", id="", redis_host=None, copy_cache=False) : + name = "remote-api-database" + data = [server + "/db"] + filename = "remote-api.db" + type = "json" + regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$" + json_data = {"version": version, "id": id} + super().__init__(name, data, filename, type=type, redis_host=redis_host, redis_ex=redis_ex, regex=regex, copy_cache=copy_cache, json_data=json_data, method=method) + + def _json(self, data) : + return data["data"] diff --git a/jobs/RemoteApiRegister.py b/jobs/RemoteApiRegister.py new file mode 100644 index 00000000..822273c2 --- /dev/null +++ b/jobs/RemoteApiRegister.py @@ -0,0 +1,16 @@ +from Job import Job + +class RemoteApiRegister(Job) : + + def __init__(self, server="", version="") : + name = "remote-api-register" + data = [server + "/register"] + filename = "machine.id" + type = "json" + regex = r"^[0-9a-f]{256}$" + json_data = {"version": version} + method = "POST" + super().__init__(name, data, filename, type=type, regex=regex, copy_cache=True, json_data=json_data, method=method) + + def _json(self, data) : + return data["data"] diff --git a/jobs/main.py b/jobs/main.py index 55fa7774..e096bb48 100644 --- a/jobs/main.py +++ b/jobs/main.py @@ -17,8 +17,11 @@ JOBS = { "geoip": GeoIP.GeoIP, "proxies": Proxies.Proxies, "referrers": Referrers.Referrers, + "remote-api-database": RemoteApiDatabase.RemoteApiDatabase, + "remote-api-register": RemoteApiRegister.RemoteApiRegister, "self-signed-cert": SelfSignedCert.SelfSignedCert, "user-agents": UserAgents.UserAgents + } if __name__ == "__main__" : @@ -36,6 +39,9 @@ if __name__ == "__main__" : parser.add_argument("--dst_key", default="", type=str, help="key path for self-signed-cert job (e.g. : /etc/nginx/default-key.pem)") parser.add_argument("--expiry", default="", type=str, help="number of validity days for self-signed-cert job (e.g. : 365)") parser.add_argument("--subj", default="", type=str, help="certificate subject for self-signed-cert job (e.g. : OU=X/CN=Y...)") + parser.add_argument("--server", default="", type=str, help="address of the server for remote-api jobs") + parser.add_argument("--id", default="", type=str, help="machine id for remote-api jobs") + parser.add_argument("--version", default="", type=str, help="bunkerized-nginx version for remote-api jobs") args = parser.parse_args() # Check job name @@ -68,6 +74,10 @@ if __name__ == "__main__" : instance = JOBS[job](redis_host=redis_host, copy_cache=args.cache, domain=args.domain, email=args.email, staging=args.staging) elif job == "self-signed-cert" : instance = JOBS[job](redis_host=redis_host, copy_cache=args.cache, dst_cert=args.dst_cert, dst_key=args.dst_key, expiry=args.expiry, subj=args.subj) + elif job == "remote-api-database" : + instance = JOBS[job](server=args.server, version=args.version, id=args.id, redis_host=redis_host, copy_cache=args.cache) + elif job == "remote-api-register" : + instance = JOBS[job](server=args.server, version=args.version) else : instance = JOBS[job](redis_host=redis_host, copy_cache=args.cache) ret = instance.run() diff --git a/misc/cron b/misc/cron index d1bafbe1..c63ebcdf 100644 --- a/misc/cron +++ b/misc/cron @@ -3,5 +3,6 @@ 45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1 0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1 0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1 +30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1 0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1 0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1 diff --git a/misc/cron-autoconf b/misc/cron-autoconf index 9d597d4f..f5296b7c 100644 --- a/misc/cron-autoconf +++ b/misc/cron-autoconf @@ -3,5 +3,6 @@ 45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name referrers" nginx >> /var/log/nginx/jobs.log 2>&1 0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name abusers" nginx >> /var/log/nginx/jobs.log 2>&1 0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name proxies" nginx >> /var/log/nginx/jobs.log 2>&1 +30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server $(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2) --version $(cat /opt/bunkerized-nginx/VERSION) --id $(cat /opt/bunkerized-nginx/machine.id)" nginx >> /var/log/nginx/jobs.log 2>&1 0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name exit-nodes" nginx >> /var/log/nginx/jobs.log 2>&1 0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name geoip" nginx >> /var/log/nginx/jobs.log 2>&1 diff --git a/misc/cron-linux b/misc/cron-linux index f4274795..85f86cc7 100644 --- a/misc/cron-linux +++ b/misc/cron-linux @@ -3,5 +3,6 @@ 45 0 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1 0 1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1 0 2 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1 +30 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1 0 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1 0 3 2 * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1