jobs - edit referrers and user-agents data and init work on autoconf integration
This commit is contained in:
parent
d12369c900
commit
5f845680ff
|
@ -9,9 +9,9 @@ class Config :
|
|||
self.__swarm = swarm
|
||||
self.__api = api
|
||||
|
||||
def __jobs(self, type) :
|
||||
utils.log("[*] Starting jobs (type = " + type + ") ...")
|
||||
proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/entrypoint/" + type + "-jobs.sh", "nginx"], capture_output=True)
|
||||
def __jobs(self) :
|
||||
utils.log("[*] Starting jobs")
|
||||
proc = subprocess.run(["/bin/su", "-c", "/opt/bunkerized-nginx/entrypoint/jobs.sh", "nginx"], capture_output=True)
|
||||
stdout = proc.stdout.decode("ascii")
|
||||
stderr = proc.stderr.decode("ascii")
|
||||
if len(stdout) > 1 :
|
||||
|
@ -71,7 +71,7 @@ class Config :
|
|||
# We're done
|
||||
if proc.returncode == 0 :
|
||||
if self.__swarm :
|
||||
return self.__jobs("pre")
|
||||
return self.__jobs()
|
||||
return True
|
||||
utils.log("[!] Error while generating site config for " + env["SERVER_NAME"] + " : return code = " + str(proc.returncode))
|
||||
|
||||
|
@ -80,11 +80,7 @@ class Config :
|
|||
return False
|
||||
|
||||
def reload(self, instances) :
|
||||
if self.__api_call(instances, "/reload") :
|
||||
if self.__swarm :
|
||||
return self.__jobs("post")
|
||||
return True
|
||||
return False
|
||||
return self.__api_call(instances, "/reload")
|
||||
|
||||
def __ping(self, instances) :
|
||||
return self.__api_call(instances, "/ping")
|
||||
|
|
|
@ -1,20 +1,19 @@
|
|||
FROM alpine
|
||||
|
||||
COPY autoconf/dependencies.sh /tmp
|
||||
RUN chmod +x /tmp/dependencies.sh && \
|
||||
/tmp/dependencies.sh && \
|
||||
rm -f /tmp/dependencies.sh && \
|
||||
mkdir /opt/bunkerized-nginx
|
||||
|
||||
COPY gen/ /opt/bunkerized-nginx/gen
|
||||
COPY entrypoint/ /opt/bunkerized-nginx/entrypoint
|
||||
COPY confs/global/ /opt/bunkerized-nginx/confs/global
|
||||
COPY confs/site/ /opt/bunkerized-nginx/confs/site
|
||||
COPY scripts/ /opt/bunkerized-nginx/scripts
|
||||
COPY jobs/ /opt/bunkerized-nginx/jobs
|
||||
COPY settings.json /opt/bunkerized-nginx/
|
||||
COPY misc/cron /etc/crontabs/nginx
|
||||
COPY autoconf/* /opt/bunkerized-nginx/entrypoint/
|
||||
|
||||
RUN apk add --no-cache py3-pip bash certbot curl openssl && \
|
||||
pip3 install -r /opt/bunkerized-nginx/gen/requirements.txt && \
|
||||
pip3 install -r /opt/bunkerized-nginx/entrypoint/requirements.txt && \
|
||||
pip3 install -r /opt/bunkerized-nginx/jobs/requirements.txt
|
||||
|
||||
COPY autoconf/prepare.sh /tmp
|
||||
RUN chmod +x /tmp/prepare.sh && \
|
||||
/tmp/prepare.sh && \
|
||||
|
|
|
@ -1,5 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# install dependencies
|
||||
apk add py3-pip bash certbot curl openssl
|
||||
pip3 install docker requests jinja2
|
|
@ -14,6 +14,8 @@ find /opt/bunkerized-nginx -type f -exec chmod 0740 {} \;
|
|||
find /opt/bunkerized-nginx -type d -exec chmod 0750 {} \;
|
||||
chmod ugo+x /opt/bunkerized-nginx/entrypoint/* /opt/bunkerized-nginx/scripts/*
|
||||
chmod ugo+x /opt/bunkerized-nginx/gen/main.py
|
||||
chmod ugo+x /opt/bunkerized-nginx/jobs/main.py
|
||||
chmod ugo+x /opt/bunkerized-nginx/jobs/reload.py
|
||||
chmod 770 /opt/bunkerized-nginx
|
||||
chmod 440 /opt/bunkerized-nginx/settings.json
|
||||
|
||||
|
|
|
@ -1,19 +0,0 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import sys, socket, os
|
||||
|
||||
if not os.path.exists("/tmp/autoconf.sock") :
|
||||
sys.exit(1)
|
||||
|
||||
try :
|
||||
client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
client.connect("/tmp/autoconf.sock")
|
||||
client.send("reload".encode("utf-8"))
|
||||
data = client.recv(512)
|
||||
client.close()
|
||||
if not data or data.decode("utf-8") != "ok" :
|
||||
sys.exit(3)
|
||||
except Exception as e :
|
||||
sys.exit(2)
|
||||
|
||||
sys.exit(0)
|
|
@ -0,0 +1,3 @@
|
|||
docker
|
||||
requests
|
||||
jinja2
|
11
jobs/Job.py
11
jobs/Job.py
|
@ -50,15 +50,17 @@ class Job(abc.ABC) :
|
|||
for url in self.__data :
|
||||
data = self.__download_data(url)
|
||||
for chunk in data :
|
||||
if self.__type == "line" and not re.match(self.__regex, chunk.decode("utf-8")) :
|
||||
continue
|
||||
count += 1
|
||||
if self.__type == "line" :
|
||||
if not re.match(self.__regex, chunk.decode("utf-8")) :
|
||||
continue
|
||||
chunk = self.__edit(chunk)
|
||||
if self.__redis == None :
|
||||
if self.__type == "line" :
|
||||
chunk += b"\n"
|
||||
file.write(chunk)
|
||||
else :
|
||||
pipe.set(self.__name + "_" + chunk, "1")
|
||||
count += 1
|
||||
|
||||
if self.__redis == None :
|
||||
file.close()
|
||||
|
@ -89,6 +91,9 @@ class Job(abc.ABC) :
|
|||
if proc.returncode != 0 :
|
||||
raise Exception("error code " + str(proc.returncode))
|
||||
|
||||
def __edit(self, chunk) :
|
||||
return chunk
|
||||
|
||||
def __from_cache(self) :
|
||||
if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
|
||||
return False
|
||||
|
|
|
@ -9,3 +9,6 @@ class Referrers(Job) :
|
|||
type = "line"
|
||||
regex = r"^.+$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def __edit(self, chunk) :
|
||||
return chunk.replace(b".", b"%.").replace(b"-", b"%-")
|
||||
|
|
|
@ -9,3 +9,6 @@ class UserAgents(Job) :
|
|||
type = "line"
|
||||
regex = r"^.+$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def __edit(self, chunk) :
|
||||
return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")
|
||||
|
|
|
@ -59,10 +59,14 @@ if __name__ == "__main__" :
|
|||
# TODO : only reload if needed
|
||||
do_reload = True
|
||||
if do_reload :
|
||||
if not reload() :
|
||||
ret = reload()
|
||||
if ret == 0 :
|
||||
print("[*] Reload operation successfully executed")
|
||||
elif ret == 1 :
|
||||
print("[!] Error while doing reload operation")
|
||||
sys.exit(1)
|
||||
print("[*] Reload operation successfully executed")
|
||||
elif ret == 2 :
|
||||
print("[*] Skipped reload operation because nginx is not running")
|
||||
|
||||
# Done
|
||||
sys.exit(0)
|
||||
|
|
|
@ -11,8 +11,8 @@ def reload() :
|
|||
print(proc.stdout.decode("ascii"))
|
||||
if len(proc.stderr.decode("ascii")) > 1 :
|
||||
print(proc.stderr.decode("ascii"))
|
||||
return False
|
||||
return True
|
||||
return 0
|
||||
return 1
|
||||
|
||||
# Autoconf case (Docker, Swarm and Ingress)
|
||||
if os.path.exists("/tmp/autoconf.sock") and stat.S_ISSOCK(os.stat("/tmp/autoconf.sock")) :
|
||||
|
@ -23,17 +23,21 @@ def reload() :
|
|||
client.close()
|
||||
if not data or data.decode("utf-8") != "ok" :
|
||||
print("[!] Can't reload nginx (data not ok)")
|
||||
return False
|
||||
return True
|
||||
return 0
|
||||
return 1
|
||||
|
||||
return False
|
||||
return 2
|
||||
|
||||
if __name__ == "__main__" :
|
||||
try :
|
||||
print("[*] Starting reload operation ...")
|
||||
if not reload() :
|
||||
ret = reload()
|
||||
if ret == 0 :
|
||||
sys.exit(1)
|
||||
print("[*] Reload operation successfully executed")
|
||||
elif ret == 1 :
|
||||
print("[*] Reload operation successfully executed")
|
||||
elif ret == 2 :
|
||||
print("[*] Skipped reload operation because nginx is not running")
|
||||
sys.exit(0)
|
||||
except :
|
||||
print("[!] Can't reload nginx (exception)")
|
||||
|
|
Loading…
Reference in New Issue