librewolf
/
bunkerized-nginx
mirror of https://github.com/bunkerity/bunkerized-nginx
4
0
Fork 1
Code Issues Projects Releases Wiki Activity

jobs - improved log and reload management

This commit is contained in:
bunkerity 2021-07-23 17:28:05 +02:00
parent ef8969e2cf
commit 50c279617b
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
8 changed files with 72 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
entrypoint/jobs.sh
View File

@ -20,11 +20,6 @@ if [ "$files" != "" ] ; then
SELF_SIGNED_SSL_OU="$(sed -nE 's/^SELF_SIGNED_SSL_OU=(.*)$/\1/p' $file)"
SELF_SIGNED_SSL_CN="$(sed -nE 's/^SELF_SIGNED_SSL_CN=(.*)$/\1/p' $file)"
/opt/bunkerized-nginx/jobs/main.py --name self-signed-cert --dst_cert "${dest}self-cert.pem" --dst_key "${dest}self-key.pem" --expiry "$SELF_SIGNED_SSL_EXPIRY" --subj "/C=$SELF_SIGNED_SSL_COUNTRY/ST=$SELF_SIGNED_SSL_STATE/L=$SELF_SIGNED_SSL_CITY/O=$SELF_SIGNED_SSL_ORG/OU=$SELF_SIGNED_SSL_OU/CN=$SELF_SIGNED_SSL_CN"
if [ $? -eq 0 ] ; then
echo "[*] Generated self-signed certificate ${dest}self-cert.pem with key ${dest}self-key.pem"
else
echo "[!] Error while generating self-signed certificate"
fi
done
fi
@ -38,11 +33,6 @@ if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$(has_value GENERATE_SEL
SELF_SIGNED_SSL_OU="IT"
SELF_SIGNED_SSL_CN="www.yourdomain.com"
/opt/bunkerized-nginx/jobs/main.py --name self-signed-cert --dst_cert "/etc/nginx/default-cert.pem" --dst_key "/etc/nginx/default-key.pem" --expiry "$SELF_SIGNED_SSL_EXPIRY" --subj "/C=$SELF_SIGNED_SSL_COUNTRY/ST=$SELF_SIGNED_SSL_STATE/L=$SELF_SIGNED_SSL_CITY/O=$SELF_SIGNED_SSL_ORG/OU=$SELF_SIGNED_SSL_OU/CN=$SELF_SIGNED_SSL_CN"
if [ $? -eq 0 ] ; then
echo "[*] Generated self-signed certificate for default server"
else
echo "[!] Error while generating self-signed certificate for default server"
fi
fi
# certbot
@ -64,11 +54,6 @@ if [ "$files" != "" ] ; then
else
/opt/bunkerized-nginx/jobs/main.py --name certbot-new --domain "$(echo -n $SERVER_NAME | sed 's/ /,/g')" --email "$EMAIL_LETS_ENCRYPT" --staging
fi
if [ $? -eq 0 ] ; then
echo "[*] Certbot new successfully executed for domain(s) $(echo -n $SERVER_NAME | sed 's/ /,/g')"
else
echo "[*] Error while executing certbot new : $certbot_output"
fi
done
fi

10
jobs/CertbotNew.py
View File

@ -1,5 +1,7 @@
from Job import Job
from logger import log
class CertbotNew(Job) :
def __init__(self, redis_host=None, copy_cache=False, domain="", email="", staging=False) :
@ -8,4 +10,12 @@ class CertbotNew(Job) :
if staging :
data.append("--staging")
type = "exec"
self.__domain = domain
super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache)
def _callback(self, success) :
if success :
log("certbot-new", "INFO", "generated certificate for domain(s) " + self.__domain)
else :
log("certbot-new", "ERROR", "can't generate certificate for domain(s) " + self.__domain)

25
jobs/Job.py
View File

@ -1,5 +1,7 @@
import abc, requests, redis, os, datetime, traceback, re, shutil, enum, filecmp, subprocess
from logger import log
class JobRet(enum.Enum) :
KO = 0
OK_RELOAD = 1
@ -17,17 +19,11 @@ class Job(abc.ABC) :
try :
self._redis.echo("test")
except :
self._log("can't connect to redis host " + redis_host)
log(self._name, "ERROR", "can't connect to redis host " + redis_host)
self._type = type
self._regex = regex
self._copy_cache = copy_cache
def _log(self, data) :
when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]")
what = self._name + " - " + data + "\n"
with open("/var/log/nginx/jobs.log", "a") as f :
f.write(when + " " + what)
def run(self) :
ret = JobRet.KO
try :
@ -41,7 +37,7 @@ class Job(abc.ABC) :
elif self._type == "exec" :
return self.__exec()
except Exception as e :
self._log("exception while running job : " + traceback.format_exc())
log(self._name, "ERROR", "exception while running job : " + traceback.format_exc())
return JobRet.KO
return ret
@ -102,18 +98,23 @@ class Job(abc.ABC) :
proc = subprocess.run(self._data, capture_output=True)
stdout = proc.stdout.decode("ascii")
stderr = proc.stderr.decode("ascii")
if len(stdout) > 1 :
self._log("stdout = " + stdout)
if len(stderr) > 1 :
self._log("stderr = " + stderr)
if proc.returncode != 0 :
if len(stdout) > 1 :
log(self._name, "ERROR", "stdout = " + stdout)
if len(stderr) > 1 :
log(self._name, "ERROR", "stderr = " + stderr)
self._callback(False)
return JobRet.KO
# TODO : check if reload is needed ?
self._callback(True)
return JobRet.OK_RELOAD
def _edit(self, chunk) :
return [chunk]
def _callback(self, success) :
pass
def __from_cache(self) :
if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) :
return JobRet.KO

11
jobs/SelfSignedCert.py
View File

@ -1,9 +1,20 @@
from Job import Job
from logger import log
class SelfSignedCert(Job) :
def __init__(self, redis_host=None, copy_cache=False, dst_cert="/etc/nginx/default-cert.pem", dst_key="/etc/nginx/default-key.pem", expiry="999", subj="CN=www.example.com") :
name = "self-signed-cert"
data = ["openssl", "req", "-nodes", "-x509", "-newkey", "rsa:4096", "-keyout", dst_key, "-out", dst_cert, "-days", expiry, "-subj", subj]
type = "exec"
self.__dst_cert = dst_cert
self.__dst_key = dst_key
super().__init__(name, data, filename=None, redis_host=redis_host, type=type, copy_cache=copy_cache)
def _callback(self, success) :
if success :
log("self-signed-cert", "INFO", "generated certificate " + self.__dst_cert + " with private key " + self.__dst_key)
else :
log("self-signed-cert", "ERROR", "can't generate certificate " + self.__dst_cert + " with private key " + self.__dst_key)

6
jobs/logger.py Normal file
View File

@ -0,0 +1,6 @@
import datetime
def log(title, severity, msg) :
when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]")
what = title + " - " + severity + " - " + msg
print(when + " " + what, flush=True)

22
jobs/main.py
View File

@ -8,6 +8,7 @@ import Abusers, CertbotNew, CertbotRenew, ExitNodes, GeoIP, Proxies, Referrers,
from Job import JobRet
from reload import reload
from logger import log
JOBS = {
"abusers": Abusers.Abusers,
@ -28,6 +29,7 @@ if __name__ == "__main__" :
parser.add_argument("--name", default="", type=str, help="job to run (e.g : abusers or certbot-new or certbot-renew ...)")
parser.add_argument("--redis", default=None, type=str, help="hostname of the redis server if any")
parser.add_argument("--cache", action="store_true", help="copy data from cache if available")
parser.add_argument("--reload", action="store_true", help="reload nginx if necessary and the job is successful")
parser.add_argument("--domain", default="", type=str, help="domain(s) for certbot-new job (e.g. : www.example.com or app1.example.com,app2.example.com)")
parser.add_argument("--email", default="", type=str, help="email for certbot-new job (e.g. : contact@example.com)")
parser.add_argument("--staging", action="store_true", help="use staging server for let's encrypt instead of the production one")
@ -39,12 +41,12 @@ if __name__ == "__main__" :
# Check job name
if not args.name in JOBS :
print("[!] unknown job " + args.name)
log("job", "ERROR", "unknown job " + args.name)
sys.exit(1)
job = args.name
# Run job
print("[*] Executing job " + job)
log("job", "INFO", "executing job " + job)
ret = 0
if job == "certbot-new" :
instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache, domain=args.domain, email=args.email, staging=args.staging)
@ -54,22 +56,22 @@ if __name__ == "__main__" :
instance = JOBS[job](redis_host=args.redis, copy_cache=args.cache)
ret = instance.run()
if ret == JobRet.KO :
print("[!] Error while running job " + job)
log("job", "ERROR", "error while running job " + job)
sys.exit(1)
print("[*] Job " + job + " successfully executed")
log("job", "INFO", "job " + job + " successfully executed")
# Reload
if ret == JobRet.OK_RELOAD :
if ret == JobRet.OK_RELOAD and args.reload :
ret = reload()
if ret == 0 :
print("[*] Reload operation successfully executed")
elif ret == 1 :
print("[!] Error while doing reload operation")
log("job", "ERROR", "error while doing reload operation (job = " + job + ")")
sys.exit(1)
elif ret == 1 :
log("job", "INFO", "reload operation successfully executed (job = " + job + ")")
elif ret == 2 :
print("[*] Skipped reload operation because nginx is not running")
log("job", "INFO", "skipped reload operation because nginx is not running (job = " + job + ")")
else :
print("[*] Skipped reload operation because it's not needed")
log("job", "INFO", "skipped reload operation because it's not needed (job = " + job + ")")
# Done
sys.exit(0)

24
jobs/reload.py
View File

@ -1,16 +1,18 @@
import docker, subprocess, os, stat, sys, traceback
from logger import log
def reload() :
# Linux or single Docker use case
if os.path.isfile("/usr/sbin/nginx") and os.path.isfile("/tmp/nginx.pid") :
proc = subprocess.run(["/usr/sbin/nginx", "-s", "reload"], capture_output=True)
if proc.returncode != 0 :
print("[!] Can't reload nginx (status code = " + str(proc.returncode) + ")")
log("reload", "ERROR", "can't reload nginx (status code = " + str(proc.returncode) + ")")
if len(proc.stdout.decode("ascii")) > 1 :
print(proc.stdout.decode("ascii"))
log("reload", "ERROR", proc.stdout.decode("ascii"))
if len(proc.stderr.decode("ascii")) > 1 :
print(proc.stderr.decode("ascii"))
log("reload", "ERROR", proc.stderr.decode("ascii"))
return 0
return 1
@ -22,7 +24,7 @@ def reload() :
data = client.recv(512)
client.close()
if not data or data.decode("utf-8") != "ok" :
print("[!] Can't reload nginx (data not ok)")
log("reload", "ERROR", "can't reload nginx (data not ok)")
return 0
return 1
@ -30,16 +32,16 @@ def reload() :
if __name__ == "__main__" :
try :
print("[*] Starting reload operation ...")
#print("[*] Starting reload operation ...")
ret = reload()
if ret == 0 :
sys.exit(1)
elif ret == 1 :
print("[*] Reload operation successfully executed")
elif ret == 2 :
print("[*] Skipped reload operation because nginx is not running")
#elif ret == 1 :
#print("[*] Reload operation successfully executed")
#elif ret == 2 :
#print("[*] Skipped reload operation because nginx is not running")
sys.exit(0)
except :
print("[!] Can't reload nginx (exception)")
print(traceback.format_exc())
log("reload", "ERROR", "can't reload nginx (exception)")
log("reload", "ERROR", traceback.format_exc())
sys.exit(2)

14
misc/cron
View File

@ -1,7 +1,7 @@
15 0 * * * /opt/bunkerized-nginx/jobs/main.py --name certbot-renew >> /var/log/nginx/jobs.log 2>&1
30 0 * * * /opt/bunkerized-nginx/jobs/main.py --name user-agents >> /var/log/nginx/jobs.log 2>&1
45 0 * * * /opt/bunkerized-nginx/jobs/main.py --name referrers >> /var/log/nginx/jobs.log 2>&1
0 1 * * * /opt/bunkerized-nginx/jobs/main.py --name abusers >> /var/log/nginx/jobs.log 2>&1
0 2 * * * /opt/bunkerized-nginx/jobs/main.py --name proxies >> /var/log/nginx/jobs.log 2>&1
0 */1 * * * /opt/bunkerized-nginx/jobs/main.py --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
0 3 2 * * /opt/bunkerized-nginx/jobs/main.py --name geoip >> /var/log/nginx/jobs.log 2>&1
15 0 * * * /opt/bunkerized-nginx/jobs/main.py --reload --name certbot-renew >> /var/log/nginx/jobs.log 2>&1
30 0 * * * /opt/bunkerized-nginx/jobs/main.py --reload --name user-agents >> /var/log/nginx/jobs.log 2>&1
45 0 * * * /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
0 1 * * * /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
0 2 * * * /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
0 */1 * * * /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
0 3 2 * * /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1