jobs - cache management
This commit is contained in:
bunkerity
parent
2fca4cd014
commit
71741b2d34
|
|
@ -2,10 +2,10 @@ from Job import Job
|
|||
|
||||
class Abusers(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=False) :
|
||||
name = "abusers"
|
||||
data = ["https://iplists.firehol.org/files/firehol_abusers_30d.netset"]
|
||||
filename = "abusers.list"
|
||||
type = "line"
|
||||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ from Job import Job
|
|||
|
||||
class CertbotRenew(Job) :
|
||||
|
||||
def __init__(self, redis_host=None, domain="", email="") :
|
||||
def __init__(self, redis_host=None, copy_cache=False, domain="", email="") :
|
||||
name = "certbot-new"
|
||||
data = ["certbot", "certonly", "--webroot", "-w", "/opt/bunkerized-nginx/acme-challenge", "-n", "-d", domain, "--email", email, "--agree-tos"]
|
||||
type = "exec"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -2,8 +2,8 @@ from Job import Job
|
|||
|
||||
class CertbotRenew(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=False) :
|
||||
name = "certbot-renew"
|
||||
data = ["certbot", "renew", "--deploy-hook", "/opt/bunkerized-nginx/jobs/reload.py"]
|
||||
type = "exec"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ from Job import Job
|
|||
|
||||
class ExitNodes(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=copy_cache) :
|
||||
name = "exit-nodes"
|
||||
data = ["https://iplists.firehol.org/files/tor_exits.ipset"]
|
||||
filename = "tor-exit-nodes.list"
|
||||
type = "line"
|
||||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@ import datetime, gzip
|
|||
|
||||
class GeoIP(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=False) :
|
||||
name = "geoip"
|
||||
data = ["https://download.db-ip.com/free/dbip-country-lite-" + datetime.datetime.today().strftime("%Y-%m") + ".mmdb.gz"]
|
||||
filename = "geoip.mmdb.gz"
|
||||
type = "file"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def run(self) :
|
||||
super().run()
|
||||
|
|
|
|||
34
jobs/Job.py
34
jobs/Job.py
|
|
@ -2,7 +2,7 @@ import abc, requests, redis, os, datetime, traceback
|
|||
|
||||
class Job(abc.ABC) :
|
||||
|
||||
def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$") :
|
||||
def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) :
|
||||
self.__name = name
|
||||
self.__data = data
|
||||
self.__filename = filename
|
||||
|
|
@ -15,6 +15,7 @@ class Job(abc.ABC) :
|
|||
self.__log("can't connect to redis host " + redis_host)
|
||||
self.__type = type
|
||||
self.__regex = regex
|
||||
self.__copy_cache = copy_cache
|
||||
|
||||
def __log(self, data) :
|
||||
when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]")
|
||||
|
|
@ -25,7 +26,10 @@ class Job(abc.ABC) :
|
|||
def run(self) :
|
||||
try :
|
||||
if self.__type == "line" or self.__type == "file" :
|
||||
if self.__copy_cache and self.__from_cache() :
|
||||
return True
|
||||
self.__external()
|
||||
self.__to_cache()
|
||||
elif self.__type == "exec" :
|
||||
self.__exec()
|
||||
except Exception as e :
|
||||
|
|
@ -84,3 +88,31 @@ class Job(abc.ABC) :
|
|||
self.__log("stderr = " + stderr)
|
||||
if proc.returncode != 0 :
|
||||
raise Exception("error code " + str(proc.returncode))
|
||||
|
||||
def __from_cache(self) :
|
||||
if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
|
||||
return False
|
||||
if self.__redis == None or self.__type == "file" :
|
||||
shutil.copyfile("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename)
|
||||
elif self.__redis != None and self.__type == "line" :
|
||||
self.__redis.del(self.__redis.keys(self.__name + "_*"))
|
||||
with open("/opt/bunkerized-nginx/cache/" + self.__filename) as f :
|
||||
pipe = self.__redis.pipeline()
|
||||
while True :
|
||||
line = f.readline()
|
||||
if not line :
|
||||
break
|
||||
line = line.strip()
|
||||
pipe.set(self.__name + "_" + line, "1")
|
||||
pipe.execute()
|
||||
return True
|
||||
|
||||
def __to_cache(self) :
|
||||
if self.__redis == None or self.__type == "file" :
|
||||
shutil.copyfile("/etc/nginx/" + self.__filename, "/opt/bunkerized-nginx/cache/" + self.__filename)
|
||||
elif self.__redis != None and self.__type == "line" :
|
||||
if os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
|
||||
os.remove("/opt/bunkerized-nginx/cache/" + self.__filename)
|
||||
with open("/opt/bunkerized-nginx/cache/" + self.__filename, "a") as f :
|
||||
for key in self.__redis.keys(self.__name + "_*") :
|
||||
f.write(self.__redis.get(key) + "\n")
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ from Job import Job
|
|||
|
||||
class Proxies(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=False) :
|
||||
name = "proxies"
|
||||
data = ["https://iplists.firehol.org/files/firehol_proxies.netset"]
|
||||
filename = "proxies.list"
|
||||
type = "line"
|
||||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex; copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ from Job import Job
|
|||
|
||||
class Referrers(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=False) :
|
||||
name = "referrers"
|
||||
data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list"]
|
||||
filename = "referrers.list"
|
||||
type = "line"
|
||||
regex = r"^.+$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -2,10 +2,10 @@ from Job import Job
|
|||
|
||||
class UserAgents(Job) :
|
||||
|
||||
def __init__(self, redis_host=None) :
|
||||
def __init__(self, redis_host=None, copy_cache=False) :
|
||||
name = "user-agents"
|
||||
data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list", "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt"]
|
||||
filename = "user-agents.list"
|
||||
type = "line"
|
||||
regex = r"^.+$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
requests
|
||||
redis
|
||||
Loading…
Reference in New Issue