jobs - python stubs

jobs/Abusers.py

@@ -0,0 +1,11 @@
+from Job import Job
+
+class Abusers(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "abusers"
+		data = ["https://iplists.firehol.org/files/firehol_abusers_30d.netset"]
+		filename = "abusers.list"
+		type = "line"
+		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)

jobs/CertbotNew.py

@@ -0,0 +1,9 @@
+from Job import Job
+
+class CertbotRenew(Job) :
+
+	def __init__(self, redis_host=None, domain="", email="") :
+		name = "certbot-new"
+		data = ["certbot", "certonly", "--webroot", "-w", "/opt/bunkerized-nginx/acme-challenge", "-n", "-d", domain, "--email", email, "--agree-tos"]
+		type = "exec"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type)

jobs/CertbotRenew.py

@@ -0,0 +1,9 @@
+from Job import Job
+
+class CertbotRenew(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "certbot-renew"
+		data = ["certbot", "renew", "--deploy-hook", "/opt/bunkerized-nginx/jobs/reload.py"]
+		type = "exec"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type)

jobs/ExitNodes.py

@@ -3,9 +3,9 @@ from Job import Job
 class ExitNodes(Job) :
 
 	def __init__(self, redis_host=None) :
-		name = "ExitNodes"
-		urls = ["https://iplists.firehol.org/files/tor_exits.ipset"]
+		name = "exit-nodes"
+		data = ["https://iplists.firehol.org/files/tor_exits.ipset"]
 		filename = "tor-exit-nodes.list"
 		type = "line"
 		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
-		super().__init__(name, urls, filename, redis_host=redis_host, type=type, regex=regex)
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)

jobs/GeoIP.py

@@ -0,0 +1,27 @@
+from Job import Job
+
+import datetime, gzip
+
+class GeoIP(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "geoip"
+		data = ["https://download.db-ip.com/free/dbip-country-lite-" + datetime.datetime.today().strftime("%Y-%m") + ".mmdb.gz"]
+		filename = "geoip.mmdb.gz"
+		type = "file"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)
+
+	def run(self) :
+		super().run()
+		count = 0
+		with gzip.open("/etc/nginx/geoip.mmdb.gz", "rb") as f :
+			with open("/tmp/geoip.mmdb", "w") as f2
+				while True :
+					chunk = f.read(8192)
+					if not chunk :
+						break
+					f2.write(chunk)
+					count += 1
+		shutil.copyfile("/tmp/geoip.mmdb", "/etc/nginx/geoip.mmdb")
+		os.remove("/tmp/geoip.mmdb")
+		os.remove("/etc/nginx/geoip.mmdb.gz")

jobs/Job.py

@@ -2,9 +2,9 @@ import abc, requests, redis, os
 
 class Job(abc.ABC) :
 
-	def __init__(self, name, urls, filename, redis_host=None, type="line", regex=r"^.*$") :
+	def __init__(self, name, data, filename, redis_host=None, type="line", regex=r"^.+$") :
 		self.__name = name
-		self.__urls = urls
+		self.__data = data
 		self.__filename = filename
 		self.__redis = None
 		if redis_host != None :
@@ -13,6 +13,12 @@ class Job(abc.ABC) :
 		self.__regex = regex
 
 	def run(self) :
+		if self.__type == "line" or self.__type == "file" :
+			self.__external()
+		elif self.__type == "exec" :
+			self.__exec()
+
+	def __external(self) :
 		if self.__redis == None :
 			if os.path.isfile("/tmp/" + self.__filename) :
 				os.remove("/tmp/" + self.__filename)
@@ -22,7 +28,7 @@ class Job(abc.ABC) :
 			pipe = self.__redis.pipeline()
 
 		count = 0
-		for url in self.__urls :
+		for url in self.__data :
 			data = self.__download_data(url)
 			for chunk in data :
 				if self.__type == "line" and not re.match(self.__regex, chunk) :
@@ -52,3 +58,6 @@ class Job(abc.ABC) :
 		if self.__type == "line" :
 			return r.iter_lines()
 		return r.iter_content(chunk_size=8192)
+
+	def __exec(self) :
+		proc = subprocess.run(self.__data, capture_output=True)

jobs/Proxies.py

@@ -0,0 +1,11 @@
+from Job import Job
+
+class Proxies(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "proxies"
+		data = ["https://iplists.firehol.org/files/firehol_proxies.netset"]
+		filename = "proxies.list"
+		type = "line"
+		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)

jobs/Referrers.py

@@ -0,0 +1,11 @@
+from Job import Job
+
+class Referrers(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "referrers"
+		data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list"]
+		filename = "referrers.list"
+		type = "line"
+		regex = r"^.+$"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)

jobs/UserAgents.py

@@ -0,0 +1,11 @@
+from Job import Job
+
+class UserAgents(Job) :
+
+	def __init__(self, redis_host=None) :
+		name = "user-agents"
+		data = ["https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list", "https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt"]
+		filename = "user-agents.list"
+		type = "line"
+		regex = r"^.+$"
+		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex)