fix missing import in generator, expand networks to ips in jobs and init work on a generic checker with shared dict and redis support

gen/main.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-import argparse, os, sys, shutil
+import argparse, os, sys, shutil, glob
 
 import utils
 from Configurator import Configurator

jobs/Abusers.py

@@ -1,5 +1,7 @@
 from Job import Job
 
+import re, ipaddress
+
 class Abusers(Job) :
 
 	def __init__(self, redis_host=None, copy_cache=False) :
@@ -9,3 +11,13 @@ class Abusers(Job) :
 		type = "line"
 		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def _Job__edit(self, chunk) :
+		if self.__redis != None :
+			network = chunk.decode("utf-8")
+			if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
+				ips = []
+				for ip in ipaddress.IPv4Network(network) :
+					ips.append(str(ip).encode("utf-8"))
+		return [chunk]
+

jobs/ExitNodes.py

@@ -1,5 +1,7 @@
 from Job import Job
 
+import re, ipaddress
+
 class ExitNodes(Job) :
 
 	def __init__(self, redis_host=None, copy_cache=False) :
@@ -9,3 +11,12 @@ class ExitNodes(Job) :
 		type = "line"
 		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def _Job__edit(self, chunk) :
+		if self.__redis != None :
+			network = chunk.decode("utf-8")
+			if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
+				ips = []
+				for ip in ipaddress.IPv4Network(network) :
+					ips.append(str(ip).encode("utf-8"))
+		return [chunk]

jobs/Job.py

@@ -61,13 +61,17 @@ class Job(abc.ABC) :
 				if self.__type == "line" :
 					if not re.match(self.__regex, chunk.decode("utf-8")) :
 						continue
-					chunk = self.__edit(chunk)
+					chunks = self.__edit(chunk)
 				if self.__redis == None :
 					if self.__type == "line" :
 						chunk += b"\n"
 					file.write(chunk)
 				else :
-					pipe.set(self.__name + "_" + chunk, "1")
+					if self.__type == "line" :
+						for chunk in chunks :
+							pipe.set(self.__name + "_" + chunk, "1")
+					else :
+						pipe.set(self.__name + "_" + chunk, "1")
 				count += 1
 
 		if self.__redis == None :
@@ -106,7 +110,7 @@ class Job(abc.ABC) :
 		return JobRet.OK_RELOAD
 
 	def __edit(self, chunk) :
-		return chunk
+		return [chunk]
 
 	def __from_cache(self) :
 		if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :

jobs/Proxies.py

@@ -1,5 +1,7 @@
 from Job import Job
 
+import re, ipaddress
+
 class Proxies(Job) :
 
 	def __init__(self, redis_host=None, copy_cache=False) :
@@ -9,3 +11,12 @@ class Proxies(Job) :
 		type = "line"
 		regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
 		super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
+
+	def _Job__edit(self, chunk) :
+		if self.__redis != None :
+			network = chunk.decode("utf-8")
+			if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
+				ips = []
+				for ip in ipaddress.IPv4Network(network) :
+					ips.append(str(ip).encode("utf-8"))
+		return [chunk]

lua/checker.lua

@@ -0,0 +1,48 @@
+local M		= {}
+local redis	= require "resty.redis"
+
+function M.new(self, name, data_dict, redis_client, type)
+	return selfmetatable({
+		__name = name,
+		__data_dict = data_dict,
+		__redis_client = redis_client,
+		__type = type
+	})
+end
+
+function M.check(self, data) :
+	-- without redis
+	if self.__data_dict ~= nil and redis_client == nil then
+		if self.__type == "simple" then
+			local value, flags = self.__data_dict:get(data)
+			return ~= nil
+		else if self.__type == "match" then
+			local patterns = self.__data_dict:get_keys(0)
+			for i, pattern in ipairs(patterns) do
+				if string.match(data, pattern) then
+					return true
+				end
+			end
+			return false
+		end
+
+	-- with redis
+	else if data_dict == nil and redis_client ~= nil then
+		if self.__type == "simple" then
+			local res, err = self.__redis_client:get(self.__name .. "_" .. data)
+			return res and res ~= ngx.null
+		else if self.__type == "match" then
+			local patterns = self.__redis_client:keys(self.__name .. "_*")
+			if patterns then
+				for i, pattern in ipairs(patterns) do
+					if string.match(data, pattern) do
+						return true
+					end
+				end
+			end
+		end
+	end
+
+	return false
+
+return M