edit visibility of Job members and integration of a generic checker for nginx
This commit is contained in:
bunkerity
parent
9a207dfdc5
commit
0dc2a5ec25
|
|
@ -69,6 +69,7 @@ local iputils = require "resty.iputils"
|
|||
local behavior = require "behavior"
|
||||
local logger = require "logger"
|
||||
local redis = require "resty.redis"
|
||||
local checker = require "checker"
|
||||
|
||||
-- user variables
|
||||
local antibot_uri = "{{ ANTIBOT_URI }}"
|
||||
|
|
@ -157,8 +158,8 @@ end
|
|||
|
||||
-- check if IP is in proxies list
|
||||
if use_proxies then
|
||||
local value, flags = ngx.shared.proxies_data:get(iputils.ip2bin(ngx.var.remote_addr))
|
||||
if value ~= nil then
|
||||
local checker = checker:new("proxies", ngx.shared.proxies_data, redis_client, "simple")
|
||||
if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
|
||||
logger.log(ngx.WARN, "PROXIES", "IP " .. ngx.var.remote_addr .. " is in proxies list")
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
|
|
@ -166,8 +167,8 @@ end
|
|||
|
||||
-- check if IP is in abusers list
|
||||
if use_abusers then
|
||||
local value, flags = ngx.shared.abusers_data:get(iputils.ip2bin(ngx.var.remote_addr))
|
||||
if value ~= nil then
|
||||
local checker = checker:new("abusers", ngx.shared.abusers_data, redis_client, "simple")
|
||||
if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
|
||||
logger.log(ngx.WARN, "ABUSERS", "IP " .. ngx.var.remote_addr .. " is in abusers list")
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
|
|
@ -175,8 +176,8 @@ end
|
|||
|
||||
-- check if IP is in TOR exit nodes list
|
||||
if use_tor_exit_nodes then
|
||||
local value, flags = ngx.shared.tor_exit_nodes_data:get(iputils.ip2bin(ngx.var.remote_addr))
|
||||
if value ~= nil then
|
||||
local checker = checker:new("exit-nodes", ngx.shared.tor_exit_nodes_data, redis_client, "simple")
|
||||
if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
|
||||
logger.log(ngx.WARN, "TOR", "IP " .. ngx.var.remote_addr .. " is in TOR exit nodes list")
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
|
|
@ -193,23 +194,9 @@ if use_user_agents and ngx.var.http_user_agent ~= nil then
|
|||
end
|
||||
end
|
||||
if not whitelisted then
|
||||
local value, flags = ngx.shared.user_agents_cache:get(ngx.var.http_user_agent)
|
||||
if value == nil then
|
||||
local patterns = ngx.shared.user_agents_data:get_keys(0)
|
||||
for i, pattern in ipairs(patterns) do
|
||||
if string.match(ngx.var.http_user_agent, pattern) then
|
||||
value = "ko"
|
||||
ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ko", 86400)
|
||||
break
|
||||
end
|
||||
end
|
||||
if value == nil then
|
||||
value = "ok"
|
||||
ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ok", 86400)
|
||||
end
|
||||
end
|
||||
if value == "ko" then
|
||||
logger.log(ngx.WARN, "USER-AGENT", "User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
|
||||
local checker = checker:new("user-agents", ngx.shared.user_agents_data, redis_client, "match")
|
||||
if checker:check(ngx.var.http_user_agent) then
|
||||
logger.log(ngx.WARN, "USER-AGENTS", "User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
end
|
||||
|
|
@ -217,23 +204,9 @@ end
|
|||
|
||||
-- check if referrer is allowed
|
||||
if use_referrer and ngx.var.http_referer ~= nil then
|
||||
local value, flags = ngx.shared.referrers_cache:get(ngx.var.http_referer)
|
||||
if value == nil then
|
||||
local patterns = ngx.shared.referrers_data:get_keys(0)
|
||||
for i, pattern in ipairs(patterns) do
|
||||
if string.match(ngx.var.http_referer, pattern) then
|
||||
value = "ko"
|
||||
ngx.shared.referrers_cache:set(ngx.var.http_referer, "ko", 86400)
|
||||
break
|
||||
end
|
||||
end
|
||||
if value == nil then
|
||||
value = "ok"
|
||||
ngx.shared.referrers_cache:set(ngx.var.http_referer, "ok", 86400)
|
||||
end
|
||||
end
|
||||
if value == "ko" then
|
||||
logger.log(ngx.WARN, "REFERRER", "Referrer " .. ngx.var.http_referer .. " is blacklisted")
|
||||
local checker = checker:new("referrers", ngx.shared.referrers_data, redis_client, "match")
|
||||
if checker:check(ngx.var.http_referer) then
|
||||
logger.log(ngx.WARN, "REFERRERS", "Referrer " .. ngx.var.http_referer .. " is blacklisted")
|
||||
ngx.exit(ngx.HTTP_FORBIDDEN)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ class Abusers(Job) :
|
|||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def _Job__edit(self, chunk) :
|
||||
if self.__redis != None :
|
||||
def _edit(self, chunk) :
|
||||
if self._redis != None :
|
||||
network = chunk.decode("utf-8")
|
||||
if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
|
||||
ips = []
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ class ExitNodes(Job) :
|
|||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def _Job__edit(self, chunk) :
|
||||
if self.__redis != None :
|
||||
def _edit(self, chunk) :
|
||||
if self._redis != None :
|
||||
network = chunk.decode("utf-8")
|
||||
if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
|
||||
ips = []
|
||||
|
|
|
|||
122
jobs/Job.py
122
jobs/Job.py
|
|
@ -8,37 +8,37 @@ class JobRet(enum.Enum) :
|
|||
class Job(abc.ABC) :
|
||||
|
||||
def __init__(self, name, data, filename=None, redis_host=None, type="line", regex=r"^.+$", copy_cache=False) :
|
||||
self.__name = name
|
||||
self.__data = data
|
||||
self.__filename = filename
|
||||
self.__redis = None
|
||||
self._name = name
|
||||
self._data = data
|
||||
self._filename = filename
|
||||
self._redis = None
|
||||
if redis_host != None :
|
||||
self.__redis = redis.Redis(host=redis_host, port=6379, db=0)
|
||||
self._redis = redis.Redis(host=redis_host, port=6379, db=0)
|
||||
try :
|
||||
self.__redis.echo("test")
|
||||
self._redis.echo("test")
|
||||
except :
|
||||
self.__log("can't connect to redis host " + redis_host)
|
||||
self.__type = type
|
||||
self.__regex = regex
|
||||
self.__copy_cache = copy_cache
|
||||
self._log("can't connect to redis host " + redis_host)
|
||||
self._type = type
|
||||
self._regex = regex
|
||||
self._copy_cache = copy_cache
|
||||
|
||||
def __log(self, data) :
|
||||
def _log(self, data) :
|
||||
when = datetime.datetime.today().strftime("[%Y-%m-%d %H:%M:%S]")
|
||||
what = self.__name + " - " + data + "\n"
|
||||
what = self._name + " - " + data + "\n"
|
||||
with open("/var/log/nginx/jobs.log", "a") as f :
|
||||
f.write(when + " " + what)
|
||||
|
||||
def run(self) :
|
||||
ret = JobRet.KO
|
||||
try :
|
||||
if self.__type == "line" or self.__type == "file" :
|
||||
if self.__copy_cache :
|
||||
if self._type == "line" or self._type == "file" :
|
||||
if self._copy_cache :
|
||||
ret = self.__from_cache()
|
||||
if ret != JobRet.KO :
|
||||
return ret
|
||||
ret = self.__external()
|
||||
self.__to_cache()
|
||||
elif self.__type == "exec" :
|
||||
elif self._type == "exec" :
|
||||
return self.__exec()
|
||||
except Exception as e :
|
||||
self.__log("exception while running job : " + traceback.format_exc())
|
||||
|
|
@ -46,43 +46,45 @@ class Job(abc.ABC) :
|
|||
return ret
|
||||
|
||||
def __external(self) :
|
||||
if self.__redis == None :
|
||||
if os.path.isfile("/tmp/" + self.__filename) :
|
||||
os.remove("/tmp/" + self.__filename)
|
||||
file = open("/tmp/" + self.__filename, "ab")
|
||||
if self._redis == None :
|
||||
if os.path.isfile("/tmp/" + self._filename) :
|
||||
os.remove("/tmp/" + self._filename)
|
||||
file = open("/tmp/" + self._filename, "ab")
|
||||
|
||||
elif self.__redis != None :
|
||||
pipe = self.__redis.pipeline()
|
||||
elif self._redis != None :
|
||||
pipe = self._redis.pipeline()
|
||||
|
||||
count = 0
|
||||
for url in self.__data :
|
||||
for url in self._data :
|
||||
data = self.__download_data(url)
|
||||
for chunk in data :
|
||||
if self.__type == "line" :
|
||||
if not re.match(self.__regex, chunk.decode("utf-8")) :
|
||||
if self._type == "line" :
|
||||
if not re.match(self._regex, chunk.decode("utf-8")) :
|
||||
continue
|
||||
chunks = self.__edit(chunk)
|
||||
if self.__redis == None :
|
||||
if self.__type == "line" :
|
||||
chunk += b"\n"
|
||||
file.write(chunk)
|
||||
else :
|
||||
if self.__type == "line" :
|
||||
chunks = self._edit(chunk)
|
||||
if self._redis == None :
|
||||
if self._type == "line" :
|
||||
for chunk in chunks :
|
||||
pipe.set(self.__name + "_" + chunk, "1")
|
||||
file.write(chunk + b"\n")
|
||||
else :
|
||||
pipe.set(self.__name + "_" + chunk, "1")
|
||||
file.write(chunk)
|
||||
else :
|
||||
if self._type == "line" :
|
||||
for chunk in chunks :
|
||||
pipe.set(self._name + "_" + chunk, "1")
|
||||
else :
|
||||
pipe.set(self._name + "_" + chunk, "1")
|
||||
count += 1
|
||||
|
||||
if self.__redis == None :
|
||||
if self._redis == None :
|
||||
file.close()
|
||||
if count > 0 :
|
||||
shutil.copyfile("/tmp/" + self.__filename, "/etc/nginx/" + self.__filename)
|
||||
os.remove("/tmp/" + self.__filename)
|
||||
shutil.copyfile("/tmp/" + self._filename, "/etc/nginx/" + self._filename)
|
||||
os.remove("/tmp/" + self._filename)
|
||||
return JobRet.OK_RELOAD
|
||||
|
||||
elif self.__redis != None and count > 0 :
|
||||
self.__redis.delete(self.__redis.keys(self.__name + "_*"))
|
||||
elif self._redis != None and count > 0 :
|
||||
self._redis.delete(self._redis.keys(self._name + "_*"))
|
||||
pipe.execute()
|
||||
return JobRet.OK_RELOAD
|
||||
|
||||
|
|
@ -92,57 +94,57 @@ class Job(abc.ABC) :
|
|||
r = requests.get(url, stream=True)
|
||||
if not r or r.status_code != 200 :
|
||||
raise Exception("can't download data at " + url)
|
||||
if self.__type == "line" :
|
||||
if self._type == "line" :
|
||||
return r.iter_lines()
|
||||
return r.iter_content(chunk_size=8192)
|
||||
|
||||
def __exec(self) :
|
||||
proc = subprocess.run(self.__data, capture_output=True)
|
||||
proc = subprocess.run(self._data, capture_output=True)
|
||||
stdout = proc.stdout.decode("ascii")
|
||||
stderr = proc.stderr.decode("err")
|
||||
if len(stdout) > 1 :
|
||||
self.__log("stdout = " + stdout)
|
||||
self._log("stdout = " + stdout)
|
||||
if len(stderr) > 1 :
|
||||
self.__log("stderr = " + stderr)
|
||||
self._log("stderr = " + stderr)
|
||||
if proc.returncode != 0 :
|
||||
return JobRet.KO
|
||||
# TODO : check if reload is needed ?
|
||||
return JobRet.OK_RELOAD
|
||||
|
||||
def __edit(self, chunk) :
|
||||
def _edit(self, chunk) :
|
||||
return [chunk]
|
||||
|
||||
def __from_cache(self) :
|
||||
if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
|
||||
if not os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) :
|
||||
return JobRet.KO
|
||||
|
||||
if self.__redis == None or self.__type == "file" :
|
||||
if not os.path.isfile("/etc/nginx/" + self.__filename) or not filecmp.cmp("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename, shallow=False) :
|
||||
shutil.copyfile("/opt/bunkerized-nginx/cache/" + self.__filename, "/etc/nginx/" + self.__filename)
|
||||
if self._redis == None or self._type == "file" :
|
||||
if not os.path.isfile("/etc/nginx/" + self._filename) or not filecmp.cmp("/opt/bunkerized-nginx/cache/" + self._filename, "/etc/nginx/" + self._filename, shallow=False) :
|
||||
shutil.copyfile("/opt/bunkerized-nginx/cache/" + self._filename, "/etc/nginx/" + self._filename)
|
||||
return JobRet.OK_RELOAD
|
||||
return JobRet.OK_NO_RELOAD
|
||||
|
||||
if self.__redis != None and self.__type == "line" :
|
||||
self.__redis.delete(self.__redis.keys(self.__name + "_*"))
|
||||
with open("/opt/bunkerized-nginx/cache/" + self.__filename) as f :
|
||||
pipe = self.__redis.pipeline()
|
||||
if self._redis != None and self._type == "line" :
|
||||
self._redis.delete(self._redis.keys(self._name + "_*"))
|
||||
with open("/opt/bunkerized-nginx/cache/" + self._filename) as f :
|
||||
pipe = self._redis.pipeline()
|
||||
while True :
|
||||
line = f.readline()
|
||||
if not line :
|
||||
break
|
||||
line = line.strip()
|
||||
pipe.set(self.__name + "_" + line, "1")
|
||||
pipe.set(self._name + "_" + line, "1")
|
||||
pipe.execute()
|
||||
return JobRet.OK_NO_RELOAD
|
||||
|
||||
return JobRet.KO
|
||||
|
||||
def __to_cache(self) :
|
||||
if self.__redis == None or self.__type == "file" :
|
||||
shutil.copyfile("/etc/nginx/" + self.__filename, "/opt/bunkerized-nginx/cache/" + self.__filename)
|
||||
elif self.__redis != None and self.__type == "line" :
|
||||
if os.path.isfile("/opt/bunkerized-nginx/cache/" + self.__filename) :
|
||||
os.remove("/opt/bunkerized-nginx/cache/" + self.__filename)
|
||||
with open("/opt/bunkerized-nginx/cache/" + self.__filename, "a") as f :
|
||||
for key in self.__redis.keys(self.__name + "_*") :
|
||||
f.write(self.__redis.get(key) + "\n")
|
||||
if self._redis == None or self._type == "file" :
|
||||
shutil.copyfile("/etc/nginx/" + self._filename, "/opt/bunkerized-nginx/cache/" + self._filename)
|
||||
elif self._redis != None and self._type == "line" :
|
||||
if os.path.isfile("/opt/bunkerized-nginx/cache/" + self._filename) :
|
||||
os.remove("/opt/bunkerized-nginx/cache/" + self._filename)
|
||||
with open("/opt/bunkerized-nginx/cache/" + self._filename, "a") as f :
|
||||
for key in self._redis.keys(self._name + "_*") :
|
||||
f.write(self._redis.get(key) + "\n")
|
||||
|
|
|
|||
|
|
@ -12,8 +12,8 @@ class Proxies(Job) :
|
|||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def _Job__edit(self, chunk) :
|
||||
if self.__redis != None :
|
||||
def _edit(self, chunk) :
|
||||
if self._redis != None :
|
||||
network = chunk.decode("utf-8")
|
||||
if re.match(network, r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\/?[0-9]+$") :
|
||||
ips = []
|
||||
|
|
|
|||
|
|
@ -10,5 +10,5 @@ class Referrers(Job) :
|
|||
regex = r"^.+$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def _Job__edit(self, chunk) :
|
||||
return chunk.replace(b".", b"%.").replace(b"-", b"%-")
|
||||
def _edit(self, chunk) :
|
||||
return [chunk.replace(b".", b"%.").replace(b"-", b"%-")]
|
||||
|
|
|
|||
|
|
@ -10,5 +10,5 @@ class UserAgents(Job) :
|
|||
regex = r"^.+$"
|
||||
super().__init__(name, data, filename, redis_host=redis_host, type=type, regex=regex, copy_cache=copy_cache)
|
||||
|
||||
def _Job__edit(self, chunk) :
|
||||
return chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")
|
||||
def _edit(self, chunk) :
|
||||
return [chunk.replace(b"\\ ", b" ").replace(b"\\.", b"%.").replace(b"\\\\", b"\\").replace(b"-", b"%-")]
|
||||
|
|
|
|||
|
|
@ -1,41 +1,42 @@
|
|||
local M = {}
|
||||
local redis = require "resty.redis"
|
||||
|
||||
local mt = { __index = M }
|
||||
|
||||
function M.new(self, name, data_dict, redis_client, type)
|
||||
return selfmetatable({
|
||||
return setmetatable({
|
||||
__name = name,
|
||||
__data_dict = data_dict,
|
||||
__redis_client = redis_client,
|
||||
__type = type
|
||||
})
|
||||
}, mt)
|
||||
end
|
||||
|
||||
function M.check(self, data) :
|
||||
function M.check(self, data)
|
||||
-- without redis
|
||||
if self.__data_dict ~= nil and redis_client == nil then
|
||||
if self.__data_dict ~= nil and self.__redis_client == nil then
|
||||
if self.__type == "simple" then
|
||||
local value, flags = self.__data_dict:get(data)
|
||||
return ~= nil
|
||||
else if self.__type == "match" then
|
||||
return value ~= nil
|
||||
elseif self.__type == "match" then
|
||||
local patterns = self.__data_dict:get_keys(0)
|
||||
for i, pattern in ipairs(patterns) do
|
||||
if string.match(data, pattern) then
|
||||
return true
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
-- with redis
|
||||
else if data_dict == nil and redis_client ~= nil then
|
||||
elseif self.__data_dict == nil and self.__redis_client ~= nil then
|
||||
if self.__type == "simple" then
|
||||
local res, err = self.__redis_client:get(self.__name .. "_" .. data)
|
||||
return res and res ~= ngx.null
|
||||
else if self.__type == "match" then
|
||||
elseif self.__type == "match" then
|
||||
local patterns = self.__redis_client:keys(self.__name .. "_*")
|
||||
if patterns then
|
||||
for i, pattern in ipairs(patterns) do
|
||||
if string.match(data, pattern) do
|
||||
if string.match(data, pattern) then
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
|
@ -45,4 +46,6 @@ function M.check(self, data) :
|
|||
|
||||
return false
|
||||
|
||||
end
|
||||
|
||||
return M
|
||||
|
|
|
|||
Loading…
Reference in New Issue