remote API - ban IP from distributed DB

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## v1.3.2 -
 
 - Use API instead of a shared folder for Swarm and Kubernetes integrations
+- Beta integration of distributed bad IPs database through a remote API
 - Various bug fixes related to antibot feature
 - Fix Moodle example
 

confs/global/init-lua.conf

@@ -4,6 +4,7 @@ local dataloader		= require "dataloader"
 local logger			= require "logger"
 local cjson			= require "cjson"
 local remoteapi			= require "remoteapi"
+local iputils			= require "resty.iputils"
 
 local use_redis			= {% if USE_REDIS == "yes" %}true{% else %}false{% endif +%}
 
@@ -15,6 +16,30 @@ local use_referrers		= {% if has_value("BLOCK_REFERRER", "yes") %}true{% else %}
 
 local use_remote_api		= {% if has_value("USE_REMOTE_API", "yes") %}true{% else %}false{% endif +%}
 
+
+-- Load reserved IPs
+local reserved_ips = {
+	"0.0.0.0/8",
+	"10.0.0.0/8",
+	"100.64.0.0/10",
+	"127.0.0.0/8",
+	"169.254.0.0/16",
+	"172.16.0.0/12",
+	"192.0.0.0/24",
+	"192.0.2.0/24",
+	"192.88.99.0/24",
+	"192.168.0.0/16",
+	"198.18.0.0/15",
+	"198.51.100.0/24",
+	"203.0.113.0/24",
+	"224.0.0.0/4",
+	"233.252.0.0/24",
+	"240.0.0.0/4",
+	"255.255.255.255/32"
+}
+ngx.shared.reserved_ips:safe_set("cidrs", iputils.parse_cidrs(reserved_ips), 0)
+
+-- Load blacklists
 if not use_redis then
 	if use_proxies then
 		dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data)
@@ -87,19 +112,33 @@ if use_remote_api then
 	f:close()
 
 	-- Save machine ID
+	local id = "empty"
 	local f = io.open("/etc/nginx/machine.id", "r")
 	if f == nil then
-		id = "empty"
 		logger.log(ngx.ERR, "REMOTE API", "USE_REMOTE_API is set to yes but machine ID is not generated - communication with {{ REMOTE_API_SERVER }} won't work")
 	else
 		id = f:read("*all"):gsub("[\r\n]", "")
-		logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Machine ID = " .. id)	
+		logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Using existing machine ID (" .. id .. ")")	
 		f:close()
 	end
 	ngx.shared.remote_api:set("id", id, 0)
 
-	-- TODO : ping (blocking socket)
-	-- TODO : load database
+	-- Ping the remote API
+	local ping = "ko"
+	if id ~= "empty" then
+		if remoteapi.ping2() then
+			ping = "ok"
+			logger.log(ngx.ERR, "REMOTE API", "*NOT AN ERROR* Successfully requested the remote API")
+		else
+			logger.log(ngx.ERR, "REMOTE API", "Can't contact the remote API, feature will be disabled")
+		end
+	end
+	ngx.shared.remote_api:set("ping", ping, 0)
+
+	-- Load the database
+	if ping ~= "ko" then
+		dataloader.load_ip("/etc/nginx/remote-api.db", ngx.shared.remote_api_db)
+	end
 end
 
 }

confs/global/nginx.conf

@@ -91,7 +91,9 @@ http {
 	{% if has_value("USE_BAD_BEHAVIOR", "yes") %}lua_shared_dict behavior_ban 10m;{% endif +%}
 	{% if has_value("USE_BAD_BEHAVIOR", "yes") %}lua_shared_dict behavior_count 10m;{% endif +%}
 	lua_shared_dict plugins_data 10m;
+	lua_shared_dict reserved_ips 1m;
 	{% if has_value("USE_REMOTE_API", "yes") %}lua_shared_dict remote_api 1m;{% endif +%}
+	{% if has_value("USE_REMOTE_API", "yes") %}lua_shared_dict remote_api_db 10m;{% endif +%}
 
 	# shared memory zone for limit_req
 	{% if has_value("USE_LIMIT_REQ", "yes") %}limit_req_zone $binary_remote_addr$uri zone=limit:{{ LIMIT_REQ_CACHE }} rate={{ LIMIT_REQ_RATE }};{% endif +%}

confs/site/log-lua.conf

@@ -20,8 +20,9 @@ end
 -- remote API
 local use_remote_api            = {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%}
 local remoteapi			= require "remoteapi"
+local iputils			= require "resty.iputils"
 
-if use_remote_api and ngx.shared.remote_api:get("id") ~= "empty" then
+if use_remote_api and not iputils.ip_in_cidrs(ngx.var.remote_addr, ngx.shared.reserved_ips:get("data")) and ngx.shared.remote_api:get("id") ~= "empty" and ngx.shared.remote_api:get("ping") ~= "ko" then
 	if ngx.status == ngx.HTTP_FORBIDDEN then
 		local reason = "other"
 		if use_bad_behavior and new_bad_behavior_ban then
@@ -34,16 +35,13 @@ if use_remote_api and ngx.shared.remote_api:get("id") ~= "empty" then
 			local remoteapi = require "remoteapi"
 			local logger = require "logger"
 			local res, data = remoteapi.ip(ip, reason)
-			-- TODO : find a way to log ?
---			if res then
---				logger.log(ngx.ERR, "REMOTE API", "Successfully reported ip " .. ngx.var.remote_addr)			
---			else
---				logger.log(ngx.ERR, "REMOTE API", "Error while reporting ip " .. ngx.var.remote_addr .. " : " .. data)
---			end
+			-- TODO : find a way to log
 		end
 		local ok, err = ngx.timer.at(0, report_ip, ngx.var.remote_addr, reason)
 		if not ok then
 			logger.log(ngx.ERR, "REMOTE API", "Error while creating report timer " .. err)
+		else
+			logger.log(ngx.NOTICE, "REMOTE API", "Reporting " .. ngx.var.remote_addr .. "(reason: " .. reason .. ") to the remote API")
 		end
 	end
 end

confs/site/main-lua.conf

@@ -57,6 +57,9 @@ local dnsbl_list		= {% raw %}{{% endraw %}{% if DNSBL_LIST != "" %}{% set elemen
 -- bad behavior
 local use_bad_behavior		= {% if USE_BAD_BEHAVIOR == "yes" %}true{% else %}false{% endif +%}
 
+-- remote API
+local use_remote_api		= {% if USE_REMOTE_API == "yes" %}true{% else %}false{% endif +%}
+
 -- include LUA code
 local whitelist			= require "whitelist"
 local blacklist			= require "blacklist"
@@ -224,6 +227,15 @@ if use_dnsbl and not dnsbl.cached() then
 	end
 end
 
+-- check if IP is in distributed DB
+if use_remote_api then
+	local checker = checker:new("remote-api-db", ngx.shared.remote_api_db, redis_client, "simple")
+	if checker:check(iputils.ip2bin(ngx.var.remote_addr)) then
+		logger.log(ngx.WARN, "REMOTE API", "IP " .. ngx.var.remote_addr .. " is in the distributed DB")
+		ngx.exit(ngx.HTTP_FORBIDDEN)
+	end
+end
+
 -- cookie check
 if use_antibot_cookie and ngx.var.uri ~= "/favicon.ico" then
 	if not cookie.is_set("uri") then

jobs/Job.py

@@ -132,7 +132,7 @@ class Job(abc.ABC) :
 					chunk = chunk.decode("utf-8")
 				if self._type in ["line", "json"] :
 					if not re.match(self._regex, chunk) :
-						log(self._name, "WARN", chunk + " doesn't match regex " + self._regex)
+						#log(self._name, "WARN", chunk + " doesn't match regex " + self._regex)
 						continue
 				if self._redis == None :
 					if self._type in ["line", "json"] :

lua/remoteapi.lua

@@ -39,6 +39,28 @@ function M.gen_data(use_id, data)
 	return all_data
 end
 
+function M.ping2()
+	local https = require "ssl.https"
+	local ltn12 = require "ltn12"
+	local request_body = cjson.encode(M.gen_data(true, {}))
+	local response_body = {}
+	local res, code, headers, status = https.request {
+		url = ngx.shared.remote_api:get("server") .. "/ping",
+		method = "GET",
+		headers = {
+			["Content-Type"] = "application/json",
+			["User-Agent"] = "bunkerized-nginx/" .. ngx.shared.remote_api:get("version"),
+			["Content-Length"] = request_body:len()
+		},
+		source = ltn12.source.string(request_body),
+		sink = ltn12.sink.table(response_body)
+	}
+	if res and status == 200 and response_body["data"] == "pong" then
+		return true
+	end
+	return false
+end
+
 function M.register()
 	local request = {}
 	local res, status, data = M.send("POST", "/register", M.gen_data(false, request))
@@ -58,7 +80,6 @@ function M.ping()
 end
 
 function M.ip(ip, reason)
-	-- TODO : check if IP is global
 	local request = {
 		["ip"] = ip,
 		["reason"] = reason