improved blacklist/whitelist/dnsbl with lua

2020-10-09 23:20:40 +02:00 · 2020-10-09 23:20:40 +02:00 · fc3d911ff7
parent ef7d842ff0
commit fc3d911ff7
14 changed files with 318 additions and 96 deletions
--- a/1
+++ b/1
@ -10,6 +10,7 @@ COPY confs/ /opt/confs
 COPY scripts/ /opt/scripts
 COPY fail2ban/ /opt/fail2ban
 COPY logs/ /opt/logs
+COPY lua/ /opt/lua

 RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
    chmod +x /opt/entrypoint.sh /opt/scripts/* && \
--- a/1
+++ b/1
@ -10,6 +10,7 @@ COPY confs/ /opt/confs
 COPY scripts/ /opt/scripts
 COPY fail2ban/ /opt/fail2ban
 COPY logs/ /opt/logs
+COPY lua/ /opt/lua

 RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
    chmod +x /opt/entrypoint.sh /opt/scripts/* && \
--- a/1
+++ b/1
@ -17,6 +17,7 @@ COPY confs/ /opt/confs
 COPY scripts/ /opt/scripts
 COPY fail2ban/ /opt/fail2ban
 COPY logs/ /opt/logs
+COPY lua/ /opt/lua

 RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
    chmod +x /opt/entrypoint.sh /opt/scripts/* && \
--- a/1
+++ b/1
@ -17,6 +17,7 @@ COPY confs/ /opt/confs
 COPY scripts/ /opt/scripts
 COPY fail2ban/ /opt/fail2ban
 COPY logs/ /opt/logs
+COPY lua/ /opt/lua

 RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
    chmod +x /opt/entrypoint.sh /opt/scripts/* && \
--- a/1
+++ b/1
@ -10,6 +10,7 @@ COPY confs/ /opt/confs
 COPY scripts/ /opt/scripts
 COPY fail2ban/ /opt/fail2ban
 COPY logs/ /opt/logs
+COPY lua/ /opt/lua

 RUN apk --no-cache add php7-fpm certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog && \
    chmod +x /opt/entrypoint.sh /opt/scripts/* && \
--- a/confs/dnsbl.conf
+++ b/confs/dnsbl.conf
@ -1,84 +0,0 @@
-access_by_lua_block {
-
-	-- get client IP
-	local ip = ngx.var.remote_addr
-	
-	-- check if IP is in cache
-	local cached = ngx.shared.dnsblcache:get(ip)
-	if cached ~= nil then
-		if cached == "ok" then
-			ngx.exit(ngx.OK)
-		else
-			ngx.exit(ngx.HTTP_FORBIDDEN)
-		end
-	end
-	
-	-- get the reverse DNS
-	local rdns = ""
-	local both = false
-	local resolver = require "resty.dns.resolver"
-	local resolvers = {%DNSBL_RESOLVERS%}
-	local r, err = resolver:new{nameservers=resolvers, retrans=2, timeout=2000}
-	if not r then
-		ngx.exit(ngx.OK)
-	end
-	local answers, err = r:reverse_query(ip)
-	if not answers.errcode then
-		for ak, av in ipairs(answers) do
-			if av.ptrdname then
-				rdns = av.ptrdname
-				break
-			end
-		end
-	end
-	if rdns ~= "" then
-		local answers, err, tries = r:query(rdns, nil, {})
-		for ak, av in ipairs(answers) do
-			if av.address and av.address == ip then
-				both = true
-				break
-			end
-		end
-	end
-	
-	-- check if it's a legitimate SE crawler 
-	local ips = {"23.21.227.69", "40.88.21.235", "50.16.241.113", "50.16.241.114", "50.16.241.117", "50.16.247.234", "52.204.97.54", "52.5.190.19", "54.197.234.188", "54.208.100.253", "54.208.102.37", "107.21.1.8"}
-	local domains = {".googlebot.com", ".google.com", ".search.msn.com", ".crawl.yahoot.net", ".crawl.baidu.jp", ".crawl.baidu.com", ".yandex.com", ".yandex.ru", ".yandex.net"}
-	for k, v in pairs(ips) do
-		if v == ip then
-			ngx.shared.dnsblcache:set(ip, "ok", 86400)
-			ngx.exit(ngx.OK)
-		end
-	end
-	if both and rdns ~= "" then
-		for k, v in pairs(domains) do
-			if rdns:sub(-#v) == v then
-				ngx.shared.dnsblcache:set(ip, "ok", 86400)
-				ngx.exit(ngx.OK)
-			end
-		end
-	end
-	
-	-- dnsbl check
-	local dnsbls = {%DNSBL_LIST%}
-	for k, v in pairs(dnsbls) do
-		local name = resolver.arpa_str(ip)
-		name = name:gsub("%.in%-addr%.arpa", ""):gsub("%.ip6%.arpa", "") .. "." .. v
-		local answers, err, tries = r:query(name, nil, {})
-		if not answers.errcode then 
-			for ak, av in ipairs(answers) do
-				if av.address then
-					a,b,c,d = av.address:match("([%d]+).([%d]+).([%d]+).([%d]+)")
-					if a == "127" then
-						ngx.shared.dnsblcache:set(ip, "dnsbl", 86400)
-						ngx.exit(ngx.HTTP_FORBIDDEN)
-					end
-				end
-			end
-		end
-	end
-	
-	-- legitimate user
-	ngx.shared.dnsblcache:set(ip, "ok", 86400)
-	ngx.exit(ngx.OK)
-}
--- a/confs/main-lua.conf
+++ b/confs/main-lua.conf
@ -0,0 +1,72 @@
+access_by_lua_block {
+
+local use_whitelist_ip			= %USE_WHITELIST_IP%
+local use_whitelist_reverse		= %USE_WHITELIST_REVERSE%
+local use_blacklist_ip			= %USE_BLACKLIST_IP%
+local use_blacklist_reverse		= %USE_BLACKLIST_REVERSE%
+local use_dnsbl				= %USE_DNS%
+
+-- include LUA code
+local whitelist = require "whitelist"
+local blacklist = require "blacklist"
+local dnsbl	= require "dnsbl"
+
+-- check if already in whitelist cache
+if use_whitelist_ip and whitelist.ip_cached_ok() then
+	ngx.exit(ngx.OK)
+end
+if use_whitelist_reverse and whitelist.reverse_cached_ok() then
+	ngx.exit(ngx.OK)
+end
+
+-- check if already in blacklist cache
+if use_blacklist_ip and blacklist.ip_cached_ko() then
+	ngx.exit(ngx.HTTP_FORBIDDEN)
+end
+if use_blacklist_reverse and blacklist.reverse_cached_ko() then
+	ngx.exit(ngx.HTTP_FORBIDDEN)
+end
+
+-- check if already in dnsbl cache
+if use_dnsbl and dnsbl.cached_ko() then
+	ngx.exit(ngx.HTTP_FORBIDDEN)
+end
+
+-- check if IP is whitelisted (only if not in cache)
+if use_whitelist_ip and not whitelist.ip_cached() then
+	if whitelist.check_ip() then
+		ngx.exit(ngx.OK)
+	end
+end
+
+-- check if reverse is whitelisted (only if not in cache)
+if use_whitelist_reverse and not whitelist.reverse_cached() then
+	if whitelist.check_reverse() then
+		ngx.exit(ngx.OK)
+	end
+end
+
+-- check if IP is blacklisted (only if not in cache)
+if use_blacklist_ip and not blacklist.ip_cached() then
+	if blacklist.check_ip() then
+		ngx.exit(ngx.HTTP_FORBIDDEN)
+	end
+end
+
+-- check if reverse is blacklisted (only if not in cache)
+if use_blacklist_reverse and not blacklist.reverse_cached() then
+	if blacklist.check_reverse() then
+		ngx.exit(ngx.HTTP_FORBIDDEN)
+	end
+end
+
+-- check if IP is in DNSBLs (only if not in cache)
+if use_dnsbl and not dnsbl.cached() then
+	if dnsbl.check() then
+		ngx.exit(ngx.HTTP_FORBIDDEN)
+	end
+end
+
+ngx.exit(ngx.OK)
+
+}
--- a/confs/nginx.conf
+++ b/confs/nginx.conf
@ -65,10 +65,14 @@ http {
 	access_log syslog:server=unix:/dev/log,nohostname,facility=local0,severity=notice combined;
 	error_log syslog:server=unix:/dev/log,nohostname,facility=local0 warn;

-	# lua path
+	# lua path and dicts
 	lua_package_path "/usr/local/lib/lua/?.lua;;";
+	%WHITELIST_IP_CACHE%
+	%WHITELIST_REVERSE_CACHE%
+	%BLACKLIST_IP_CACHE%
+	%BLACKLIST_REVERSE_CACHE%
 	%DNSBL_CACHE%
-
+	
 	# shared memory zone for limit_req
 	%LIMIT_REQ_ZONE%

--- a/confs/server.conf
+++ b/confs/server.conf
@ -1,5 +1,6 @@
 server {
 	include /server-confs/*.conf;
+	include /etc/nginx/main-lua.conf;
 	%LISTEN_HTTP%
 	%USE_HTTPS%
 	%REDIRECT_HTTP_TO_HTTPS%
@ -11,7 +12,6 @@ server {
 		return 405;
 	}
 	%LIMIT_REQ%
-	%DNSBL%
 	%AUTH_BASIC%
 	%USE_PHP%
 	%HEADER_SERVER%
--- a/entrypoint.sh
+++ b/entrypoint.sh
@ -52,6 +52,7 @@ cp -r /opt/confs/owasp-crs /etc/nginx
 cp /opt/confs/php.ini /etc/php7/php.ini
 cp /opt/logs/rsyslog.conf /etc/rsyslog.conf
 cp /opt/logs/logrotate.conf /etc/logrotate.conf
+cp /opt/lua/*.lua /usr/local/lib/lua

 # remove cron jobs
 echo "" > /etc/crontabs/root
@ -123,9 +124,16 @@ USE_CUSTOM_HTTPS="${USE_CUSTOM_HTTPS-no}"
 ROOT_FOLDER="${ROOT_FOLDER-/www}"
 LOGROTATE_MINSIZE="${LOGROTATE_MINSIZE-10M}"
 LOGROTATE_MAXAGE="${LOGROTATE_MAXAGE-7}"
+DNS_RESOLVERS="${DNS_RESOLVERS-8.8.8.8 8.8.4.4}"
+USE_WHITELIST_IP="${USE_WHITELIST_IP-yes}"
+WHITELIST_IP_LIST="${WHITELIST_IP_LIST-23.21.227.69 40.88.21.235 50.16.241.113 50.16.241.114 50.16.241.117 50.16.247.234 52.204.97.54 52.5.190.19 54.197.234.188 54.208.100.253 54.208.102.37 107.21.1.8}"
+USE_WHITELIST_REVERSE="${USE_WHITELIST_REVERSE-yes}"
+WHITELIST_REVERSE_LIST="${WHITELIST_REVERSE_LIST-.googlebot.com .google.com .search.msn.com .crawl.yahoot.net .crawl.baidu.jp .crawl.baidu.com .yandex.com .yandex.ru .yandex.net}"
+USE_BLACKLIST_IP="${USE_BLACKLIST_IP-yes}"
+BLACKLIST_IP_LIST="${BLACKLIST_IP_LIST-}"
+USE_BLACKLIST_REVERSE="${USE_BLACKLIST_REVERSE-yes}"
+BLACKLIST_REVERSE_LIST="${BLACKLIST_REVERSE_LIST-.shodan.io}"
 USE_DNSBL="${USE_DNSBL-yes}"
-DNSBL_CACHE="${DNSBL_CACHE-10m}"
-DNSBL_RESOLVERS="${DNSBL_RESOLVERS-8.8.8.8 8.8.4.4}"
 DNSBL_LIST="${DNSBL_LIST-bl.blocklist.de problems.dnsbl.sorbs.net sbl.spamhaus.org xbl.spamhaus.org}"
 USE_LIMIT_REQ="${USE_LIMIT_REQ-yes}"
 LIMIT_REQ_RATE="${LIMIT_REQ_RATE-20r/s}"
@ -423,17 +431,66 @@ if [ "$USE_AUTH_BASIC" = "yes" ] ; then
 else
 	replace_in_file "/etc/nginx/server.conf" "%AUTH_BASIC%" ""
 fi
+
+# lua resolvers
+resolvers=$(spaces_to_lua "$DNS_RESOLVERS")
+replace_in_file "/usr/local/lib/lua/dns.lua" "%DNS_RESOLVERS%" "$resolvers"
+
+# whitelist IP
+if [ "$USE_WHITELIST_IP" = "yes" ] ; then
+	replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_IP_CACHE%" "lua_shared_dict whitelist_ip_cache 10m;"
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_IP%" "true"
+else
+	replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_IP_CACHE%" ""
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_IP%" "false"
+fi
+list=$(spaces_to_lua "$WHITELIST_IP_LIST")
+replace_in_file "/usr/local/lib/lua/whitelist.lua" "%WHITELIST_IP_LIST%" "$list"
+
+# whitelist rDNS
+if [ "$USE_WHITELIST_REVERSE" = "yes" ] ; then
+	replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_REVERSE_CACHE%" "lua_shared_dict whitelist_reverse_cache 10m;"
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_REVERSE%" "true"
+else
+	replace_in_file "/etc/nginx/nginx.conf" "%WHITELIST_REVERSE_CACHE%" ""
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_WHITELIST_REVERSE%" "false"
+fi
+list=$(spaces_to_lua "$WHITELIST_REVERSE_LIST")
+replace_in_file "/usr/local/lib/lua/whitelist.lua" "%WHITELIST_REVERSE_LIST%" "$list"
+
+# blacklist IP
+if [ "$USE_BLACKLIST_IP" = "yes" ] ; then
+	replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_IP_CACHE%" "lua_shared_dict blacklist_ip_cache 10m;"
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_IP%" "true"
+else
+	replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_IP_CACHE%" ""
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_IP%" "false"
+fi
+list=$(spaces_to_lua "$BLACKLIST_IP_LIST")
+replace_in_file "/usr/local/lib/lua/blacklist.lua" "%BLACKLIST_IP_LIST%" "$list"
+
+# blacklist rDNS
+if [ "$USE_BLACKLIST_REVERSE" = "yes" ] ; then
+	replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_REVERSE_CACHE%" "lua_shared_dict blacklist_reverse_cache 10m;"
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_REVERSE%" "true"
+else
+	replace_in_file "/etc/nginx/nginx.conf" "%BLACKLIST_REVERSE_CACHE%" ""
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_BLACKLIST_REVERSE%" "false"
+fi
+list=$(spaces_to_lua "$BLACKLIST_REVERSE_LIST")
+replace_in_file "/usr/local/lib/lua/blacklist.lua" "%BLACKLIST_REVERSE_LIST%" "$list"
+
+# DNSBL
 if [ "$USE_DNSBL" = "yes" ] ; then
-	replace_in_file "/etc/nginx/nginx.conf" "%DNSBL_CACHE%" "lua_shared_dict dnsblcache $DNSBL_CACHE;"
-	replace_in_file "/etc/nginx/server.conf" "%DNSBL%" "include /etc/nginx/dnsbl.conf;"
-	resolvers=$(spaces_to_lua "$DNSBL_RESOLVERS")
-	list=$(spaces_to_lua "$DNSBL_LIST")
-	replace_in_file "/etc/nginx/dnsbl.conf" "%DNSBL_RESOLVERS%" "$resolvers"
-	replace_in_file "/etc/nginx/dnsbl.conf" "%DNSBL_LIST%" "$list"
+	replace_in_file "/etc/nginx/nginx.conf" "%DNSBL_CACHE%" "lua_shared_dict dnsbl_cache 10m;"
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_DNSBL%" "true"
 else
 	replace_in_file "/etc/nginx/nginx.conf" "%DNSBL_CACHE%" ""
-	replace_in_file "/etc/nginx/server.conf" "%DNSBL%" ""
+	replace_in_file "/etc/nginx/main-lua.conf" "%USE_DNSBL%" "false"
 fi
+list=$(spaces_to_lua "$DNSBL_LIST")
+replace_in_file "/usr/local/lib/lua/dnsbl.lua" "%DNSBL_LIST%" "$list"
+
 if [ "$USE_LIMIT_REQ" = "yes" ] ; then
 	replace_in_file "/etc/nginx/nginx.conf" "%LIMIT_REQ_ZONE%" "limit_req_zone \$binary_remote_addr zone=limit:${LIMIT_REQ_CACHE} rate=${LIMIT_REQ_RATE};"
 	replace_in_file "/etc/nginx/server.conf" "%LIMIT_REQ%" "include /etc/nginx/limit-req.conf;"
--- a/lua/blacklist.lua
+++ b/lua/blacklist.lua
@ -0,0 +1,45 @@
+local dns		= require "dns"
+local ip_list 		= {%BLACKLIST_IP_LIST%}
+local reverse_list	= {%BLACKLIST_REVERSE_LIST%}
+local ip		= ngx.var.remote_addr
+
+function ip_cached_ko ()
+	return ngx.shared.blacklist_ip_cache:get(ip) == "ko"
+end
+
+function reverse_cached_ko ()
+	return ngx.shared.blacklist_reverse_cache:get(ip) == "ko"
+end
+
+function ip_cached ()
+	return ngx.shared.blacklist_ip_cache:get(ip) ~= nil
+end
+
+function reverse_cached ()
+	return ngx.shared.blacklist_reverse_cache:get(ip) ~= nil
+end
+
+function check_ip ()
+	for k, v in ipairs(ip_list) do
+		if v == ip then
+			ngx.shared.blacklist_ip_cache:set(ip, "ko", 86400)
+			return true
+		end
+	end
+	ngx.shared.blacklist_ip_cache:set(ip, "ok", 86400)
+	return false
+end
+
+function check_reverse ()
+	local rdns = dns.get_reverse()
+	if rdns ~= "" then
+		for k, v in ipairs(reverse_list) do
+			if rdns:sub(-#v) == v then
+				ngx.shared.blacklist_reverse_cache:set(ip, "ko", 86400)
+				return true
+			end
+		end
+	end
+	ngx.shared.blacklist_reverse_cache:set(ip, "ok", 86400)
+	return false
+end
--- a/lua/dns.lua
+++ b/lua/dns.lua
@ -0,0 +1,40 @@
+local resolver	= require "resty.dns.resolver"
+local resolvers	= {%DNS_RESOLVERS%}
+local ip	= ngx.var.remote_addr
+
+function get_reverse()
+	local r, err = resolver:new{nameservers=resolvers, retrans=2, timeout=2000}
+	if not r then
+		return ""
+	end
+	local rdns = ""
+	local answers, err = r:reverse_query(ip)
+	if not answers.errcode then
+		for ak, av in ipairs(answers) do
+			if av.ptrdname then
+				rdns = av.ptrdname
+				break
+			end
+		end
+	end
+	return rdns
+end
+
+function get_ips(fqdn)
+	local r, err = resolver:new{nameservers=resolvers, retrans=2, timeout=2000}
+	if not r then
+		return ""
+	end
+	local ips = {}
+	local answers, err, tries = r:query(fqdn, nil, {})
+	for ak, av in ipairs(answers) do
+		if av.address then
+			table.insert(ips, av.address)
+		end
+	end
+	return ips
+end
+
+function ip_to_arpa()
+	return resolver.arpa_str(ip):gsub("%.in%-addr%.arpa", ""):gsub("%.ip6%.arpa", "")
+end
--- a/lua/dnsbl.lua
+++ b/lua/dnsbl.lua
@ -0,0 +1,28 @@
+local dns	= require "dns"
+local dnsbls	= {%DNSBL_LIST%}
+local ip	= ngx.var.remote_addr
+
+function cached_ko ()
+	return ngx.shared.dnsbl_cache:get(ip) == "ko"
+end
+
+function cached ()
+	return ngx.shared.dnsbl_cache:get(ip) ~= nil
+end
+
+function check ()
+	local rip = dns.ip_to_arpa()
+	for k, v in ipairs(dnsbls) do
+		local req = rip .. "." .. v
+		local ips = dns.get_ips(req)
+		for k2, v2 in ipairs(ips) do
+			a,b,c,d = v2:match("([%d]+).([%d]+).([%d]+).([%d]+)")
+			if a == "127" then
+				ngx.shared.dnsbl_cache:set(ip, "ko", 86400)
+				return true
+			end
+		end
+	end
+	ngx.shared.dnsbl_cache:set(ip, "ok", 86400)
+	return false
+end
--- a/lua/whitelist.lua
+++ b/lua/whitelist.lua
@ -0,0 +1,55 @@
+local dns		= require "dns"
+local ip_list 		= {%WHITELIST_IP_LIST%}
+local reverse_list	= {%WHITELIST_REVERSE_LIST%}
+local ip		= ngx.var.remote_addr
+
+function ip_cached_ok ()
+	return ngx.shared.whitelist_ip_cache:get(ip) == "ok"
+end
+
+function reverse_cached_ok ()
+	return ngx.shared.whitelist_reverse_cache:get(ip) == "ok"
+end
+
+function ip_cached ()
+	return ngx.shared.whitelist_ip_cache:get(ip) ~= nil
+end
+
+function reverse_cached ()
+	return ngx.shared.whitelist_reverse_cache:get(ip) ~= nil
+end
+
+function check_ip ()
+	for k, v in ipairs(ip_list) do
+		if v == ip then
+			ngx.shared.whitelist_ip_cache:set(ip, "ok", 86400)
+			return true
+		end
+	end
+	ngx.shared.whitelist_ip_cache:set(ip, "ko", 86400)
+	return false
+end
+
+function check_reverse ()
+	local rdns = dns.get_reverse()
+	if rdns ~= "" then
+		local whitelisted = false
+		for k, v in ipairs(reverse_list) do
+			if rdns:sub(-#v) == v then
+				whitelisted = true
+				break
+			end
+		end
+		if whitelisted then
+			local ips = dns.get_ips(rdns)
+			for k, v in ipairs(ips) do
+				if v == ip then
+					ngx.shared.whitelist_reverse_cache:set(ip, "ok", 86400)
+					return true
+				end
+			end
+		end
+	end
+	ngx.shared.whitelist_reverse_cache:set(ip, "ko", 86400)
+	return false
+end