performance - move bad user-agents and referrers checks from nginx to LUA with caching

confs/global/init-lua.conf

@@ -1,10 +1,12 @@
 init_by_lua_block {
 
-local dataloader	= require "dataloader"
+local dataloader		= require "dataloader"
 
 local use_proxies		= %USE_PROXIES%
 local use_abusers		= %USE_ABUSERS%
 local use_tor_exit_nodes	= %USE_TOR_EXIT_NODES%
+local use_user_agents		= %USE_USER_AGENTS%
+local use_referrers		= %USE_REFERRERS%
 
 if use_proxies then
 	dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data)
@@ -18,4 +20,12 @@ if use_tor_exit_nodes then
 	dataloader.load_ip("/etc/nginx/tor-exit-nodes.list", ngx.shared.tor_exit_nodes_data)
 end
 
+if use_user_agents then
+	dataloader.load_raw("/etc/nginx/user-agents.list", ngx.shared.user_agents_data)
+end
+
+if use_referrers then
+	dataloader.load_raw("/etc/nginx/referrers.list", ngx.shared.referrers_data)
+end
+
 }

confs/global/map-referrer.conf

@@ -1 +0,0 @@
-map $http_referer $bad_referrer { hostnames; default no; }

confs/global/map-user-agent.conf

@@ -1 +0,0 @@
-map $http_user_agent $bad_user_agent { default no; }

confs/global/nginx.conf

@@ -84,6 +84,11 @@ http {
 	%BLACKLIST_IP_CACHE%
 	%BLACKLIST_REVERSE_CACHE%
 	%DNSBL_CACHE%
+	%BLOCK_PROXIES%
+	%BLOCK_ABUSERS%
+	%BLOCK_TOR_EXIT_NODES%
+	%BLOCK_USER_AGENTS%
+	%BLOCK_REFERRERS%
 
 	# crowdsec init
 	%USE_CROWDSEC%
@@ -94,24 +99,9 @@ http {
 	# shared memory zone for limit_conn
 	%LIMIT_CONN_ZONE%
 
-	# shared memory zone for proxies
-	%BLOCK_PROXIES%
-
-	# shared memory zone for abusers
-	%BLOCK_ABUSERS%
-
-	# shared memory zone for TOR exit nodes
-	%BLOCK_TOR_EXIT_NODES%
-
 	# whitelist or blacklist country
 	%USE_COUNTRY%
 
-	# list of blocked user agents
-	%BLOCK_USER_AGENT%
-
-	# list of blocked referrers
-	%BLOCK_REFERRER%
-
 	# zone for proxy_cache
 	%PROXY_CACHE_PATH%
 

confs/site/main-lua.conf

@@ -6,11 +6,11 @@ access_by_lua_block {
 local use_lets_encrypt		= %USE_LETS_ENCRYPT%
 local use_whitelist_ip		= %USE_WHITELIST_IP%
 local use_whitelist_reverse	= %USE_WHITELIST_REVERSE%
-local use_user_agent		= %USE_USER_AGENT%
+local use_user_agents		= %USE_USER_AGENTS%
 local use_proxies		= %USE_PROXIES%
 local use_abusers		= %USE_ABUSERS%
 local use_tor_exit_nodes	= %USE_TOR_EXIT_NODES%
-local use_referrer		= %USE_REFERRER%
+local use_referrers		= %USE_REFERRERS%
 local use_country		= %USE_COUNTRY%
 local use_blacklist_ip		= %USE_BLACKLIST_IP%
 local use_blacklist_reverse	= %USE_BLACKLIST_REVERSE%
@@ -126,25 +126,59 @@ if use_tor_exit_nodes then
 end
 
 -- check if user-agent is allowed
-if use_user_agent and ngx.var.bad_user_agent == "yes" then
+if use_user_agents then
-	local block = true
+	local whitelisted = false
 	for k, v in pairs(whitelist_user_agent) do
 		if string.match(ngx.var.http_user_agent, v) then
 			ngx.log(ngx.NOTICE, "[ALLOW] User-Agent " .. ngx.var.http_user_agent .. " is whitelisted")
-			block = false
+			whitelisted = true
 			break
 		end
 	end
-	if block then
+	if not whitelisted then
-		ngx.log(ngx.NOTICE, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
+		local value, flags = ngx.shared.user_agents_cache:get(ngx.var.http_user_agent)
-		ngx.exit(ngx.HTTP_FORBIDDEN)
+		if value == nil then
+			local patterns = ngx.shared.user_agents_data:get_keys(0)
+			for i, pattern in ipairs(patterns) do
+				if string.match(ngx.var.http_user_agent, pattern) then
+					value = "ko"
+					ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ko", 86400)
+					break
+				end
+			end
+			if value == nil then
+				value = "ok"
+				ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ok", 86400)
+			end
+		end
+		if value == "ko" then
+			ngx.log(ngx.NOTICE, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
+			ngx.exit(ngx.HTTP_FORBIDDEN)
+		end
 	end
 end
 
 -- check if referrer is allowed
-if use_referrer and ngx.var.bad_referrer == "yes" then
+if use_referrer then
-	ngx.log(ngx.NOTICE, "[BLOCK] Referrer " .. ngx.var.http_referer .. " is blacklisted")
+	local value, flags = ngx.shared.referrers_cache:get(ngx.var.http_referer)
-	ngx.exit(ngx.HTTP_FORBIDDEN)
+	if value == nil then
+		local patterns = ngx.shared.referrers_data:get_keys(0)
+		for i, pattern in ipairs(patterns) do
+			if string.match(ngx.var.http_referer, pattern) then
+				value = "ko"
+				ngx.shared.referrers_cache:set(ngx.var.http_referer, "ko", 86400)
+				break
+			end
+		end
+		if value == nil then
+			value = "ok"
+			ngx.shared.referrers_cache:set(ngx.var.http_referer, "ok", 86400)
+		end
+	end
+	if value == "ko" then
+		ngx.log(ngx.NOTICE, "[BLOCK] Referrer " .. ngx.var.http_referer .. " is blacklisted")
+		ngx.exit(ngx.HTTP_FORBIDDEN)
+	end
 end
 
 -- check if country is allowed

entrypoint/global-config.sh

@@ -113,18 +113,22 @@ fi
 
 # block bad UA
 if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then
-	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENT%" "include /etc/nginx/map-user-agent.conf;"
 	echo "$BLOCK_USER_AGENT_CRON /opt/scripts/user-agents.sh" >> /etc/crontabs/nginx
+	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENTS%" "lua_shared_dict user_agents_data 1m; lua_shared_dict user_agents_cache 10m;"
+	replace_in_file "/etc/nginx/init-lua.conf" "%USE_USER_AGENTS%" "true"
 else
-	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENT%" ""
+	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENTS%" ""
+	replace_in_file "/etc/nginx/init-lua.conf" "%USE_USER_AGENTS%" "false"
 fi
 
 # block bad refferer
 if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then
-	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRER%" "include /etc/nginx/map-referrer.conf;"
 	echo "$BLOCK_REFERRER_CRON /opt/scripts/referrers.sh" >> /etc/crontabs/nginx
+	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRERS%" "lua_shared_dict referrers_data 1m; lua_shared_dict referrers_cache 10m;"
+	replace_in_file "/etc/nginx/init-lua.conf" "%USE_REFERRERS%" "true"
 else
-	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRER%" ""
+	replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRERS%" ""
+	replace_in_file "/etc/nginx/init-lua.conf" "%USE_REFERRERS%" "false"
 fi
 
 # block TOR exit nodes

entrypoint/jobs.sh

@@ -19,9 +19,9 @@ fi
 
 # User-Agents
 if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then
-	if [ -f "/cache/map-user-agent.conf" ] ; then
+	if [ -f "/cache/user-agents.list" ] ; then
-		echo "[*] Copying cached map-user-agent.conf ..."
+		echo "[*] Copying cached user-agents.list ..."
-		cp /cache/map-user-agent.conf /etc/nginx/map-user-agent.conf
+		cp /cache/user-agents.list /etc/nginx/user-agents.list
 	else
 		echo "[*] Downloading bad user-agent list (in background) ..."
 		/opt/scripts/user-agents.sh > /dev/null 2>&1 &
@@ -30,9 +30,9 @@ fi
 
 # Referrers
 if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then
-	if [ -f "/cache/map-referrer.conf" ] ; then
+	if [ -f "/cache/referrers.list" ] ; then
-		echo "[*] Copying cached map-referrer.conf ..."
+		echo "[*] Copying cached referrers.list ..."
-		cp /cache/map-referrer.conf /etc/nginx/map-referrer.conf
+		cp /cache/referrers.list /etc/nginx/referrers.list
 	else
 		echo "[*] Downloading bad referrer list (in background) ..."
 		/opt/scripts/referrers.sh > /dev/null 2>&1 &
@@ -41,9 +41,9 @@ fi
 
 # exit nodes
 if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then
-	if [ -f "/cache/block-tor-exit-node.conf" ] ; then
+	if [ -f "/cache/tor-exit-nodes.list" ] ; then
-		echo "[*] Copying cached block-tor-exit-node.conf ..."
+		echo "[*] Copying cached tor-exit-nodes.list ..."
-		cp /cache/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf
+		cp /cache/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list
 	else
 		echo "[*] Downloading tor exit nodes list (in background) ..."
 		/opt/scripts/exit-nodes.sh > /dev/null 2>&1 &
@@ -52,9 +52,9 @@ fi
 
 # proxies
 if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then
-	if [ -f "/cache/block-proxies.conf" ] ; then
+	if [ -f "/cache/proxies.list" ] ; then
-		echo "[*] Copying cached block-proxies.conf ..."
+		echo "[*] Copying cached proxies.list ..."
-		cp /cache/block-proxies.conf /etc/nginx/block-proxies.conf
+		cp /cache/proxies.list /etc/nginx/proxies.list
 	else
 		echo "[*] Downloading proxies list (in background) ..."
 		/opt/scripts/proxies.sh > /dev/null 2>&1 &
@@ -63,9 +63,9 @@ fi
 
 # abusers
 if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then
-	if [ -f "/cache/block-abusers.conf" ] ; then
+	if [ -f "/cache/abusers.list" ] ; then
-		echo "[*] Copying cached block-abusers.conf ..."
+		echo "[*] Copying cached abusers.list ..."
-		cp /cache/block-abusers.conf /etc/nginx/block-abusers.conf
+		cp /cache/abusers.list /etc/nginx/abusers.list
 	else
 		echo "[*] Downloading abusers list (in background) ..."
 		/opt/scripts/abusers.sh > /dev/null 2>&1 &

entrypoint/site-config.sh

@@ -278,7 +278,7 @@ fi
 
 # block bad UA
 if [ "$BLOCK_USER_AGENT" = "yes" ] ; then
-	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENT%" "true"
+	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENTS%" "true"
 	if [ "$WHITELIST_USER_AGENT" != "" ] ; then
 		list=$(spaces_to_lua "$WHITELIST_USER_AGENT")
 		replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" "$list"
@@ -286,7 +286,7 @@ if [ "$BLOCK_USER_AGENT" = "yes" ] ; then
 		replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" ""
 	fi
 else
-	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENT%" "false"
+	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENTS%" "false"
 	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" ""
 fi
 
@@ -300,7 +300,7 @@ fi
 
 # block bad referrer
 if [ "$BLOCK_REFERRER" = "yes" ] ; then
-	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_REFERRER%" "true"
+	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_REFERRERS%" "true"
 else
 	replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_REFERRER%" "false"
 fi

lua/dataloader.lua

@@ -1,7 +1,15 @@
 local M		= {}
 local iputils	= require "resty.iputils"
 
+function M.flush_dict (dict)
+	local keys = dict:get_keys(0)
+	for i, key in ipairs(keys) do
+		dict:delete(key)
+	end
+end
+
 function M.load_ip (path, dict)
+	M.flush_dict(dict)
 	local file = io.open(path, "r")
 	if not file then
 		ngx.log(ngx.ERR, "[INIT] can't open " .. path)
@@ -9,11 +17,17 @@ function M.load_ip (path, dict)
 		io.input(file)
 		local i = 0
 		for line in io.lines() do
+			local continue = true
 			if string.match(line, "/") then
 				local lower, upper = iputils.parse_cidr(line)
 				local bin_ip = lower
 				while bin_ip <= upper do
-					dict:set(bin_ip, true, 0)
+					local ok, err = dict:safe_set(bin_ip, true, 0)
+					if not ok then
+						ngx.log(ngx.ERR, "[INIT] not enough memory allocated to load data from " .. path) 
+						continue = false
+						break
+					end
 					bin_ip = bin_ip + 1
 					i = i + 1
 				end
@@ -22,8 +36,32 @@ function M.load_ip (path, dict)
 				dict:set(bin_ip, true, 0)
 				i = i + 1
 			end
+			if not continue then
+				break
+			end
 		end
-		ngx.log(ngx.ERR, "[INIT] loaded " .. tostring(i) .. " IPs from " .. path)
+		ngx.log(ngx.ERR, "[INIT] *NOT AN ERROR* loaded " .. tostring(i) .. " IPs from " .. path)
+		io.close(file)
+	end
+end
+
+function M.load_raw (path, dict)
+	M.flush_dict(dict)
+	local file = io.open(path, "r")
+	if not file then
+		ngx.log(ngx.ERR, "[INIT] can't open " .. path)
+	else
+		io.input(file)
+		local i = 0
+		for line in io.lines() do
+			local ok, err = dict:safe_set(line, true, 0)
+			if not ok then
+				ngx.log(ngx.ERR, "[INIT] not enough memory allocated to load data from " .. path) 
+				break
+			end
+			i = i + 1
+		end
+		ngx.log(ngx.ERR, "[INIT] *NOT AN ERROR* loaded " .. tostring(i) .. " entries from " .. path)
 		io.close(file)
 	end
 end

scripts/referrers.sh

@@ -4,14 +4,16 @@
 . /opt/entrypoint/utils.sh
 
 # save old conf
-cp /etc/nginx/map-referrer.conf /cache
+cp /etc/nginx/referrers.list /cache
 
 # generate new conf
-BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)"
+BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list | sed 's:\.:%\.:g;s:\-:%\-:g')"
 if [ "$?" -ne 0 ] ; then
 	job_log "[BLACKLIST] can't update referrers list"
+	echo -n "" > /tmp/referrers.list
+else
+	echo -e "$BLACKLIST" > /tmp/referrers.list
 fi
-echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $(echo -e "$BLACKLIST" | sed 's/^/"~/;s/$/" yes;/') }" > /tmp/map-referrer.conf
 
 # if we are running nginx
 if [ -f /tmp/nginx.pid ] ; then
@@ -22,25 +24,25 @@ elif [ -S /tmp/autoconf.sock ] ; then
 fi
 
 # check number of lines
-lines="$(wc -l /tmp/map-referrer.conf | cut -d ' ' -f 1)"
+lines="$(wc -l /tmp/referrers.list | cut -d ' ' -f 1)"
 if [ "$lines" -gt 1 ] ; then
-	mv /tmp/map-referrer.conf /etc/nginx/map-referrer.conf
+	mv /tmp/referrers.list /etc/nginx/referrers.list
 	job_log "[BLACKLIST] referrers list updated ($lines entries)"
 	if [ "$RELOAD" != "" ] ; then
 		$RELOAD > /dev/null 2>&1
 		if [ "$?" -eq 0 ] ; then
-			cp /etc/nginx/map-referrer.conf /cache
+			cp /etc/nginx/referrers.list /cache
 			job_log "[NGINX] successfull nginx reload after referrers list update"
 		else
-			cp /cache/map-referrer.conf /etc/nginx
+			cp /cache/referrers.list /etc/nginx
 			job_log "[NGINX] failed nginx reload after referrers list update fallback to old list"
 			$RELOAD > /dev/null 2>&1
 		fi
 	else
-		cp /etc/nginx/map-referrer.conf /cache
+		cp /etc/nginx/referrers.list /cache
 	fi
 else
 	job_log "[BLACKLIST] can't update referrers list"
 fi
 
-rm -f /tmp/map-referrer.conf 2> /dev/null
+rm -f /tmp/referrers.list 2> /dev/null

scripts/user-agents.sh

@@ -4,15 +4,16 @@
 . /opt/entrypoint/utils.sh
 
 # save old conf
-cp /etc/nginx/map-user-agent.conf /cache
+cp /etc/nginx/user-agents.list /cache
 
 # generate new conf
-IFS= BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list;
+BLACKLIST="$( (curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list ; curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt) | sort -u | sed 's:\\ : :g;s:\\\.:%\.:g;s:\\\\:\\:g;s:\\/:/:g;s:\-:%\-:g')"
-curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt | sort -u)"
 if [ "$?" -ne 0 ] ; then
 	job_log "[BLACKLIST] can't update user-agent list"
+	echo -n "" > /tmp/user-agents.list
+else
+	echo -e "$BLACKLIST" > /tmp/user-agents.list
 fi
-echo -e "map \$http_user_agent \$bad_user_agent { default no; $(echo $BLACKLIST | sed 's: :\\ :g;s:^:~*:;s:$: yes;:') }" > /tmp/map-user-agent.conf
 
 # if we are running nginx
 if [ -f /tmp/nginx.pid ] ; then
@@ -23,25 +24,25 @@ elif [ -S /tmp/autoconf.sock ] ; then
 fi
 
 # check number of lines
-lines="$(wc -l /tmp/map-user-agent.conf | cut -d ' ' -f 1)"
+lines="$(wc -l /tmp/user-agents.list | cut -d ' ' -f 1)"
 if [ "$lines" -gt 1 ] ; then
-	mv /tmp/map-user-agent.conf /etc/nginx/map-user-agent.conf
+	mv /tmp/user-agents.list /etc/nginx/user-agents.list
 	job_log "[BLACKLIST] user-agent list updated ($lines entries)"
 	if [ "$RELOAD" != "" ] ; then
 		$RELOAD > /dev/null 2>&1
 		if [ "$?" -eq 0 ] ; then
-			cp /etc/nginx/map-user-agent.conf /cache
+			cp /etc/nginx/user-agents.list /cache
 			job_log "[NGINX] successfull nginx reload after user-agent list update"
 		else
-			cp /cache/map-user-agent.conf /etc/nginx
+			cp /cache/user-agents.list /etc/nginx
 			job_log "[NGINX] failed nginx reload after user-agent list update fallback to old list"
 			$RELOAD > /dev/null 2>&1
 		fi
 	else
-		cp /etc/nginx/map-user-agent.conf /cache
+		cp /etc/nginx/user-agents.list /cache
 	fi
 else
 	job_log "[BLACKLIST] can't update user-agent list"
 fi
 
-rm -f /tmp/map-user-agent.conf 2> /dev/null
+rm -f /tmp/user-agents.list 2> /dev/null