librewolf
/
bunkerized-nginx
mirror of https://github.com/bunkerity/bunkerized-nginx
4
0
Fork 1
Code Issues Projects Releases Wiki Activity

performance - move bad user-agents and referrers checks from nginx to LUA with caching

This commit is contained in:
bunkerity 2021-05-11 15:30:16 +02:00
parent 42c3fb8740
commit b1d03cd11c
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
13 changed files with 149 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
confs/global/init-lua.conf
View File

@ -1,10 +1,12 @@
init_by_lua_block {
local dataloader = require "dataloader"
local dataloader = require "dataloader"
local use_proxies = %USE_PROXIES%
local use_abusers = %USE_ABUSERS%
local use_tor_exit_nodes = %USE_TOR_EXIT_NODES%
local use_user_agents = %USE_USER_AGENTS%
local use_referrers = %USE_REFERRERS%
if use_proxies then
dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data)
@ -18,4 +20,12 @@ if use_tor_exit_nodes then
dataloader.load_ip("/etc/nginx/tor-exit-nodes.list", ngx.shared.tor_exit_nodes_data)
end
if use_user_agents then
dataloader.load_raw("/etc/nginx/user-agents.list", ngx.shared.user_agents_data)
end
if use_referrers then
dataloader.load_raw("/etc/nginx/referrers.list", ngx.shared.referrers_data)
end
}

1
confs/global/map-referrer.conf
View File

@ -1 +0,0 @@
map $http_referer $bad_referrer { hostnames; default no; }

1
confs/global/map-user-agent.conf
View File

@ -1 +0,0 @@
map $http_user_agent $bad_user_agent { default no; }

20
confs/global/nginx.conf
View File

@ -84,6 +84,11 @@ http {
%BLACKLIST_IP_CACHE%
%BLACKLIST_REVERSE_CACHE%
%DNSBL_CACHE%
%BLOCK_PROXIES%
%BLOCK_ABUSERS%
%BLOCK_TOR_EXIT_NODES%
%BLOCK_USER_AGENTS%
%BLOCK_REFERRERS%
# crowdsec init
%USE_CROWDSEC%
@ -94,24 +99,9 @@ http {
# shared memory zone for limit_conn
%LIMIT_CONN_ZONE%
# shared memory zone for proxies
%BLOCK_PROXIES%
# shared memory zone for abusers
%BLOCK_ABUSERS%
# shared memory zone for TOR exit nodes
%BLOCK_TOR_EXIT_NODES%
# whitelist or blacklist country
%USE_COUNTRY%
# list of blocked user agents
%BLOCK_USER_AGENT%
# list of blocked referrers
%BLOCK_REFERRER%
# zone for proxy_cache
%PROXY_CACHE_PATH%

0
confs/global/referrers.list Normal file
View File

0
confs/global/user-agents.list Normal file
View File

56
confs/site/main-lua.conf
View File

@ -6,11 +6,11 @@ access_by_lua_block {
local use_lets_encrypt = %USE_LETS_ENCRYPT%
local use_whitelist_ip = %USE_WHITELIST_IP%
local use_whitelist_reverse = %USE_WHITELIST_REVERSE%
local use_user_agent = %USE_USER_AGENT%
local use_user_agents = %USE_USER_AGENTS%
local use_proxies = %USE_PROXIES%
local use_abusers = %USE_ABUSERS%
local use_tor_exit_nodes = %USE_TOR_EXIT_NODES%
local use_referrer = %USE_REFERRER%
local use_referrers = %USE_REFERRERS%
local use_country = %USE_COUNTRY%
local use_blacklist_ip = %USE_BLACKLIST_IP%
local use_blacklist_reverse = %USE_BLACKLIST_REVERSE%
@ -126,25 +126,59 @@ if use_tor_exit_nodes then
end
-- check if user-agent is allowed
if use_user_agent and ngx.var.bad_user_agent == "yes" then
local block = true
if use_user_agents then
local whitelisted = false
for k, v in pairs(whitelist_user_agent) do
if string.match(ngx.var.http_user_agent, v) then
ngx.log(ngx.NOTICE, "[ALLOW] User-Agent " .. ngx.var.http_user_agent .. " is whitelisted")
block = false
whitelisted = true
break
end
end
if block then
ngx.log(ngx.NOTICE, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
ngx.exit(ngx.HTTP_FORBIDDEN)
if not whitelisted then
local value, flags = ngx.shared.user_agents_cache:get(ngx.var.http_user_agent)
if value == nil then
local patterns = ngx.shared.user_agents_data:get_keys(0)
for i, pattern in ipairs(patterns) do
if string.match(ngx.var.http_user_agent, pattern) then
value = "ko"
ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ko", 86400)
break
end
end
if value == nil then
value = "ok"
ngx.shared.user_agents_cache:set(ngx.var.http_user_agent, "ok", 86400)
end
end
if value == "ko" then
ngx.log(ngx.NOTICE, "[BLOCK] User-Agent " .. ngx.var.http_user_agent .. " is blacklisted")
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
end
-- check if referrer is allowed
if use_referrer and ngx.var.bad_referrer == "yes" then
ngx.log(ngx.NOTICE, "[BLOCK] Referrer " .. ngx.var.http_referer .. " is blacklisted")
ngx.exit(ngx.HTTP_FORBIDDEN)
if use_referrer then
local value, flags = ngx.shared.referrers_cache:get(ngx.var.http_referer)
if value == nil then
local patterns = ngx.shared.referrers_data:get_keys(0)
for i, pattern in ipairs(patterns) do
if string.match(ngx.var.http_referer, pattern) then
value = "ko"
ngx.shared.referrers_cache:set(ngx.var.http_referer, "ko", 86400)
break
end
end
if value == nil then
value = "ok"
ngx.shared.referrers_cache:set(ngx.var.http_referer, "ok", 86400)
end
end
if value == "ko" then
ngx.log(ngx.NOTICE, "[BLOCK] Referrer " .. ngx.var.http_referer .. " is blacklisted")
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- check if country is allowed

12
entrypoint/global-config.sh
View File

@ -113,18 +113,22 @@ fi
# block bad UA
if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENT%" "include /etc/nginx/map-user-agent.conf;"
echo "$BLOCK_USER_AGENT_CRON /opt/scripts/user-agents.sh" >> /etc/crontabs/nginx
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENTS%" "lua_shared_dict user_agents_data 1m; lua_shared_dict user_agents_cache 10m;"
replace_in_file "/etc/nginx/init-lua.conf" "%USE_USER_AGENTS%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENT%" ""
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_USER_AGENTS%" ""
replace_in_file "/etc/nginx/init-lua.conf" "%USE_USER_AGENTS%" "false"
fi
# block bad refferer
if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRER%" "include /etc/nginx/map-referrer.conf;"
echo "$BLOCK_REFERRER_CRON /opt/scripts/referrers.sh" >> /etc/crontabs/nginx
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRERS%" "lua_shared_dict referrers_data 1m; lua_shared_dict referrers_cache 10m;"
replace_in_file "/etc/nginx/init-lua.conf" "%USE_REFERRERS%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRER%" ""
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_REFERRERS%" ""
replace_in_file "/etc/nginx/init-lua.conf" "%USE_REFERRERS%" "false"
fi
# block TOR exit nodes

30
entrypoint/jobs.sh
View File

@ -19,9 +19,9 @@ fi
# User-Agents
if [ "$(has_value BLOCK_USER_AGENT yes)" != "" ] ; then
if [ -f "/cache/map-user-agent.conf" ] ; then
echo "[*] Copying cached map-user-agent.conf ..."
cp /cache/map-user-agent.conf /etc/nginx/map-user-agent.conf
if [ -f "/cache/user-agents.list" ] ; then
echo "[*] Copying cached user-agents.list ..."
cp /cache/user-agents.list /etc/nginx/user-agents.list
else
echo "[*] Downloading bad user-agent list (in background) ..."
/opt/scripts/user-agents.sh > /dev/null 2>&1 &
@ -30,9 +30,9 @@ fi
# Referrers
if [ "$(has_value BLOCK_REFERRER yes)" != "" ] ; then
if [ -f "/cache/map-referrer.conf" ] ; then
echo "[*] Copying cached map-referrer.conf ..."
cp /cache/map-referrer.conf /etc/nginx/map-referrer.conf
if [ -f "/cache/referrers.list" ] ; then
echo "[*] Copying cached referrers.list ..."
cp /cache/referrers.list /etc/nginx/referrers.list
else
echo "[*] Downloading bad referrer list (in background) ..."
/opt/scripts/referrers.sh > /dev/null 2>&1 &
@ -41,9 +41,9 @@ fi
# exit nodes
if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then
if [ -f "/cache/block-tor-exit-node.conf" ] ; then
echo "[*] Copying cached block-tor-exit-node.conf ..."
cp /cache/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf
if [ -f "/cache/tor-exit-nodes.list" ] ; then
echo "[*] Copying cached tor-exit-nodes.list ..."
cp /cache/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list
else
echo "[*] Downloading tor exit nodes list (in background) ..."
/opt/scripts/exit-nodes.sh > /dev/null 2>&1 &
@ -52,9 +52,9 @@ fi
# proxies
if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then
if [ -f "/cache/block-proxies.conf" ] ; then
echo "[*] Copying cached block-proxies.conf ..."
cp /cache/block-proxies.conf /etc/nginx/block-proxies.conf
if [ -f "/cache/proxies.list" ] ; then
echo "[*] Copying cached proxies.list ..."
cp /cache/proxies.list /etc/nginx/proxies.list
else
echo "[*] Downloading proxies list (in background) ..."
/opt/scripts/proxies.sh > /dev/null 2>&1 &
@ -63,9 +63,9 @@ fi
# abusers
if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then
if [ -f "/cache/block-abusers.conf" ] ; then
echo "[*] Copying cached block-abusers.conf ..."
cp /cache/block-abusers.conf /etc/nginx/block-abusers.conf
if [ -f "/cache/abusers.list" ] ; then
echo "[*] Copying cached abusers.list ..."
cp /cache/abusers.list /etc/nginx/abusers.list
else
echo "[*] Downloading abusers list (in background) ..."
/opt/scripts/abusers.sh > /dev/null 2>&1 &

6
entrypoint/site-config.sh
View File

@ -278,7 +278,7 @@ fi
# block bad UA
if [ "$BLOCK_USER_AGENT" = "yes" ] ; then
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENT%" "true"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENTS%" "true"
if [ "$WHITELIST_USER_AGENT" != "" ] ; then
list=$(spaces_to_lua "$WHITELIST_USER_AGENT")
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" "$list"
@ -286,7 +286,7 @@ if [ "$BLOCK_USER_AGENT" = "yes" ] ; then
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" ""
fi
else
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENT%" "false"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_USER_AGENTS%" "false"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%WHITELIST_USER_AGENT%" ""
fi
@ -300,7 +300,7 @@ fi
# block bad referrer
if [ "$BLOCK_REFERRER" = "yes" ] ; then
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_REFERRER%" "true"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_REFERRERS%" "true"
else
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_REFERRER%" "false"
fi

42
lua/dataloader.lua
View File

@ -1,7 +1,15 @@
local M = {}
local iputils = require "resty.iputils"
function M.flush_dict (dict)
local keys = dict:get_keys(0)
for i, key in ipairs(keys) do
dict:delete(key)
end
end
function M.load_ip (path, dict)
M.flush_dict(dict)
local file = io.open(path, "r")
if not file then
ngx.log(ngx.ERR, "[INIT] can't open " .. path)
@ -9,11 +17,17 @@ function M.load_ip (path, dict)
io.input(file)
local i = 0
for line in io.lines() do
local continue = true
if string.match(line, "/") then
local lower, upper = iputils.parse_cidr(line)
local bin_ip = lower
while bin_ip <= upper do
dict:set(bin_ip, true, 0)
local ok, err = dict:safe_set(bin_ip, true, 0)
if not ok then
ngx.log(ngx.ERR, "[INIT] not enough memory allocated to load data from " .. path)
continue = false
break
end
bin_ip = bin_ip + 1
i = i + 1
end
@ -22,8 +36,32 @@ function M.load_ip (path, dict)
dict:set(bin_ip, true, 0)
i = i + 1
end
if not continue then
break
end
end
ngx.log(ngx.ERR, "[INIT] loaded " .. tostring(i) .. " IPs from " .. path)
ngx.log(ngx.ERR, "[INIT] *NOT AN ERROR* loaded " .. tostring(i) .. " IPs from " .. path)
io.close(file)
end
end
function M.load_raw (path, dict)
M.flush_dict(dict)
local file = io.open(path, "r")
if not file then
ngx.log(ngx.ERR, "[INIT] can't open " .. path)
else
io.input(file)
local i = 0
for line in io.lines() do
local ok, err = dict:safe_set(line, true, 0)
if not ok then
ngx.log(ngx.ERR, "[INIT] not enough memory allocated to load data from " .. path)
break
end
i = i + 1
end
ngx.log(ngx.ERR, "[INIT] *NOT AN ERROR* loaded " .. tostring(i) .. " entries from " .. path)
io.close(file)
end
end

20
scripts/referrers.sh
View File

@ -4,14 +4,16 @@
. /opt/entrypoint/utils.sh
# save old conf
cp /etc/nginx/map-referrer.conf /cache
cp /etc/nginx/referrers.list /cache
# generate new conf
BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)"
BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list | sed 's:\.:%\.:g;s:\-:%\-:g')"
if [ "$?" -ne 0 ] ; then
job_log "[BLACKLIST] can't update referrers list"
echo -n "" > /tmp/referrers.list
else
echo -e "$BLACKLIST" > /tmp/referrers.list
fi
echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $(echo -e "$BLACKLIST" | sed 's/^/"~/;s/$/" yes;/') }" > /tmp/map-referrer.conf
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@ -22,25 +24,25 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check number of lines
lines="$(wc -l /tmp/map-referrer.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/referrers.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
mv /tmp/map-referrer.conf /etc/nginx/map-referrer.conf
mv /tmp/referrers.list /etc/nginx/referrers.list
job_log "[BLACKLIST] referrers list updated ($lines entries)"
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/map-referrer.conf /cache
cp /etc/nginx/referrers.list /cache
job_log "[NGINX] successfull nginx reload after referrers list update"
else
cp /cache/map-referrer.conf /etc/nginx
cp /cache/referrers.list /etc/nginx
job_log "[NGINX] failed nginx reload after referrers list update fallback to old list"
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/map-referrer.conf /cache
cp /etc/nginx/referrers.list /cache
fi
else
job_log "[BLACKLIST] can't update referrers list"
fi
rm -f /tmp/map-referrer.conf 2> /dev/null
rm -f /tmp/referrers.list 2> /dev/null

21
scripts/user-agents.sh
View File

@ -4,15 +4,16 @@
. /opt/entrypoint/utils.sh
# save old conf
cp /etc/nginx/map-user-agent.conf /cache
cp /etc/nginx/user-agents.list /cache
# generate new conf
IFS= BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list;
curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt | sort -u)"
BLACKLIST="$( (curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list ; curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt) | sort -u | sed 's:\\ : :g;s:\\\.:%\.:g;s:\\\\:\\:g;s:\\/:/:g;s:\-:%\-:g')"
if [ "$?" -ne 0 ] ; then
job_log "[BLACKLIST] can't update user-agent list"
echo -n "" > /tmp/user-agents.list
else
echo -e "$BLACKLIST" > /tmp/user-agents.list
fi
echo -e "map \$http_user_agent \$bad_user_agent { default no; $(echo $BLACKLIST | sed 's: :\\ :g;s:^:~*:;s:$: yes;:') }" > /tmp/map-user-agent.conf
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@ -23,25 +24,25 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check number of lines
lines="$(wc -l /tmp/map-user-agent.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/user-agents.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
mv /tmp/map-user-agent.conf /etc/nginx/map-user-agent.conf
mv /tmp/user-agents.list /etc/nginx/user-agents.list
job_log "[BLACKLIST] user-agent list updated ($lines entries)"
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/map-user-agent.conf /cache
cp /etc/nginx/user-agents.list /cache
job_log "[NGINX] successfull nginx reload after user-agent list update"
else
cp /cache/map-user-agent.conf /etc/nginx
cp /cache/user-agents.list /etc/nginx
job_log "[NGINX] failed nginx reload after user-agent list update fallback to old list"
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/map-user-agent.conf /cache
cp /etc/nginx/user-agents.list /cache
fi
else
job_log "[BLACKLIST] can't update user-agent list"
fi
rm -f /tmp/map-user-agent.conf 2> /dev/null
rm -f /tmp/user-agents.list 2> /dev/null