librewolf
/
bunkerized-nginx
mirror of https://github.com/bunkerity/bunkerized-nginx
4
0
Fork 1
Code Issues Projects Releases Wiki Activity

performance - move external blacklists checks from nginx to LUA

This commit is contained in:
bunkerity 2021-05-10 17:51:07 +02:00
parent 009d6fb5ae
commit fd61df205f
No known key found for this signature in database
GPG Key ID: 3D80806F12602A7C
14 changed files with 141 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
confs/global/block-abusers.conf → confs/global/abusers.list
View File

21
confs/global/init-lua.conf Normal file
View File

@ -0,0 +1,21 @@
init_by_lua_block {
local dataloader = require "dataloader"
local use_proxies = %USE_PROXIES%
local use_abusers = %USE_ABUSERS%
local use_tor_exit_nodes = %USE_TOR_EXIT_NODES%
if use_proxies then
dataloader.load_ip("/etc/nginx/proxies.list", ngx.shared.proxies_data)
end
if use_abusers then
dataloader.load_ip("/etc/nginx/abusers.list", ngx.shared.abusers_data)
end
if use_tor_exit_nodes then
dataloader.load_ip("/etc/nginx/tor-exit-nodes.list", ngx.shared.tor_exit_nodes_data)
end
}

12
confs/global/nginx.conf
View File

@ -94,6 +94,15 @@ http {
# shared memory zone for limit_conn
%LIMIT_CONN_ZONE%
# shared memory zone for proxies
%BLOCK_PROXIES%
# shared memory zone for abusers
%BLOCK_ABUSERS%
# shared memory zone for TOR exit nodes
%BLOCK_TOR_EXIT_NODES%
# whitelist or blacklist country
%USE_COUNTRY%
@ -109,6 +118,9 @@ http {
# custom http confs
include /http-confs/*.conf;
# LUA init block
include /etc/nginx/init-lua.conf;
# default server when MULTISITE=yes
%MULTISITE_DEFAULT_SERVER%

0
confs/global/block-proxies.conf → confs/global/proxies.list
View File

0
confs/global/block-tor-exit-node.conf → confs/global/tor-exit-nodes.list
View File

31
confs/site/main-lua.conf
View File

@ -7,6 +7,9 @@ local use_lets_encrypt = %USE_LETS_ENCRYPT%
local use_whitelist_ip = %USE_WHITELIST_IP%
local use_whitelist_reverse = %USE_WHITELIST_REVERSE%
local use_user_agent = %USE_USER_AGENT%
local use_proxies = %USE_PROXIES%
local use_abusers = %USE_ABUSERS%
local use_tor_exit_nodes = %USE_TOR_EXIT_NODES%
local use_referrer = %USE_REFERRER%
local use_country = %USE_COUNTRY%
local use_blacklist_ip = %USE_BLACKLIST_IP%
@ -26,6 +29,7 @@ local cookie = require "cookie"
local javascript = require "javascript"
local captcha = require "captcha"
local recaptcha = require "recaptcha"
local iputils = require "resty.iputils"
-- user variables
local antibot_uri = "%ANTIBOT_URI%"
@ -94,6 +98,33 @@ if use_blacklist_reverse and not blacklist.reverse_cached() then
end
end
-- check if IP is in proxies list
if use_proxies then
local value, flags = ngx.shared.proxies_data:get(iputils.ip2bin(ngx.var.remote_addr))
if value ~= nil then
ngx.log(ngx.NOTICE, "[BLOCK] IP " .. ngx.var.remote_addr .. " is in proxies list")
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- check if IP is in abusers list
if use_abusers then
local value, flags = ngx.shared.abusers_data:get(iputils.ip2bin(ngx.var.remote_addr))
if value ~= nil then
ngx.log(ngx.NOTICE, "[BLOCK] IP " .. ngx.var.remote_addr .. " is in abusers list")
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- check if IP is in TOR exit nodes list
if use_tor_exit_nodes then
local value, flags = ngx.shared.tor_exit_nodes_data:get(iputils.ip2bin(ngx.var.remote_addr))
if value ~= nil then
ngx.log(ngx.NOTICE, "[BLOCK] IP " .. ngx.var.remote_addr .. " is in TOR exit nodes list")
ngx.exit(ngx.HTTP_FORBIDDEN)
end
end
-- check if user-agent is allowed
if use_user_agent and ngx.var.bad_user_agent == "yes" then
local block = true

3
confs/site/server.conf
View File

@ -27,9 +27,6 @@ server {
%REFERRER_POLICY%
%FEATURE_POLICY%
%PERMISSIONS_POLICY%
%BLOCK_TOR_EXIT_NODE%
%BLOCK_PROXIES%
%BLOCK_ABUSERS%
%COOKIE_FLAGS%
%ERRORS%
%USE_FAIL2BAN%

2
entrypoint/entrypoint.sh
View File

@ -132,7 +132,7 @@ else
fi
# list of log files to display
LOGS="/var/log/access.log /var/log/error.log /var/log/jobs.log /var/log/nginx/modsec_audit.log"
LOGS="/var/log/access.log /var/log/error.log /var/log/jobs.log /var/log/nginx/error.log /var/log/nginx/modsec_audit.log"
# start fail2ban
if [ "$USE_FAIL2BAN" = "yes" ] ; then

15
entrypoint/global-config.sh
View File

@ -130,16 +130,31 @@ fi
# block TOR exit nodes
if [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] ; then
echo "$BLOCK_TOR_EXIT_NODE_CRON /opt/scripts/exit-nodes.sh" >> /etc/crontabs/nginx
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_TOR_EXIT_NODES%" "lua_shared_dict tor_exit_nodes_data 1m;"
replace_in_file "/etc/nginx/init-lua.conf" "%USE_TOR_EXIT_NODES%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_TOR_EXIT_NODES%" ""
replace_in_file "/etc/nginx/init-lua.conf" "%USE_TOR_EXIT_NODES%" "false"
fi
# block proxies
if [ "$(has_value BLOCK_PROXIES yes)" != "" ] ; then
echo "$BLOCK_PROXIES_CRON /opt/scripts/proxies.sh" >> /etc/crontabs/nginx
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_PROXIES%" "lua_shared_dict proxies_data 250m;"
replace_in_file "/etc/nginx/init-lua.conf" "%USE_PROXIES%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_PROXIES%" ""
replace_in_file "/etc/nginx/init-lua.conf" "%USE_PROXIES%" "false"
fi
# block abusers
if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then
echo "$BLOCK_ABUSERS_CRON /opt/scripts/abusers.sh" >> /etc/crontabs/nginx
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_ABUSERS%" "lua_shared_dict abusers_data 50m;"
replace_in_file "/etc/nginx/init-lua.conf" "%USE_ABUSERS%" "true"
else
replace_in_file "/etc/nginx/nginx.conf" "%BLOCK_ABUSERS%" ""
replace_in_file "/etc/nginx/init-lua.conf" "%USE_ABUSERS%" "false"
fi
# DNS resolvers

12
entrypoint/site-config.sh
View File

@ -307,23 +307,23 @@ fi
# block TOR exit nodes
if [ "$BLOCK_TOR_EXIT_NODE" = "yes" ] ; then
replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_TOR_EXIT_NODE%" "include /etc/nginx/block-tor-exit-node.conf;"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_TOR_EXIT_NODES%" "true"
else
replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_TOR_EXIT_NODE%" ""
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_TOR_EXIT_NODES%" "false"
fi
# block proxies
if [ "$BLOCK_PROXIES" = "yes" ] ; then
replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_PROXIES%" "include /etc/nginx/block-proxies.conf;"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_PROXIES%" "true"
else
replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_PROXIES%" ""
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_PROXIES%" "false"
fi
# block abusers
if [ "$BLOCK_ABUSERS" = "yes" ] ; then
replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_ABUSERS%" "include /etc/nginx/block-abusers.conf;"
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_ABUSERS%" "true"
else
replace_in_file "${NGINX_PREFIX}server.conf" "%BLOCK_ABUSERS%" ""
replace_in_file "${NGINX_PREFIX}main-lua.conf" "%USE_ABUSERS%" "false"
fi
# HTTPS config

31
lua/dataloader.lua Normal file
View File

@ -0,0 +1,31 @@
local M = {}
local iputils = require "resty.iputils"
function M.load_ip (path, dict)
local file = io.open(path, "r")
if not file then
ngx.log(ngx.ERR, "[INIT] can't open " .. path)
else
io.input(file)
local i = 0
for line in io.lines() do
if string.match(line, "/") then
local lower, upper = iputils.parse_cidr(line)
local bin_ip = lower
while bin_ip <= upper do
dict:set(bin_ip, true, 0)
bin_ip = bin_ip + 1
i = i + 1
end
else
local bin_ip, bin_octets = iputils.ip2bin(line)
dict:set(bin_ip, true, 0)
i = i + 1
end
end
ngx.log(ngx.ERR, "[INIT] loaded " .. tostring(i) .. " IPs from " .. path)
io.close(file)
end
end
return M

17
scripts/abusers.sh
View File

@ -4,12 +4,11 @@
. /opt/entrypoint/utils.sh
# copy old conf to cache
cp /etc/nginx/block-abusers.conf /cache
cp /etc/nginx/abusers.list /cache
# generate the new conf
curl -s "https://iplists.firehol.org/files/firehol_abusers_30d.netset" | \
grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" | \
sed 's/^/deny /;s/$/;/' > /tmp/block-abusers.conf
grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" > /tmp/abusers.list
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@ -20,28 +19,28 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check if we have at least 1 line
lines="$(wc -l /tmp/block-abusers.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/abusers.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
job_log "[BLACKLIST] abusers list updated ($lines entries)"
# reload nginx with the new config
mv /tmp/block-abusers.conf /etc/nginx/block-abusers.conf
mv /tmp/abusers.list /etc/nginx/abusers.list
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
# new config is ok : save it in the cache
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/block-abusers.conf /cache
cp /etc/nginx/abusers.list /cache
job_log "[NGINX] successfull nginx reload after abusers list update"
else
job_log "[NGINX] failed nginx reload after abusers list update fallback to old list"
cp /cache/block-abusers.conf /etc/nginx
cp /cache/abusers.list /etc/nginx
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/block-abusers.conf /cache
cp /etc/nginx/abusers.list /cache
fi
else
job_log "[BLACKLIST] can't update abusers list"
fi
rm -f /tmp/block-abusers.conf 2> /dev/null
rm -f /tmp/abusers.list 2> /dev/null

18
scripts/exit-nodes.sh
View File

@ -4,12 +4,11 @@
. /opt/entrypoint/utils.sh
# copy old conf to cache
cp /etc/nginx/block-tor-exit-node.conf /cache
cp /etc/nginx/tor-exit-nodes.list /cache
# generate the new conf
curl -s "https://iplists.firehol.org/files/tor_exits.ipset" | \
grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" | \
sed 's/^/deny /;s/$/;/' > /tmp/block-tor-exit-node.conf
grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" > /tmp/tor-exit-nodes.list
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@ -20,28 +19,27 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check if we have at least 1 line
lines="$(wc -l /tmp/block-tor-exit-node.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/tor-exit-nodes.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
job_log "[BLACKLIST] TOR exit node list updated ($lines entries)"
# reload nginx with the new config
mv /tmp/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf
mv /tmp/tor-exit-nodes.list /etc/nginx/tor-exit-nodes.list
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
# new config is ok : save it in the cache
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/block-tor-exit-node.conf /cache
cp /etc/nginx/tor-exit-nodes.list /cache
job_log "[NGINX] successfull nginx reload after TOR exit node list update"
else
job_log "[NGINX] failed nginx reload after TOR exit node list update fallback to old list"
cp /cache/block-tor-exit-node.conf /etc/nginx
cp /cache/tor-exit-nodes.list /etc/nginx
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/block-tor-exit-node.conf /cache
cp /etc/nginx/tor-exit-nodes.list /cache
fi
else
job_log "[BLACKLIST] can't update TOR exit node list"
fi
rm -f /tmp/block-tor-exit-node.conf 2> /dev/null
rm -f /tmp/tor-exit-nodes.list 2> /dev/null

17
scripts/proxies.sh
View File

@ -4,12 +4,11 @@
. /opt/entrypoint/utils.sh
# copy old conf to cache
cp /etc/nginx/block-proxies.conf /cache
cp /etc/nginx/proxies.list /cache
# generate the new conf
curl -s "https://iplists.firehol.org/files/firehol_proxies.netset" | \
grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" | \
sed 's/^/deny /;s/$/;/' > /tmp/block-proxies.conf
grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$" > /tmp/proxies.list
# if we are running nginx
if [ -f /tmp/nginx.pid ] ; then
@ -20,28 +19,28 @@ elif [ -S /tmp/autoconf.sock ] ; then
fi
# check if we have at least 1 line
lines="$(wc -l /tmp/block-proxies.conf | cut -d ' ' -f 1)"
lines="$(wc -l /tmp/proxies.list | cut -d ' ' -f 1)"
if [ "$lines" -gt 1 ] ; then
job_log "[BLACKLIST] proxies list updated ($lines entries)"
# reload nginx with the new config
mv /tmp/block-proxies.conf /etc/nginx/block-proxies.conf
mv /tmp/proxies.list /etc/nginx/proxies.list
if [ "$RELOAD" != "" ] ; then
$RELOAD > /dev/null 2>&1
# new config is ok : save it in the cache
if [ "$?" -eq 0 ] ; then
cp /etc/nginx/block-proxies.conf /cache
cp /etc/nginx/proxies.list /cache
job_log "[NGINX] successfull nginx reload after proxies list update"
else
job_log "[NGINX] failed nginx reload after proxies list update fallback to old list"
cp /cache/block-proxies.conf /etc/nginx
cp /cache/proxies.list /etc/nginx
$RELOAD > /dev/null 2>&1
fi
else
cp /etc/nginx/block-proxies.conf /cache
cp /etc/nginx/proxies.list /cache
fi
else
job_log "[BLACKLIST] can't update proxies list"
fi
rm -f /tmp/block-proxies.conf 2> /dev/null
rm -f /tmp/proxies.list 2> /dev/null