various redis fixes and improvements
This commit is contained in:
florian
parent
e80965ca9a
commit
d3d0136bf5
|
|
@ -18,7 +18,7 @@ function M:connect()
|
|||
["REDIS_KEEPALIVE_POOL"] = ""
|
||||
}
|
||||
for k, v in pairs(variables) do
|
||||
local value, err = utils.get_variable(k)
|
||||
local value, err = utils.get_variable(k, false)
|
||||
if value == nil then
|
||||
return false, err
|
||||
end
|
||||
|
|
@ -47,7 +47,7 @@ function M:close(redis_client)
|
|||
["REDIS_KEEPALIVE_POOL"] = ""
|
||||
}
|
||||
for k, v in pairs(variables) do
|
||||
local value, err = utils.get_variable(k)
|
||||
local value, err = utils.get_variable(k, false)
|
||||
if value == nil then
|
||||
return false, err
|
||||
end
|
||||
|
|
|
|||
|
|
@ -0,0 +1,73 @@
|
|||
local clusterstore = require "clusterstore"
|
||||
local datastore = require "datastore"
|
||||
local utils = require "utils"
|
||||
|
||||
local redisutils = {}
|
||||
|
||||
redisutils.ban = function(ip)
|
||||
-- Connect
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
return nil, "can't connect to redis server : " .. err
|
||||
end
|
||||
-- Start transaction
|
||||
local ok, err = redis_client:multi()
|
||||
if not ok then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "MULTI failed : " .. err
|
||||
end
|
||||
-- Get ban
|
||||
ok, err = redis_client:get("ban_" .. ip)
|
||||
if not ok then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "GET failed : " .. err
|
||||
end
|
||||
-- Get ttl
|
||||
ok, err = redis_client:ttl("ban_" .. ip)
|
||||
if not ok then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "TTL failed : " .. err
|
||||
end
|
||||
-- Exec transaction
|
||||
local exec, err = redis_client:exec()
|
||||
if err then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "EXEC failed : " .. err
|
||||
end
|
||||
if type(exec) ~= "table" then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "EXEC result is not a table"
|
||||
end
|
||||
-- Extract ban reason
|
||||
local reason = exec[1]
|
||||
if type(reason) == "table" then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "GET failed : " .. reason[2]
|
||||
end
|
||||
if reason == ngx.null then
|
||||
clusterstore:close(redis_client)
|
||||
datastore:delete("bans_ip_" .. ip)
|
||||
return false
|
||||
end
|
||||
-- Extract ttl
|
||||
local ttl = exec[2]
|
||||
if type(ttl) == "table" then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "TTL failed : " .. ttl[2]
|
||||
end
|
||||
if ttl <= 0 then
|
||||
clusterstore:close(redis_client)
|
||||
return nil, "TTL returned invalid value : " .. tostring(ttl)
|
||||
end
|
||||
ok, err = datastore:set("bans_ip_" .. ip, reason, ttl)
|
||||
if not ok then
|
||||
clusterstore:close(redis_client)
|
||||
datastore:delete("bans_ip_" .. ip)
|
||||
return nil, "can't save ban to local datastore : " .. err
|
||||
end
|
||||
-- Return reason
|
||||
clusterstore:close(redis_client)
|
||||
return true, reason
|
||||
end
|
||||
|
||||
return redisutils
|
||||
|
|
@ -1,9 +1,9 @@
|
|||
local datastore = require "datastore"
|
||||
local ipmatcher = require "resty.ipmatcher"
|
||||
local cjson = require "cjson"
|
||||
local resolver = require "resty.dns.resolver"
|
||||
local mmdb = require "mmdb"
|
||||
local logger = require "logger"
|
||||
local datastore = require "datastore"
|
||||
local ipmatcher = require "resty.ipmatcher"
|
||||
local cjson = require "cjson"
|
||||
local resolver = require "resty.dns.resolver"
|
||||
local mmdb = require "mmdb"
|
||||
local logger = require "logger"
|
||||
|
||||
local utils = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ local logger = require "logger"
|
|||
local datastore = require "datastore"
|
||||
local plugins = require "plugins"
|
||||
local utils = require "utils"
|
||||
local clusterstore = require "clusterstore"
|
||||
local redisutils = require "redisutils"
|
||||
|
||||
-- Don't process internal requests
|
||||
if ngx.req.is_internal() then
|
||||
|
|
@ -15,39 +15,33 @@ end
|
|||
logger.log(ngx.INFO, "ACCESS", "Access phase started")
|
||||
|
||||
-- Process bans as soon as possible
|
||||
local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr)
|
||||
if banned then
|
||||
logger.log(ngx.WARN, "ACCESS", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned)
|
||||
ngx.exit(utils.get_deny_status())
|
||||
end
|
||||
local banned = nil
|
||||
-- Redis case
|
||||
local use_redis = utils.get_variable("USE_REDIS")
|
||||
if use_redis == "yes" then
|
||||
-- Connect
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
logger.log(ngx.ERR, "ACCESS", "can't connect to redis server : " .. err)
|
||||
local redis_banned, reason = redisutils.ban(ngx.var.remote_addr)
|
||||
if redis_banned == nil then
|
||||
logger.log(ngx.ERR, "ACCESS", "Error while checking ban from redis, falling back to local : " .. reason)
|
||||
elseif not redis_banned then
|
||||
banned = false
|
||||
else
|
||||
-- Get ban
|
||||
local ban, err = redis_client:get("ban_" .. ngx.var.remote_addr)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "ACCESS", "GET failed : " .. err)
|
||||
elseif ban then
|
||||
-- Get TTL
|
||||
local ttl, err = redis_client:ttl("ban_" .. ngx.var.remote_addr)
|
||||
if not ttl then
|
||||
logger.log(ngx.ERR, "ACCESS", "TTL failed : " .. err)
|
||||
else
|
||||
local ok, err = datastore:set("bans_ip_" .. ngx.var.remote_addr, ban, ttl)
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "ACCESS", "can't save ban to the datastore : " .. err)
|
||||
return
|
||||
end
|
||||
end
|
||||
end
|
||||
redis_client:close()
|
||||
banned = reason
|
||||
end
|
||||
end
|
||||
-- Local case
|
||||
if banned == nil then
|
||||
local reason, err = datastore:get("bans_ip_" .. ngx.var.remote_addr)
|
||||
if reason then
|
||||
banned = reason
|
||||
else
|
||||
banned = false
|
||||
end
|
||||
end
|
||||
-- Deny request
|
||||
if banned then
|
||||
logger.log(ngx.WARN, "ACCESS", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned)
|
||||
ngx.exit(utils.get_deny_status())
|
||||
end
|
||||
|
||||
-- List all plugins
|
||||
local list, err = plugins:list()
|
||||
|
|
|
|||
|
|
@ -1,46 +1,41 @@
|
|||
preread_by_lua_block {
|
||||
|
||||
local logger = require "logger"
|
||||
local datastore = require "datastore"
|
||||
local plugins = require "plugins"
|
||||
local utils = require "utils"
|
||||
local logger = require "logger"
|
||||
local datastore = require "datastore"
|
||||
local plugins = require "plugins"
|
||||
local utils = require "utils"
|
||||
local redisutils = require "redisutils"
|
||||
|
||||
logger.log(ngx.INFO, "PREREAD", "Preread phase started")
|
||||
|
||||
-- Process bans as soon as possible
|
||||
local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr)
|
||||
if banned then
|
||||
logger.log(ngx.WARN, "PREREAD", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned)
|
||||
ngx.exit(utils.get_deny_status())
|
||||
end
|
||||
local banned = nil
|
||||
-- Redis case
|
||||
local use_redis = utils.get_variable("USE_REDIS")
|
||||
if use_redis == "yes" then
|
||||
-- Connect
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
logger.log(ngx.ERR, "PREREAD", "can't connect to redis server : " .. err)
|
||||
local redis_banned, reason = redisutils.ban(ngx.var.remote_addr)
|
||||
if redis_banned == nil then
|
||||
logger.log(ngx.ERR, "ACCESS", "Error while checking ban from redis, falling back to local : " .. reason)
|
||||
elseif not redis_banned then
|
||||
banned = false
|
||||
else
|
||||
-- Get ban
|
||||
local ban, err = redis_client:get("ban_" .. ngx.var.remote_addr)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "PREREAD", "GET failed : " .. err)
|
||||
elseif ban then
|
||||
-- Get TTL
|
||||
local ttl, err = redis_client:ttl("ban_" .. ngx.var.remote_addr)
|
||||
if not ttl then
|
||||
logger.log(ngx.ERR, "PREREAD", "TTL failed : " .. err)
|
||||
else
|
||||
local ok, err = datastore:set("bans_ip_" .. ip, ban, ttl)
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "PREREAD", "can't save ban to the datastore : " .. err)
|
||||
return
|
||||
end
|
||||
end
|
||||
end
|
||||
redis_client:close()
|
||||
banned = reason
|
||||
end
|
||||
end
|
||||
-- Local case
|
||||
if banned == nil then
|
||||
local reason, err = datastore:get("bans_ip_" .. ngx.var.remote_addr)
|
||||
if reason then
|
||||
banned = reason
|
||||
else
|
||||
banned = false
|
||||
end
|
||||
end
|
||||
-- Deny request
|
||||
if banned then
|
||||
logger.log(ngx.WARN, "ACCESS", "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. banned)
|
||||
ngx.exit(utils.get_deny_status())
|
||||
end
|
||||
|
||||
-- List all plugins
|
||||
local list, err = plugins:list()
|
||||
|
|
|
|||
|
|
@ -13,69 +13,32 @@ function _M.new()
|
|||
end
|
||||
|
||||
function _M.increase(premature, use_redis, ip, count_time, ban_time, threshold)
|
||||
-- Local case
|
||||
local counter, err = datastore:get("plugin_badbehavior_count_" .. ip)
|
||||
if not counter and err ~= "not found" then
|
||||
return false, "can't get counts from the datastore : " .. err
|
||||
end
|
||||
if counter == nil then
|
||||
counter = 0
|
||||
end
|
||||
counter = counter + 1
|
||||
-- Declare counter
|
||||
local counter = false
|
||||
-- Redis case
|
||||
if use_redis then
|
||||
-- Connect to server
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't connect to redis server : " .. err)
|
||||
return
|
||||
local redis_counter = _M.redis_increase(ip, count_time, ban_time, threshold)
|
||||
if not redis_counter then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) redis_increase failed, falling back to local")
|
||||
else
|
||||
counter = redis_counter
|
||||
end
|
||||
-- Start transaction
|
||||
local ok, err = redis_client:multi()
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't start transaction : " .. err)
|
||||
redis_client:close()
|
||||
return
|
||||
end
|
||||
-- Local case
|
||||
if not counter then
|
||||
local local_counter, err = datastore:get("plugin_badbehavior_count_" .. ip)
|
||||
if not local_counter and err ~= "not found" then
|
||||
return false, "can't get counts from the datastore : " .. err
|
||||
end
|
||||
-- Increment counter
|
||||
counter, err = redis_client:incr("bad_behavior_" .. ip)
|
||||
if not counter then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) INCR failed : " .. err)
|
||||
redis_client:close()
|
||||
return
|
||||
if local_counter == nil then
|
||||
counter = 0
|
||||
end
|
||||
-- Expires counter
|
||||
local expire, err = redis_client:expire("bad_behavior_" .. ip, count_time)
|
||||
if not counter then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXPIRE failed : " .. err)
|
||||
redis_client:close()
|
||||
return
|
||||
end
|
||||
-- Add IP to redis bans if needed
|
||||
if counter > threshold then
|
||||
local ban, err = redis_client:set("ban_" .. ip, "bad behavior", "EX", ban_time)
|
||||
if not ban then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) SET failed : " .. err)
|
||||
redis_client:close()
|
||||
return
|
||||
end
|
||||
end
|
||||
-- Exec transaction
|
||||
local exec, err = redis_client:exec()
|
||||
if not exec then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXEC failed : " .. err)
|
||||
redis_client:close()
|
||||
return
|
||||
end
|
||||
for i, v in ipairs(exec) do
|
||||
if v[1] == false then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Transaction failed : " .. v[2])
|
||||
redis_client:close()
|
||||
return
|
||||
end
|
||||
end
|
||||
-- End connection
|
||||
redis_client:close()
|
||||
counter = counter + 1
|
||||
end
|
||||
-- Call decrease later
|
||||
local ok, err = ngx.timer.at(count_time, _M.decrease, use_redis, ip)
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) can't create decrease timer : " .. err)
|
||||
end
|
||||
-- Store local counter
|
||||
local ok, err = datastore:set("plugin_badbehavior_count_" .. ip, counter)
|
||||
|
|
@ -92,26 +55,35 @@ function _M.increase(premature, use_redis, ip, count_time, ban_time, threshold)
|
|||
end
|
||||
logger.log(ngx.WARN, "BAD-BEHAVIOR", "IP " .. ip .. " is banned for " .. ban_time .. "s (" .. tostring(counter) .. "/" .. tostring(threshold) .. ")")
|
||||
end
|
||||
-- Call decrease later
|
||||
local ok, err = ngx.timer.at(count_time, _M.decrease, use_redis, ip)
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) can't create decrease timer : " .. err)
|
||||
end
|
||||
end
|
||||
|
||||
function _M.decrease(premature, use_redis, ip)
|
||||
-- Decrease from local store
|
||||
local count, err = datastore:get("plugin_badbehavior_count_" .. ip)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't get counts from the datastore : " .. err)
|
||||
return
|
||||
-- Declare counter
|
||||
local counter = false
|
||||
-- Redis case
|
||||
if use_redis then
|
||||
local redis_counter = _M.redis_decrease(ip)
|
||||
if not redis_counter then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) redis_decrease failed, falling back to local")
|
||||
else
|
||||
counter = redis_counter
|
||||
end
|
||||
end
|
||||
if not count then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Count is null")
|
||||
return
|
||||
-- Local case
|
||||
if not counter then
|
||||
local local_counter, err = datastore:get("plugin_badbehavior_count_" .. ip)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't get counts from the datastore : " .. err)
|
||||
return
|
||||
end
|
||||
if not local_counter then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Count is null")
|
||||
return
|
||||
end
|
||||
counter = local_counter - 1
|
||||
end
|
||||
local new_count = count - 1
|
||||
if new_count <= 0 then
|
||||
-- Update local counter
|
||||
if counter <= 0 then
|
||||
datastore:delete("plugin_badbehavior_count_" .. ip)
|
||||
return
|
||||
end
|
||||
|
|
@ -120,23 +92,6 @@ function _M.decrease(premature, use_redis, ip)
|
|||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't save counts to the datastore : " .. err)
|
||||
return
|
||||
end
|
||||
-- Decrease from redis
|
||||
if use_redis then
|
||||
-- Connect to server
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't connect to redis server : " .. err)
|
||||
return
|
||||
end
|
||||
-- Decrement counter
|
||||
local counter, err = redis_client:decr("bad_behavior_" .. ip)
|
||||
if not counter then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) DECR failed : " .. err)
|
||||
redis_client:close()
|
||||
return
|
||||
end
|
||||
redis_client:close()
|
||||
end
|
||||
end
|
||||
|
||||
function _M:log()
|
||||
|
|
@ -159,6 +114,11 @@ function _M:log()
|
|||
if ngx.var.is_whitelisted == "yes" then
|
||||
return true, "client is whitelisted"
|
||||
end
|
||||
-- Check if we are already banned
|
||||
local banned, err = datastore:get("bans_ip_" .. ngx.var.remote_addr)
|
||||
if banned then
|
||||
return true, "already banned"
|
||||
end
|
||||
-- Call increase function later and with cosocket enabled
|
||||
local use_redis = false
|
||||
if self.use_redis == "yes" then
|
||||
|
|
@ -166,7 +126,7 @@ function _M:log()
|
|||
end
|
||||
local ok, err = ngx.timer.at(0, _M.increase, use_redis, ngx.var.remote_addr, tonumber(self.count_time), tonumber(self.ban_time), tonumber(self.threshold))
|
||||
if not ok then
|
||||
return false, "can't create decrease timer : " .. err
|
||||
return false, "can't create increase timer : " .. err
|
||||
end
|
||||
return true, "success"
|
||||
end
|
||||
|
|
@ -175,4 +135,103 @@ function _M:log_default()
|
|||
return _M:log()
|
||||
end
|
||||
|
||||
function _M.redis_increase(ip, count_time, ban_time, threshold)
|
||||
-- Connect to server
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't connect to redis server : " .. err)
|
||||
return false
|
||||
end
|
||||
-- Start transaction
|
||||
local ok, err = redis_client:multi()
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) Can't start transaction : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Increment counter
|
||||
ok, err = redis_client:incr("bad_behavior_" .. ip)
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) INCR failed : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Expires counter
|
||||
ok, err = redis_client:expire("bad_behavior_" .. ip, count_time)
|
||||
if not ok then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXPIRE failed : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Exec transaction
|
||||
local exec, err = redis_client:exec()
|
||||
if err then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXEC failed : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
if type(exec) ~= "table" then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXEC result is not a table")
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Extract counter
|
||||
local counter = exec[1]
|
||||
if type(counter) == "table" then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) INCR error : " .. counter[2])
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Check expire result
|
||||
local expire = exec[2]
|
||||
if type(expire) == "table" then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) EXPIRE error : " .. expire[2])
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Add IP to redis bans if needed
|
||||
if counter > threshold then
|
||||
local ban, err = redis_client:set("ban_" .. ip, "bad behavior", "EX", ban_time)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(increase) SET failed : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
end
|
||||
-- End connection
|
||||
clusterstore:close(redis_client)
|
||||
return counter
|
||||
end
|
||||
|
||||
function _M.redis_decrease(ip)
|
||||
-- Connect to server
|
||||
local redis_client, err = clusterstore:connect()
|
||||
if not redis_client then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) Can't connect to redis server : " .. err)
|
||||
return false
|
||||
end
|
||||
-- Decrement counter
|
||||
local counter, err = redis_client:decr("bad_behavior_" .. ip)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) DECR failed : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
-- Delete counter
|
||||
if counter < 0 then
|
||||
counter = 0
|
||||
end
|
||||
if counter == 0 then
|
||||
local ok, err = redis_client:del("bad_behavior_" .. ip)
|
||||
if err then
|
||||
logger.log(ngx.ERR, "BAD-BEHAVIOR", "(decrease) DEL failed : " .. err)
|
||||
clusterstore:close(redis_client)
|
||||
return false
|
||||
end
|
||||
end
|
||||
-- End connection
|
||||
clusterstore:close(redis_client)
|
||||
return counter
|
||||
end
|
||||
|
||||
return _M
|
||||
|
|
|
|||
Loading…
Reference in New Issue