save work

This commit is contained in:
bunkerity 2023-04-11 16:45:46 +02:00
parent afc0ac1988
commit 8c29081577
15 changed files with 603 additions and 146 deletions

View File

@ -0,0 +1,180 @@
local mlcache = require "resty.mlcache"
local clogger = require "bunkerweb.logger"
local class = require "middleclass"
local cachestore = class("cachestore")
-- Instantiate mlcache object at module level (which will be cached when running init phase)
-- TODO : custom settings
local shm = "cachestore"
local ipc_shm = "cachestore_ipc"
local shm_miss = "cachestore_miss"
local shm_locks = "cachestore_locks"
if not ngx.shared.cachestore then
shm = "cachestore_stream"
ipc_shm = "cachestore_ipc_stream"
shm_miss = "cachestore_miss_stream"
shm_locks = "cachestore_locks_stream"
end
local cache, err = mlcache.new(
"cachestore",
shm,
{
lru_size = 100,
ttl = 30,
neg_ttl = 0.1,
shm_set_tries = 3,
shm_miss = shm_miss,
shm_locks = shm_locks,
resty_lock_opts = {
exptime = 30,
timeout = 5,
step = 0.001,
ratio = 2,
max_step = 0.5
},
ipc_shm = ipc_shm
}
)
local logger = clogger:new("CACHESTORE")
if not store then
logger:log(ngx.ERR, "can't instantiate mlcache : " .. err)
end
function cachestore:new(use_redis)
self.cache = cache
self.use_redis = use_redis or false
self.logger = logger
end
function cachestore:get(key)
local function callback(key)
-- Connect to redis
local clusterstore = require "clusterstore"
local redis, err = clusterstore:connect()
if not redis then
return nil, "can't connect to redis : " .. err, nil
end
-- Start transaction
local ok, err = redis:multi()
if not ok then
clusterstore:close(redis)
return nil, "multi() failed : " .. err, nil
end
-- GET
local ok, err = redis:get(key)
if not ok then
clusterstore:close(redis)
return nil, "get() failed : " .. err, nil
end
-- TTL
local ok, err = redis:ttl(key)
if not ok then
clusterstore:close(redis)
return nil, "ttl() failed : " .. err, nil
end
-- Exec transaction
local exec, err = redis:exec()
if err then
clusterstore:close(redis)
return nil, "exec() failed : " .. err, nil
end
-- Get results
if type(exec) ~= "table" then
clusterstore:close(redis)
return nil, "exec() result is not a table", nil
end
local value = exec[1]
if type(value) == "table" then
clusterstore:close(redis)
return nil, "GET error : " .. value[2], nil
end
local ttl = exec[2]
if type(ttl) == "table" then
clusterstore:close(redis)
return nil, "TTL error : " .. ttl[2], nil
end
-- Return value
clusterstore:close(redis)
if value == ngx.null then
value = nil
end
if ttl < 0 then
ttl = ttl + 1
end
return value, nil, ttl
end
local value, err, hit_level
if self.use_redis then
value, err, hit_level = self.cache:get(key, nil, callback, key)
else
value, err, hit_level = self.cache:get(key)
end
if err then
return false, err
end
self.logger:log(ngx.INFO, "hit level for " .. key .. " = " .. tostring(hit_level))
return true, value
end
function cachestore:set(key, value, ex)
if self.use_redis then
local ok, err = self.set_redis(key, value, ex)
if not ok then
self.logger:log(ngx.ERR, err)
end
end
local ok, err = self.cache:set(key, nil, value)
if not ok then
return false, err
end
return true
end
function cachestore:set_redis(key, value, ex)
-- Connect to redis
local redis, err = clusterstore:connect()
if not redis then
return false, "can't connect to redis : " .. err
end
-- Set value with ttl
local default_ex = ttl or 30
local ok, err = redis:set(key, value, "EX", ex)
if err then
clusterstore:close(redis)
return false, "GET failed : " .. err
end
clusterstore:close(redis)
return true
end
function cachestore:delete(key, value, ex)
if self.use_redis then
local ok, err = self.del_redis(key)
if not ok then
self.logger:log(ngx.ERR, err)
end
end
local ok, err = self.cache:delete(key)
if not ok then
return false, err
end
return true
end
function cachestore:del_redis(key)
-- Connect to redis
local redis, err = clusterstore:connect()
if not redis then
return false, "can't connect to redis : " .. err
end
-- Set value with ttl
local ok, err = redis:del(key)
if err then
clusterstore:close(redis)
return false, "DEL failed : " .. err
end
clusterstore:close(redis)
return true
end
return cachestore

View File

@ -0,0 +1,51 @@
local class = require "middleclass"
local datastore = class("datastore")
function datastore:new()
self.dict = ngx.shared.datastore
if not self.dict then
self.dict = ngx.shared.datastore_stream
end
end
function datastore:get(key)
local value, err = self.dict:get(key)
if not value and not err then
err = "not found"
end
return value, err
end
function datastore:set(self, key, value, exptime)
exptime = exptime or 0
return self.dict:safe_set(key, value, exptime)
end
function datastore:delete(self, key)
self.dict:delete(key)
return true, "success"
end
function datastore:keys(self)
return self.dict:get_keys(0)
end
function datastore:exp(self, key)
local ttl, err = self.dict:ttl(key)
if not ttl then
return false, err
end
return true, ttl
end
function datastore:delete_all(self, pattern)
local keys = self.dict:get_keys(0)
for i, key in ipairs(keys) do
if key:match(pattern) then
self.dict:delete(key)
end
end
return true, "success"
end
return datastore

View File

@ -76,4 +76,8 @@ helpers.call_plugin = function(plugin, method)
return true, ret
end
helpers.get_plugins = function()
end
return helpers

View File

@ -1,98 +0,0 @@
local mlcache = require "resty.mlcache"
local clogger = require "bunkerweb.logger"
local class = require "middleclass"
local datastore = class("datastore")
-- Instantiate mlcache objects at module level (which will be cached when running init phase)
-- TODO : shm_miss, shm_locks
local shm = "datastore"
local ipc_shm = "datastore_ipc"
if not ngx.shared.datastore then
shm = "datastore_stream"
ipc_shm = "datastore_ipc_stream"
end
local store, err = mlcache.new(
"datastore",
shm,
{
lru_size = 100,
ttl = 0,
neg_ttl = 0,
shm_set_tries = 1,
ipc_shm = ipc_shm
}
)
local logger = clogger:new("DATASTORE")
if not store then
logger:log(ngx.ERR, "can't instantiate mlcache : " .. err)
end
function datastore:new()
self.store = store
self.logger = logger
end
function datastore:get(key)
local value, err, hit_level = self.store:get(key)
if err then
return false, err
end
self.logger:log(ngx.INFO, "hit level for " .. key .. " = " .. tostring(hit_level))
return true, value
end
function datastore:set(key, value)
local ok, err = self.store:set(key, nil, value)
if not ok then
return false, err
end
return true
end
local datastore = { dict = ngx.shared.datastore }
if not datastore.dict then
datastore.dict = ngx.shared.datastore_stream
end
datastore.get = function(self, key)
local value, err = self.dict:get(key)
if not value and not err then
err = "not found"
end
return value, err
end
datastore.set = function(self, key, value, exptime)
exptime = exptime or 0
return self.dict:safe_set(key, value, exptime)
end
datastore.keys = function(self)
return self.dict:get_keys(0)
end
datastore.delete = function(self, key)
self.dict:delete(key)
return true, "success"
end
datastore.exp = function(self, key)
local ttl, err = self.dict:ttl(key)
if not ttl then
return false, err
end
return true, ttl
end
datastore.delete_all = function(self, pattern)
local keys = self.dict:get_keys(0)
for i, key in ipairs(keys) do
if key:match(pattern) then
self.dict:delete(key)
end
end
return true, "success"
end
return datastore

194
src/bw/lua/middleclass.lua Normal file
View File

@ -0,0 +1,194 @@
local middleclass = {
_VERSION = 'middleclass v4.1.1',
_DESCRIPTION = 'Object Orientation for Lua',
_URL = 'https://github.com/kikito/middleclass',
_LICENSE = [[
MIT LICENSE
Copyright (c) 2011 Enrique García Cota
Permission is hereby granted, free of charge, to any person obtaining a
copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be included
in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
]]
}
local function _createIndexWrapper(aClass, f)
if f == nil then
return aClass.__instanceDict
elseif type(f) == "function" then
return function(self, name)
local value = aClass.__instanceDict[name]
if value ~= nil then
return value
else
return (f(self, name))
end
end
else -- if type(f) == "table" then
return function(self, name)
local value = aClass.__instanceDict[name]
if value ~= nil then
return value
else
return f[name]
end
end
end
end
local function _propagateInstanceMethod(aClass, name, f)
f = name == "__index" and _createIndexWrapper(aClass, f) or f
aClass.__instanceDict[name] = f
for subclass in pairs(aClass.subclasses) do
if rawget(subclass.__declaredMethods, name) == nil then
_propagateInstanceMethod(subclass, name, f)
end
end
end
local function _declareInstanceMethod(aClass, name, f)
aClass.__declaredMethods[name] = f
if f == nil and aClass.super then
f = aClass.super.__instanceDict[name]
end
_propagateInstanceMethod(aClass, name, f)
end
local function _tostring(self) return "class " .. self.name end
local function _call(self, ...) return self:new(...) end
local function _createClass(name, super)
local dict = {}
dict.__index = dict
local aClass = { name = name, super = super, static = {},
__instanceDict = dict, __declaredMethods = {},
subclasses = setmetatable({}, {__mode='k'}) }
if super then
setmetatable(aClass.static, {
__index = function(_,k)
local result = rawget(dict,k)
if result == nil then
return super.static[k]
end
return result
end
})
else
setmetatable(aClass.static, { __index = function(_,k) return rawget(dict,k) end })
end
setmetatable(aClass, { __index = aClass.static, __tostring = _tostring,
__call = _call, __newindex = _declareInstanceMethod })
return aClass
end
local function _includeMixin(aClass, mixin)
assert(type(mixin) == 'table', "mixin must be a table")
for name,method in pairs(mixin) do
if name ~= "included" and name ~= "static" then aClass[name] = method end
end
for name,method in pairs(mixin.static or {}) do
aClass.static[name] = method
end
if type(mixin.included)=="function" then mixin:included(aClass) end
return aClass
end
local DefaultMixin = {
__tostring = function(self) return "instance of " .. tostring(self.class) end,
initialize = function(self, ...) end,
isInstanceOf = function(self, aClass)
return type(aClass) == 'table'
and type(self) == 'table'
and (self.class == aClass
or type(self.class) == 'table'
and type(self.class.isSubclassOf) == 'function'
and self.class:isSubclassOf(aClass))
end,
static = {
allocate = function(self)
assert(type(self) == 'table', "Make sure that you are using 'Class:allocate' instead of 'Class.allocate'")
return setmetatable({ class = self }, self.__instanceDict)
end,
new = function(self, ...)
assert(type(self) == 'table', "Make sure that you are using 'Class:new' instead of 'Class.new'")
local instance = self:allocate()
instance:initialize(...)
return instance
end,
subclass = function(self, name)
assert(type(self) == 'table', "Make sure that you are using 'Class:subclass' instead of 'Class.subclass'")
assert(type(name) == "string", "You must provide a name(string) for your class")
local subclass = _createClass(name, self)
for methodName, f in pairs(self.__instanceDict) do
if not (methodName == "__index" and type(f) == "table") then
_propagateInstanceMethod(subclass, methodName, f)
end
end
subclass.initialize = function(instance, ...) return self.initialize(instance, ...) end
self.subclasses[subclass] = true
self:subclassed(subclass)
return subclass
end,
subclassed = function(self, other) end,
isSubclassOf = function(self, other)
return type(other) == 'table' and
type(self.super) == 'table' and
( self.super == other or self.super:isSubclassOf(other) )
end,
include = function(self, ...)
assert(type(self) == 'table', "Make sure you that you are using 'Class:include' instead of 'Class.include'")
for _,mixin in ipairs({...}) do _includeMixin(self, mixin) end
return self
end
}
}
function middleclass.class(name, super)
assert(type(name) == 'string', "A name (string) is needed for the new class")
return super and super:subclass(name) or _includeMixin(_createClass(name), DefaultMixin)
end
setmetatable(middleclass, { __call = function(_, ...) return middleclass.class(...) end })
return middleclass

View File

@ -45,7 +45,10 @@ lua_package_cpath "/usr/share/bunkerweb/deps/lib/?.so;/usr/share/bunkerweb/deps/
lua_ssl_trusted_certificate "/usr/share/bunkerweb/misc/root-ca.pem";
lua_ssl_verify_depth 2;
lua_shared_dict datastore {{ DATASTORE_MEMORY_SIZE }};
lua_shared_dict datastore_ipc {{ DATASTORE_IPC_MEMORY_SIZE }};
lua_shared_dict cachestore {{ CACHESTORE_MEMORY_SIZE }};
lua_shared_dict cachestore_ipc {{ CACHESTORE_IPC_MEMORY_SIZE }};
lua_shared_dict cachestore_miss {{ CACHESTORE_MISS_MEMORY_SIZE }};
lua_shared_dict cachestore_locks {{ CACHESTORE_LOCKS_MEMORY_SIZE }};
# LUA init block
include /etc/nginx/init-lua.conf;

View File

@ -87,13 +87,13 @@ logger:log(ngx.NOTICE, "saved misc values into datastore")
logger:log(ngx.NOTICE, "saving API values into datastore ...")
local value, err = datastore:get("variable_USE_API")
if not value then
logger.log(ngx.ERR, "can't get variable USE_API from the datastore : " .. err)
logger:log(ngx.ERR, "can't get variable USE_API from the datastore : " .. err)
return false
end
if value == "yes" then
local value, err = datastore:get("variable_API_WHITELIST_IP")
if not value then
logger.log(ngx.ERR, "can't get variable API_WHITELIST_IP from the datastore : " .. err)
logger:log(ngx.ERR, "can't get variable API_WHITELIST_IP from the datastore : " .. err)
return false
end
local whitelists = {}
@ -102,7 +102,7 @@ if value == "yes" then
end
local ok, err = datastore:set("api_whitelist_ip", cjson.encode(whitelists))
if not ok then
logger.log(ngx.ERR, "can't save API whitelist_ip to datastore : " .. err)
logger:log(ngx.ERR, "can't save API whitelist_ip to datastore : " .. err)
return false
end
logger:log(ngx.INFO, "saved API whitelist_ip into datastore")
@ -118,19 +118,19 @@ for i, plugin_path in ipairs(plugin_paths) do
for path in paths:lines() do
local ok, plugin = helpers.load_plugin(path .. "/plugin.json")
if ok then
logger.log(ngx.ERR, err)
logger:log(ngx.ERR, err)
else
table.insert(plugins, plugin)
table.sort(plugins, function (a, b)
return a.order < b.order
end)
logger.log(ngx.NOTICE, "loaded plugin " .. plugin.id .. " v" .. plugin.version)
logger:log(ngx.NOTICE, "loaded plugin " .. plugin.id .. " v" .. plugin.version)
end
end
end
local ok, err = datastore:set("plugins", cjson.encode(plugins))
if not ok then
logger.log(ngx.ERR, "can't save plugins into datastore : " .. err)
logger:log(ngx.ERR, "can't save plugins into datastore : " .. err)
return false
end
logger:log(ngx.NOTICE, "saved plugins into datastore")
@ -140,21 +140,21 @@ logger:log(ngx.NOTICE, "calling init() methods of plugins ...")
for i, plugin in ipairs(plugins) do
local plugin_lua, err = helpers.new_plugin(plugin.id)
if plugin_lua == false then
logger.log(ngx.ERR, err)
logger:log(ngx.ERR, err)
else
logger.log(ngx.NOTICE, err)
logger:log(ngx.NOTICE, err)
end
if plugin_lua ~= nil then
local ok, ret = helpers.call_plugin(plugin_lua)
local ok, ret = helpers.call_plugin(plugin_lua, "init")
if ok == false then
logger.log(ngx.ERR, ret)
logger:log(ngx.ERR, ret)
elseif ok == nil then
logger.log(ngx.NOTICE, ret)
logger:log(ngx.NOTICE, ret)
else
if ret.ret then
logger.log(ngx.NOTICE, plugin.id .. ":init() call successful : " .. ret.msg)
logger:log(ngx.NOTICE, plugin.id .. ":init() call successful : " .. ret.msg)
else
logger.log(ngx.ERR, plugin.id .. ":init() call failed : " .. ret.msg)
logger:log(ngx.ERR, plugin.id .. ":init() call failed : " .. ret.msg)
end
end
end

View File

@ -1,5 +1,74 @@
access_by_lua_block {
local class = require "middleclass"
local clogger = require "bunkerweb.logger"
local helpers = require "bunkerweb.helpers"
local datastore = require "bunkerweb.datastore"
-- Don't process internal requests
logger:new("ACCESS")
if ngx.req.is_internal() then
logger:log(ngx.INFO, "skipped access phase because request is internal")
return true
end
-- Start access phase
datastore:new()
logger:log(ngx.INFO, "access phase started")
-- Process bans as soon as possible
local ok, reason = cachestore:get("bans_ip_" .. ngx.var.remote_addr)
if not ok and reason then
logger:log(ngx.INFO, "error while checking if client is banned : " .. reason)
return false
else reason then
logger:log(ngx.WARN, "IP " .. ngx.var.remote_addr .. " is banned with reason : " .. reason)
return ngx.exit(utils.get_deny_status())
end
-- Get plugins
local plugins, err = datastore:get("plugins")
if not plugins then
logger:log(ngx.ERR, "can't get plugins from datastore : " .. err)
return false
end
-- Call access() methods
logger:log(ngx.INFO, "calling access() methods of plugins ...")
for i, plugin in ipairs(plugins) do
local plugin_lua, err = helpers.new_plugin(plugin.id)
if plugin_lua == false then
logger:log(ngx.ERR, err)
else
logger:log(ngx.INFO, err)
end
if plugin_lua ~= nil then
local ok, ret = helpers.call_plugin(plugin_lua, "access")
if ok == false then
logger:log(ngx.ERR, ret)
elseif ok == nil then
logger:log(ngx.INFO, ret)
else
if ret.ret then
logger:log(ngx.INFO, plugin.id .. ":access() call successful : " .. ret.msg)
if ret.status then
if ret.status == utils.get_deny_status() then
ngx.ctx.reason = plugin.id
return ngx.exit(ret.status)
end
end
else
logger:log(ngx.ERR, plugin.id .. ":access() call failed : " .. ret.msg)
end
end
end
end
logger:log(ngx.NOTICE, "called set() methods of plugins")
return true
local logger = require "logger"
local datastore = require "datastore"
local plugins = require "plugins"
@ -82,13 +151,7 @@ for i, plugin in ipairs(list) do
end
end
-- Save session
local ok, err = utils.save_session()
if not ok then
logger.log(ngx.ERR, "ACCESS", "Can't save session : " .. err)
else
logger.log(ngx.INFO, "ACCESS", "Session save status : " .. err)
end
logger.log(ngx.INFO, "ACCESS", "Access phase ended")

View File

@ -1,39 +1,63 @@
set $dummy_set "";
set_by_lua_block $dummy_set {
local utils = require "utils"
local logger = require "logger"
local datastore = require "datastore"
local plugins = require "plugins"
local class = require "middleclass"
local clogger = require "bunkerweb.logger"
local helpers = require "bunkerweb.helpers"
local datastore = require "bunkerweb.datastore"
local cachestore = require "bunkerweb.cachestore"
logger.log(ngx.INFO, "SET", "Set phase started")
-- List all plugins
local list, err = plugins:list()
if not list then
logger.log(ngx.ERR, "SET", "Can't list loaded plugins : " .. err)
list = {}
-- Don't process internal requests
logger:new("SET")
if ngx.req.is_internal() then
logger:log(ngx.INFO, "skipped access phase because request is internal")
return true
end
-- Call set method of plugins
for i, plugin in ipairs(list) do
local ret, plugin_lua = pcall(require, plugin.id .. "/" .. plugin.id)
if ret then
local plugin_obj = plugin_lua.new()
if plugin_obj.set ~= nil then
logger.log(ngx.INFO, "SET", "Executing set() of " .. plugin.id)
local ok, err = plugin_obj:set()
if not ok then
logger.log(ngx.ERR, "SET", "Error while calling set() on plugin " .. plugin.id .. " : " .. err)
else
logger.log(ngx.INFO, "SET", "Return value from " .. plugin.id .. ".set() is : " .. err)
end
-- Start set phase
datastore:new()
logger:log(ngx.INFO, "set phase started")
-- Update cachestore only once and before any other code
cachestore:new()
local ok, err = cachestore.cache:update()
if not ok then
logger:log(ngx.ERR, "can't update cachestore : " .. err)
end
-- Get plugins
local plugins, err = datastore:get("plugins")
if not plugins then
logger:log(ngx.ERR, "can't get plugins from datastore : " .. err)
return false
end
-- Call set() methods
logger:log(ngx.INFO, "calling set() methods of plugins ...")
for i, plugin in ipairs(plugins) do
local plugin_lua, err = helpers.new_plugin(plugin.id)
if plugin_lua == false then
logger:log(ngx.ERR, err)
else
logger:log(ngx.INFO, err)
end
if plugin_lua ~= nil then
local ok, ret = helpers.call_plugin(plugin_lua, "set")
if ok == false then
logger:log(ngx.ERR, ret)
elseif ok == nil then
logger:log(ngx.INFO, ret)
else
logger.log(ngx.INFO, "SET", "set() method not found in " .. plugin.id .. ", skipped execution")
if ret.ret then
logger:log(ngx.INFO, plugin.id .. ":set() call successful : " .. ret.msg)
else
logger:log(ngx.ERR, plugin.id .. ":set() call failed : " .. ret.msg)
end
end
end
end
logger:log(ngx.INFO, "called set() methods of plugins")
logger.log(ngx.INFO, "SET", "Set phase ended")
return true
}

View File

@ -119,13 +119,49 @@
},
"DATASTORE_MEMORY_SIZE": {
"context": "global",
"default": "256m",
"default": "64m",
"help": "Size of the internal datastore.",
"id": "datastore-memory-size",
"label": "Datastore memory size",
"regex": "^\\d+[kKmMgG]?$",
"type": "text"
},
"CACHESTORE_MEMORY_SIZE": {
"context": "global",
"default": "64m",
"help": "Size of the internal cachestore.",
"id": "cachestore-memory-size",
"label": "Cachestore memory size",
"regex": "^\\d+[kKmMgG]?$",
"type": "text"
},
"CACHESTORE_IPC_MEMORY_SIZE": {
"context": "global",
"default": "16m",
"help": "Size of the internal cachestore (ipc).",
"id": "cachestore-ipc-memory-size",
"label": "Cachestore ipc memory size",
"regex": "^\\d+[kKmMgG]?$",
"type": "text"
},
"CACHESTORE_MISS_MEMORY_SIZE": {
"context": "global",
"default": "16m",
"help": "Size of the internal cachestore (miss).",
"id": "cachestore-miss-memory-size",
"label": "Cachestore miss memory size",
"regex": "^\\d+[kKmMgG]?$",
"type": "text"
},
"CACHESTORE_LOCKS_MEMORY_SIZE": {
"context": "global",
"default": "16m",
"help": "Size of the internal cachestore (locks).",
"id": "cachestore-locks-memory-size",
"label": "Cachestore locks memory size",
"regex": "^\\d+[kKmMgG]?$",
"type": "text"
},
"USE_API": {
"context": "global",
"default": "yes",