jobs - add remote API
This commit is contained in:
bunkerity
parent
31ed4ff834
commit
7b9722fac4
|
|
@ -87,3 +87,11 @@ fi
|
|||
if [ "$(has_value BLOCK_ABUSERS yes)" != "" ] ; then
|
||||
/opt/bunkerized-nginx/jobs/main.py --name abusers --cache
|
||||
fi
|
||||
|
||||
# remote API
|
||||
if [ "$(has_value USE_REMOTE_API yes)" != "" ] ; then
|
||||
/opt/bunkerized-nginx/jobs/main.py --name remote-api-register --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)"
|
||||
if [ $? -eq 0 ] ; then
|
||||
/opt/bunkerized-nginx/jobs/main.py --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)"
|
||||
fi
|
||||
fi
|
||||
|
|
|
|||
22
jobs/Job.py
22
jobs/Job.py
|
|
@ -77,7 +77,7 @@ class JobManagement() :
|
|||
|
||||
class Job(abc.ABC) :
|
||||
|
||||
def __init__(self, name, data, filename=None, redis_host=None, redis_ex=86400, type="line", regex=r"^.+$", copy_cache=False) :
|
||||
def __init__(self, name, data, filename=None, redis_host=None, redis_ex=86400, type="line", regex=r"^.+$", copy_cache=False, json_data=None, method="GET") :
|
||||
self._name = name
|
||||
self._data = data
|
||||
self._filename = filename
|
||||
|
|
@ -92,11 +92,13 @@ class Job(abc.ABC) :
|
|||
self._type = type
|
||||
self._regex = regex
|
||||
self._copy_cache = copy_cache
|
||||
self._json_data = json_data
|
||||
self._method = method
|
||||
|
||||
def run(self) :
|
||||
ret = JobRet.KO
|
||||
try :
|
||||
if self._type == "line" or self._type == "file" :
|
||||
if self._type in ["line", "file", "json"] :
|
||||
if self._copy_cache :
|
||||
ret = self.__from_cache()
|
||||
if ret != JobRet.KO :
|
||||
|
|
@ -123,18 +125,18 @@ class Job(abc.ABC) :
|
|||
for url in self._data :
|
||||
data = self.__download_data(url)
|
||||
for chunk in data :
|
||||
if self._type == "line" :
|
||||
if self._type == ["line", "json"] :
|
||||
if not re.match(self._regex, chunk.decode("utf-8")) :
|
||||
continue
|
||||
chunks = self._edit(chunk)
|
||||
if self._redis == None :
|
||||
if self._type == "line" :
|
||||
if self._type in ["line", "json"] :
|
||||
for chunk in chunks :
|
||||
file.write(chunk + b"\n")
|
||||
else :
|
||||
file.write(chunk)
|
||||
else :
|
||||
if self._type == "line" :
|
||||
if self._type in ["line", "json"] :
|
||||
for chunk in chunks :
|
||||
pipe.set(self._name + "_" + chunk, "1", ex=self._redis_ex)
|
||||
else :
|
||||
|
|
@ -155,11 +157,16 @@ class Job(abc.ABC) :
|
|||
return JobRet.KO
|
||||
|
||||
def __download_data(self, url) :
|
||||
r = requests.get(url, stream=True)
|
||||
r = requests.request(self._method, url, stream=True, json=self._json_data)
|
||||
if not r or r.status_code != 200 :
|
||||
raise Exception("can't download data at " + url)
|
||||
if self._type == "line" :
|
||||
return r.iter_lines()
|
||||
if self._type == "json" :
|
||||
try :
|
||||
return self._json(r.json())
|
||||
except :
|
||||
raise Exception("can't decode json from " + url)
|
||||
return r.iter_content(chunk_size=8192)
|
||||
|
||||
def __exec(self) :
|
||||
|
|
@ -177,6 +184,9 @@ class Job(abc.ABC) :
|
|||
self._callback(True)
|
||||
return JobRet.OK_RELOAD
|
||||
|
||||
def _json(self, data) :
|
||||
return data
|
||||
|
||||
def _edit(self, chunk) :
|
||||
return [chunk]
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,15 @@
|
|||
from Job import Job
|
||||
|
||||
class RemoteApiDatabase(Job) :
|
||||
|
||||
def __init__(self, server="", version="", id="", redis_host=None, copy_cache=False) :
|
||||
name = "remote-api-database"
|
||||
data = [server + "/db"]
|
||||
filename = "remote-api.db"
|
||||
type = "json"
|
||||
regex = r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$"
|
||||
json_data = {"version": version, "id": id}
|
||||
super().__init__(name, data, filename, type=type, redis_host=redis_host, redis_ex=redis_ex, regex=regex, copy_cache=copy_cache, json_data=json_data, method=method)
|
||||
|
||||
def _json(self, data) :
|
||||
return data["data"]
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
from Job import Job
|
||||
|
||||
class RemoteApiRegister(Job) :
|
||||
|
||||
def __init__(self, server="", version="") :
|
||||
name = "remote-api-register"
|
||||
data = [server + "/register"]
|
||||
filename = "machine.id"
|
||||
type = "json"
|
||||
regex = r"^[0-9a-f]{256}$"
|
||||
json_data = {"version": version}
|
||||
method = "POST"
|
||||
super().__init__(name, data, filename, type=type, regex=regex, copy_cache=True, json_data=json_data, method=method)
|
||||
|
||||
def _json(self, data) :
|
||||
return data["data"]
|
||||
10
jobs/main.py
10
jobs/main.py
|
|
@ -17,8 +17,11 @@ JOBS = {
|
|||
"geoip": GeoIP.GeoIP,
|
||||
"proxies": Proxies.Proxies,
|
||||
"referrers": Referrers.Referrers,
|
||||
"remote-api-database": RemoteApiDatabase.RemoteApiDatabase,
|
||||
"remote-api-register": RemoteApiRegister.RemoteApiRegister,
|
||||
"self-signed-cert": SelfSignedCert.SelfSignedCert,
|
||||
"user-agents": UserAgents.UserAgents
|
||||
|
||||
}
|
||||
|
||||
if __name__ == "__main__" :
|
||||
|
|
@ -36,6 +39,9 @@ if __name__ == "__main__" :
|
|||
parser.add_argument("--dst_key", default="", type=str, help="key path for self-signed-cert job (e.g. : /etc/nginx/default-key.pem)")
|
||||
parser.add_argument("--expiry", default="", type=str, help="number of validity days for self-signed-cert job (e.g. : 365)")
|
||||
parser.add_argument("--subj", default="", type=str, help="certificate subject for self-signed-cert job (e.g. : OU=X/CN=Y...)")
|
||||
parser.add_argument("--server", default="", type=str, help="address of the server for remote-api jobs")
|
||||
parser.add_argument("--id", default="", type=str, help="machine id for remote-api jobs")
|
||||
parser.add_argument("--version", default="", type=str, help="bunkerized-nginx version for remote-api jobs")
|
||||
args = parser.parse_args()
|
||||
|
||||
# Check job name
|
||||
|
|
@ -68,6 +74,10 @@ if __name__ == "__main__" :
|
|||
instance = JOBS[job](redis_host=redis_host, copy_cache=args.cache, domain=args.domain, email=args.email, staging=args.staging)
|
||||
elif job == "self-signed-cert" :
|
||||
instance = JOBS[job](redis_host=redis_host, copy_cache=args.cache, dst_cert=args.dst_cert, dst_key=args.dst_key, expiry=args.expiry, subj=args.subj)
|
||||
elif job == "remote-api-database" :
|
||||
instance = JOBS[job](server=args.server, version=args.version, id=args.id, redis_host=redis_host, copy_cache=args.cache)
|
||||
elif job == "remote-api-register" :
|
||||
instance = JOBS[job](server=args.server, version=args.version)
|
||||
else :
|
||||
instance = JOBS[job](redis_host=redis_host, copy_cache=args.cache)
|
||||
ret = instance.run()
|
||||
|
|
|
|||
|
|
@ -3,5 +3,6 @@
|
|||
45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
|
||||
0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
|
||||
0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
|
||||
30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1
|
||||
0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
|
||||
0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1
|
||||
|
|
|
|||
|
|
@ -3,5 +3,6 @@
|
|||
45 0 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name referrers" nginx >> /var/log/nginx/jobs.log 2>&1
|
||||
0 1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name abusers" nginx >> /var/log/nginx/jobs.log 2>&1
|
||||
0 2 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name proxies" nginx >> /var/log/nginx/jobs.log 2>&1
|
||||
30 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server $(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2) --version $(cat /opt/bunkerized-nginx/VERSION) --id $(cat /opt/bunkerized-nginx/machine.id)" nginx >> /var/log/nginx/jobs.log 2>&1
|
||||
0 */1 * * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name exit-nodes" nginx >> /var/log/nginx/jobs.log 2>&1
|
||||
0 3 2 * * . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /bin/su -c "/opt/bunkerized-nginx/jobs/main.py --reload --lock --name geoip" nginx >> /var/log/nginx/jobs.log 2>&1
|
||||
|
|
|
|||
|
|
@ -3,5 +3,6 @@
|
|||
45 0 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_REFERRER yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name referrers >> /var/log/nginx/jobs.log 2>&1
|
||||
0 1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_ABUSERS yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name abusers >> /var/log/nginx/jobs.log 2>&1
|
||||
0 2 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_PROXIES yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name proxies >> /var/log/nginx/jobs.log 2>&1
|
||||
30 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value USE_REMOTE_API yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name remote-api-database --server "$(grep '^REMOTE_API_SERVER=' /etc/nginx/global.env | cut -d '=' -f 2)" --version "$(cat /opt/bunkerized-nginx/VERSION)" --id "$(cat /opt/bunkerized-nginx/machine.id)" >> /var/log/nginx/jobs.log 2>&1
|
||||
0 */1 * * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ "$(has_value BLOCK_TOR_EXIT_NODE yes)" != "" ] && /opt/bunkerized-nginx/jobs/main.py --reload --name exit-nodes >> /var/log/nginx/jobs.log 2>&1
|
||||
0 3 2 * * nginx . /opt/bunkerized-nginx/entrypoint/utils.sh && [ [ "$(has_value BLACKLIST_COUNTRY ".\+")" != "" ] || [ "$(has_value WHITELIST_COUNTRY ".\+")" != "" ] ] && /opt/bunkerized-nginx/jobs/main.py --reload --name geoip >> /var/log/nginx/jobs.log 2>&1
|
||||
|
|
|
|||
Loading…
Reference in New Issue