mirror of
https://github.com/bunkerity/bunkerized-nginx
synced 2023-12-13 21:30:18 +01:00
Update mmdb download to check the checksum at start
This commit is contained in:
parent
4378f18cc8
commit
b58798746d
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from gzip import decompress
|
from gzip import decompress
|
||||||
|
from hashlib import sha1
|
||||||
from os import _exit, getenv
|
from os import _exit, getenv
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from sys import exit as sys_exit, path as sys_path
|
from sys import exit as sys_exit, path as sys_path
|
||||||
|
@ -24,62 +25,91 @@ from jobs import cache_file, cache_hash, file_hash, is_cached_file
|
||||||
|
|
||||||
logger = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO"))
|
logger = setup_logger("JOBS.mmdb-asn", getenv("LOG_LEVEL", "INFO"))
|
||||||
status = 0
|
status = 0
|
||||||
db = Database(
|
|
||||||
logger,
|
|
||||||
sqlalchemy_string=getenv("DATABASE_URI", None),
|
|
||||||
)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Don't go further if the cache is fresh
|
dl_mmdb = True
|
||||||
if is_cached_file("/var/cache/bunkerweb/asn.mmdb", "month", db):
|
tmp_path = "/var/tmp/bunkerweb/asn.mmdb"
|
||||||
logger.info("asn.mmdb is already in cache, skipping download...")
|
new_hash = None
|
||||||
_exit(0)
|
|
||||||
|
|
||||||
# Compute the mmdb URL
|
# Don't go further if the cache match the latest version
|
||||||
mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
|
if Path("/var/tmp/bunkerweb/asn.mmdb").exists():
|
||||||
|
response = get("https://db-ip.com/db/download/ip-to-asn-lite")
|
||||||
|
|
||||||
# Download the mmdb file and save it to tmp
|
if response.status_code == 200:
|
||||||
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
|
_sha1 = sha1()
|
||||||
file_content = b""
|
with open("/var/tmp/bunkerweb/asn.mmdb", "rb") as f:
|
||||||
with get(mmdb_url, stream=True) as resp:
|
while True:
|
||||||
resp.raise_for_status()
|
data = f.read(1024)
|
||||||
for chunk in resp.iter_content(chunk_size=4 * 1024):
|
if not data:
|
||||||
if chunk:
|
break
|
||||||
file_content += chunk
|
_sha1.update(data)
|
||||||
|
|
||||||
try:
|
if response.content.decode().find(_sha1.hexdigest()) != -1:
|
||||||
assert file_content
|
logger.info(
|
||||||
except AssertionError:
|
"asn.mmdb is already the latest version, skipping download..."
|
||||||
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
|
)
|
||||||
_exit(2)
|
dl_mmdb = False
|
||||||
|
tmp_path = "/var/tmp/bunkerweb/asn.mmdb"
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Unable to check if asn.mmdb is the latest version, downloading it anyway..."
|
||||||
|
)
|
||||||
|
|
||||||
# Decompress it
|
db = Database(
|
||||||
logger.info("Decompressing mmdb file ...")
|
logger,
|
||||||
Path(f"/var/tmp/bunkerweb/asn.mmdb").write_bytes(decompress(file_content))
|
sqlalchemy_string=getenv("DATABASE_URI", None),
|
||||||
|
)
|
||||||
|
|
||||||
|
if dl_mmdb:
|
||||||
|
# Don't go further if the cache is fresh
|
||||||
|
if is_cached_file("/var/cache/bunkerweb/asn.mmdb", "month", db):
|
||||||
|
logger.info("asn.mmdb is already in cache, skipping download...")
|
||||||
|
_exit(0)
|
||||||
|
|
||||||
|
# Compute the mmdb URL
|
||||||
|
mmdb_url = f"https://download.db-ip.com/free/dbip-asn-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
|
||||||
|
|
||||||
|
# Download the mmdb file and save it to tmp
|
||||||
|
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
|
||||||
|
file_content = b""
|
||||||
|
with get(mmdb_url, stream=True) as resp:
|
||||||
|
resp.raise_for_status()
|
||||||
|
for chunk in resp.iter_content(chunk_size=4 * 1024):
|
||||||
|
if chunk:
|
||||||
|
file_content += chunk
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert file_content
|
||||||
|
except AssertionError:
|
||||||
|
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
|
||||||
|
_exit(2)
|
||||||
|
|
||||||
|
# Decompress it
|
||||||
|
logger.info("Decompressing mmdb file ...")
|
||||||
|
Path(tmp_path).write_bytes(decompress(file_content))
|
||||||
|
|
||||||
|
# Check if file has changed
|
||||||
|
new_hash = file_hash(tmp_path)
|
||||||
|
old_hash = cache_hash("/var/cache/bunkerweb/asn.mmdb", db)
|
||||||
|
if new_hash == old_hash:
|
||||||
|
logger.info("New file is identical to cache file, reload is not needed")
|
||||||
|
_exit(0)
|
||||||
|
|
||||||
# Try to load it
|
# Try to load it
|
||||||
logger.info("Checking if mmdb file is valid ...")
|
logger.info("Checking if mmdb file is valid ...")
|
||||||
with open_database("/var/tmp/bunkerweb/asn.mmdb") as reader:
|
with open_database(tmp_path or "/var/cache/bunkerweb/asn.mmdb") as reader:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Check if file has changed
|
|
||||||
new_hash = file_hash("/var/tmp/bunkerweb/asn.mmdb")
|
|
||||||
old_hash = cache_hash("/var/cache/bunkerweb/asn.mmdb", db)
|
|
||||||
if new_hash == old_hash:
|
|
||||||
logger.info("New file is identical to cache file, reload is not needed")
|
|
||||||
_exit(0)
|
|
||||||
|
|
||||||
# Move it to cache folder
|
# Move it to cache folder
|
||||||
logger.info("Moving mmdb file to cache ...")
|
logger.info("Moving mmdb file to cache ...")
|
||||||
cached, err = cache_file(
|
cached, err = cache_file(tmp_path, "/var/cache/bunkerweb/asn.mmdb", new_hash, db)
|
||||||
"/var/tmp/bunkerweb/asn.mmdb", "/var/cache/bunkerweb/asn.mmdb", new_hash, db
|
|
||||||
)
|
|
||||||
if not cached:
|
if not cached:
|
||||||
logger.error(f"Error while caching mmdb file : {err}")
|
logger.error(f"Error while caching mmdb file : {err}")
|
||||||
_exit(2)
|
_exit(2)
|
||||||
|
|
||||||
# Success
|
# Success
|
||||||
logger.info(f"Downloaded new mmdb from {mmdb_url}")
|
if dl_mmdb:
|
||||||
|
logger.info(f"Downloaded new mmdb from {mmdb_url}")
|
||||||
|
|
||||||
status = 1
|
status = 1
|
||||||
|
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
|
|
||||||
from datetime import date
|
from datetime import date
|
||||||
from gzip import decompress
|
from gzip import decompress
|
||||||
|
from hashlib import sha1
|
||||||
from os import _exit, getenv
|
from os import _exit, getenv
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from sys import exit as sys_exit, path as sys_path
|
from sys import exit as sys_exit, path as sys_path
|
||||||
|
@ -26,86 +27,91 @@ logger = setup_logger("JOBS.mmdb-country", getenv("LOG_LEVEL", "INFO"))
|
||||||
status = 0
|
status = 0
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# Only download mmdb if the country blacklist or whitelist is enabled
|
dl_mmdb = True
|
||||||
dl_mmdb = False
|
tmp_path = "/var/tmp/bunkerweb/country.mmdb"
|
||||||
# Multisite case
|
new_hash = None
|
||||||
if getenv("MULTISITE", "no") == "yes":
|
|
||||||
for first_server in getenv("SERVER_NAME", "").split(" "):
|
|
||||||
if getenv(
|
|
||||||
f"{first_server}_BLACKLIST_COUNTRY", getenv("BLACKLIST_COUNTRY")
|
|
||||||
) or getenv(
|
|
||||||
f"{first_server}_WHITELIST_COUNTRY", getenv("WHITELIST_COUNTRY")
|
|
||||||
):
|
|
||||||
dl_mmdb = True
|
|
||||||
break
|
|
||||||
# Singlesite case
|
|
||||||
elif getenv("BLACKLIST_COUNTRY") or getenv("WHITELIST_COUNTRY"):
|
|
||||||
dl_mmdb = True
|
|
||||||
|
|
||||||
if not dl_mmdb:
|
# Don't go further if the cache match the latest version
|
||||||
logger.info(
|
if Path("/var/tmp/bunkerweb/country.mmdb").exists():
|
||||||
"Country blacklist or whitelist is not enabled, skipping download..."
|
response = get("https://db-ip.com/db/download/ip-to-country-lite")
|
||||||
)
|
|
||||||
_exit(0)
|
if response.status_code == 200:
|
||||||
|
_sha1 = sha1()
|
||||||
|
with open("/var/tmp/bunkerweb/country.mmdb", "rb") as f:
|
||||||
|
while True:
|
||||||
|
data = f.read(1024)
|
||||||
|
if not data:
|
||||||
|
break
|
||||||
|
_sha1.update(data)
|
||||||
|
|
||||||
|
if response.content.decode().find(_sha1.hexdigest()) != -1:
|
||||||
|
logger.info(
|
||||||
|
"country.mmdb is already the latest version, skipping download..."
|
||||||
|
)
|
||||||
|
dl_mmdb = False
|
||||||
|
tmp_path = "/var/tmp/bunkerweb/country.mmdb"
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"Unable to check if country.mmdb is the latest version, downloading it anyway..."
|
||||||
|
)
|
||||||
|
|
||||||
db = Database(
|
db = Database(
|
||||||
logger,
|
logger,
|
||||||
sqlalchemy_string=getenv("DATABASE_URI", None),
|
sqlalchemy_string=getenv("DATABASE_URI", None),
|
||||||
)
|
)
|
||||||
|
|
||||||
# Don't go further if the cache is fresh
|
if dl_mmdb:
|
||||||
if is_cached_file("/var/cache/bunkerweb/country.mmdb", "month", db):
|
# Don't go further if the cache is fresh
|
||||||
logger.info("country.mmdb is already in cache, skipping download...")
|
if is_cached_file("/var/cache/bunkerweb/country.mmdb", "month", db):
|
||||||
_exit(0)
|
logger.info("country.mmdb is already in cache, skipping download...")
|
||||||
|
_exit(0)
|
||||||
|
|
||||||
# Compute the mmdb URL
|
# Compute the mmdb URL
|
||||||
mmdb_url = f"https://download.db-ip.com/free/dbip-country-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
|
mmdb_url = f"https://download.db-ip.com/free/dbip-country-lite-{date.today().strftime('%Y-%m')}.mmdb.gz"
|
||||||
|
|
||||||
# Download the mmdb file and save it to tmp
|
# Download the mmdb file and save it to tmp
|
||||||
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
|
logger.info(f"Downloading mmdb file from url {mmdb_url} ...")
|
||||||
file_content = b""
|
file_content = b""
|
||||||
with get(mmdb_url, stream=True) as resp:
|
with get(mmdb_url, stream=True) as resp:
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
for chunk in resp.iter_content(chunk_size=4 * 1024):
|
for chunk in resp.iter_content(chunk_size=4 * 1024):
|
||||||
if chunk:
|
if chunk:
|
||||||
file_content += chunk
|
file_content += chunk
|
||||||
|
|
||||||
try:
|
try:
|
||||||
assert file_content
|
assert file_content
|
||||||
except AssertionError:
|
except AssertionError:
|
||||||
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
|
logger.error(f"Error while downloading mmdb file from {mmdb_url}")
|
||||||
_exit(2)
|
_exit(2)
|
||||||
|
|
||||||
# Decompress it
|
# Decompress it
|
||||||
logger.info("Decompressing mmdb file ...")
|
logger.info("Decompressing mmdb file ...")
|
||||||
Path(f"/var/tmp/bunkerweb/country.mmdb").write_bytes(decompress(file_content))
|
Path(tmp_path).write_bytes(decompress(file_content))
|
||||||
|
|
||||||
|
# Check if file has changed
|
||||||
|
new_hash = file_hash(tmp_path)
|
||||||
|
old_hash = cache_hash("/var/cache/bunkerweb/country.mmdb", db)
|
||||||
|
if new_hash == old_hash:
|
||||||
|
logger.info("New file is identical to cache file, reload is not needed")
|
||||||
|
_exit(0)
|
||||||
|
|
||||||
# Try to load it
|
# Try to load it
|
||||||
logger.info("Checking if mmdb file is valid ...")
|
logger.info("Checking if mmdb file is valid ...")
|
||||||
with open_database("/var/tmp/bunkerweb/country.mmdb") as reader:
|
with open_database(tmp_path or "/var/cache/bunkerweb/country.mmdb") as reader:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
# Check if file has changed
|
|
||||||
new_hash = file_hash("/var/tmp/bunkerweb/country.mmdb")
|
|
||||||
old_hash = cache_hash("/var/cache/bunkerweb/country.mmdb", db)
|
|
||||||
if new_hash == old_hash:
|
|
||||||
logger.info("New file is identical to cache file, reload is not needed")
|
|
||||||
_exit(0)
|
|
||||||
|
|
||||||
# Move it to cache folder
|
# Move it to cache folder
|
||||||
logger.info("Moving mmdb file to cache ...")
|
logger.info("Moving mmdb file to cache ...")
|
||||||
cached, err = cache_file(
|
cached, err = cache_file(
|
||||||
"/var/tmp/bunkerweb/country.mmdb",
|
tmp_path, "/var/cache/bunkerweb/country.mmdb", new_hash, db
|
||||||
"/var/cache/bunkerweb/country.mmdb",
|
|
||||||
new_hash,
|
|
||||||
db,
|
|
||||||
)
|
)
|
||||||
if not cached:
|
if not cached:
|
||||||
logger.error(f"Error while caching mmdb file : {err}")
|
logger.error(f"Error while caching mmdb file : {err}")
|
||||||
_exit(2)
|
_exit(2)
|
||||||
|
|
||||||
# Success
|
# Success
|
||||||
logger.info(f"Downloaded new mmdb from {mmdb_url}")
|
if dl_mmdb:
|
||||||
|
logger.info(f"Downloaded new mmdb from {mmdb_url}")
|
||||||
|
|
||||||
status = 1
|
status = 1
|
||||||
|
|
||||||
|
|
|
@ -135,7 +135,12 @@ def cache_hash(cache: str, db=None) -> Optional[str]:
|
||||||
|
|
||||||
|
|
||||||
def cache_file(
|
def cache_file(
|
||||||
file: str, cache: str, _hash: str, db=None, *, service_id: Optional[str] = None
|
file: str,
|
||||||
|
cache: str,
|
||||||
|
_hash: Optional[str],
|
||||||
|
db=None,
|
||||||
|
*,
|
||||||
|
service_id: Optional[str] = None,
|
||||||
) -> Tuple[bool, str]:
|
) -> Tuple[bool, str]:
|
||||||
ret, err = True, "success"
|
ret, err = True, "success"
|
||||||
try:
|
try:
|
||||||
|
@ -143,6 +148,9 @@ def cache_file(
|
||||||
Path(cache).write_bytes(content)
|
Path(cache).write_bytes(content)
|
||||||
Path(file).unlink()
|
Path(file).unlink()
|
||||||
|
|
||||||
|
if not _hash:
|
||||||
|
_hash = file_hash(cache)
|
||||||
|
|
||||||
if db:
|
if db:
|
||||||
with lock:
|
with lock:
|
||||||
err = db.update_job_cache(
|
err = db.update_job_cache(
|
||||||
|
|
|
@ -68,6 +68,11 @@ RUN apk add --no-cache bash libgcc libstdc++ openssl && \
|
||||||
chmod 660 /usr/share/bunkerweb/INTEGRATION && \
|
chmod 660 /usr/share/bunkerweb/INTEGRATION && \
|
||||||
chown root:scheduler /usr/share/bunkerweb/INTEGRATION
|
chown root:scheduler /usr/share/bunkerweb/INTEGRATION
|
||||||
|
|
||||||
|
COPY --chown=root:scheduler src/bw/misc/asn.mmdb /var/tmp/bunkerweb/asn.mmdb
|
||||||
|
COPY --chown=root:scheduler src/bw/misc/country.mmdb /var/tmp/bunkerweb/country.mmdb
|
||||||
|
|
||||||
|
RUN chmod 770 /var/tmp/bunkerweb/asn.mmdb /var/tmp/bunkerweb/country.mmdb
|
||||||
|
|
||||||
# Fix CVEs
|
# Fix CVEs
|
||||||
RUN apk add "libcrypto3>=3.0.8-r4" "libssl3>=3.0.8-r4"
|
RUN apk add "libcrypto3>=3.0.8-r4" "libssl3>=3.0.8-r4"
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,3 @@
|
||||||
schedule==1.2.0
|
schedule==1.2.0
|
||||||
certbot==2.6.0
|
certbot==2.6.0
|
||||||
maxminddb==2.3.0
|
maxminddb==2.3.0
|
||||||
|
|
Loading…
Reference in a new issue