librewolf
/
bunkerized-nginx
mirror of https://github.com/bunkerity/bunkerized-nginx
4
0
Fork 1
Code Issues Projects Releases Wiki Activity

Update scheduler changes check to reduce CPU usage

This commit is contained in:
Théophile Diot 2023-05-26 14:31:51 -04:00
parent bb7dcda48d
commit 953128be6e
No known key found for this signature in database
GPG Key ID: E752C80DB72BB014
3 changed files with 104 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
src/common/db/Database.py
View File

@ -257,6 +257,48 @@ class Database:
return ""
def check_changes(self) -> Dict[str, bool]:
"""Check if either the config, the custom configs or plugins have changed inside the database"""
with self.__db_session() as session:
try:
metadata = (
session.query(Metadata)
.with_entities(
Metadata.custom_configs_changed,
Metadata.external_plugins_changed,
Metadata.config_changed,
)
.filter_by(id=1)
.first()
)
return dict(
custom_configs_changed=metadata is not None
and metadata.custom_configs_changed,
external_plugins_changed=metadata is not None
and metadata.external_plugins_changed,
config_changed=metadata is not None and metadata.config_changed,
)
except BaseException:
return format_exc()
def checked_changes(self) -> str:
"""Set that the config, the custom configs and the plugins didn't change"""
with self.__db_session() as session:
try:
metadata = session.query(Metadata).get(1)
if not metadata:
return "The metadata are not set yet, try again"
metadata.config_changed = False
metadata.custom_configs_changed = False
metadata.external_plugins_changed = False
session.commit()
except BaseException:
return format_exc()
return ""
def init_tables(self, default_plugins: List[dict]) -> Tuple[bool, str]:
"""Initialize the database tables and return the result"""
inspector = inspect(self.__sql_engine)
@ -615,8 +657,10 @@ class Database:
with suppress(ProgrammingError, OperationalError):
metadata = session.query(Metadata).get(1)
if metadata is not None and not metadata.first_config_saved:
metadata.first_config_saved = True
if metadata is not None:
if not metadata.first_config_saved:
metadata.first_config_saved = True
metadata.config_changed = bool(to_put)
try:
session.add_all(to_put)
@ -709,6 +753,12 @@ class Database:
)
)
if to_put:
with suppress(ProgrammingError, OperationalError):
metadata = session.query(Metadata).get(1)
if metadata is not None:
metadata.custom_configs_changed = True
try:
session.add_all(to_put)
session.commit()
@ -1400,6 +1450,13 @@ class Database:
session.query(Plugin_pages).filter(
Plugin_pages.plugin_id == plugin["id"]
).update(updates)
if to_put:
with suppress(ProgrammingError, OperationalError):
metadata = session.query(Metadata).get(1)
if metadata is not None:
metadata.external_plugins_changed = True
try:
session.add_all(to_put)
session.commit()

3
src/common/db/model.py
View File

@ -279,5 +279,8 @@ class Metadata(Base):
is_initialized = Column(Boolean, nullable=False)
first_config_saved = Column(Boolean, nullable=False)
autoconf_loaded = Column(Boolean, default=False, nullable=True)
custom_configs_changed = Column(Boolean, default=False, nullable=True)
external_plugins_changed = Column(Boolean, default=False, nullable=True)
config_changed = Column(Boolean, default=False, nullable=True)
integration = Column(INTEGRATIONS_ENUM, default="Unknown", nullable=False)
version = Column(String(32), default="1.5.0", nullable=False)

95
src/scheduler/main.py
View File

@ -387,6 +387,14 @@ if __name__ == "__main__":
first_run = True
while True:
ret = db.checked_changes()
if ret:
logger.error(
f"An error occurred when setting the changes to checked in the database : {changes}"
)
stop(1)
# Instantiate scheduler
scheduler = JobScheduler(
env=env.copy() | environ.copy(),
@ -511,6 +519,8 @@ if __name__ == "__main__":
generate = True
scheduler.setup()
need_reload = False
configs_need_generation = False
plugins_need_generation = False
first_run = False
# infinite schedule for the jobs
@ -520,13 +530,34 @@ if __name__ == "__main__":
scheduler.run_pending()
sleep(1)
changes = db.check_changes()
if isinstance(changes, str):
logger.error(
f"An error occurred when checking for changes in the database : {changes}"
)
stop(1)
# check if the custom configs have changed since last time
tmp_db_configs: Dict[str, Any] = db.get_custom_configs()
if db_configs != tmp_db_configs:
if changes["custom_configs_changed"]:
logger.info("Custom configs changed, generating ...")
logger.debug(f"{tmp_db_configs=}")
logger.debug(f"{db_configs=}")
db_configs = tmp_db_configs.copy()
configs_need_generation = True
need_reload = True
# check if the plugins have changed since last time
if changes["external_plugins_changed"]:
logger.info("External plugins changed, generating ...")
plugins_need_generation = True
need_reload = True
# check if the config have changed since last time
if changes["config_changed"]:
logger.info("Config changed, generating ...")
need_reload = True
if need_reload:
if configs_need_generation:
db_configs = db.get_custom_configs()
# Remove old custom configs files
logger.info("Removing old custom configs files ...")
@ -544,40 +575,10 @@ if __name__ == "__main__":
original_path=configs_path,
)
# reload nginx
logger.info("Reloading nginx ...")
if integration not in (
"Autoconf",
"Swarm",
"Kubernetes",
"Docker",
):
# Reloading the nginx server.
proc = subprocess_run(
# Reload nginx
["sudo", join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
stdin=DEVNULL,
stderr=STDOUT,
env=env.copy(),
)
if proc.returncode == 0:
logger.info("Successfully reloaded nginx")
else:
logger.error(
f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stderr.decode('utf-8') if proc.stderr else 'Missing stderr'}",
)
else:
need_reload = True
# check if the plugins have changed since last time
tmp_external_plugins: List[Dict[str, Any]] = db.get_plugins(
external=True
)
if external_plugins != tmp_external_plugins:
logger.info("External plugins changed, generating ...")
logger.debug(f"{tmp_external_plugins=}")
logger.debug(f"{external_plugins=}")
external_plugins = tmp_external_plugins.copy()
if plugins_need_generation:
external_plugins: List[Dict[str, Any]] = db.get_plugins(
external=True, with_data=True
)
# Remove old external plugins files
logger.info("Removing old external plugins files ...")
@ -588,26 +589,14 @@ if __name__ == "__main__":
elif file.is_dir():
rmtree(str(file), ignore_errors=True)
logger.info("Generating new external plugins ...")
generate_external_plugins(
db.get_plugins(external=True, with_data=True),
external_plugins,
integration,
api_caller,
original_path=plugins_dir,
)
need_reload = True
# check if the config have changed since last time
tmp_env: Dict[str, Any] = db.get_config()
tmp_env["DATABASE_URI"] = environ.get(
"DATABASE_URI", tmp_env["DATABASE_URI"]
)
if env != tmp_env:
logger.info("Config changed, generating ...")
logger.debug(f"{tmp_env=}")
logger.debug(f"{env=}")
env = tmp_env.copy()
need_reload = True
env = db.get_config()
except:
logger.error(
f"Exception while executing scheduler : {format_exc()}",