mirror of
https://github.com/bunkerity/bunkerized-nginx
synced 2023-12-13 21:30:18 +01:00
Update scheduler changes check to reduce CPU usage
This commit is contained in:
parent
bb7dcda48d
commit
953128be6e
3 changed files with 104 additions and 55 deletions
|
@ -257,6 +257,48 @@ class Database:
|
|||
|
||||
return ""
|
||||
|
||||
def check_changes(self) -> Dict[str, bool]:
|
||||
"""Check if either the config, the custom configs or plugins have changed inside the database"""
|
||||
with self.__db_session() as session:
|
||||
try:
|
||||
metadata = (
|
||||
session.query(Metadata)
|
||||
.with_entities(
|
||||
Metadata.custom_configs_changed,
|
||||
Metadata.external_plugins_changed,
|
||||
Metadata.config_changed,
|
||||
)
|
||||
.filter_by(id=1)
|
||||
.first()
|
||||
)
|
||||
return dict(
|
||||
custom_configs_changed=metadata is not None
|
||||
and metadata.custom_configs_changed,
|
||||
external_plugins_changed=metadata is not None
|
||||
and metadata.external_plugins_changed,
|
||||
config_changed=metadata is not None and metadata.config_changed,
|
||||
)
|
||||
except BaseException:
|
||||
return format_exc()
|
||||
|
||||
def checked_changes(self) -> str:
|
||||
"""Set that the config, the custom configs and the plugins didn't change"""
|
||||
with self.__db_session() as session:
|
||||
try:
|
||||
metadata = session.query(Metadata).get(1)
|
||||
|
||||
if not metadata:
|
||||
return "The metadata are not set yet, try again"
|
||||
|
||||
metadata.config_changed = False
|
||||
metadata.custom_configs_changed = False
|
||||
metadata.external_plugins_changed = False
|
||||
session.commit()
|
||||
except BaseException:
|
||||
return format_exc()
|
||||
|
||||
return ""
|
||||
|
||||
def init_tables(self, default_plugins: List[dict]) -> Tuple[bool, str]:
|
||||
"""Initialize the database tables and return the result"""
|
||||
inspector = inspect(self.__sql_engine)
|
||||
|
@ -615,8 +657,10 @@ class Database:
|
|||
|
||||
with suppress(ProgrammingError, OperationalError):
|
||||
metadata = session.query(Metadata).get(1)
|
||||
if metadata is not None and not metadata.first_config_saved:
|
||||
metadata.first_config_saved = True
|
||||
if metadata is not None:
|
||||
if not metadata.first_config_saved:
|
||||
metadata.first_config_saved = True
|
||||
metadata.config_changed = bool(to_put)
|
||||
|
||||
try:
|
||||
session.add_all(to_put)
|
||||
|
@ -709,6 +753,12 @@ class Database:
|
|||
)
|
||||
)
|
||||
|
||||
if to_put:
|
||||
with suppress(ProgrammingError, OperationalError):
|
||||
metadata = session.query(Metadata).get(1)
|
||||
if metadata is not None:
|
||||
metadata.custom_configs_changed = True
|
||||
|
||||
try:
|
||||
session.add_all(to_put)
|
||||
session.commit()
|
||||
|
@ -1400,6 +1450,13 @@ class Database:
|
|||
session.query(Plugin_pages).filter(
|
||||
Plugin_pages.plugin_id == plugin["id"]
|
||||
).update(updates)
|
||||
|
||||
if to_put:
|
||||
with suppress(ProgrammingError, OperationalError):
|
||||
metadata = session.query(Metadata).get(1)
|
||||
if metadata is not None:
|
||||
metadata.external_plugins_changed = True
|
||||
|
||||
try:
|
||||
session.add_all(to_put)
|
||||
session.commit()
|
||||
|
|
|
@ -279,5 +279,8 @@ class Metadata(Base):
|
|||
is_initialized = Column(Boolean, nullable=False)
|
||||
first_config_saved = Column(Boolean, nullable=False)
|
||||
autoconf_loaded = Column(Boolean, default=False, nullable=True)
|
||||
custom_configs_changed = Column(Boolean, default=False, nullable=True)
|
||||
external_plugins_changed = Column(Boolean, default=False, nullable=True)
|
||||
config_changed = Column(Boolean, default=False, nullable=True)
|
||||
integration = Column(INTEGRATIONS_ENUM, default="Unknown", nullable=False)
|
||||
version = Column(String(32), default="1.5.0", nullable=False)
|
||||
|
|
|
@ -387,6 +387,14 @@ if __name__ == "__main__":
|
|||
|
||||
first_run = True
|
||||
while True:
|
||||
ret = db.checked_changes()
|
||||
|
||||
if ret:
|
||||
logger.error(
|
||||
f"An error occurred when setting the changes to checked in the database : {changes}"
|
||||
)
|
||||
stop(1)
|
||||
|
||||
# Instantiate scheduler
|
||||
scheduler = JobScheduler(
|
||||
env=env.copy() | environ.copy(),
|
||||
|
@ -511,6 +519,8 @@ if __name__ == "__main__":
|
|||
generate = True
|
||||
scheduler.setup()
|
||||
need_reload = False
|
||||
configs_need_generation = False
|
||||
plugins_need_generation = False
|
||||
first_run = False
|
||||
|
||||
# infinite schedule for the jobs
|
||||
|
@ -520,13 +530,34 @@ if __name__ == "__main__":
|
|||
scheduler.run_pending()
|
||||
sleep(1)
|
||||
|
||||
changes = db.check_changes()
|
||||
|
||||
if isinstance(changes, str):
|
||||
logger.error(
|
||||
f"An error occurred when checking for changes in the database : {changes}"
|
||||
)
|
||||
stop(1)
|
||||
|
||||
# check if the custom configs have changed since last time
|
||||
tmp_db_configs: Dict[str, Any] = db.get_custom_configs()
|
||||
if db_configs != tmp_db_configs:
|
||||
if changes["custom_configs_changed"]:
|
||||
logger.info("Custom configs changed, generating ...")
|
||||
logger.debug(f"{tmp_db_configs=}")
|
||||
logger.debug(f"{db_configs=}")
|
||||
db_configs = tmp_db_configs.copy()
|
||||
configs_need_generation = True
|
||||
need_reload = True
|
||||
|
||||
# check if the plugins have changed since last time
|
||||
if changes["external_plugins_changed"]:
|
||||
logger.info("External plugins changed, generating ...")
|
||||
plugins_need_generation = True
|
||||
need_reload = True
|
||||
|
||||
# check if the config have changed since last time
|
||||
if changes["config_changed"]:
|
||||
logger.info("Config changed, generating ...")
|
||||
need_reload = True
|
||||
|
||||
if need_reload:
|
||||
if configs_need_generation:
|
||||
db_configs = db.get_custom_configs()
|
||||
|
||||
# Remove old custom configs files
|
||||
logger.info("Removing old custom configs files ...")
|
||||
|
@ -544,40 +575,10 @@ if __name__ == "__main__":
|
|||
original_path=configs_path,
|
||||
)
|
||||
|
||||
# reload nginx
|
||||
logger.info("Reloading nginx ...")
|
||||
if integration not in (
|
||||
"Autoconf",
|
||||
"Swarm",
|
||||
"Kubernetes",
|
||||
"Docker",
|
||||
):
|
||||
# Reloading the nginx server.
|
||||
proc = subprocess_run(
|
||||
# Reload nginx
|
||||
["sudo", join(sep, "usr", "sbin", "nginx"), "-s", "reload"],
|
||||
stdin=DEVNULL,
|
||||
stderr=STDOUT,
|
||||
env=env.copy(),
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
logger.info("Successfully reloaded nginx")
|
||||
else:
|
||||
logger.error(
|
||||
f"Error while reloading nginx - returncode: {proc.returncode} - error: {proc.stderr.decode('utf-8') if proc.stderr else 'Missing stderr'}",
|
||||
)
|
||||
else:
|
||||
need_reload = True
|
||||
|
||||
# check if the plugins have changed since last time
|
||||
tmp_external_plugins: List[Dict[str, Any]] = db.get_plugins(
|
||||
external=True
|
||||
)
|
||||
if external_plugins != tmp_external_plugins:
|
||||
logger.info("External plugins changed, generating ...")
|
||||
logger.debug(f"{tmp_external_plugins=}")
|
||||
logger.debug(f"{external_plugins=}")
|
||||
external_plugins = tmp_external_plugins.copy()
|
||||
if plugins_need_generation:
|
||||
external_plugins: List[Dict[str, Any]] = db.get_plugins(
|
||||
external=True, with_data=True
|
||||
)
|
||||
|
||||
# Remove old external plugins files
|
||||
logger.info("Removing old external plugins files ...")
|
||||
|
@ -588,26 +589,14 @@ if __name__ == "__main__":
|
|||
elif file.is_dir():
|
||||
rmtree(str(file), ignore_errors=True)
|
||||
|
||||
logger.info("Generating new external plugins ...")
|
||||
generate_external_plugins(
|
||||
db.get_plugins(external=True, with_data=True),
|
||||
external_plugins,
|
||||
integration,
|
||||
api_caller,
|
||||
original_path=plugins_dir,
|
||||
)
|
||||
need_reload = True
|
||||
|
||||
# check if the config have changed since last time
|
||||
tmp_env: Dict[str, Any] = db.get_config()
|
||||
tmp_env["DATABASE_URI"] = environ.get(
|
||||
"DATABASE_URI", tmp_env["DATABASE_URI"]
|
||||
)
|
||||
if env != tmp_env:
|
||||
logger.info("Config changed, generating ...")
|
||||
logger.debug(f"{tmp_env=}")
|
||||
logger.debug(f"{env=}")
|
||||
env = tmp_env.copy()
|
||||
need_reload = True
|
||||
env = db.get_config()
|
||||
except:
|
||||
logger.error(
|
||||
f"Exception while executing scheduler : {format_exc()}",
|
||||
|
|
Loading…
Reference in a new issue