add instances changes check to scheduler and auto push dev container images

.github/workflows/dev.yml

@@ -93,3 +93,28 @@ jobs:
     secrets:
       PRIVATE_REGISTRY: ${{ secrets.PRIVATE_REGISTRY }}
       PRIVATE_REGISTRY_TOKEN: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+  
+  # Push with dev tag
+  push-dev:
+    needs: [tests-core]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
+      - name: Login to private repository
+        uses: docker/login-action@v2
+        with:
+          registry: ${{ secrets.PRIVATE_REGISTRY }}
+          username: registry
+          password: ${{ secrets.PRIVATE_REGISTRY_TOKEN }}
+      - name: Push BW image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/bunkerweb-tests:dev bunkerity/bunkerweb:dev && docker push bunkerity/bunkerweb:dev
+      - name: Push scheduler image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/scheduler-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/scheduler-tests:dev bunkerity/bunkerweb-scheduler:dev && docker push bunkerity/bunkerweb-scheduler:dev
+      - name: Push UI image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/ui-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/ui-tests:dev bunkerity/bunkerweb-ui:dev && docker push bunkerity/bunkerweb-ui:dev
+      - name: Push autoconf image
+        run: docker pull ${{ secrets.PRIVATE_REGISTRY }}/infra/autoconf-tests:dev && docker tag ${{ secrets.PRIVATE_REGISTRY }}/infra/autoconf-tests:dev bunkerity/bunkerweb-autoconf:dev && docker push bunkerity/bunkerweb-autoconf:dev

src/common/db/Database.py

@@ -296,7 +296,7 @@ class Database:
         return ""
 
     def check_changes(self) -> Union[Dict[str, bool], bool, str]:
-        """Check if either the config, the custom configs or plugins have changed inside the database"""
+        """Check if either the config, the custom configs, plugins or instances have changed inside the database"""
         with self.__db_session() as session:
             try:
                 metadata = (
@@ -305,6 +305,7 @@ class Database:
                         Metadata.custom_configs_changed,
                         Metadata.external_plugins_changed,
                         Metadata.config_changed,
+                        Metadata.instances_changed
                     )
                     .filter_by(id=1)
                     .first()
@@ -316,13 +317,14 @@ class Database:
                     external_plugins_changed=metadata is not None
                     and metadata.external_plugins_changed,
                     config_changed=metadata is not None and metadata.config_changed,
+                    instances_changed=metadata is not None and metadata.instances_changed
                 )
             except BaseException:
                 return format_exc()
 
     def checked_changes(self, changes: Optional[List[str]] = None) -> str:
-        """Set that the config, the custom configs and the plugins didn't change"""
-        changes = changes or ["config", "custom_configs", "external_plugins"]
+        """Set that the config, the custom configs, the plugins and instances didn't change"""
+        changes = changes or ["config", "custom_configs", "external_plugins", "instances"]
         with self.__db_session() as session:
             try:
                 metadata = session.query(Metadata).get(1)
@@ -336,6 +338,8 @@ class Database:
                     metadata.custom_configs_changed = False
                 if "external_plugins" in changes:
                     metadata.external_plugins_changed = False
+                if "instances" in changes:
+                    metadata.instances_changed = False
                 session.commit()
             except BaseException:
                 return format_exc()
@@ -1739,6 +1743,11 @@ class Database:
                     )
                 )
 
+            with suppress(ProgrammingError, OperationalError):
+                metadata = session.query(Metadata).get(1)
+                if metadata is not None:
+                    metadata.instances_changed = True
+
             try:
                 session.add_all(to_put)
                 session.commit()

src/common/db/model.py

@@ -274,5 +274,6 @@ class Metadata(Base):
     custom_configs_changed = Column(Boolean, default=False, nullable=True)
     external_plugins_changed = Column(Boolean, default=False, nullable=True)
     config_changed = Column(Boolean, default=False, nullable=True)
+    instances_changed = Column(Boolean, default=False, nullable=True)
     integration = Column(INTEGRATIONS_ENUM, default="Unknown", nullable=False)
     version = Column(String(32), default="1.5.0", nullable=False)

src/scheduler/JobScheduler.py

@@ -68,6 +68,23 @@ class JobScheduler(ApiCaller):
     def auto_setup(self):
         super().auto_setup(bw_integration=self.__integration)
 
+    def update_instances(self):
+        super().apis(self.__get_apis())
+
+    def __get_apis(self):
+        apis = []
+        try:
+            with self.__thread_lock:
+                instances = self.__db.get_instances()
+            for instance in instances:
+                api = API(f"http://{instance["hostname"]}:{instance["port"]}", host=instance["server_name"])
+                apis.append(api)
+        except:
+            self.__logger.warning(
+                f"Exception while getting jobs instances : {format_exc()}",
+            )
+        return apis
+
     def update_jobs(self):
         self.__jobs = self.__get_jobs()
 

src/scheduler/main.py

@@ -587,6 +587,7 @@ if __name__ == "__main__":
             CONFIG_NEED_GENERATION = False
             CONFIGS_NEED_GENERATION = False
             PLUGINS_NEED_GENERATION = False
+            INSTANCES_NEED_GENERATION = False
 
             # infinite schedule for the jobs
             logger.info("Executing job scheduler ...")
@@ -659,6 +660,11 @@ if __name__ == "__main__":
                     CONFIG_NEED_GENERATION = True
                     NEED_RELOAD = True
 
+                # check if the instances have changed since last time
+                if changes["instances_changed"]:
+                    logger.info("Instances changed, generating ...")
+                    INSTANCES_NEED_GENERATION = True
+
             FIRST_RUN = False
 
             if NEED_RELOAD:
@@ -683,6 +689,10 @@ if __name__ == "__main__":
                 if CONFIG_NEED_GENERATION:
                     CHANGES.append("config")
                     env = db.get_config()
+
+                if INSTANCES_NEED_GENERATION:
+                    CHANGES.append("instances")
+                    SCHEDULER.update_instances()
     except:
         logger.error(
             f"Exception while executing scheduler : {format_exc()}",