diff --git a/autoconf/src/Config.py b/autoconf/src/Config.py index c1002e2d..79dd7265 100644 --- a/autoconf/src/Config.py +++ b/autoconf/src/Config.py @@ -149,7 +149,14 @@ class Config : except : ret = False elif self.__type == Controller.Type.KUBERNETES : - log("config", "ERROR", "TODO get urls for k8s") + for instance in instances : + name = instance.metadata.name + try : + dns_result = dns.resolver.query(name + ".default.svc.cluster.local") + for ip in dns_result : + urls.append("http://" + ip.to_text() + ":8080" + self.__api_uri + path) + except : + ret = False for url in urls : req = None diff --git a/autoconf/src/IngressController.py b/autoconf/src/IngressController.py index 9b63818a..b0db0674 100644 --- a/autoconf/src/IngressController.py +++ b/autoconf/src/IngressController.py @@ -1,5 +1,6 @@ from kubernetes import client, config, watch from threading import Thread, Lock +import time import Controller @@ -13,20 +14,35 @@ class IngressController(Controller.Controller) : self.__api = client.CoreV1Api() self.__extensions_api = client.ExtensionsV1beta1Api() self.__old_env = {} + self.__internal_lock = Lock() + + def __get_pods(self) : + return self.__api.list_pod_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items def __get_ingresses(self) : - return self.__extensions_api.list_ingress_for_all_namespaces(watch=False).items + return self.__extensions_api.list_ingress_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items - def __get_services(self) : - return self.__api.list_service_for_all_namespaces(watch=False).items + def __get_services(self, autoconf=False) : + services = self.__api.list_service_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items + if not autoconf : + return services + services_autoconf = [] + for service in services : + if service.metadata.annotations != None and "bunkerized-nginx.AUTOCONF" in service.metadata.annotations : + services_autoconf.append(service) + return services_autoconf - def __annotations_to_env(self, annotations, service=False) : + def __pod_to_env(self, pod_env) : env = {} - prefix = "" - if service : - if not "bunkerized-nginx.SERVER_NAME" in annotations : - raise Exception("Missing bunkerized-nginx.SERVER_NAME annotation in Service.") - prefix = annotations["bunkerized-nginx.SERVER_NAME"].split(" ")[0] + "_" + for env_var in pod_env : + env[env_var.name] = env_var.value + if env_var.value == None : + env[env_var.name] = "" + return env + + def __annotations_to_env(self, annotations) : + env = {} + prefix = annotations["bunkerized-nginx.SERVER_NAME"].split(" ")[0] + "_" for annotation in annotations : if annotation.startswith("bunkerized-nginx.") and annotation.replace("bunkerized-nginx.", "", 1) != "" and annotation.replace("bunkerized-nginx.", "", 1) != "AUTOCONF" : env[prefix + annotation.replace("bunkerized-nginx.", "", 1)] = annotations[annotation] @@ -34,64 +50,124 @@ class IngressController(Controller.Controller) : def __rules_to_env(self, rules) : env = {} + first_servers = [] for rule in rules : + rule = rule.to_dict() prefix = "" if "host" in rule : prefix = rule["host"] + "_" + first_servers.append(rule["host"]) if not "http" in rule or not "paths" in rule["http"] : continue for path in rule["http"]["paths"] : env[prefix + "USE_REVERSE_PROXY"] = "yes" env[prefix + "REVERSE_PROXY_URL"] = path["path"] - env[prefix + "REVERSE_PROXY_HOST"] = "http://" + path["backend"]["serviceName"] + ":" + str(path["backend"]["servicePort"]) + env[prefix + "REVERSE_PROXY_HOST"] = "http://" + path["backend"]["service_name"] + ":" + str(path["backend"]["service_port"]) + env["SERVER_NAME"] = " ".join(first_servers) return env def get_env(self) : + pods = self.__get_pods() ingresses = self.__get_ingresses() services = self.__get_services() env = {} + first_servers = [] + for pod in pods : + env.update(self.__pod_to_env(pod.spec.containers[0].env)) + if "SERVER_NAME" in env and env["SERVER_NAME"] != "" : + first_servers.extend(env["SERVER_NAME"].split(" ")) for ingress in ingresses : - if ingress.metadata.annotations == None : - continue - if "bunkerized-nginx.AUTOCONF" in ingress.metadata.annotations : - env.update(self.__annotations_to_env(ingress.metadata.annotations)) - env.update(self.__rules_to_env(ingress.spec.rules)) + env.update(self.__rules_to_env(ingress.spec.rules)) + if "SERVER_NAME" in env and env["SERVER_NAME"] != "" : + first_servers.extend(env["SERVER_NAME"].split(" ")) for service in services : - if service.metadata.annotations == None : - continue - if "bunkerized-nginx.AUTOCONF" in service.metadata.annotations : - env.update(self.__annotations_to_env(service.metadata.annotations, service=True)) + if service.metadata.annotations != None and "bunkerized-nginx.SERVER_NAME" in service.metadata.annotations : + env.update(self.__annotations_to_env(service.metadata.annotations)) + first_servers.append(service.metadata.annotations["SERVER_NAME"]) + first_servers = list(dict.fromkeys(first_servers)) + if len(first_servers) == 0 : + env["SERVER_NAME"] = "" + else : + env["SERVER_NAME"] = " ".join(first_servers) return self._fix_env(env) def process_events(self, current_env) : self.__old_env = current_env + t_pod = Thread(target=self.__watch_pod) t_ingress = Thread(target=self.__watch_ingress) t_service = Thread(target=self.__watch_service) + t_pod.start() t_ingress.start() t_service.start() + t_pod.join() t_ingress.join() t_service.join() - def __watch_ingress(self) : + def __watch_pod(self) : w = watch.Watch() - for event in w.stream(self.__extensions_api.list_ingress_for_all_namespaces) : + for event in w.stream(self.__api.list_pod_for_all_namespaces, label_selector="bunkerized-nginx") : + self.__internal_lock.acquire() new_env = self.get_env() - if new_env != self.__old_env() : - if self.gen_conf(new_env, lock=False) : + if new_env != self.__old_env : + if self.gen_conf(new_env) : self.__old_env = new_env.copy() log("CONTROLLER", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") + self.__internal_lock.release() + + def __watch_ingress(self) : + w = watch.Watch() + for event in w.stream(self.__extensions_api.list_ingress_for_all_namespaces, label_selector="bunkerized-nginx") : + self.__internal_lock.acquire() + new_env = self.get_env() + if new_env != self.__old_env : + if self.gen_conf(new_env) : + self.__old_env = new_env.copy() + log("CONTROLLER", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") + self.__internal_lock.release() def __watch_service(self) : w = watch.Watch() - for event in w.stream(self.__api.list_service_for_all_namespaces) : + for event in w.stream(self.__api.list_service_for_all_namespaces, label_selector="bunkerized-nginx") : + self.__internal_lock.acquire() new_env = self.get_env() - if new_env != self.__old_env() : - if self.gen_conf(new_env, lock=False) : + if new_env != self.__old_env : + if self.gen_conf(new_env) : self.__old_env = new_env.copy() log("CONTROLLER", "INFO", "successfully generated new configuration") + if self.reload() : + log("controller", "INFO", "successful reload") + else : + log("controller", "ERROR", "failed reload") + self.__internal_lock.release() def reload(self) : - return self._reload(self.__get_ingresses()) + return self._reload(self.__get_services(autoconf=True)) def wait(self) : - return self._config.wait(self.__get_ingresses()) + # Wait for at least one bunkerized-nginx pod + pods = self.__get_pods() + while len(pods) == 0 : + time.sleep(1) + pods = self.__get_pods() + + # Wait for at least one bunkerized-nginx service + services = self.__get_services(autoconf=True) + while len(services) == 0 : + time.sleep(1) + services = self.__get_services(autoconf=True) + + # Generate first config + env = self.get_env() + if not self.gen_conf(env) : + return False, env + + # Wait for bunkerized-nginx + return self._config.wait(services), env diff --git a/entrypoint/entrypoint.sh b/entrypoint/entrypoint.sh index c395cde6..8bffa862 100644 --- a/entrypoint/entrypoint.sh +++ b/entrypoint/entrypoint.sh @@ -17,7 +17,7 @@ trap "trap_exit" TERM INT QUIT # trap SIGHUP function trap_reload() { log "reload" "INFO" "catched reload operation" - if [ "$SWARM_MODE" != "yes" ] ; then + if [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] ; then /opt/bunkerized-nginx/entrypoint/jobs.sh fi if [ -f /tmp/nginx.pid ] ; then @@ -40,10 +40,10 @@ if [ ! -f "/etc/nginx/global.env" ] ; then log "entrypoint" "INFO" "configuring bunkerized-nginx ..." # check permissions - if [ "$SWARM_MODE" != "yes" ] ; then + if [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] ; then /opt/bunkerized-nginx/entrypoint/permissions.sh else - /opt/bunkerized-nginx/entrypoint/permissions-swarm.sh + /opt/bunkerized-nginx/entrypoint/permissions-cluster.sh fi if [ "$?" -ne 0 ] ; then exit 1 diff --git a/entrypoint/nginx-temp.sh b/entrypoint/nginx-temp.sh index f6565de8..1f818da2 100644 --- a/entrypoint/nginx-temp.sh +++ b/entrypoint/nginx-temp.sh @@ -4,7 +4,7 @@ . /opt/bunkerized-nginx/entrypoint/utils.sh # start nginx with temp conf for let's encrypt challenges and API -if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$SWARM_MODE" = "yes" ] || [ "$AUTO_LETS_ENCRYPT" = "yes" ] ; then +if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$SWARM_MODE" = "yes" ] || [ "$AUTO_LETS_ENCRYPT" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then cp /opt/bunkerized-nginx/confs/global/nginx-temp.conf /tmp/nginx-temp.conf cp /opt/bunkerized-nginx/confs/global/api-temp.conf /tmp/api.conf if [ "$SWARM_MODE" = "yes" ] ; then diff --git a/entrypoint/permissions-swarm.sh b/entrypoint/permissions-cluster.sh similarity index 100% rename from entrypoint/permissions-swarm.sh rename to entrypoint/permissions-cluster.sh diff --git a/gen/Configurator.py b/gen/Configurator.py index b07cbb76..2addee74 100644 --- a/gen/Configurator.py +++ b/gen/Configurator.py @@ -1,4 +1,4 @@ -import json, re +import json, re, sys class Configurator : diff --git a/helpers/kubernetes-ingress.yml b/helpers/kubernetes-ingress.yml index e18588c6..b54e6970 100644 --- a/helpers/kubernetes-ingress.yml +++ b/helpers/kubernetes-ingress.yml @@ -2,11 +2,10 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: bunkerized-nginx-ingress + labels: + bunkerized-nginx: "yes" annotations: - # mandatory, keep this annotation - bunkerized-nginx.AUTOCONF: "yes" # add any global and default environment variables here as annotations with the "bunkerized-nginx." prefix - # if the scope is "multisite", they will be applied to all services unless overriden by the service # examples : #bunkerized-nginx.AUTO_LETS_ENCRYPT: "yes" #bunkerized-nginx.USE_ANTIBOT: "javascript" @@ -25,7 +24,39 @@ spec: path: "/" backend: service: - name: app-service + name: myapp port: number: 80 - +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: myapp + labels: + app: myapp +spec: + replicas: 1 + selector: + matchLabels: + app: myapp + template: + metadata: + labels: + app: myapp + spec: + containers: + - name: myapp + image: containous/whoami +--- +apiVersion: v1 +kind: Service +metadata: + name: myapp +spec: + type: ClusterIP + selector: + app: myapp + ports: + - protocol: TCP + port: 80 + targetPort: 80 diff --git a/helpers/kubernetes-nginx.yml b/helpers/kubernetes-nginx.yml index b21fed7a..10656aba 100644 --- a/helpers/kubernetes-nginx.yml +++ b/helpers/kubernetes-nginx.yml @@ -17,51 +17,56 @@ spec: serviceAccountName: bunkerized-nginx-ingress-controller containers: - name: bunkerized-nginx-autoconf - image: bunkerity/bunkerized-nginx-autoconf:1.3.0 + image: bunkerity/bunkerized-nginx-autoconf:testing + imagePullPolicy: Always env: - name: KUBERNETES_MODE value: "yes" - name: API_URI value: "/ChangeMeToSomethingHardToGuess" + - name: SERVER_NAME + value: "" + - name: MULTISITE + value: "yes" volumeMounts: - - name: config + - name: confs mountPath: /etc/nginx - - name: certs + - name: letsencrypt mountPath: /etc/letsencrypt - - name: challenges + - name: acme-challenge mountPath: /acme-challenge - name: cache mountPath: /cache - - name: custom-modsec + - name: modsec-confs mountPath: /modsec-confs readOnly: true - - name: custom-modsec-crs + - name: modsec-crs-confs mountPath: /modsec-crs-confs readOnly: true volumes: - - name: config + - name: confs hostPath: - path: /shared/config + path: /shared/confs type: Directory - - name: certs + - name: letsencrypt hostPath: - path: /shared/certs + path: /shared/letsencrypt type: Directory - - name: challenges + - name: acme-challenge hostPath: - path: /shared/challenges + path: /shared/acme-challenge type: Directory - name: cache hostPath: path: /shared/cache type: Directory - - name: custom-modsec + - name: modsec-confs hostPath: - path: /shared/custom-modsec + path: /shared/modsec-confs type: Directory - - name: custom-modsec-crs + - name: modsec-crs-confs hostPath: - path: /shared/custom-modsec-crs + path: /shared/modsec-crs-confs type: Directory --- apiVersion: apps/v1 @@ -78,12 +83,14 @@ spec: metadata: labels: name: bunkerized-nginx + bunkerized-nginx: "yes" spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet containers: - name: bunkerized-nginx - image: bunkerity/bunkerized-nginx:1.3.0 + image: bunkerity/bunkerized-nginx:testing + imagePullPolicy: Always env: - name: KUBERNETES_MODE value: "yes" @@ -91,61 +98,78 @@ spec: value: "yes" - name: API_URI value: "/ChangeMeToSomethingHardToGuess" + - name: SERVER_NAME + value: "" + - name: MULTISITE + value: "yes" volumeMounts: - - name: config + - name: confs mountPath: /etc/nginx readOnly: true - - name: certs + - name: letsencrypt mountPath: /etc/letsencrypt readOnly: true - - name: challenges + - name: acme-challenge mountPath: /acme-challenge readOnly: true - name: www mountPath: /www readOnly: true - - name: custom-http + - name: http-confs mountPath: /http-confs readOnly: true - - name: custom-server + - name: server-confs mountPath: /server-confs readOnly: true - - name: custom-modsec + - name: modsec-confs mountPath: /modsec-confs readOnly: true - - name: custom-modsec-crs + - name: modsec-crs-confs mountPath: /modsec-crs-confs readOnly: true volumes: - - name: config + - name: confs hostPath: - path: /shared/config + path: /shared/confs type: Directory - - name: certs + - name: letsencrypt hostPath: - path: /shared/certs + path: /shared/letsencrypt type: Directory - - name: challenges + - name: acme-challenge hostPath: - path: /shared/challenges + path: /shared/acme-challenge type: Directory - name: www hostPath: path: /shared/www type: Directory - - name: custom-http + - name: http-confs hostPath: - path: /shared/custom-http + path: /shared/http-confs type: Directory - - name: custom-server + - name: server-confs hostPath: - path: /shared/custom-server + path: /shared/server-confs type: Directory - - name: custom-modsec + - name: modsec-confs hostPath: - path: /shared/custom-modsec + path: /shared/modsec-confs type: Directory - - name: custom-modsec-crs + - name: modsec-crs-confs hostPath: - path: /shared/custom-modsec-crs - type: Directory \ No newline at end of file + path: /shared/modsec-crs-confs + type: Directory +--- +apiVersion: v1 +kind: Service +metadata: + name: bunkerized-nginx-service + labels: + bunkerized-nginx: "yes" + annotations: + bunkerized-nginx.AUTOCONF: "yes" +spec: + clusterIP: None + selector: + name: bunkerized-nginx diff --git a/helpers/kubernetes-rbac.yml b/helpers/kubernetes-rbac.yml index 51f33690..3811b1df 100644 --- a/helpers/kubernetes-rbac.yml +++ b/helpers/kubernetes-rbac.yml @@ -4,7 +4,7 @@ metadata: name: bunkerized-nginx-ingress-controller rules: - apiGroups: [""] - resources: ["services"] + resources: ["services", "pods"] verbs: ["get", "watch", "list"] - apiGroups: ["extensions"] resources: ["ingresses"] @@ -27,4 +27,4 @@ subjects: roleRef: kind: ClusterRole name: bunkerized-nginx-ingress-controller - apiGroup: rbac.authorization.k8s.io \ No newline at end of file + apiGroup: rbac.authorization.k8s.io diff --git a/settings.json b/settings.json index 7e0b9bef..e9ad29b0 100644 --- a/settings.json +++ b/settings.json @@ -1201,6 +1201,15 @@ "regex": "^(yes|no)$", "type": "checkbox" }, + { + "context": "global", + "default": "no", + "env": "KUBERNETES_MODE", + "id": "kubernetes-mode", + "label": "Kubernetes mode", + "regex": "^(yes|no)$", + "type": "checkbox" + }, { "context": "global", "default": "no",