autoconf - basic ingress controller support for kubernetes

autoconf/src/Config.py

@@ -149,7 +149,14 @@ class Config :
 				except :
 					ret = False
 		elif self.__type == Controller.Type.KUBERNETES :
-			log("config", "ERROR", "TODO get urls for k8s")
+			for instance in instances :
+				name = instance.metadata.name
+				try :
+					dns_result = dns.resolver.query(name + ".default.svc.cluster.local")
+					for ip in dns_result :
+						urls.append("http://" + ip.to_text() + ":8080" + self.__api_uri + path)
+				except :
+					ret = False
 
 		for url in urls :
 			req = None

autoconf/src/IngressController.py

@@ -1,5 +1,6 @@
 from kubernetes import client, config, watch
 from threading import Thread, Lock
+import time
 
 import Controller
 
@@ -13,20 +14,35 @@ class IngressController(Controller.Controller) :
 		self.__api = client.CoreV1Api()
 		self.__extensions_api = client.ExtensionsV1beta1Api()
 		self.__old_env = {}
+		self.__internal_lock = Lock()
+
+	def __get_pods(self) :
+		return self.__api.list_pod_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items
 
 	def __get_ingresses(self) :
-		return self.__extensions_api.list_ingress_for_all_namespaces(watch=False).items
+		return self.__extensions_api.list_ingress_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items
 
-	def __get_services(self) :
+	def __get_services(self, autoconf=False) :
-		return self.__api.list_service_for_all_namespaces(watch=False).items
+		services = self.__api.list_service_for_all_namespaces(watch=False, label_selector="bunkerized-nginx").items
+		if not autoconf :
+			return services
+		services_autoconf = []
+		for service in services :
+			if service.metadata.annotations != None and "bunkerized-nginx.AUTOCONF" in service.metadata.annotations :
+				services_autoconf.append(service)
+		return services_autoconf
 
-	def __annotations_to_env(self, annotations, service=False) :
+	def __pod_to_env(self, pod_env) :
 		env = {}
-		prefix = ""
+		for env_var in pod_env :
-		if service :
+			env[env_var.name] = env_var.value
-			if not "bunkerized-nginx.SERVER_NAME" in annotations :
+			if env_var.value == None :
-				raise Exception("Missing bunkerized-nginx.SERVER_NAME annotation in Service.")
+				env[env_var.name] = ""
-			prefix = annotations["bunkerized-nginx.SERVER_NAME"].split(" ")[0] + "_"
+		return env
+
+	def __annotations_to_env(self, annotations) :
+		env = {}
+		prefix = annotations["bunkerized-nginx.SERVER_NAME"].split(" ")[0] + "_"
 		for annotation in annotations :
 			if annotation.startswith("bunkerized-nginx.") and annotation.replace("bunkerized-nginx.", "", 1) != "" and annotation.replace("bunkerized-nginx.", "", 1) != "AUTOCONF" :
 				env[prefix + annotation.replace("bunkerized-nginx.", "", 1)] = annotations[annotation]
@@ -34,64 +50,124 @@ class IngressController(Controller.Controller) :
 
 	def __rules_to_env(self, rules) :
 		env = {}
+		first_servers = []
 		for rule in rules :
+			rule = rule.to_dict()
 			prefix = ""
 			if "host" in rule :
 				prefix = rule["host"] + "_"
+				first_servers.append(rule["host"])
 			if not "http" in rule or not "paths" in rule["http"] :
 				continue
 			for path in rule["http"]["paths"] :
 				env[prefix + "USE_REVERSE_PROXY"] = "yes"
 				env[prefix + "REVERSE_PROXY_URL"] = path["path"]
-				env[prefix + "REVERSE_PROXY_HOST"] = "http://" + path["backend"]["serviceName"] + ":" + str(path["backend"]["servicePort"])
+				env[prefix + "REVERSE_PROXY_HOST"] = "http://" + path["backend"]["service_name"] + ":" + str(path["backend"]["service_port"])
+		env["SERVER_NAME"] = " ".join(first_servers)
 		return env
 
 	def get_env(self) :
+		pods = self.__get_pods()
 		ingresses = self.__get_ingresses()
 		services = self.__get_services()
 		env = {}
+		first_servers = []
+		for pod in pods :
+			env.update(self.__pod_to_env(pod.spec.containers[0].env))
+			if "SERVER_NAME" in env and env["SERVER_NAME"] != "" :
+				first_servers.extend(env["SERVER_NAME"].split(" "))
 		for ingress in ingresses :
-			if ingress.metadata.annotations == None :
+			env.update(self.__rules_to_env(ingress.spec.rules))
-				continue
+			if "SERVER_NAME" in env and env["SERVER_NAME"] != "" :
-			if "bunkerized-nginx.AUTOCONF" in ingress.metadata.annotations :
+				first_servers.extend(env["SERVER_NAME"].split(" "))
-				env.update(self.__annotations_to_env(ingress.metadata.annotations))
-				env.update(self.__rules_to_env(ingress.spec.rules))
 		for service in services :
-			if service.metadata.annotations == None :
+			if service.metadata.annotations != None and "bunkerized-nginx.SERVER_NAME" in service.metadata.annotations :
-				continue
+				env.update(self.__annotations_to_env(service.metadata.annotations))
-			if "bunkerized-nginx.AUTOCONF" in service.metadata.annotations :
+				first_servers.append(service.metadata.annotations["SERVER_NAME"])
-				env.update(self.__annotations_to_env(service.metadata.annotations, service=True))
+		first_servers = list(dict.fromkeys(first_servers))
+		if len(first_servers) == 0 :
+			env["SERVER_NAME"] = ""
+		else :
+			env["SERVER_NAME"] = " ".join(first_servers)
 		return self._fix_env(env)
 
 	def process_events(self, current_env) :
 		self.__old_env = current_env
+		t_pod = Thread(target=self.__watch_pod)
 		t_ingress = Thread(target=self.__watch_ingress)
 		t_service = Thread(target=self.__watch_service)
+		t_pod.start()
 		t_ingress.start()
 		t_service.start()
+		t_pod.join()
 		t_ingress.join()
 		t_service.join()
 
-	def __watch_ingress(self) :
+	def __watch_pod(self) :
 		w = watch.Watch()
-		for event in w.stream(self.__extensions_api.list_ingress_for_all_namespaces) :
+		for event in w.stream(self.__api.list_pod_for_all_namespaces, label_selector="bunkerized-nginx") :
+			self.__internal_lock.acquire()
 			new_env = self.get_env()
-			if new_env != self.__old_env() :
+			if new_env != self.__old_env :
-				if self.gen_conf(new_env, lock=False) :
+				if self.gen_conf(new_env) :
 					self.__old_env = new_env.copy()
 					log("CONTROLLER", "INFO", "successfully generated new configuration")
+					if self.reload() :
+						log("controller", "INFO", "successful reload")
+					else :
+						log("controller", "ERROR", "failed reload")
+			self.__internal_lock.release()
+
+	def __watch_ingress(self) :
+		w = watch.Watch()
+		for event in w.stream(self.__extensions_api.list_ingress_for_all_namespaces, label_selector="bunkerized-nginx") :
+			self.__internal_lock.acquire()
+			new_env = self.get_env()
+			if new_env != self.__old_env :
+				if self.gen_conf(new_env) :
+					self.__old_env = new_env.copy()
+					log("CONTROLLER", "INFO", "successfully generated new configuration")
+					if self.reload() :
+						log("controller", "INFO", "successful reload")
+					else :
+						log("controller", "ERROR", "failed reload")
+			self.__internal_lock.release()
 
 	def __watch_service(self) :
 		w = watch.Watch()
-		for event in w.stream(self.__api.list_service_for_all_namespaces) :
+		for event in w.stream(self.__api.list_service_for_all_namespaces, label_selector="bunkerized-nginx") :
+			self.__internal_lock.acquire()
 			new_env = self.get_env()
-			if new_env != self.__old_env() :
+			if new_env != self.__old_env :
-				if self.gen_conf(new_env, lock=False) :
+				if self.gen_conf(new_env) :
 					self.__old_env = new_env.copy()
 					log("CONTROLLER", "INFO", "successfully generated new configuration")
+					if self.reload() :
+						log("controller", "INFO", "successful reload")
+					else :
+						log("controller", "ERROR", "failed reload")
+			self.__internal_lock.release()
 
 	def reload(self) :
-		return self._reload(self.__get_ingresses())
+		return self._reload(self.__get_services(autoconf=True))
 
 	def wait(self) :
-		return self._config.wait(self.__get_ingresses())
+		# Wait for at least one bunkerized-nginx pod
+		pods = self.__get_pods()
+		while len(pods) == 0 :
+			time.sleep(1)
+			pods = self.__get_pods()
+
+		# Wait for at least one bunkerized-nginx service
+		services = self.__get_services(autoconf=True)
+		while len(services) == 0 :
+			time.sleep(1)
+			services = self.__get_services(autoconf=True)
+
+		# Generate first config
+		env = self.get_env()
+		if not self.gen_conf(env) :
+			return False, env
+
+		# Wait for bunkerized-nginx
+		return self._config.wait(services), env

entrypoint/entrypoint.sh

@@ -17,7 +17,7 @@ trap "trap_exit" TERM INT QUIT
 # trap SIGHUP
 function trap_reload() {
 	log "reload" "INFO" "catched reload operation"
-	if [ "$SWARM_MODE" != "yes" ] ; then
+	if [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] ; then
 		/opt/bunkerized-nginx/entrypoint/jobs.sh
 	fi
 	if [ -f /tmp/nginx.pid ] ; then
@@ -40,10 +40,10 @@ if [ ! -f "/etc/nginx/global.env" ] ; then
 	log "entrypoint" "INFO" "configuring bunkerized-nginx ..."
 
 	# check permissions
-	if [ "$SWARM_MODE" != "yes" ] ; then
+	if [ "$SWARM_MODE" != "yes" ] && [ "$KUBERNETES_MODE" != "yes" ] ; then
 		/opt/bunkerized-nginx/entrypoint/permissions.sh
 	else
-		/opt/bunkerized-nginx/entrypoint/permissions-swarm.sh
+		/opt/bunkerized-nginx/entrypoint/permissions-cluster.sh
 	fi
 	if [ "$?" -ne 0 ] ; then
 		exit 1

entrypoint/nginx-temp.sh

@@ -4,7 +4,7 @@
 . /opt/bunkerized-nginx/entrypoint/utils.sh
 
 # start nginx with temp conf for let's encrypt challenges and API
-if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$SWARM_MODE" = "yes" ] || [ "$AUTO_LETS_ENCRYPT" = "yes" ] ; then
+if [ "$(has_value AUTO_LETS_ENCRYPT yes)" != "" ] || [ "$SWARM_MODE" = "yes" ] || [ "$AUTO_LETS_ENCRYPT" = "yes" ] || [ "$KUBERNETES_MODE" = "yes" ] ; then
 	cp /opt/bunkerized-nginx/confs/global/nginx-temp.conf /tmp/nginx-temp.conf
 	cp /opt/bunkerized-nginx/confs/global/api-temp.conf /tmp/api.conf
 	if [ "$SWARM_MODE" = "yes" ] ; then

gen/Configurator.py

@@ -1,4 +1,4 @@
-import json, re
+import json, re, sys
 
 class Configurator :
 

helpers/kubernetes-ingress.yml

@@ -2,11 +2,10 @@ apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
   name: bunkerized-nginx-ingress
+  labels:
+    bunkerized-nginx: "yes"
   annotations:
-    # mandatory, keep this annotation
-    bunkerized-nginx.AUTOCONF: "yes"
     # add any global and default environment variables here as annotations with the "bunkerized-nginx." prefix
-    # if the scope is "multisite", they will be applied to all services unless overriden by the service
     # examples :
     #bunkerized-nginx.AUTO_LETS_ENCRYPT: "yes"
     #bunkerized-nginx.USE_ANTIBOT: "javascript"
@@ -25,7 +24,39 @@ spec:
         path: "/"
         backend:
           service:
-            name: app-service
+            name: myapp
             port:
               number: 80
-
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: myapp
+  labels:
+    app: myapp
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: myapp
+  template:
+    metadata:
+      labels:
+        app: myapp
+    spec:
+      containers:
+      - name: myapp
+        image: containous/whoami
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: myapp
+spec:
+  type: ClusterIP
+  selector:
+    app: myapp
+  ports:
+    - protocol: TCP
+      port: 80
+      targetPort: 80

helpers/kubernetes-nginx.yml

@@ -17,51 +17,56 @@ spec:
       serviceAccountName: bunkerized-nginx-ingress-controller
       containers:
       - name: bunkerized-nginx-autoconf
-        image: bunkerity/bunkerized-nginx-autoconf:1.3.0
+        image: bunkerity/bunkerized-nginx-autoconf:testing
+        imagePullPolicy: Always
         env:
         - name: KUBERNETES_MODE
           value: "yes"
         - name: API_URI
           value: "/ChangeMeToSomethingHardToGuess"
+        - name: SERVER_NAME
+          value: ""
+        - name: MULTISITE
+          value: "yes"
         volumeMounts:
-        - name: config
+        - name: confs
           mountPath: /etc/nginx
-        - name: certs
+        - name: letsencrypt
           mountPath: /etc/letsencrypt
-        - name: challenges
+        - name: acme-challenge
           mountPath: /acme-challenge
         - name: cache
           mountPath: /cache
-        - name: custom-modsec
+        - name: modsec-confs
           mountPath: /modsec-confs
           readOnly: true
-        - name: custom-modsec-crs
+        - name: modsec-crs-confs
           mountPath: /modsec-crs-confs
           readOnly: true
       volumes:
-      - name: config
+      - name: confs
         hostPath:
-          path: /shared/config
+          path: /shared/confs
           type: Directory
-      - name: certs
+      - name: letsencrypt
         hostPath:
-          path: /shared/certs
+          path: /shared/letsencrypt
           type: Directory
-      - name: challenges
+      - name: acme-challenge
         hostPath:
-          path: /shared/challenges
+          path: /shared/acme-challenge
           type: Directory
       - name: cache
         hostPath:
           path: /shared/cache
           type: Directory
-      - name: custom-modsec
+      - name: modsec-confs
         hostPath:
-          path: /shared/custom-modsec
+          path: /shared/modsec-confs
           type: Directory
-      - name: custom-modsec-crs
+      - name: modsec-crs-confs
         hostPath:
-          path: /shared/custom-modsec-crs
+          path: /shared/modsec-crs-confs
           type: Directory
 ---
 apiVersion: apps/v1
@@ -78,12 +83,14 @@ spec:
     metadata:
       labels:
         name: bunkerized-nginx
+        bunkerized-nginx: "yes"
     spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
       containers:
       - name: bunkerized-nginx
-        image: bunkerity/bunkerized-nginx:1.3.0
+        image: bunkerity/bunkerized-nginx:testing
+        imagePullPolicy: Always
         env:
         - name: KUBERNETES_MODE
           value: "yes"
@@ -91,61 +98,78 @@ spec:
           value: "yes"
         - name: API_URI
           value: "/ChangeMeToSomethingHardToGuess"
+        - name: SERVER_NAME
+          value: ""
+        - name: MULTISITE
+          value: "yes"
         volumeMounts:
-        - name: config
+        - name: confs
           mountPath: /etc/nginx
           readOnly: true
-        - name: certs
+        - name: letsencrypt
           mountPath: /etc/letsencrypt
           readOnly: true
-        - name: challenges
+        - name: acme-challenge
           mountPath: /acme-challenge
           readOnly: true
         - name: www
           mountPath: /www
           readOnly: true
-        - name: custom-http
+        - name: http-confs
           mountPath: /http-confs
           readOnly: true
-        - name: custom-server
+        - name: server-confs
           mountPath: /server-confs
           readOnly: true
-        - name: custom-modsec
+        - name: modsec-confs
           mountPath: /modsec-confs
           readOnly: true
-        - name: custom-modsec-crs
+        - name: modsec-crs-confs
           mountPath: /modsec-crs-confs
           readOnly: true
       volumes:
-      - name: config
+      - name: confs
         hostPath:
-          path: /shared/config
+          path: /shared/confs
           type: Directory
-      - name: certs
+      - name: letsencrypt
         hostPath:
-          path: /shared/certs
+          path: /shared/letsencrypt
           type: Directory
-      - name: challenges
+      - name: acme-challenge
         hostPath:
-          path: /shared/challenges
+          path: /shared/acme-challenge
           type: Directory
       - name: www
         hostPath:
           path: /shared/www
           type: Directory
-      - name: custom-http
+      - name: http-confs
         hostPath:
-          path: /shared/custom-http
+          path: /shared/http-confs
           type: Directory
-      - name: custom-server
+      - name: server-confs
         hostPath:
-          path: /shared/custom-server
+          path: /shared/server-confs
           type: Directory
-      - name: custom-modsec
+      - name: modsec-confs
         hostPath:
-          path: /shared/custom-modsec
+          path: /shared/modsec-confs
           type: Directory
-      - name: custom-modsec-crs
+      - name: modsec-crs-confs
         hostPath:
-          path: /shared/custom-modsec-crs
+          path: /shared/modsec-crs-confs
-          type: Directory
+          type: Directory
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: bunkerized-nginx-service
+  labels:
+    bunkerized-nginx: "yes"
+  annotations:
+    bunkerized-nginx.AUTOCONF: "yes"
+spec:
+  clusterIP: None
+  selector:
+    name: bunkerized-nginx

helpers/kubernetes-rbac.yml

@@ -4,7 +4,7 @@ metadata:
   name: bunkerized-nginx-ingress-controller
 rules:
 - apiGroups: [""]
-  resources: ["services"]
+  resources: ["services", "pods"]
   verbs: ["get", "watch", "list"]
 - apiGroups: ["extensions"]
   resources: ["ingresses"]
@@ -27,4 +27,4 @@ subjects:
 roleRef:
   kind: ClusterRole
   name: bunkerized-nginx-ingress-controller
-  apiGroup: rbac.authorization.k8s.io
+  apiGroup: rbac.authorization.k8s.io

settings.json

@@ -1201,6 +1201,15 @@
                 "regex": "^(yes|no)$",
                 "type": "checkbox"
             },
+            {
+                "context": "global",
+                "default": "no",
+                "env": "KUBERNETES_MODE",
+                "id": "kubernetes-mode",
+                "label": "Kubernetes mode",
+                "regex": "^(yes|no)$",
+                "type": "checkbox"
+            },
             {
                 "context": "global",
                 "default": "no",