road to swarm - automatic reload after jobs

autoconf/AutoConf.py

@@ -12,7 +12,7 @@ class AutoConf :
 		self.__config = Config(self.__swarm, api)
 
 	def reload(self) :
-		return self.__config.reload(self.instances)
+		return self.__config.reload(self.__instances)
 
 	def pre_process(self, objs) :
 		for instance in objs :

autoconf/Config.py

@@ -23,7 +23,7 @@ class Config :
 			if self.globalconf(instances) :
 				i = 0
 				started = False
-				while i < 5 :
+				while i < 10 :
 					if self.__ping(instances) :
 						started = True
 						break

autoconf/Dockerfile

@@ -17,7 +17,13 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     chmod 770 /var/log/letsencrypt && \
     mkdir /var/lib/letsencrypt && \
     chown root:nginx /var/lib/letsencrypt && \
-    chmod 770 /var/lib/letsencrypt
+    chmod 770 /var/lib/letsencrypt && \
+    mkdir /cache && \
+    chown root:nginx /cache && \
+    chmod 770 /cache && \
+    touch /var/log/jobs.log && \
+    chown root:nginx /var/log/jobs.log && \
+    chmod 770 /var/log/jobs.log
 
 COPY scripts/* /opt/scripts/
 COPY confs/site/ /opt/confs/site

autoconf/Dockerfile-amd64

@@ -1,5 +1,9 @@
+FROM nginx:stable-alpine AS builder
+
 FROM amd64/alpine
 
+COPY --from=builder /etc/nginx/ /opt/confs/nginx
+
 RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     pip3 install docker requests && \
     mkdir /opt/entrypoint && \
@@ -14,10 +18,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     chown root:nginx /var/lib/letsencrypt && \
     chmod 770 /var/lib/letsencrypt
 
+COPY scripts/* /opt/scripts/
+COPY confs/global/ /opt/confs/global
 COPY confs/site/ /opt/confs/site
 COPY entrypoint/* /opt/entrypoint/
 COPY autoconf/* /opt/entrypoint/
-RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh
+RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh
 
 VOLUME /etc/nginx
 

autoconf/Dockerfile-arm32v7

@@ -3,9 +3,12 @@ FROM alpine AS builder
 ENV QEMU_URL https://github.com/balena-io/qemu/releases/download/v4.0.0%2Bbalena2/qemu-4.0.0.balena2-arm.tar.gz
 RUN apk add curl && curl -L ${QEMU_URL} | tar zxvf - -C . --strip-components 1
 
+FROM nginx:stable-alpine AS builder2
+
 FROM arm32v7/alpine
 
 COPY --from=builder qemu-arm-static /usr/bin
+COPY --from=builder2 /etc/nginx/ /opt/confs/nginx
 
 RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     pip3 install docker requests && \
@@ -21,10 +24,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     chown root:nginx /var/lib/letsencrypt && \
     chmod 770 /var/lib/letsencrypt
 
+COPY scripts/* /opt/scripts/
+COPY confs/global/ /opt/confs/global
 COPY confs/site/ /opt/confs/site
 COPY entrypoint/* /opt/entrypoint/
 COPY autoconf/* /opt/entrypoint/
-RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh
+RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh
 
 VOLUME /etc/nginx
 

autoconf/Dockerfile-arm64v8

@@ -3,9 +3,12 @@ FROM alpine AS builder
 ENV QEMU_URL https://github.com/balena-io/qemu/releases/download/v4.0.0%2Bbalena2/qemu-4.0.0.balena2-aarch64.tar.gz
 RUN apk add curl && curl -L ${QEMU_URL} | tar zxvf - -C . --strip-components 1
 
+FROM nginx:stable-alpine AS builder2
+
 FROM arm64v8/alpine
 
 COPY --from=builder qemu-aarch64-static /usr/bin
+COPY --from=builder2 /etc/nginx/ /opt/confs/nginx
 
 RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     pip3 install docker requests && \
@@ -21,10 +24,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     chown root:nginx /var/lib/letsencrypt && \
     chmod 770 /var/lib/letsencrypt
 
+COPY scripts/* /opt/scripts/
+COPY confs/global/ /opt/confs/global
 COPY confs/site/ /opt/confs/site
 COPY entrypoint/* /opt/entrypoint/
 COPY autoconf/* /opt/entrypoint/
-RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh
+RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh
 
 VOLUME /etc/nginx
 

autoconf/Dockerfile-i386

@@ -1,5 +1,9 @@
+FROM nginx:stable-alpine AS builder
+
 FROM i386/alpine
 
+COPY --from=builder /etc/nginx/ /opt/confs/nginx
+
 RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     pip3 install docker requests && \
     mkdir /opt/entrypoint && \
@@ -14,10 +18,12 @@ RUN apk add py3-pip apache2-utils bash certbot curl logrotate openssl && \
     chown root:nginx /var/lib/letsencrypt && \
     chmod 770 /var/lib/letsencrypt
 
+COPY scripts/* /opt/scripts/
+COPY confs/global/ /opt/confs/global
 COPY confs/site/ /opt/confs/site
 COPY entrypoint/* /opt/entrypoint/
 COPY autoconf/* /opt/entrypoint/
-RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh
+RUN chmod +x /opt/entrypoint/*.py /opt/entrypoint/*.sh /opt/scripts/*.sh
 
 VOLUME /etc/nginx
 

autoconf/ReloadServer.py

@@ -1,20 +1,25 @@
-import socketserver, threading
+import socketserver, threading, utils, os, stat
 
-class ReloadServerHandler(socketserver.BaseRequestHandler):
+class ReloadServerHandler(socketserver.StreamRequestHandler):
 
 	def handle(self) :
-		data = self.request.recv(512)
-		if not data :
-			return
-		with self.server.lock :
-			ret = self.server.autoconf.reload()
-		if ret :
-			self.request.sendall("ok")
-		else :
-			self.request.sendall("ko")
+		try :
+			data = self.request.recv(512)
+			if not data :
+				return
+			with self.server.lock :
+				ret = self.server.autoconf.reload()
+			if ret :
+				self.request.sendall("ok".encode("utf-8"))
+			else :
+				self.request.sendall("ko".encode("utf-8"))
+		except Exception as e :
+			utils.log("Exception " + str(e))
 
 def run_reload_server(autoconf, lock) :
-	server = socketserver.UnixStreamServer("/tmp/autoconf.pid", ReloadServerHandler)
+	server = socketserver.UnixStreamServer("/tmp/autoconf.sock", ReloadServerHandler)
+	os.chown("/tmp/autoconf.sock", 0, 101)
+	os.chmod("/tmp/autoconf.sock", 0o770)
 	server.autoconf = autoconf
 	server.lock = lock
 	thread = threading.Thread(target=server.serve_forever)

scripts/abusers.sh

@@ -10,7 +10,7 @@ cp /etc/nginx/block-abusers.conf /cache
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="/opt/entrypoint/reload.py"
 fi
 

scripts/certbot-renew-hook.sh

@@ -9,7 +9,7 @@ job_log "[CERTBOT] certificates have been renewed"
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="echo reload > /tmp/autoconf.sock"
 fi
 

scripts/exit-nodes.sh

@@ -10,7 +10,7 @@ cp /etc/nginx/block-tor-exit-node.conf /cache
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="/opt/entrypoint/reload.py"
 fi
 

scripts/geoip.sh

@@ -7,7 +7,7 @@
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="/opt/entrypoint/reload.py"
 fi
 

scripts/proxies.sh

@@ -10,7 +10,7 @@ cp /etc/nginx/block-proxies.conf /cache
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="/opt/entrypoint/reload.py"
 fi
 

scripts/referrers.sh

@@ -10,7 +10,7 @@ cp /etc/nginx/map-referrer.conf /cache
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="/opt/entrypoint/reload.py"
 fi
 

scripts/user-agents.sh

@@ -10,7 +10,7 @@ cp /etc/nginx/map-user-agent.conf /cache
 if [ -f /tmp/nginx.pid ] ; then
 	RELOAD="/usr/sbin/nginx -s reload > /dev/null 2>&1"
 # if we are in autoconf
-elif [ -f /tmp/autoconf.sock ] ; then
+elif [ -S /tmp/autoconf.sock ] ; then
 	RELOAD="/opt/entrypoint/reload.py"
 fi