JOBS - fallback to old conf in case reload failed

Dockerfile

@@ -19,17 +19,8 @@ COPY logs/ /opt/logs
 COPY lua/ /opt/lua
 COPY crowdsec/ /opt/crowdsec
 
-RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli && \
-    chmod +x /opt/entrypoint/* /opt/scripts/* && \
-    mkdir /opt/entrypoint.d && \
-    rm -f /var/log/nginx/* && \
-    chown root:nginx /var/log/nginx && \
-    chmod 750 /var/log/nginx && \
-    touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
-    chown nginx:nginx /var/log/nginx/*.log && \
-    mkdir /acme-challenge && \
-    chown root:nginx /acme-challenge && \
-    chmod 750 /acme-challenge
+COPY prepare.sh /tmp/prepare.sh
+RUN chmod +x /tmp/prepares.sh && /tmp/prepare.sh && rm -f /tmp/prepare.sh
 
 # Fix CVE-2020-28928 & CVE-2020-8231
 RUN apk --no-cache add "musl-utils>1.1.24-r2" "curl>7.67.0-r1"

Dockerfile-amd64

@@ -19,17 +19,8 @@ COPY logs/ /opt/logs
 COPY lua/ /opt/lua
 COPY crowdsec/ /opt/crowdsec
 
-RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli && \
-    chmod +x /opt/entrypoint/* /opt/scripts/* && \
-    mkdir /opt/entrypoint.d && \
-    rm -f /var/log/nginx/* && \
-    chown root:nginx /var/log/nginx && \
-    chmod 750 /var/log/nginx && \
-    touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
-    chown nginx:nginx /var/log/nginx/*.log && \
-    mkdir /acme-challenge && \
-    chown root:nginx /acme-challenge && \
-    chmod 750 /acme-challenge
+COPY prepare.sh /tmp/prepare.sh
+RUN chmod +x /tmp/prepares.sh && /tmp/prepare.sh && rm -f /tmp/prepare.sh
 
 # Fix CVE-2020-28928 & CVE-2020-8231
 RUN apk --no-cache add "musl-utils>1.1.24-r2" "curl>7.67.0-r1"

Dockerfile-arm32v7

@@ -26,17 +26,8 @@ COPY logs/ /opt/logs
 COPY lua/ /opt/lua
 COPY crowdsec/ /opt/crowdsec
 
-RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli && \
-    chmod +x /opt/entrypoint/* /opt/scripts/* && \
-    mkdir /opt/entrypoint.d && \
-    rm -f /var/log/nginx/* && \
-    chown root:nginx /var/log/nginx && \
-    chmod 750 /var/log/nginx && \
-    touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
-    chown nginx:nginx /var/log/nginx/*.log && \
-    mkdir /acme-challenge && \
-    chown root:nginx /acme-challenge && \
-    chmod 750 /acme-challenge
+COPY prepare.sh /tmp/prepare.sh
+RUN chmod +x /tmp/prepares.sh && /tmp/prepare.sh && rm -f /tmp/prepare.sh
 
 # Fix CVE-2020-28928 & CVE-2020-8231
 RUN apk --no-cache add "musl-utils>1.1.24-r2" "curl>7.67.0-r1"

Dockerfile-arm64v8

@@ -26,17 +26,8 @@ COPY logs/ /opt/logs
 COPY lua/ /opt/lua
 COPY crowdsec/ /opt/crowdsec
 
-RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli && \
-    chmod +x /opt/entrypoint/* /opt/scripts/* && \
-    mkdir /opt/entrypoint.d && \
-    rm -f /var/log/nginx/* && \
-    chown root:nginx /var/log/nginx && \
-    chmod 750 /var/log/nginx && \
-    touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
-    chown nginx:nginx /var/log/nginx/*.log && \
-    mkdir /acme-challenge && \
-    chown root:nginx /acme-challenge && \
-    chmod 750 /acme-challenge
+COPY prepare.sh /tmp/prepare.sh
+RUN chmod +x /tmp/prepares.sh && /tmp/prepare.sh && rm -f /tmp/prepare.sh
 
 # Fix CVE-2020-28928 & CVE-2020-8231
 RUN apk --no-cache add "musl-utils>1.1.24-r2" "curl>7.67.0-r1"

Dockerfile-i386

@@ -19,17 +19,8 @@ COPY logs/ /opt/logs
 COPY lua/ /opt/lua
 COPY crowdsec/ /opt/crowdsec
 
-RUN apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli && \
-    chmod +x /opt/entrypoint/* /opt/scripts/* && \
-    mkdir /opt/entrypoint.d && \
-    rm -f /var/log/nginx/* && \
-    chown root:nginx /var/log/nginx && \
-    chmod 750 /var/log/nginx && \
-    touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log && \
-    chown nginx:nginx /var/log/nginx/*.log && \
-    mkdir /acme-challenge && \
-    chown root:nginx /acme-challenge && \
-    chmod 750 /acme-challenge
+COPY prepare.sh /tmp/prepare.sh
+RUN chmod +x /tmp/prepares.sh && /tmp/prepare.sh && rm -f /tmp/prepare.sh
 
 # Fix CVE-2020-28928 & CVE-2020-8231
 RUN apk --no-cache add "musl-utils>1.1.24-r2" "curl>7.67.0-r1"

confs/global/nginx-temp.conf

@@ -1,5 +1,5 @@
 daemon on;
-pid /tmp/nginx.pid;
+pid /tmp/nginx-temp.pid;
 
 events {
         worker_connections 1024;

prepare.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# install dependencies
+apk --no-cache add certbot libstdc++ libmaxminddb geoip pcre yajl fail2ban clamav apache2-utils rsyslog openssl lua libgd go jq mariadb-connector-c bash brotli
+
+# make scripts executable
+chmod +x /opt/entrypoint/* /opt/scripts/*
+mkdir /opt/entrypoint.d
+
+# log files/folders rights
+rm -f /var/log/nginx/*
+chown root:nginx /var/log/nginx
+chmod 750 /var/log/nginx
+touch /var/log/nginx/error.log /var/log/nginx/modsec_audit.log /var/log/jobs.log
+chown nginx:nginx /var/log/nginx/*.log
+
+# let's encrypt webroot
+mkdir /acme-challenge
+chown root:nginx /acme-challenge
+chmod 750 /acme-challenge

scripts/abusers.sh

@@ -1,32 +1,43 @@
 #!/bin/sh
 
+# load some functions
 . /opt/scripts/utils.sh
 
-if [ ! -f /etc/nginx/block-abusers.conf ] ; then
-	echo "" > /etc/nginx/block-abusers.conf
-fi
-echo "" > /cache/block-abusers.conf
+# copy old conf to cache
+cp /etc/nginx/block-abusers.conf /cache
+
+# generate the new conf
 curl -s "https://iplists.firehol.org/files/firehol_abusers_30d.netset" | grep -v "^\#.*" |
 while read entry ; do
 	check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$")
 	if [ "$check" != "" ] ; then
-		echo "deny ${entry};" >> /cache/block-abusers.conf
+		echo "deny ${entry};" >> /tmp/block-abusers.conf
 	fi
 done
 
-cp /cache/block-abusers.conf /etc/nginx/block-abusers.conf
-lines="$(wc -l /etc/nginx/block-abusers.conf | cut -d ' ' - f1)"
+# check if we have at least 1 line
+lines="$(wc -l /tmp/block-abusers.conf | cut -d ' ' -f 1)"
 if [ "$lines" -gt 1 ] ; then
 	job_log "[BLACKLIST] abusers list updated ($lines entries)"
+	# reload nginx with the new config
+	mv /tmp/block-abusers.conf /etc/nginx/block-abusers.conf
+	if [ -f /tmp/nginx.pid ] ; then
+		/usr/sbin/nginx -s reload > /dev/null 2>&1
+		# new config is ok : save it in the cache
+		if [ "$?" -eq 0 ] ; then
+			cp /etc/nginx/block-abusers.conf /cache
+			job_log "[NGINX] successfull nginx reload after abusers list update"
+		else
+			job_log "[NGINX] failed nginx reload after abusers list update fallback to old list"
+			cp /cache/block-abusers.conf /etc/nginx
+			/usr/sbin/nginx -s reload > /dev/null 2>&1
+		fi
+	else
+		cp /etc/nginx/block-abusers.conf /cache
+	fi
 else
 	job_log "[BLACKLIST] can't update abusers list"
 fi
 
-if [ -f /tmp/nginx.pid ] ; then
-	/usr/sbin/nginx -s reload > /dev/null 2>&1
-	if [ "$?" -eq 0 ] ; then
-		job_log "[NGINX] successfull nginx reload after abusers list update"
-	else
-		job_log "[NGINX] failed nginx reload after abusers list update"
-	fi
-fi
+rm -f /tmp/block-abusers.conf 2> /dev/null
+

scripts/exit-nodes.sh

@@ -1,32 +1,43 @@
 #!/bin/sh
 
+# load some functions
 . /opt/scripts/utils.sh
 
-if [ ! -f /etc/nginx/block-tor-exit-node.conf ] ; then
-	echo "" > /etc/nginx/block-tor-exit-node.conf
-fi
-echo "" > /cache/block-tor-exit-node.conf
+# copy old conf to cache
+cp /etc/nginx/block-tor-exit-node.conf /cache
+
+# generate the new conf
 curl -s "https://iplists.firehol.org/files/tor_exits.ipset" | grep -v "^\#.*" |
 while read entry ; do
 	check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$")
 	if [ "$check" != "" ] ; then
-		echo "deny ${entry};" >> /cache/block-tor-exit-node.conf
+		echo "deny ${entry};" >> /tmp/block-tor-exit-node.conf
 	fi
 done
 
-cp /cache/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf
-lines="$(wc -l /etc/nginx/block-tor-exit-node.conf | cut -d ' ' - f1)"
+# check if we have at least 1 line
+lines="$(wc -l /tmp/block-tor-exit-node.conf | cut -d ' ' -f 1)"
 if [ "$lines" -gt 1 ] ; then
 	job_log "[BLACKLIST] TOR exit node list updated ($lines entries)"
+	# reload nginx with the new config
+	mv /tmp/block-tor-exit-node.conf /etc/nginx/block-tor-exit-node.conf
+	if [ -f /tmp/nginx.pid ] ; then
+		/usr/sbin/nginx -s reload > /dev/null 2>&1
+		# new config is ok : save it in the cache
+		if [ "$?" -eq 0 ] ; then
+			cp /etc/nginx/block-tor-exit-node.conf /cache
+			job_log "[NGINX] successfull nginx reload after TOR exit node list update"
+		else
+			job_log "[NGINX] failed nginx reload after TOR exit node list update fallback to old list"
+			cp /cache/block-tor-exit-node.conf /etc/nginx
+			/usr/sbin/nginx -s reload > /dev/null 2>&1
+		fi
+	else
+		cp /etc/nginx/block-tor-exit-node.conf /cache
+	fi
 else
 	job_log "[BLACKLIST] can't update TOR exit node list"
 fi
 
-if [ -f /tmp/nginx.pid ] ; then
-	/usr/sbin/nginx -s reload > /dev/null 2>&1
-	if [ "$?" -eq 0 ] ; then
-		job_log "[NGINX] successfull nginx reload after TOR exit node list update"
-	else
-		job_log "[NGINX] failed nginx reload after TOR exit node list update"
-	fi
-fi
+rm -f /tmp/block-tor-exit-node.conf 2> /dev/null
+

scripts/geoip.sh

@@ -1,25 +1,35 @@
 #!/bin/sh
 
+# load some functions
 . /opt/scripts/utils.sh
 
 # MMDB from https://db-ip.com/db/download/ip-to-country-lite
 URL="https://download.db-ip.com/free/dbip-country-lite-$(date +%Y-%m).mmdb.gz"
-wget -O /cache/geoip.mmdb.gz "$URL" > /dev/null 2>&1
-if [ -f /cache/geoip.mmdb.gz ] ; then
-	gunzip -f /cache/geoip.mmdb.gz > /dev/null 2>&1
+wget -O /tmp/geoip.mmdb.gz "$URL" > /dev/null 2>&1
+if [ "$?" -eq 0 ] && [ -f /tmp/geoip.mmdb.gz ] ; then
+	gunzip -f /tmp/geoip.mmdb.gz > /dev/null 2>&1
 	if [ "$?" -ne 0 ] ; then
 		job_log "[GEOIP] can't extract DB from $URL"
 		exit 1
 	fi
-	cp /cache/geoip.mmdb /etc/nginx/geoip.mmdb
+	mv /tmp/geoip.mmdb /etc/nginx
 	if [ -f /tmp/nginx.pid ] ; then
 		/usr/sbin/nginx -s reload > /dev/null 2>&1
 		if [ "$?" -eq 0 ] ; then
+			cp /etc/nginx/geoip.mmdb /cache
 			job_log "[NGINX] successfull nginx reload after GeoIP DB update"
 		else
 			job_log "[NGINX] failed nginx reload after GeoIP DB update"
+			if [ -f /cache/geoip.mmdb ] ; then
+				cp /cache/geoip.mmdb /etc/nginx/geoip.mmdb
+				/usr/sbin/nginx -s reload > /dev/null 2>&1
+			fi
 		fi
+	else
+		cp /etc/nginx/geoip.mmdb /cache
 	fi
 else
 	job_log "[GEOIP] can't download DB from $URL"
 fi
+
+rm -f /tmp/geoip* 2> /dev/null

scripts/logrotate.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+# load some functions
+. /opt/scripts/utils.sh
+
 logrotate -f /etc/logrotate.conf > /dev/null 2>&1
 
 pkill -HUP rsyslogd

scripts/proxies.sh

@@ -1,32 +1,43 @@
 #!/bin/sh
 
+# load some functions
 . /opt/scripts/utils.sh
 
-if [ ! -f /etc/nginx/block-proxies.conf ] ; then
-	echo "" > /etc/nginx/block-proxies.conf
-fi
-echo "" > /cache/block-proxies.conf
+# copy old conf to cache
+cp /etc/nginx/block-proxies.conf /cache
+
+# generate the new conf
 curl -s "https://iplists.firehol.org/files/firehol_proxies.netset" | grep -v "^\#.*" |
 while read entry ; do
 	check=$(echo $entry | grep -E "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/?[0-9]*$")
 	if [ "$check" != "" ] ; then
-		echo "deny ${entry};" >> /cache/block-proxies.conf
+		echo "deny ${entry};" >> /tmp/block-proxies.conf
 	fi
 done
 
-cp /cache/block-proxies.conf /etc/nginx/block-proxies.conf
-lines="$(wc -l /etc/nginx/block-proxies.conf | cut -d ' ' - f1)"
+# check if we have at least 1 line
+lines="$(wc -l /tmp/block-proxies.conf | cut -d ' ' -f 1)"
 if [ "$lines" -gt 1 ] ; then
 	job_log "[BLACKLIST] proxies list updated ($lines entries)"
+	# reload nginx with the new config
+	mv /tmp/block-proxies.conf /etc/nginx/block-proxies.conf
+	if [ -f /tmp/nginx.pid ] ; then
+		/usr/sbin/nginx -s reload > /dev/null 2>&1
+		# new config is ok : save it in the cache
+		if [ "$?" -eq 0 ] ; then
+			cp /etc/nginx/block-proxies.conf /cache
+			job_log "[NGINX] successfull nginx reload after proxies list update"
+		else
+			job_log "[NGINX] failed nginx reload after proxies list update fallback to old list"
+			cp /cache/block-proxies.conf /etc/nginx
+			/usr/sbin/nginx -s reload > /dev/null 2>&1
+		fi
+	else
+		cp /etc/nginx/block-proxies.conf /cache
+	fi
 else
 	job_log "[BLACKLIST] can't update proxies list"
 fi
 
-if [ -f /tmp/nginx.pid ] ; then
-	/usr/sbin/nginx -s reload > /dev/null 2>&1
-	if [ "$?" -eq 0 ] ; then
-		job_log "[NGINX] successfull nginx reload after proxies list update"
-	else
-		job_log "[NGINX] failed nginx reload after proxies list update"
-	fi
-fi
+rm -f /tmp/block-proxies.conf 2> /dev/null
+

scripts/referrers.sh

@@ -1,10 +1,12 @@
 #!/bin/sh
 
+# load some functions
 . /opt/scripts/utils.sh
 
-echo "map \$http_referer \$bad_referrer { hostnames; default no; }" > /etc/nginx/map-referrer.conf
-echo "map \$http_referer \$bad_referrer { hostnames; default no; }" > /cache/map-referrer.conf
+# save old conf
+cp /etc/nginx/map-referrer.conf /cache
 
+# generate new conf
 BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list)"
 if [ "$?" -ne 0 ] ; then
 	job_log "[BLACKLIST] can't update referrers list"
@@ -14,17 +16,29 @@ IFS=$'\n'
 for ref in $BLACKLIST ; do
         DATA="${DATA}\"~${ref}\" yes;\n"
 done
+echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $DATA }" > /tmp/map-referrer.conf
 
-echo -e "map \$http_referer \$bad_referrer { hostnames; default no; $DATA }" > /cache/map-referrer.conf
-cp /cache/map-referrer.conf /etc/nginx/map-referrer.conf
+# check number of lines
 lines="$(wc -l /etc/nginx/map-referrer.conf | cut -d ' ' -f 1)"
-job_log "[BLACKLIST] referrers list updated ($lines entries)"
-
-if [ -f /tmp/nginx.pid ] ; then
-	/usr/sbin/nginx -s reload > /dev/null 2>&1
-	if [ "$?" -eq 0 ] ; then
-		job_log "[NGINX] successfull nginx reload after referrers list update"
+if [ "$lines" -gt 1 ] ; then
+	mv /tmp/map-referrer.conf /etc/nginx/map-referrer.conf
+	job_log "[BLACKLIST] referrers list updated ($lines entries)"
+	if [ -f /tmp/nginx.pid ] ; then
+		/usr/sbin/nginx -s reload > /dev/null 2>&1
+		if [ "$?" -eq 0 ] ; then
+			cp /etc/nginx/map-referrer.conf /cache
+			job_log "[NGINX] successfull nginx reload after referrers list update"
+		else
+			cp /cache/map-referrer.conf /etc/nginx
+			job_log "[NGINX] failed nginx reload after referrers list update fallback to old list"
+			/usr/sbin/nginx -s reload > /dev/null 2>&1
+		fi
 	else
-		job_log "[NGINX] failed nginx reload after referrers list update"
+		cp /etc/nginx/map-referrer.conf /cache
 	fi
+else
+	job_log "[BLACKLIST] can't update referrers list"
+
 fi
+
+rm -f /tmp/map-referrer.conf 2> /dev/null

scripts/user-agents.sh

@@ -1,9 +1,12 @@
 #!/bin/sh
 
-echo "map \$http_user_agent \$bad_user_agent { default no; }" > /etc/nginx/map-user-agent.conf
-echo "map \$http_user_agent \$bad_user_agent { default no; }" > /cache/map-user-agent.conf
+# load some functions
+. /opt/scripts/utils.sh
 
+# save old conf
+cp /etc/nginx/map-user-agent.conf /cache
 
+# generate new conf
 BLACKLIST="$(curl -s https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list)
 $(curl -s https://raw.githubusercontent.com/JayBizzle/Crawler-Detect/master/raw/Crawlers.txt)"
 if [ "$?" -ne 0 ] ; then
@@ -15,17 +18,29 @@ for ua in $BLACKLIST ; do
         DATA="${DATA}~*${ua} yes;\n"
 done
 DATA_ESCAPED=$(echo "$DATA" | sed 's: :\\\\ :g' | sed 's:\\\\ yes;: yes;:g' | sed 's:\\\\\\ :\\\\ :g')
+echo -e "map \$http_user_agent \$bad_user_agent { default no; $DATA_ESCAPED }" > /tmp/map-user-agent.conf
 
-echo -e "map \$http_user_agent \$bad_user_agent { default no; $DATA_ESCAPED }" > /cache/map-user-agent.conf
-cp /cache/map-user-agent.conf /etc/nginx/map-user-agent.conf
+# check number of lines
 lines="$(wc -l /etc/nginx/map-user-agent.conf | cut -d ' ' -f 1)"
-job_log "[BLACKLIST] user-agent list updated ($lines entries)"
-
-if [ -f /tmp/nginx.pid ] ; then
-	/usr/sbin/nginx -s reload > /dev/null 2>&1
-	if [ "$?" -eq 0 ] ; then
-		job_log "[NGINX] successfull nginx reload after user-agent list update"
+if [ "$lines" -gt 1 ] ; then
+	mv /tmp/map-user-agent.conf /etc/nginx/map-user-agent.conf
+	job_log "[BLACKLIST] user-agent list updated ($lines entries)"
+	if [ -f /tmp/nginx.pid ] ; then
+		/usr/sbin/nginx -s reload > /dev/null 2>&1
+		if [ "$?" -eq 0 ] ; then
+			cp /etc/nginx/map-user-agent.conf /cache
+			job_log "[NGINX] successfull nginx reload after user-agent list update"
+		else
+			cp /cache/map-user-agent.conf /etc/nginx
+			job_log "[NGINX] failed nginx reload after user-agent list update fallback to old list"
+			/usr/sbin/nginx -s reload > /dev/null 2>&1
+		fi
 	else
-		job_log "[NGINX] failed nginx reload after user-agent list update"
+		cp /etc/nginx/map-user-agent.conf /cache
 	fi
+else
+	job_log "[BLACKLIST] can't update user-agent list"
+
 fi
+
+rm -f /tmp/map-user-agent.conf 2> /dev/null

scripts/utils.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 function job_log() {
-	when="$(date +[%d/%m/%Y %H:%M:%S])
+	when="$(date +[%d/%m/%Y %H:%M:%S])"
 	what="$1"
 	echo "$when $what" >> /var/log/jobs.log
 }