Update Dockerfiles to install pip and its deps before the project ones

src/autoconf/Dockerfile

@@ -1,26 +1,27 @@
 FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded AS builder
 
 # Copy python requirements
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.1
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.2
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.1
 
 WORKDIR /usr/share/bunkerweb
 
 RUN mkdir -p deps/python && \
-    cat /tmp/req/requirements.txt* > deps/requirements.txt && \
-    rm -rf /tmp/req
+  cat /tmp/req/requirements.txt* > deps/requirements.txt && \
+  rm -rf /tmp/req
 
 # Install python dependencies
 RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev
 
 # Install python requirements
 RUN export MAKEFLAGS="-j$(nproc)" && \
-    pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
+  pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
+  pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Remove build dependencies
 RUN apk del .build-deps && \
-    rm -rf /var/cache/apk/*
+  rm -rf /var/cache/apk/*
 
 # Copy files
 # can't exclude specific files/dir from . so we are copying everything by hand
@@ -45,22 +46,22 @@ WORKDIR /usr/share/bunkerweb
 
 # Add autoconf user, drop bwcli, install runtime dependencies, create data folders and set permissions
 RUN apk add --no-cache bash && \
-    addgroup -g 101 autoconf && \
-    adduser -h /var/cache/autoconf -g autoconf -s /bin/sh -G autoconf -D -H -u 101 autoconf && \
-    cp helpers/bwcli /usr/bin/ && \
-    mkdir -p /var/tmp/bunkerweb && \
-    mkdir -p /var/www && \
-    mkdir -p /etc/bunkerweb && \
-    mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \
-    mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \
-    mkdir -p /data/www && ln -s /data/www /var/www/html && \
-    for dir in $(echo "configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \
-    for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \
-    chown -R root:autoconf /data && \
-    chmod -R 770 /data && \
-    chown -R root:autoconf /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /usr/bin/bwcli && \
-    chmod -R 770 /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb && \
-    chmod 750 cli/main.py helpers/*.sh /usr/bin/bwcli autoconf/main.py deps/python/bin/*
+  addgroup -g 101 autoconf && \
+  adduser -h /var/cache/autoconf -g autoconf -s /bin/sh -G autoconf -D -H -u 101 autoconf && \
+  cp helpers/bwcli /usr/bin/ && \
+  mkdir -p /var/tmp/bunkerweb && \
+  mkdir -p /var/www && \
+  mkdir -p /etc/bunkerweb && \
+  mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \
+  mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \
+  mkdir -p /data/www && ln -s /data/www /var/www/html && \
+  for dir in $(echo "configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \
+  for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \
+  chown -R root:autoconf /data && \
+  chmod -R 770 /data && \
+  chown -R root:autoconf /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /usr/bin/bwcli && \
+  chmod -R 770 /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb && \
+  chmod 750 cli/main.py helpers/*.sh /usr/bin/bwcli autoconf/main.py deps/python/bin/*
 
 # Fix CVEs
 # There are no CVE to fix for the moment

src/bw/Dockerfile

@@ -16,18 +16,15 @@ RUN mkdir -p /usr/share/bunkerweb/deps/python && \
 	chmod +x install.sh && \
 	bash install.sh
 
-# Copy python requirements
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.1
-
 WORKDIR /usr/share/bunkerweb
 
-RUN mkdir -p deps && \
-	cat /tmp/req/requirements.txt* > deps/requirements.txt && \
-	rm -rf /tmp/req
+# Copy python requirements
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/common/gen/requirements.txt deps/requirements.txt
 
 # Install python requirements
 RUN export MAKEFLAGS="-j$(nproc)" && \
+	pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
 	pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Clean up temporary dependencies

src/linux/Dockerfile-centos

@@ -29,11 +29,11 @@ RUN mkdir -p /usr/share/bunkerweb/deps/python && \
     bash install.sh
 
 # Copy dependencies sources folder
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/scheduler/requirements.txt /tmp/req/requirements.txt.1
-COPY src/ui/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.3
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.4
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
+COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
 
 WORKDIR /usr/share/bunkerweb
 
@@ -43,6 +43,7 @@ RUN mkdir -p deps/python && \
 
 # Compile and install dependencies
 RUN export MAKEFLAGS="-j$(nproc)" && \
+    pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
     pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Copy files

src/linux/Dockerfile-debian

@@ -29,11 +29,11 @@ RUN mkdir -p /usr/share/bunkerweb/deps/python && \
     bash install.sh
 
 # Copy dependencies sources folder
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/scheduler/requirements.txt /tmp/req/requirements.txt.1
-COPY src/ui/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.3
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.4
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
+COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
 
 WORKDIR /usr/share/bunkerweb
 
@@ -43,6 +43,7 @@ RUN mkdir -p deps/python && \
 
 # Compile and install dependencies
 RUN export MAKEFLAGS="-j$(nproc)" && \
+    pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
     pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Copy files

src/linux/Dockerfile-fedora

@@ -25,11 +25,11 @@ RUN mkdir -p /usr/share/bunkerweb/deps/python && \
     bash install.sh
 
 # Copy dependencies sources folder
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/scheduler/requirements.txt /tmp/req/requirements.txt.1
-COPY src/ui/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.3
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.4
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
+COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
 
 WORKDIR /usr/share/bunkerweb
 
@@ -39,6 +39,7 @@ RUN mkdir -p deps/python && \
 
 # Compile and install dependencies
 RUN export MAKEFLAGS="-j$(nproc)" && \
+    pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
     pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Copy files

src/linux/Dockerfile-rhel

@@ -40,11 +40,11 @@ RUN mkdir -p /usr/share/bunkerweb/deps/python && \
   bash install.sh
 
 # Copy dependencies sources folder
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/scheduler/requirements.txt /tmp/req/requirements.txt.1
-COPY src/ui/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.3
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.4
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
+COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
 
 WORKDIR /usr/share/bunkerweb
 
@@ -55,6 +55,7 @@ RUN mkdir -p deps/python && \
 # Compile and install dependencies
 RUN easy_install-3.9 pip && \
   export MAKEFLAGS="-j$(nproc)" && \
+  pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
   pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Copy BW files

src/linux/Dockerfile-ubuntu

@@ -29,11 +29,11 @@ RUN mkdir -p /usr/share/bunkerweb/deps/python && \
     bash install.sh
 
 # Copy dependencies sources folder
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/scheduler/requirements.txt /tmp/req/requirements.txt.1
-COPY src/ui/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.3
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.4
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
+COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
 
 WORKDIR /usr/share/bunkerweb
 
@@ -43,6 +43,7 @@ RUN mkdir -p deps/python && \
 
 # Compile and install dependencies
 RUN export MAKEFLAGS="-j$(nproc)" && \
+    pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
     pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Copy files

src/scheduler/Dockerfile

@@ -1,10 +1,10 @@
 FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded AS builder
 
 # Copy python requirements
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/scheduler/requirements.txt /tmp/req/requirements.txt.1
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/scheduler/requirements.txt /tmp/req/requirements.txt
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.2
 
 WORKDIR /usr/share/bunkerweb
 
@@ -17,6 +17,7 @@ RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev
 
 # Install python requirements
 RUN export MAKEFLAGS="-j$(nproc)" && \
+  pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
   pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Remove build dependencies

src/ui/Dockerfile

@@ -1,27 +1,28 @@
 FROM python:3.11.5-alpine@sha256:cd311c6a0164f34a7edbf364e05258b07d66d3f7bc155139dcb9bef88a186ded AS builder
 
 # Copy python requirements
-COPY src/deps/requirements.txt /tmp/req/requirements.txt
-COPY src/ui/requirements.txt /tmp/req/requirements.txt.1
-COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.2
-COPY src/common/db/requirements.txt /tmp/req/requirements.txt.3
+COPY src/deps/requirements.txt /tmp/requirements-deps.txt
+COPY src/ui/requirements.txt /tmp/req/requirements.txt
+COPY src/common/gen/requirements.txt /tmp/req/requirements.txt.1
+COPY src/common/db/requirements.txt /tmp/req/requirements.txt.2
 
 WORKDIR /usr/share/bunkerweb
 
 RUN mkdir -p deps/python && \
-    cat /tmp/req/requirements.txt* > deps/requirements.txt && \
-    rm -rf /tmp/req
+  cat /tmp/req/requirements.txt* > deps/requirements.txt && \
+  rm -rf /tmp/req
 
 # Install python dependencies
 RUN apk add --no-cache --virtual .build-deps g++ gcc musl-dev jpeg-dev zlib-dev libffi-dev cairo-dev pango-dev gdk-pixbuf-dev openssl-dev cargo postgresql-dev file make
 
 # Install python requirements
 RUN export MAKEFLAGS="-j$(nproc)" && \
-    pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
+  pip install --no-cache-dir --ignore-installed --require-hashes -r /tmp/requirements-deps.txt && \
+  pip install --no-cache-dir --require-hashes --target deps/python -r deps/requirements.txt
 
 # Remove build dependencies
 RUN apk del .build-deps && \
-    rm -rf /var/cache/apk/*
+  rm -rf /var/cache/apk/*
 
 # Copy files
 # can't exclude specific files/dir from . so we are copying everything by hand
@@ -47,26 +48,26 @@ WORKDIR /usr/share/bunkerweb
 
 # Add ui user, drop bwcli, install runtime dependencies, create data folders and set permissions
 RUN apk add --no-cache bash && \
-    addgroup -g 101 ui && \
-    adduser -h /var/cache/nginx -g ui -s /bin/sh -G ui -D -H -u 101 ui && \
-    echo "Docker" > INTEGRATION && \
-    mkdir -p /var/tmp/bunkerweb && \
-    mkdir -p /var/run/bunkerweb && \
-    mkdir -p /etc/bunkerweb && \
-    mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \
-    mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \
-    mkdir -p /var/log/bunkerweb/ && \
-    for dir in $(echo "configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \
-    for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \
-    chown -R root:ui /data && \
-    chmod -R 770 /data && \
-    chown -R root:ui INTEGRATION /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb && \
-    chmod 770 /var/cache/bunkerweb /var/lib/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb && \
-    chmod 750 gen/*.py ui/*.py ui/src/*.py deps/python/bin/* helpers/*.sh && \
-    chmod 660 INTEGRATION && \
-    chown root:ui INTEGRATION && \
-    ln -s /proc/1/fd/1 /var/log/bunkerweb/ui-access.log && \
-    ln -s /proc/1/fd/2 /var/log/bunkerweb/ui.log
+  addgroup -g 101 ui && \
+  adduser -h /var/cache/nginx -g ui -s /bin/sh -G ui -D -H -u 101 ui && \
+  echo "Docker" > INTEGRATION && \
+  mkdir -p /var/tmp/bunkerweb && \
+  mkdir -p /var/run/bunkerweb && \
+  mkdir -p /etc/bunkerweb && \
+  mkdir -p /data/cache && ln -s /data/cache /var/cache/bunkerweb && \
+  mkdir -p /data/lib && ln -s /data/lib /var/lib/bunkerweb && \
+  mkdir -p /var/log/bunkerweb/ && \
+  for dir in $(echo "configs plugins") ; do mkdir -p "/data/${dir}" && ln -s "/data/${dir}" "/etc/bunkerweb/${dir}" ; done && \
+  for dir in $(echo "configs/http configs/stream configs/server-http configs/server-stream configs/default-server-http configs/default-server-stream configs/modsec configs/modsec-crs") ; do mkdir "/data/${dir}" ; done && \
+  chown -R root:ui /data && \
+  chmod -R 770 /data && \
+  chown -R root:ui INTEGRATION /var/cache/bunkerweb /var/lib/bunkerweb /etc/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb && \
+  chmod 770 /var/cache/bunkerweb /var/lib/bunkerweb /var/tmp/bunkerweb /var/run/bunkerweb /var/log/bunkerweb && \
+  chmod 750 gen/*.py ui/*.py ui/src/*.py deps/python/bin/* helpers/*.sh && \
+  chmod 660 INTEGRATION && \
+  chown root:ui INTEGRATION && \
+  ln -s /proc/1/fd/1 /var/log/bunkerweb/ui-access.log && \
+  ln -s /proc/1/fd/2 /var/log/bunkerweb/ui.log
 
 # Fix CVEs
 # There are no CVE to fix for the moment