core - Add redis tests

This commit is contained in:
Théophile Diot 2023-05-19 16:25:01 -04:00
parent 0087ae5832
commit af19cc226d
No known key found for this signature in database
GPG Key ID: E752C80DB72BB014
9 changed files with 688 additions and 1 deletions

View File

@ -0,0 +1,27 @@
FROM python:3.11.3-alpine
# Install firefox and geckodriver
RUN apk add --no-cache --virtual .build-deps curl grep zip && \
apk add --no-cache firefox
# Installing geckodriver for firefox...
RUN GECKODRIVER_VERSION=`curl -i https://github.com/mozilla/geckodriver/releases/latest | grep -Po 'v[0-9]+.[0-9]+.[0-9]+'` && \
wget -O geckodriver.tar.gz https://github.com/mozilla/geckodriver/releases/download/$GECKODRIVER_VERSION/geckodriver-$GECKODRIVER_VERSION-linux64.tar.gz && \
tar -C /usr/local/bin -xzvf geckodriver.tar.gz && \
chmod +x /usr/local/bin/geckodriver && \
rm geckodriver.tar.gz
WORKDIR /tmp
COPY requirements.txt .
RUN MAKEFLAGS="-j $(nproc)" pip install --no-cache -r requirements.txt && \
rm -f requirements.txt
WORKDIR /opt/tests
COPY main.py .
EXPOSE 8080
ENTRYPOINT [ "python3", "main.py" ]

View File

@ -0,0 +1,9 @@
FROM redis:7-alpine
RUN apk add --no-cache bash openssl
COPY entrypoint.sh .
RUN chmod +x entrypoint.sh
ENTRYPOINT [ "./entrypoint.sh" ]

View File

@ -0,0 +1,24 @@
version: "3.5"
services:
tests:
build: .
environment:
PYTHONUNBUFFERED: "1"
USE_REVERSE_SCAN: "no"
USE_ANTIBOT: "no"
BLACKLIST_IP: ""
REDIS_HOST: "bw-redis"
REDIS_PORT: "6379"
REDIS_DATABASE: "0"
REDIS_SSL: "no"
extra_hosts:
- "www.example.com:1.0.0.2"
networks:
bw-services:
ipv4_address: 1.0.0.3
networks:
bw-services:
external: true

View File

@ -0,0 +1,84 @@
version: "3.5"
services:
bw:
image: bunkerity/bunkerweb:1.5.0-beta
pull_policy: never
depends_on:
- bw-redis
labels:
- "bunkerweb.INSTANCE"
volumes:
- ./index.html:/var/www/html/index.html
environment:
API_WHITELIST_IP: "127.0.0.0/8 10.20.30.0/24 1.0.0.3"
HTTP_PORT: "80"
USE_BUNKERNET: "no"
BLACKLIST_IP_URLS: ""
LOG_LEVEL: "info"
SESSIONS_NAME: "test"
USE_REVERSE_SCAN: "no"
USE_ANTIBOT: "no"
USE_GREYLIST: "yes"
GREYLIST_IP: "0.0.0.0/0"
WHITELIST_COUNTRY: "AU"
# ? REDIS settings
USE_REDIS: "yes"
REDIS_HOST: "bw-redis"
REDIS_PORT: "6379"
REDIS_DATABASE: "0"
REDIS_SSL: "no"
networks:
bw-universe:
bw-services:
ipv4_address: 1.0.0.2
bw-scheduler:
image: bunkerity/bunkerweb-scheduler:1.5.0-beta
pull_policy: never
depends_on:
- bw
- bw-docker
environment:
DOCKER_HOST: "tcp://bw-docker:2375"
LOG_LEVEL: "info"
networks:
- bw-universe
- bw-docker
bw-docker:
image: tecnativa/docker-socket-proxy
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
environment:
CONTAINERS: "1"
networks:
- bw-docker
bw-redis:
build:
context: .
dockerfile: Dockerfile.redis
environment:
REDIS_PORT: "6379"
REDIS_SSL: "no"
networks:
bw-services:
ipv4_address: 1.0.0.4
networks:
bw-universe:
name: bw-universe
ipam:
driver: default
config:
- subnet: 10.20.30.0/24
bw-services:
name: bw-services
ipam:
driver: default
config:
- subnet: 1.0.0.0/24
bw-docker:
name: bw-docker

View File

@ -0,0 +1,31 @@
#!/bin/bash
set -e
command="redis-server"
if [ "$REDIS_SSL" = "yes" ]; then
mkdir /tls
openssl genrsa -out /tls/ca.key 4096
openssl req \
-x509 -new -nodes -sha256 \
-key /tls/ca.key \
-days 365 \
-subj /CN=bw-redis/ \
-out /tls/ca.crt
openssl req \
-x509 -nodes -newkey rsa:4096 \
-keyout /tls/redis.key \
-out /tls/redis.pem \
-days 365 \
-subj /CN=bw-redis/
chmod -R 640 /tls
command+=" --tls-port ${REDIS_PORT:-6379} --port 0 --tls-cert-file /tls/redis.pem --tls-key-file /tls/redis.key --tls-ca-cert-file /tls/ca.crt --tls-auth-clients no"
else
command+=" --port ${REDIS_PORT:-6379}"
fi
$command

View File

371
tests/core/redis/main.py Normal file
View File

@ -0,0 +1,371 @@
from contextlib import suppress
from fastapi import FastAPI
from multiprocessing import Process
from os import getenv
from redis import Redis
from requests import get
from requests.exceptions import RequestException
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from time import sleep
from traceback import format_exc
from uvicorn import run
fastapi_proc = None
try:
redis_host = getenv("REDIS_HOST")
if not redis_host:
print("❌ Redis host is not set, exiting ...", flush=True)
exit(1)
redis_port = getenv("REDIS_PORT", "")
if not redis_port.isdigit():
print("❌ Redis port doesn't seem to be a number, exiting ...", flush=True)
exit(1)
redis_port = int(redis_port)
redis_db = getenv("REDIS_DATABASE", "")
if not redis_db.isdigit():
print("❌ Redis database doesn't seem to be a number, exiting ...", flush=True)
exit(1)
redis_db = int(redis_db)
redis_ssl = getenv("REDIS_SSL", "no") == "yes"
print(
f" Trying to connect to Redis with the following parameters:\nhost: {redis_host}\nport: {redis_port}\ndb: {redis_db}\nssl: {redis_ssl}",
flush=True,
)
redis_client = Redis(
host=redis_host,
port=redis_port,
db=redis_db,
ssl=redis_ssl,
socket_timeout=1,
ssl_cert_reqs=None,
)
if not redis_client.ping():
print("❌ Redis is not reachable, exiting ...", flush=True)
exit(1)
use_reverse_scan = getenv("USE_REVERSE_SCAN", "no") == "yes"
if use_reverse_scan:
print(" Testing Reverse Scan, starting FastAPI ...", flush=True)
app = FastAPI()
fastapi_proc = Process(
target=run, args=(app,), kwargs=dict(host="0.0.0.0", port=8080)
)
fastapi_proc.start()
sleep(2)
print(
" FastAPI started, sending a request to http://www.example.com ...",
flush=True,
)
response = get(
"http://www.example.com",
headers={"Host": "www.example.com"},
)
if response.status_code != 403:
response.raise_for_status()
print("❌ The request was not blocked, exiting ...", flush=True)
exit(1)
sleep(0.5)
print(" The request was blocked, checking Redis ...", flush=True)
key_value = redis_client.get("plugin_reverse_scan_1.0.0.3:8080")
if key_value is None:
print(
f'❌ The Reverse Scan key ("plugin_reverse_scan_1.0.0.3:8080") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
elif key_value != b"open":
print(
f'❌ The Reverse Scan key ("plugin_reverse_scan_1.0.0.3:8080") was found, but the value is not "open" ({key_value.decode()}), exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
print(
f"✅ The Reverse Scan key was found, the value is {key_value.decode()}",
flush=True,
)
exit(0)
use_antibot = getenv("USE_ANTIBOT", "no") != "no"
if use_antibot:
print(" Testing Antibot ...", flush=True)
firefox_options = Options()
firefox_options.add_argument("--headless")
print(" Starting Firefox ...", flush=True)
with webdriver.Firefox(options=firefox_options) as driver:
driver.delete_all_cookies()
driver.maximize_window()
print(" Navigating to http://www.example.com ...", flush=True)
driver.get("http://www.example.com")
sleep(0.5)
print(" Checking Redis ...", flush=True)
keys = redis_client.keys("sessions_:test:*")
if not keys:
print(
f"❌ No Antibot keys were found, exiting ...\nkeys: {redis_client.keys()}",
flush=True,
)
exit(1)
key_value = redis_client.get(keys[0])
if key_value is None:
print(
f"❌ The Antibot key ({keys[0].decode()}) was not found, exiting ...\nkeys: {redis_client.keys()}",
flush=True,
)
exit(1)
print(
f"✅ The Antibot key was found, the value is {key_value.decode()}",
flush=True,
)
exit(0)
print(
" Sending a request to http://www.example.com/?id=/etc/passwd ...",
flush=True,
)
response = get(
"http://www.example.com/?id=/etc/passwd",
headers={"Host": "www.example.com"},
)
if response.status_code != 403:
response.raise_for_status()
print("❌ The request was not blocked, exiting ...", flush=True)
exit(1)
sleep(0.5)
print(" The request was blocked, checking Redis ...", flush=True)
key_value = redis_client.get("plugin_bad_behavior_1.0.0.3")
if key_value is None:
print(
f'❌ The Bad Behavior key ("plugin_bad_behavior_1.0.0.3") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
print(
f"✅ The Bad Behavior key was found, the value is {key_value.decode()}",
flush=True,
)
print(
" Sending another request to http://www.example.com/?id=/etc/passwd ...",
flush=True,
)
response = get(
"http://www.example.com/?id=/etc/passwd",
headers={"Host": "www.example.com"},
)
if response.status_code != 403:
response.raise_for_status()
print("❌ The request was not blocked, exiting ...", flush=True)
exit(1)
sleep(0.5)
second_key_value = redis_client.get("plugin_bad_behavior_1.0.0.3")
if second_key_value <= key_value:
print(
f'❌ The Bad Behavior key ("plugin_bad_behavior_1.0.0.3") was not incremented, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
print(
f"✅ The Bad Behavior key was incremented, the value is {second_key_value.decode()}",
flush=True,
)
print(
" Sending requests to http://www.example.com until we reach the limit ...",
flush=True,
)
status_code = 0
while status_code != 429:
response = get(
"http://www.example.com",
headers={"Host": "www.example.com"},
)
if response.status_code not in (200, 429):
response.raise_for_status()
status_code = response.status_code
sleep(0.5)
key_value = redis_client.get("plugin_limit_www.example.com1.0.0.3/")
if key_value is None:
print(
f'❌ The limit key ("plugin_limit_www.example.com1.0.0.3/") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
print(
f"✅ The limit key was found, the value is {key_value.decode()}",
flush=True,
)
print(
" Checking if the country key was created and has the correct value ...",
flush=True,
)
key_value = redis_client.get("plugin_country_www.example.com1.0.0.3")
if key_value is None:
print(
f'❌ The country key ("plugin_country_www.example.com1.0.0.3") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
print(
f"✅ The country key was found, the value is {key_value.decode()}",
flush=True,
)
print(
" Checking if the whitelist key was created and has the correct value ...",
flush=True,
)
key_value = redis_client.get("plugin_whitelist_www.example.comip1.0.0.3")
if key_value is None:
print(
f'❌ The whitelist key ("plugin_whitelist_www.example.comip1.0.0.3") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
if key_value != b"ok":
print(
f'❌ The whitelist key ("plugin_whitelist_www.example.comip1.0.0.3") was found, but the value is not "ok" ({key_value.decode()}), exiting ...\nkeys: {redis_client.keys()}',
)
print(
f"✅ The whitelist key was found, the value is {key_value.decode()}",
flush=True,
)
print(
" Checking if the blacklist key was created and has the correct value ...",
flush=True,
)
key_value = redis_client.get("plugin_blacklist_www.example.comip1.0.0.3")
if key_value is None:
print(
f'❌ The blacklist key ("plugin_blacklist_www.example.comip1.0.0.3") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
if key_value != b"ok":
print(
f'❌ The blacklist key ("plugin_blacklist_www.example.comip1.0.0.3") was found, but the value is not "ok" ({key_value.decode()}), exiting ...\nkeys: {redis_client.keys()}',
)
print(
f"✅ The blacklist key was found, the value is {key_value.decode()}",
flush=True,
)
print(
" Checking if the greylist key was created and has the correct value ...",
flush=True,
)
key_value = redis_client.get("plugin_greylist_www.example.comip1.0.0.3")
if key_value is None:
print(
f'❌ The greylist key ("plugin_greylist_www.example.comip1.0.0.3") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
if key_value != b"ip":
print(
f'❌ The greylist key ("plugin_greylist_www.example.comip1.0.0.3") was found, but the value is not "ip" ({key_value.decode()}), exiting ...\nkeys: {redis_client.keys()}',
)
print(
f"✅ The greylist key was found, the value is {key_value.decode()}",
flush=True,
)
print(
" Checking if the dnsbl keys were created ...",
flush=True,
)
key_value = redis_client.get("plugin_dnsbl_www.example.com1.0.0.3")
if key_value is None:
print(
f'❌ The dnsbl key ("plugin_dnsbl_www.example.com1.0.0.3") was not found, exiting ...\nkeys: {redis_client.keys()}',
flush=True,
)
exit(1)
print(
f"✅ The dnsbl key was found, the value is {key_value.decode()}",
flush=True,
)
except SystemExit as e:
exit(e.code)
except:
print(f"❌ Something went wrong, exiting ...\n{format_exc()}", flush=True)
exit(1)
finally:
if fastapi_proc:
fastapi_proc.terminate()

View File

@ -0,0 +1,5 @@
requests==2.30.0
redis==4.5.5
fastapi==0.95.2
uvicorn[standard]==0.22.0
selenium==4.9.1

138
tests/core/redis/test.sh Normal file → Executable file
View File

@ -1 +1,137 @@
# TODO
#!/bin/bash
echo "🧰 Building redis stack ..."
# Starting stack
docker compose pull bw-docker
if [ $? -ne 0 ] ; then
echo "🧰 Pull failed ❌"
exit 1
fi
echo "🧰 Building custom redis image ..."
docker compose build bw-redis
if [ $? -ne 0 ] ; then
echo "🧰 Build failed ❌"
exit 1
fi
echo "🧰 Building tests images ..."
docker compose -f docker-compose.test.yml build
if [ $? -ne 0 ] ; then
echo "🧰 Build failed ❌"
exit 1
fi
manual=0
end=0
cleanup_stack () {
exit_code=$?
if [[ $end -eq 1 || $exit_code = 1 ]] || [[ $end -eq 0 && $exit_code = 0 ]] && [ $manual = 0 ] ; then
find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_REVERSE_SCAN: "yes"@USE_REVERSE_SCAN: "no"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: "cookie"@USE_ANTIBOT: "no"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@BLACKLIST_IP: "0\.0\.0\.0/0"@BLACKLIST_IP: ""@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIS_PORT: "[0-9]*"@REDIS_PORT: "6379"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIS_DATABASE: "1"@REDIS_DATABASE: "0"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIS_SSL: "yes"@REDIS_SSL: "no"@' {} \;
if [[ $end -eq 1 && $exit_code = 0 ]] ; then
return
fi
fi
echo "🧰 Cleaning up current stack ..."
docker compose down -v --remove-orphans 2>/dev/null
if [ $? -ne 0 ] ; then
echo "🧰 Down failed ❌"
exit 1
fi
echo "🧰 Cleaning up current stack done ✅"
}
# Cleanup stack on exit
trap cleanup_stack EXIT
for test in "activated" "reverse_scan" "antibot" "tweaked"
do
if [ "$test" = "activated" ] ; then
echo "🧰 Running tests with redis with default values ..."
elif [ "$test" = "reverse_scan" ] ; then
echo "🧰 Running tests with redis with reverse scan activated ..."
find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_REVERSE_SCAN: "no"@USE_REVERSE_SCAN: "yes"@' {} \;
elif [ "$test" = "antibot" ] ; then
echo "🧰 Running tests with redis with antibot cookie activated ..."
find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_REVERSE_SCAN: "yes"@USE_REVERSE_SCAN: "no"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: "no"@USE_ANTIBOT: "cookie"@' {} \;
elif [ "$test" = "tweaked" ] ; then
echo "🧰 Running tests with redis' settings tweaked ..."
find . -type f -name 'docker-compose.*' -exec sed -i 's@USE_ANTIBOT: "cookie"@USE_ANTIBOT: "no"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIS_PORT: "[0-9]*"@REDIS_PORT: "6380"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIS_DATABASE: "0"@REDIS_DATABASE: "1"@' {} \;
find . -type f -name 'docker-compose.*' -exec sed -i 's@REDIS_SSL: "no"@REDIS_SSL: "yes"@' {} \;
fi
echo "🧰 Starting stack ..."
docker compose up -d 2>/dev/null
if [ $? -ne 0 ] ; then
echo "🧰 Up failed, retrying ... ⚠️"
manual=1
cleanup_stack
manual=0
docker compose up -d 2>/dev/null
if [ $? -ne 0 ] ; then
echo "🧰 Up failed ❌"
exit 1
fi
fi
# Check if stack is healthy
echo "🧰 Waiting for stack to be healthy ..."
i=0
while [ $i -lt 120 ] ; do
containers=("redis-bw-1" "redis-bw-scheduler-1")
healthy="true"
for container in "${containers[@]}" ; do
check="$(docker inspect --format "{{json .State.Health }}" $container | grep "healthy")"
if [ "$check" = "" ] ; then
healthy="false"
break
fi
done
if [ "$healthy" = "true" ] ; then
echo "🧰 Docker stack is healthy ✅"
break
fi
sleep 1
i=$((i+1))
done
if [ $i -ge 120 ] ; then
docker compose logs
echo "🧰 Docker stack is not healthy ❌"
exit 1
fi
# Start tests
docker compose -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from tests 2>/dev/null
if [ $? -ne 0 ] ; then
echo "🧰 Test \"$test\" failed ❌"
echo "🛡️ Showing BunkerWeb and BunkerWeb Scheduler logs ..."
docker compose logs bw bw-scheduler
exit 1
else
echo "🧰 Test \"$test\" succeeded ✅"
fi
manual=1
cleanup_stack
manual=0
echo " "
done
end=1
echo "🧰 Tests are done ! ✅"