225 lines
9.0 KiB
Python
225 lines
9.0 KiB
Python
from Test import Test
|
|
from os.path import isdir, isfile
|
|
from os import getenv, mkdir
|
|
from shutil import rmtree, copy
|
|
from traceback import format_exc
|
|
from subprocess import run
|
|
from time import sleep
|
|
from logger import log
|
|
from yaml import safe_load, dump
|
|
|
|
|
|
class SwarmTest(Test):
|
|
def __init__(self, name, timeout, tests, delay=0):
|
|
super().__init__(name, "swarm", timeout, tests, delay=delay)
|
|
self._domains = {
|
|
r"www\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_1')}",
|
|
r"auth\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1_2')}",
|
|
r"app1\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN1')}",
|
|
r"app2\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN2')}",
|
|
r"app3\.example\.com": f"{Test.random_string(6)}.{getenv('TEST_DOMAIN3')}",
|
|
}
|
|
|
|
@staticmethod
|
|
def init():
|
|
try:
|
|
if not Test.init():
|
|
return False
|
|
proc = run("sudo chown -R root:root /tmp/bw-data", shell=True)
|
|
if proc.returncode != 0:
|
|
raise (Exception("chown failed (swarm stack)"))
|
|
if isdir("/tmp/swarm"):
|
|
rmtree("/tmp/swarm")
|
|
mkdir("/tmp/swarm")
|
|
copy("./misc/integrations/swarm.mariadb.yml", "/tmp/swarm/stack.yml")
|
|
compose = "/tmp/swarm/stack.yml"
|
|
with open(compose, "r") as f:
|
|
data = safe_load(f.read())
|
|
if "AUTO_LETS_ENCRYPT=yes" not in data["services"]["bunkerweb"]["environment"]:
|
|
data["services"]["bunkerweb"]["environment"].append("AUTO_LETS_ENCRYPT=yes")
|
|
data["services"]["bunkerweb"]["environment"].append("USE_LETS_ENCRYPT_STAGING=yes")
|
|
del data["services"]["bunkerweb"]["deploy"]["placement"]
|
|
with open(compose, "w") as f:
|
|
f.write(dump(data))
|
|
Test.replace_in_file(
|
|
compose,
|
|
r"bunkerity/bunkerweb:.*$",
|
|
"192.168.42.100:5000/bunkerweb-tests:latest",
|
|
)
|
|
Test.replace_in_file(
|
|
compose,
|
|
r"bunkerity/bunkerweb-autoconf:.*$",
|
|
"192.168.42.100:5000/autoconf-tests:latest",
|
|
)
|
|
Test.replace_in_file(
|
|
compose,
|
|
r"bunkerity/bunkerweb-scheduler:.*$",
|
|
"192.168.42.100:5000/scheduler-tests:latest",
|
|
)
|
|
# Test.replace_in_file(compose, r"bw\-data:/", "/tmp/bw-data:/")
|
|
proc = run(
|
|
"docker stack deploy -c stack.yml bunkerweb",
|
|
cwd="/tmp/swarm",
|
|
shell=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker stack deploy failed (swarm stack)"))
|
|
i = 0
|
|
healthy = False
|
|
while i < 120:
|
|
proc = run(
|
|
'docker stack ps --no-trunc --format "{{ .CurrentState }}" bunkerweb | grep -v "Running"',
|
|
cwd="/tmp/swarm",
|
|
shell=True,
|
|
capture_output=True,
|
|
)
|
|
if not proc.stdout.decode():
|
|
healthy = True
|
|
break
|
|
sleep(1)
|
|
i += 1
|
|
if not healthy:
|
|
proc = run(
|
|
"docker service logs bunkerweb_bunkerweb ;"
|
|
+ " docker service logs bunkerweb_bw-autoconf ;"
|
|
+ " docker service logs bunkerweb_bw-scheduler ;"
|
|
+ " docker service logs bunkerweb_bw-db ;"
|
|
+ " docker service logs bunkerweb_bw-redis ;"
|
|
+ " docker stack ps --no-trunc bunkerweb",
|
|
cwd="/tmp/swarm",
|
|
shell=True,
|
|
capture_output=True,
|
|
)
|
|
log("SWARM", "❌", f"stdout logs = {proc.stdout.decode()}")
|
|
log("SWARM", "❌", f"stderr logs = {proc.stderr.decode()}")
|
|
raise (Exception("swarm stack is not healthy"))
|
|
sleep(60)
|
|
except:
|
|
log(
|
|
"SWARM",
|
|
"❌",
|
|
f"exception while running SwarmTest.init()\n{format_exc()}",
|
|
)
|
|
return False
|
|
return True
|
|
|
|
@staticmethod
|
|
def end():
|
|
ret = True
|
|
try:
|
|
if not Test.end():
|
|
return False
|
|
proc = run("docker stack rm bunkerweb", shell=True)
|
|
if proc.returncode != 0:
|
|
ret = False
|
|
rmtree("/tmp/swarm")
|
|
except:
|
|
log("SWARM", "❌", f"exception while running SwarmTest.end()\n{format_exc()}")
|
|
return False
|
|
return ret
|
|
|
|
def _setup_test(self):
|
|
try:
|
|
super()._setup_test()
|
|
test = f"/tmp/tests/{self._name}"
|
|
for ex_domain, test_domain in self._domains.items():
|
|
Test.replace_in_files(test, ex_domain, test_domain)
|
|
Test.rename(test, ex_domain, test_domain)
|
|
Test.replace_in_files(test, "example.com", getenv("ROOT_DOMAIN"))
|
|
setup = f"{test}/setup-swarm.sh"
|
|
if isfile(setup):
|
|
proc = run("sudo ./setup-swarm.sh", cwd=test, shell=True)
|
|
if proc.returncode != 0:
|
|
raise (Exception("setup-swarm failed"))
|
|
proc = run(
|
|
f'docker stack deploy -c swarm.yml "{self._name}"',
|
|
shell=True,
|
|
cwd=test,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker stack deploy failed"))
|
|
i = 0
|
|
healthy = False
|
|
while i < self._timeout:
|
|
proc = run(
|
|
'docker stack services --format "{{ .Name }}" ' + self._name,
|
|
cwd="/tmp/swarm",
|
|
shell=True,
|
|
capture_output=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise (Exception("swarm stack is not healthy (cmd1 failed)"))
|
|
all_healthy = True
|
|
for service in proc.stdout.decode().splitlines():
|
|
proc2 = run(
|
|
'docker service ps --format "{{ .CurrentState }}" ' + service,
|
|
cwd="/tmp/swarm",
|
|
shell=True,
|
|
capture_output=True,
|
|
)
|
|
if proc2.returncode != 0:
|
|
raise (Exception("swarm stack is not healthy (cmd2 failed)"))
|
|
if "Running" not in proc2.stdout.decode():
|
|
all_healthy = False
|
|
break
|
|
if all_healthy:
|
|
healthy = True
|
|
break
|
|
sleep(1)
|
|
i += 1
|
|
if not healthy:
|
|
raise (Exception("swarm stack is not healthy"))
|
|
except:
|
|
log(
|
|
"SWARM",
|
|
"❌",
|
|
f"exception while running SwarmTest._setup_test()\n{format_exc()}",
|
|
)
|
|
self._cleanup_test()
|
|
return False
|
|
return True
|
|
|
|
def _cleanup_test(self):
|
|
try:
|
|
proc = run(f'docker stack rm "{self._name}"', shell=True)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker stack rm failed"))
|
|
proc = run('docker config ls --format "{{ .ID }}"', shell=True, capture_output=True)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker config ls failed"))
|
|
for config in proc.stdout.decode().splitlines():
|
|
proc = run(f'docker config rm "{config}"', shell=True)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker config rm failed"))
|
|
proc = run(
|
|
"docker service create --mode global --mount type=bind,src=/var/run/docker.sock,dst=/var/run/docker.sock --restart-condition none --name pruner -d docker docker volume prune -f",
|
|
shell=True,
|
|
)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker pruner create failed"))
|
|
sleep(10)
|
|
proc = run("docker service rm pruner", shell=True)
|
|
if proc.returncode != 0:
|
|
raise (Exception("docker pruner rm failed"))
|
|
super()._cleanup_test()
|
|
except:
|
|
log(
|
|
"SWARM",
|
|
"❌",
|
|
f"exception while running SwarmTest._cleanup_test()\n{format_exc()}",
|
|
)
|
|
return False
|
|
return True
|
|
|
|
def _debug_fail(self):
|
|
run("docker service logs bunkerweb_bunkerweb", shell=True)
|
|
run("docker service logs bunkerweb_bw-autoconf", shell=True)
|
|
run("docker service logs bunkerweb_bw-scheduler", shell=True)
|
|
proc = run(
|
|
'docker stack services --format "{{ .Name }}" "' + self._name + '"',
|
|
shell=True,
|
|
capture_output=True,
|
|
)
|
|
for service in proc.stdout.decode().splitlines():
|
|
run(f'docker service logs "{service}"', shell=True)
|