152 lines
4.3 KiB
Python
Executable File
152 lines
4.3 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Run from the current directory.
|
|
|
|
import argparse
|
|
import testconfig
|
|
import logging
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import List
|
|
|
|
|
|
def subdirs(root_dirs: List[str]) -> List[Path]:
|
|
dirs: List[Path] = []
|
|
for r in root_dirs:
|
|
dirs.extend(Path(r).rglob("**/"))
|
|
return [d for d in dirs if not (any(c.startswith("_") for c in d.parts) or any(c.startswith(".") for c in d.parts))]
|
|
|
|
|
|
def module_dict(module):
|
|
return {k: v for k, v in module.__dict__.items() if not k.startswith("_")}
|
|
|
|
|
|
def dir_config(dir):
|
|
import importlib
|
|
|
|
module_name = str(dir).replace("/", ".") + ".config"
|
|
try:
|
|
return module_dict(importlib.import_module(module_name))
|
|
except ModuleNotFoundError:
|
|
return {}
|
|
|
|
|
|
def read_file(filename):
|
|
with open(filename) as f:
|
|
return f.read()
|
|
return None
|
|
|
|
|
|
# Input file
|
|
def workflow_yaml_template_text(os):
|
|
return Path(f"runner-templates/build-test-{os}").read_text()
|
|
|
|
|
|
# Output files
|
|
def workflow_yaml_file(dir, os, test_name):
|
|
return Path(dir / f"build-test-{os}-{test_name}.yml")
|
|
|
|
|
|
# String function from test dir to test name
|
|
def test_name(dir):
|
|
return str(dir).replace("/", "-")
|
|
|
|
|
|
def transform_template(template_text, replacements):
|
|
t = template_text
|
|
for r, v in replacements.items():
|
|
t = t.replace(r, v)
|
|
return t
|
|
|
|
|
|
def test_files_in_dir(dir):
|
|
g = dir.glob("test_*.py")
|
|
return [] if g is None else [f for f in g]
|
|
|
|
|
|
# -----
|
|
|
|
default_replacements = {
|
|
"INSTALL_TIMELORD": read_file("runner-templates/install-timelord.include.yml").rstrip(),
|
|
"CHECKOUT_TEST_BLOCKS_AND_PLOTS": read_file("runner-templates/checkout-test-plots.include.yml").rstrip(),
|
|
"TEST_DIR": "",
|
|
"TEST_NAME": "",
|
|
"PYTEST_PARALLEL_ARGS": "",
|
|
}
|
|
|
|
# -----
|
|
|
|
|
|
# Replace with update_config
|
|
def generate_replacements(defaults, conf, dir, test_files):
|
|
assert len(test_files) > 0
|
|
replacements = dict(defaults)
|
|
|
|
if not conf["checkout_blocks_and_plots"]:
|
|
replacements[
|
|
"CHECKOUT_TEST_BLOCKS_AND_PLOTS"
|
|
] = "# Omitted checking out blocks and plots repo Chia-Network/test-cache"
|
|
if not conf["install_timelord"]:
|
|
replacements["INSTALL_TIMELORD"] = "# Omitted installing Timelord"
|
|
if conf["parallel"]:
|
|
replacements["PYTEST_PARALLEL_ARGS"] = "-n auto"
|
|
if conf["job_timeout"]:
|
|
replacements["JOB_TIMEOUT"] = str(conf["job_timeout"])
|
|
test_paths = ["tests/" + str(f) for f in test_files]
|
|
# We have to list the test files individually until pytest has the
|
|
# option to only collect tests in the named dir, and not those below
|
|
replacements["TEST_DIR"] = " ".join(sorted(test_paths))
|
|
replacements["TEST_NAME"] = test_name(str(dir))
|
|
if "test_name" in conf:
|
|
replacements["TEST_NAME"] = conf["test_name"]
|
|
return replacements
|
|
|
|
|
|
# Overwrite with directory specific values
|
|
def update_config(parent, child):
|
|
if child is None:
|
|
return parent
|
|
conf = child
|
|
for k, v in parent.items():
|
|
if k not in child:
|
|
conf[k] = v
|
|
return conf
|
|
|
|
|
|
def dir_path(string):
|
|
p = Path(string)
|
|
if p.is_dir():
|
|
return p
|
|
else:
|
|
raise NotADirectoryError(string)
|
|
|
|
|
|
# args
|
|
arg_parser = argparse.ArgumentParser(description="Build github workflows")
|
|
arg_parser.add_argument("--output-dir", "-d", default="../.github/workflows", type=dir_path)
|
|
arg_parser.add_argument("--verbose", "-v", action="store_true")
|
|
args = arg_parser.parse_args()
|
|
|
|
if args.verbose:
|
|
logging.basicConfig(format="%(asctime)s:%(message)s", level=logging.DEBUG)
|
|
|
|
# main
|
|
test_dirs = subdirs(testconfig.root_test_dirs)
|
|
|
|
for os in testconfig.oses:
|
|
template_text = workflow_yaml_template_text(os)
|
|
for dir in test_dirs:
|
|
test_files = test_files_in_dir(dir)
|
|
if len(test_files) == 0:
|
|
logging.info(f"Skipping {dir}: no tests collected")
|
|
continue
|
|
conf = update_config(module_dict(testconfig), dir_config(dir))
|
|
replacements = generate_replacements(default_replacements, conf, dir, test_files)
|
|
txt = transform_template(template_text, replacements)
|
|
logging.info(f"Writing {os}-{test_name(dir)}")
|
|
workflow_yaml_file(args.output_dir, os, test_name(dir)).write_text(txt)
|
|
|
|
out = subprocess.run(["git", "diff", args.output_dir])
|
|
if out.stdout:
|
|
print(out.stdout)
|