Add .clvm.hex pre-commit check (#12050)
* Add .clvm.hex pre-commit check
* report all results
* stuff
* maybe pass
* add todo
* explicitly define suffixes
* check for (and remove) extra hex and hash files
* remove garbage failure hex and hash files
* provide runner scripts
* matrix pre-commit checks
* maybe now
* more cross platform
* move it
* oops
* shell:
* maybe
* more configurable
* explicitly specify powershell
* tidy
* remove other stuff
* tidy
* remove existing pytest clvm hex etc checker
* add a trailing newline to decompress_block_spends.clvm.hex.sha256tree
* Update activated.ps1
* git config --global core.autocrlf false
* Update pre-commit.yml
* only check `chia/`
* typing_extensions
* typing_extensions
* removing unused typing import
* move top_level out of the rglob
* catchup dl clvm hex files
* Revert "catchup dl clvm hex files"
This reverts commit cd3d4f70b8.
* now
This commit is contained in:
Kyle Altendorf
GitHub
parent
e588566045
commit
789ee4f415
|
|
@ -54,6 +54,10 @@ jobs:
|
|||
|
||||
- uses: Chia-Network/actions/git-mark-workspace-safe@main
|
||||
|
||||
- name: disable git autocrlf
|
||||
run: |
|
||||
git config --global core.autocrlf false
|
||||
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
- uses: Chia-Network/actions/setup-python@main
|
||||
|
|
|
|||
|
|
@ -29,6 +29,13 @@ repos:
|
|||
- id: check-merge-conflict
|
||||
- id: check-ast
|
||||
- id: debug-statements
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: clvm_hex
|
||||
name: .clvm.hex files
|
||||
entry: ./activated.py python tests/check_clvm.py
|
||||
language: python
|
||||
pass_filenames: false
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: mypy
|
||||
|
|
|
|||
|
|
@ -2,7 +2,11 @@ $ErrorActionPreference = "Stop"
|
|||
|
||||
$script_directory = Split-Path $MyInvocation.MyCommand.Path -Parent
|
||||
|
||||
$command = $args[0]
|
||||
$parameters = [System.Collections.ArrayList]$args
|
||||
$parameters.RemoveAt(0)
|
||||
|
||||
& $script_directory/venv/Scripts/Activate.ps1
|
||||
& @args
|
||||
& $command @parameters
|
||||
|
||||
exit $LASTEXITCODE
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
can't compile ("defconstant" "AGG_SIG_UNSAFE" 49), unknown operator
|
||||
|
|
@ -1 +0,0 @@
|
|||
ff02ffff01ff04ffff04ff38ffff04ffff02ff3affff04ff02ffff04ff05ffff04ffff02ff2effff04ff02ffff04ff17ffff04ffff02ff26ffff04ff02ffff04ff2fff80808080ff8080808080ff8080808080ffff04ff27ff80808080ffff02ff32ffff04ff02ffff04ff0bffff04ff2fffff01ff01808080808080ffff04ffff01ffffff32ff0233ffff0401ff0102ffffffff02ffff03ff05ffff01ff02ff22ffff04ff02ffff04ff0dffff04ffff0bff3cffff0bff34ff2480ffff0bff3cffff0bff3cffff0bff34ff2c80ff0980ffff0bff3cff0bffff0bff34ff8080808080ff8080808080ffff010b80ff0180ff02ffff03ff0bffff01ff04ffff04ff38ffff04ffff02ff2affff04ff02ffff04ff05ffff04ffff04ffff04ff17ff1380ff8080ff8080808080ffff01ff80808080ffff04ffff04ff10ffff04ff23ffff04ffff02ff3effff04ff02ffff04ffff04ff13ffff04ff17ff808080ff80808080ff80808080ffff02ff32ffff04ff02ffff04ff05ffff04ff1bffff04ffff10ff17ffff010180ff8080808080808080ff8080ff0180ffff02ff36ffff04ff02ffff04ff05ffff04ffff02ff3effff04ff02ffff04ff0bff80808080ff8080808080ff02ff36ffff04ff02ffff04ff05ffff04ffff02ff3effff04ff02ffff04ff0bff80808080ffff04ffff02ff3effff04ff02ffff04ff05ff80808080ff808080808080ffffff02ffff03ffff07ff0580ffff01ff04ff29ffff02ff26ffff04ff02ffff04ff0dff8080808080ff8080ff0180ff0bff3cffff0bff34ff2880ffff0bff3cffff0bff3cffff0bff34ff2c80ff0580ffff0bff3cffff02ff22ffff04ff02ffff04ff07ffff04ffff0bff34ff3480ff8080808080ffff0bff34ff8080808080ffff02ffff03ffff07ff0b80ffff01ff10ff13ffff02ff2effff04ff02ffff04ff05ffff04ff1bff808080808080ffff011580ff0180ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff3effff04ff02ffff04ff09ff80808080ffff02ff3effff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080
|
||||
|
|
@ -1 +0,0 @@
|
|||
can't compile ("my-id"), unknown operator
|
||||
|
|
@ -1 +0,0 @@
|
|||
ff02ffff01ff04ffff04ffff013effff04ffff02ff02ffff04ff02ffff04ff05ff80808080ff808080ff8080ffff04ffff01ff02ffff03ffff07ff0580ffff01ff0bffff0102ffff02ff02ffff04ff02ffff04ff09ff80808080ffff02ff02ffff04ff02ffff04ff0dff8080808080ffff01ff0bffff0101ff058080ff0180ff018080
|
||||
|
|
@ -1 +0,0 @@
|
|||
3df9de54667a96f32eba322635f14d3474edadf23f396ba5a3e2e077a89a682a
|
||||
|
|
@ -0,0 +1,167 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
|
||||
import typing_extensions
|
||||
|
||||
here = pathlib.Path(__file__).parent.resolve()
|
||||
root = here.parent
|
||||
|
||||
# This is a work-around for fixing imports so they get the appropriate top level
|
||||
# packages instead of those of the same name in the same directory as this program.
|
||||
# This undoes the Python mis-feature meant to support 'scripts' that have not been
|
||||
# installed by adding the script's directory to the import search path. This is why
|
||||
# it is simpler to just have all code get installed and all things you run be
|
||||
# accessible via entry points.
|
||||
sys.path = [path for path in sys.path if path != os.fspath(here)]
|
||||
|
||||
from clvm_tools_rs import compile_clvm # noqa: E402
|
||||
|
||||
from chia.types.blockchain_format.program import SerializedProgram # noqa: E402
|
||||
|
||||
clvm_suffix = ".clvm"
|
||||
hex_suffix = ".clvm.hex"
|
||||
hash_suffix = ".clvm.hex.sha256tree"
|
||||
|
||||
|
||||
def generate_hash_bytes(hex_bytes: bytes) -> bytes:
|
||||
cleaned_blob = bytes.fromhex(hex_bytes.decode("utf-8"))
|
||||
serialize_program = SerializedProgram.from_bytes(cleaned_blob)
|
||||
result = serialize_program.get_tree_hash().hex()
|
||||
return (result + "\n").encode("utf-8")
|
||||
|
||||
|
||||
@typing_extensions.final
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class ClvmPaths:
|
||||
clvm: pathlib.Path
|
||||
hex: pathlib.Path
|
||||
hash: pathlib.Path
|
||||
|
||||
@classmethod
|
||||
def from_clvm(cls, clvm: pathlib.Path) -> ClvmPaths:
|
||||
return cls(
|
||||
clvm=clvm,
|
||||
hex=clvm.with_name(clvm.name[: -len(clvm_suffix)] + hex_suffix),
|
||||
hash=clvm.with_name(clvm.name[: -len(clvm_suffix)] + hash_suffix),
|
||||
)
|
||||
|
||||
|
||||
@typing_extensions.final
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class ClvmBytes:
|
||||
hex: bytes
|
||||
hash: bytes
|
||||
|
||||
@classmethod
|
||||
def from_clvm_paths(cls, paths: ClvmPaths) -> ClvmBytes:
|
||||
return cls(
|
||||
hex=paths.hex.read_bytes(),
|
||||
hash=paths.hash.read_bytes(),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_hex_bytes(cls, hex_bytes: bytes) -> ClvmBytes:
|
||||
return cls(
|
||||
hex=hex_bytes,
|
||||
hash=generate_hash_bytes(hex_bytes=hex_bytes),
|
||||
)
|
||||
|
||||
|
||||
# These files have the wrong extension for now so we'll just manually exclude them
|
||||
excludes = {"condition_codes.clvm", "create-lock-puzzlehash.clvm"}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
used_excludes = set()
|
||||
overall_fail = False
|
||||
|
||||
suffixes = {"clvm": clvm_suffix, "hex": hex_suffix, "hash": hash_suffix}
|
||||
top_levels = {"chia"}
|
||||
found_stems = {
|
||||
name: {
|
||||
path.with_name(path.name[: -len(suffix)])
|
||||
for top_level in top_levels
|
||||
for path in root.joinpath(top_level).rglob(f"**/*{suffix}")
|
||||
}
|
||||
for name, suffix in suffixes.items()
|
||||
}
|
||||
for name in ["hex", "hash"]:
|
||||
found = found_stems[name]
|
||||
suffix = suffixes[name]
|
||||
extra = found - found_stems["clvm"]
|
||||
|
||||
print()
|
||||
print(f"Extra {suffix} files:")
|
||||
|
||||
if len(extra) == 0:
|
||||
print(" -")
|
||||
else:
|
||||
overall_fail = True
|
||||
for stem in extra:
|
||||
print(f" {stem.with_name(stem.name + suffix)}")
|
||||
|
||||
print()
|
||||
print("Checking that all existing .clvm files compile to .clvm.hex that match existing caches:")
|
||||
for stem_path in sorted(found_stems["clvm"]):
|
||||
clvm_path = stem_path.with_name(stem_path.name + clvm_suffix)
|
||||
if clvm_path.name in excludes:
|
||||
used_excludes.add(clvm_path.name)
|
||||
continue
|
||||
|
||||
file_fail = False
|
||||
error = None
|
||||
|
||||
try:
|
||||
reference_paths = ClvmPaths.from_clvm(clvm=clvm_path)
|
||||
reference_bytes = ClvmBytes.from_clvm_paths(paths=reference_paths)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
generated_paths = ClvmPaths.from_clvm(
|
||||
clvm=pathlib.Path(temporary_directory).joinpath(f"generated{clvm_suffix}")
|
||||
)
|
||||
|
||||
compile_clvm(
|
||||
input_path=os.fspath(reference_paths.clvm),
|
||||
output_path=os.fspath(generated_paths.hex),
|
||||
search_paths=[os.fspath(reference_paths.clvm.parent)],
|
||||
)
|
||||
|
||||
generated_bytes = ClvmBytes.from_hex_bytes(hex_bytes=generated_paths.hex.read_bytes())
|
||||
|
||||
if generated_bytes != reference_bytes:
|
||||
file_fail = True
|
||||
error = f" reference: {reference_bytes!r}\n"
|
||||
error += f" generated: {generated_bytes!r}"
|
||||
except Exception:
|
||||
file_fail = True
|
||||
error = traceback.format_exc()
|
||||
|
||||
if file_fail:
|
||||
print(f"FAIL : {clvm_path}")
|
||||
if error is not None:
|
||||
print(error)
|
||||
else:
|
||||
print(f" pass: {clvm_path}")
|
||||
|
||||
if file_fail:
|
||||
overall_fail = True
|
||||
|
||||
unused_excludes = sorted(excludes - used_excludes)
|
||||
if len(unused_excludes) > 0:
|
||||
overall_fail = True
|
||||
print()
|
||||
print("Unused excludes:")
|
||||
|
||||
for exclude in unused_excludes:
|
||||
print(f" {exclude}")
|
||||
|
||||
return 1 if overall_fail else 0
|
||||
|
||||
|
||||
sys.exit(main())
|
||||
|
|
@ -1,187 +0,0 @@
|
|||
from os import remove
|
||||
from pathlib import Path
|
||||
from tempfile import NamedTemporaryFile
|
||||
from unittest import TestCase
|
||||
|
||||
import pytest
|
||||
from clvm_tools.clvmc import compile_clvm
|
||||
|
||||
from chia.types.blockchain_format.program import Program, SerializedProgram
|
||||
|
||||
pytestmark = pytest.mark.data_layer
|
||||
|
||||
|
||||
wallet_program_files = set(
|
||||
[
|
||||
"chia/wallet/puzzles/calculate_synthetic_public_key.clvm",
|
||||
"chia/wallet/puzzles/cat_v2.clvm",
|
||||
"chia/wallet/puzzles/chialisp_deserialisation.clvm",
|
||||
"chia/wallet/puzzles/rom_bootstrap_generator.clvm",
|
||||
"chia/wallet/puzzles/lock.inner.puzzle.clvm",
|
||||
"chia/wallet/puzzles/p2_conditions.clvm",
|
||||
"chia/wallet/puzzles/p2_delegated_conditions.clvm",
|
||||
"chia/wallet/puzzles/p2_delegated_puzzle.clvm",
|
||||
"chia/wallet/puzzles/p2_delegated_puzzle_or_hidden_puzzle.clvm",
|
||||
"chia/wallet/puzzles/p2_m_of_n_delegate_direct.clvm",
|
||||
"chia/wallet/puzzles/p2_puzzle_hash.clvm",
|
||||
"chia/wallet/puzzles/rl_aggregation.clvm",
|
||||
"chia/wallet/puzzles/rl.clvm",
|
||||
"chia/wallet/puzzles/sha256tree_module.clvm",
|
||||
"chia/wallet/puzzles/singleton_top_layer.clvm",
|
||||
"chia/wallet/puzzles/did_innerpuz.clvm",
|
||||
"chia/wallet/puzzles/decompress_puzzle.clvm",
|
||||
"chia/wallet/puzzles/decompress_coin_spend_entry_with_prefix.clvm",
|
||||
"chia/wallet/puzzles/decompress_coin_spend_entry.clvm",
|
||||
"chia/wallet/puzzles/block_program_zero.clvm",
|
||||
"chia/wallet/puzzles/test_generator_deserialize.clvm",
|
||||
"chia/wallet/puzzles/test_multiple_generator_input_arguments.clvm",
|
||||
"chia/wallet/puzzles/p2_singleton.clvm",
|
||||
"chia/wallet/puzzles/pool_waitingroom_innerpuz.clvm",
|
||||
"chia/wallet/puzzles/pool_member_innerpuz.clvm",
|
||||
"chia/wallet/puzzles/singleton_launcher.clvm",
|
||||
"chia/wallet/puzzles/p2_singleton_or_delayed_puzhash.clvm",
|
||||
"chia/wallet/puzzles/genesis_by_puzzle_hash.clvm",
|
||||
"chia/wallet/puzzles/everything_with_signature.clvm",
|
||||
"chia/wallet/puzzles/delegated_tail.clvm",
|
||||
"chia/wallet/puzzles/settlement_payments.clvm",
|
||||
"chia/wallet/puzzles/genesis_by_coin_id.clvm",
|
||||
"chia/wallet/puzzles/singleton_top_layer_v1_1.clvm",
|
||||
"chia/wallet/puzzles/nft_metadata_updater_default.clvm",
|
||||
"chia/wallet/puzzles/nft_metadata_updater_updateable.clvm",
|
||||
"chia/wallet/puzzles/nft_state_layer.clvm",
|
||||
"chia/wallet/puzzles/nft_ownership_layer.clvm",
|
||||
"chia/wallet/puzzles/nft_ownership_transfer_program_one_way_claim_with_royalties.clvm",
|
||||
"chia/wallet/puzzles/graftroot_dl_offers.clvm",
|
||||
"chia/wallet/puzzles/p2_parent.clvm",
|
||||
"chia/wallet/puzzles/decompress_block_spends.clvm",
|
||||
]
|
||||
)
|
||||
|
||||
clvm_include_files = set(
|
||||
["chia/wallet/puzzles/create-lock-puzzlehash.clvm", "chia/wallet/puzzles/condition_codes.clvm"]
|
||||
)
|
||||
|
||||
CLVM_PROGRAM_ROOT = "chia/wallet/puzzles"
|
||||
|
||||
|
||||
def list_files(dir, glob):
|
||||
dir = Path(dir)
|
||||
entries = dir.glob(glob)
|
||||
files = [f for f in entries if f.is_file()]
|
||||
return files
|
||||
|
||||
|
||||
def read_file(path):
|
||||
with open(path) as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
def path_with_ext(path, ext):
|
||||
return Path(str(path) + ext)
|
||||
|
||||
|
||||
class TestClvmCompilation(TestCase):
|
||||
"""
|
||||
These are tests, and not just build scripts to regenerate the bytecode, because
|
||||
the developer must be aware if the compiled output changes, for any reason.
|
||||
"""
|
||||
|
||||
def test_all_programs_listed(self):
|
||||
"""
|
||||
Checks to see if a new .clvm file was added to chia/wallet/puzzles, but not added to `wallet_program_files`
|
||||
"""
|
||||
existing_files = list_files(CLVM_PROGRAM_ROOT, "*.clvm")
|
||||
existing_file_paths = set([Path(x).relative_to(CLVM_PROGRAM_ROOT) for x in existing_files])
|
||||
|
||||
expected_files = set(clvm_include_files).union(set(wallet_program_files))
|
||||
expected_file_paths = set([Path(x).relative_to(CLVM_PROGRAM_ROOT) for x in expected_files])
|
||||
|
||||
self.assertEqual(
|
||||
expected_file_paths,
|
||||
existing_file_paths,
|
||||
msg="Please add your new program to `wallet_program_files` or `clvm_include_files.values`",
|
||||
)
|
||||
|
||||
def test_include_and_source_files_separate(self):
|
||||
self.assertEqual(clvm_include_files.intersection(wallet_program_files), set())
|
||||
|
||||
# TODO: Test recompilation with all available compiler configurations & implementations
|
||||
def test_all_programs_are_compiled(self):
|
||||
"""Checks to see if a new .clvm file was added without its .hex file"""
|
||||
all_compiled = True
|
||||
msg = "Please compile your program with:\n"
|
||||
|
||||
# Note that we cannot test all existing .clvm files - some are not
|
||||
# meant to be run as a "module" with load_clvm; some are include files
|
||||
# We test for inclusion in `test_all_programs_listed`
|
||||
for prog_path in wallet_program_files:
|
||||
try:
|
||||
output_path = path_with_ext(prog_path, ".hex")
|
||||
hex = output_path.read_text()
|
||||
self.assertTrue(len(hex) > 0)
|
||||
except Exception as ex:
|
||||
all_compiled = False
|
||||
msg += f" run -i {prog_path.parent} -d {prog_path} > {prog_path}.hex\n"
|
||||
print(ex)
|
||||
msg += "and check it in"
|
||||
self.assertTrue(all_compiled, msg=msg)
|
||||
|
||||
def test_recompilation_matches(self):
|
||||
self.maxDiff = None
|
||||
for f in wallet_program_files:
|
||||
f = Path(f)
|
||||
compile_clvm(f, path_with_ext(f, ".recompiled"), search_paths=[f.parent])
|
||||
orig_hex = path_with_ext(f, ".hex").read_text().strip()
|
||||
new_hex = path_with_ext(f, ".recompiled").read_text().strip()
|
||||
self.assertEqual(orig_hex, new_hex, msg=f"Compilation of {f} does not match {f}.hex")
|
||||
pass
|
||||
|
||||
def test_all_compiled_programs_are_hashed(self):
|
||||
"""Checks to see if a .hex file is missing its .sha256tree file"""
|
||||
all_hashed = True
|
||||
msg = "Please hash your program with:\n"
|
||||
for prog_path in wallet_program_files:
|
||||
try:
|
||||
hex = path_with_ext(prog_path, ".hex.sha256tree").read_text()
|
||||
self.assertTrue(len(hex) > 0)
|
||||
except Exception as ex:
|
||||
print(ex)
|
||||
all_hashed = False
|
||||
msg += f" opd -H {prog_path}.hex | head -1 > {prog_path}.hex.sha256tree\n"
|
||||
msg += "and check it in"
|
||||
self.assertTrue(all_hashed, msg)
|
||||
|
||||
# TODO: Test all available shatree implementations on all progams
|
||||
def test_shatrees_match(self):
|
||||
"""Checks to see that all .sha256tree files match their .hex files"""
|
||||
for prog_path in wallet_program_files:
|
||||
# load the .hex file as a program
|
||||
hex_filename = path_with_ext(prog_path, ".hex")
|
||||
clvm_hex = hex_filename.read_text() # .decode("utf8")
|
||||
clvm_blob = bytes.fromhex(clvm_hex)
|
||||
s = SerializedProgram.from_bytes(clvm_blob)
|
||||
p = Program.from_bytes(clvm_blob)
|
||||
|
||||
# load the checked-in shatree
|
||||
existing_sha = path_with_ext(prog_path, ".hex.sha256tree").read_text().strip()
|
||||
|
||||
self.assertEqual(
|
||||
s.get_tree_hash().hex(),
|
||||
existing_sha,
|
||||
msg=f"Checked-in shatree hash file does not match hash of loaded SerializedProgram: {prog_path}",
|
||||
)
|
||||
self.assertEqual(
|
||||
p.get_tree_hash().hex(),
|
||||
existing_sha,
|
||||
msg=f"Checked-in shatree hash file does not match shatree hash of loaded Program: {prog_path}",
|
||||
)
|
||||
|
||||
def test_017_encoding_bug_fixed(self):
|
||||
with NamedTemporaryFile(delete=False) as tf:
|
||||
tf.write(b"10000000")
|
||||
hexname = tf.name + ".hex"
|
||||
compile_clvm(tf.name, hexname, [])
|
||||
with open(hexname) as f:
|
||||
self.assertEqual(f.read().strip(), "8400989680")
|
||||
remove(tf.name)
|
||||
remove(hexname)
|
||||
Loading…
Reference in New Issue