mypy: Build `mypy.ini` config with exclusions dynamically (#15158)
* mypy: Build `mypy.ini` config with exclusions dynamically * Apply suggestions from code review * Revert suggested `check=True` because `mypy` is expected to fails here * Only `read_text` once * Fix exception message * Don't `touch` * More `joinpath` * Revert `newline="\n"`, its python 3.10 only See https://bugs.python.org/issue23706 * Apply suggestions * Fail if new issues get introduced * Print the newly introduces issues * Handle error codes * Add `write_file` * Use `sys.executable` Co-authored-by: Kyle Altendorf <sda@fstab.net> * `mypy_failures[:-1]` * Add an example comment * Improve failure check * Drop exclusions after rebase * Drop one more exclusion after another rebase --------- Co-authored-by: Kyle Altendorf <sda@fstab.net>
This commit is contained in:
parent
fab36d5b4e
commit
ec5f85dfc4
|
@ -74,4 +74,5 @@ jobs:
|
||||||
|
|
||||||
- env:
|
- env:
|
||||||
CHIA_MANAGE_CLVM_CHECK_USE_CACHE: "false"
|
CHIA_MANAGE_CLVM_CHECK_USE_CACHE: "false"
|
||||||
|
CHIA_MANAGE_MYPY_CHECK_EXCLUSIONS: "true"
|
||||||
run: pre-commit run --all-files --verbose
|
run: pre-commit run --all-files --verbose
|
||||||
|
|
|
@ -62,6 +62,7 @@ jobs:
|
||||||
- name: mypy
|
- name: mypy
|
||||||
command: |
|
command: |
|
||||||
echo "MYPY VERSION IS: $(mypy --version)"
|
echo "MYPY VERSION IS: $(mypy --version)"
|
||||||
|
python manage-mypy.py build-mypy-ini
|
||||||
mypy
|
mypy
|
||||||
exclude:
|
exclude:
|
||||||
- os:
|
- os:
|
||||||
|
|
|
@ -261,6 +261,7 @@ venv.bak/
|
||||||
.mypy_cache/
|
.mypy_cache/
|
||||||
.dmypy.json
|
.dmypy.json
|
||||||
dmypy.json
|
dmypy.json
|
||||||
|
mypy.ini
|
||||||
|
|
||||||
# Pyre type checker
|
# Pyre type checker
|
||||||
.pyre/
|
.pyre/
|
||||||
|
|
|
@ -44,6 +44,13 @@ repos:
|
||||||
entry: ./activated.py python tools/chialispp.py .
|
entry: ./activated.py python tools/chialispp.py .
|
||||||
language: python
|
language: python
|
||||||
pass_filenames: false
|
pass_filenames: false
|
||||||
|
- repo: local
|
||||||
|
hooks:
|
||||||
|
- id: build mypy.ini
|
||||||
|
name: build mypy.ini
|
||||||
|
entry: ./activated.py python manage-mypy.py build-mypy-ini
|
||||||
|
language: system
|
||||||
|
pass_filenames: false
|
||||||
- repo: local
|
- repo: local
|
||||||
hooks:
|
hooks:
|
||||||
- id: mypy
|
- id: mypy
|
||||||
|
|
|
@ -0,0 +1,97 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from subprocess import CalledProcessError, run
|
||||||
|
from typing import List, cast
|
||||||
|
|
||||||
|
import click
|
||||||
|
|
||||||
|
file_path = Path(__file__)
|
||||||
|
here = file_path.parent
|
||||||
|
exclusion_file = here.joinpath("mypy-exclusions.txt")
|
||||||
|
|
||||||
|
|
||||||
|
def write_file(path: Path, content: str) -> None:
|
||||||
|
with path.open(mode="w", encoding="utf-8", newline="\n") as file:
|
||||||
|
file.write(content.strip() + "\n")
|
||||||
|
|
||||||
|
|
||||||
|
def get_mypy_failures() -> List[str]:
|
||||||
|
# Get a list of all mypy failures when only running mypy with the template file `mypy.ini.template`
|
||||||
|
command = [sys.executable, "activated.py", "mypy", "--config-file", "mypy.ini.template"]
|
||||||
|
try:
|
||||||
|
run(command, capture_output=True, check=True, encoding="utf-8")
|
||||||
|
except CalledProcessError as e:
|
||||||
|
if e.returncode == 1:
|
||||||
|
return cast(List[str], e.stdout.splitlines())
|
||||||
|
raise click.ClickException(f"Unexpected mypy failure:\n{e.stderr}") from e
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def split_mypy_failure(line: str) -> List[str]:
|
||||||
|
return list(Path(line[: line.find(".py")]).parts)
|
||||||
|
|
||||||
|
|
||||||
|
def build_exclusion_list(mypy_failures: List[str]) -> List[str]:
|
||||||
|
# Create content for `mypy-exclusions.txt` from a list of mypy failures which look like:
|
||||||
|
# # chia/cmds/wallet_funcs.py:1251: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] # noqa
|
||||||
|
return sorted({".".join(split_mypy_failure(line)) for line in mypy_failures[:-1]})
|
||||||
|
|
||||||
|
|
||||||
|
@click.group()
|
||||||
|
def main() -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
@main.command()
|
||||||
|
@click.option("--check-exclusions/--no-check-exclusions", show_default=True, envvar="CHIA_MANAGE_MYPY_CHECK_EXCLUSIONS")
|
||||||
|
def build_mypy_ini(check_exclusions: bool = False) -> None:
|
||||||
|
if not exclusion_file.exists():
|
||||||
|
raise click.ClickException(f"{exclusion_file.name} missing, run `{file_path.name} build-exclusions`")
|
||||||
|
exclusion_file_content = exclusion_file.read_text(encoding="utf-8").splitlines()
|
||||||
|
exclusion_lines = [line for line in exclusion_file_content if not line.startswith("#") and len(line.strip()) > 0]
|
||||||
|
if check_exclusions:
|
||||||
|
mypy_failures = get_mypy_failures()
|
||||||
|
updated_exclusions = build_exclusion_list(mypy_failures)
|
||||||
|
# Compare the old content with the new content and fail if some file without issues is excluded.
|
||||||
|
updated_set = set(updated_exclusions)
|
||||||
|
old_set = set(exclusion_lines)
|
||||||
|
if updated_set != old_set:
|
||||||
|
fixed = "\n".join(f" -> {entry}" for entry in sorted(old_set - updated_set))
|
||||||
|
if len(fixed) > 0:
|
||||||
|
raise click.ClickException(
|
||||||
|
f"The following fixed files need to be dropped from {exclusion_file.name}:\n{fixed}"
|
||||||
|
)
|
||||||
|
new_exclusions = sorted(updated_set - old_set)
|
||||||
|
new_failures = sorted(
|
||||||
|
line.strip()
|
||||||
|
for line in mypy_failures
|
||||||
|
if any(exclusion.split(".") == split_mypy_failure(line) for exclusion in new_exclusions)
|
||||||
|
)
|
||||||
|
if len(new_failures) > 0:
|
||||||
|
new_failures_string = "\n".join(new_failures)
|
||||||
|
raise click.ClickException(f"The following new issues have been introduced:\n{new_failures_string}")
|
||||||
|
|
||||||
|
# Create the `mypy.ini` with all entries from `mypy-exclusions.txt`
|
||||||
|
exclusion_section = f"[mypy-{','.join(exclusion_lines)}]"
|
||||||
|
mypy_config_data = (
|
||||||
|
here.joinpath("mypy.ini.template")
|
||||||
|
.read_text(encoding="utf-8")
|
||||||
|
.replace("[mypy-chia-exclusions]", exclusion_section)
|
||||||
|
)
|
||||||
|
write_file(here.joinpath("mypy.ini"), mypy_config_data)
|
||||||
|
|
||||||
|
|
||||||
|
@main.command()
|
||||||
|
def build_exclusions() -> None:
|
||||||
|
updated_file_content = [
|
||||||
|
f"# File created by: python {file_path.name} build-exclusions",
|
||||||
|
*build_exclusion_list(get_mypy_failures()),
|
||||||
|
]
|
||||||
|
write_file(exclusion_file, "\n".join(updated_file_content))
|
||||||
|
|
||||||
|
|
||||||
|
sys.exit(main())
|
|
@ -0,0 +1,201 @@
|
||||||
|
# File created by: python manage-mypy.py build-exclusions
|
||||||
|
chia.cmds.plots
|
||||||
|
chia.cmds.plotters
|
||||||
|
chia.cmds.start_funcs
|
||||||
|
chia.cmds.wallet
|
||||||
|
chia.cmds.wallet_funcs
|
||||||
|
chia.daemon.server
|
||||||
|
chia.data_layer.data_layer_wallet
|
||||||
|
chia.farmer.farmer_api
|
||||||
|
chia.introducer.introducer
|
||||||
|
chia.introducer.introducer_api
|
||||||
|
chia.plotters.bladebit
|
||||||
|
chia.plotters.chiapos
|
||||||
|
chia.plotters.madmax
|
||||||
|
chia.plotters.plotters
|
||||||
|
chia.plotters.plotters_util
|
||||||
|
chia.plotting.manager
|
||||||
|
chia.plotting.util
|
||||||
|
chia.pools.pool_puzzles
|
||||||
|
chia.pools.pool_wallet
|
||||||
|
chia.pools.pool_wallet_info
|
||||||
|
chia.rpc.harvester_rpc_api
|
||||||
|
chia.rpc.harvester_rpc_client
|
||||||
|
chia.rpc.rpc_client
|
||||||
|
chia.rpc.util
|
||||||
|
chia.rpc.wallet_rpc_api
|
||||||
|
chia.rpc.wallet_rpc_client
|
||||||
|
chia.seeder.crawl_store
|
||||||
|
chia.seeder.crawler
|
||||||
|
chia.seeder.crawler_api
|
||||||
|
chia.seeder.dns_server
|
||||||
|
chia.seeder.peer_record
|
||||||
|
chia.seeder.start_crawler
|
||||||
|
chia.server.start_harvester
|
||||||
|
chia.simulator.block_tools
|
||||||
|
chia.simulator.full_node_simulator
|
||||||
|
chia.simulator.keyring
|
||||||
|
chia.simulator.setup_services
|
||||||
|
chia.simulator.start_simulator
|
||||||
|
chia.simulator.time_out_assert
|
||||||
|
chia.simulator.wallet_tools
|
||||||
|
chia.ssl.create_ssl
|
||||||
|
chia.timelord.iters_from_block
|
||||||
|
chia.timelord.timelord
|
||||||
|
chia.timelord.timelord_api
|
||||||
|
chia.timelord.timelord_launcher
|
||||||
|
chia.timelord.timelord_state
|
||||||
|
chia.types.blockchain_format.program
|
||||||
|
chia.util.block_cache
|
||||||
|
chia.util.chia_logging
|
||||||
|
chia.util.config
|
||||||
|
chia.util.db_wrapper
|
||||||
|
chia.util.hash
|
||||||
|
chia.util.json_util
|
||||||
|
chia.util.keychain
|
||||||
|
chia.util.keyring_wrapper
|
||||||
|
chia.util.log_exceptions
|
||||||
|
chia.util.make_test_constants
|
||||||
|
chia.util.merkle_set
|
||||||
|
chia.util.partial_func
|
||||||
|
chia.util.profiler
|
||||||
|
chia.util.safe_cancel_task
|
||||||
|
chia.util.service_groups
|
||||||
|
chia.util.ssl_check
|
||||||
|
chia.wallet.block_record
|
||||||
|
chia.wallet.cat_wallet.cat_wallet
|
||||||
|
chia.wallet.chialisp
|
||||||
|
chia.wallet.did_wallet.did_wallet
|
||||||
|
chia.wallet.did_wallet.did_wallet_puzzles
|
||||||
|
chia.wallet.key_val_store
|
||||||
|
chia.wallet.lineage_proof
|
||||||
|
chia.wallet.nft_wallet.nft_puzzles
|
||||||
|
chia.wallet.payment
|
||||||
|
chia.wallet.puzzles.load_clvm
|
||||||
|
chia.wallet.puzzles.p2_conditions
|
||||||
|
chia.wallet.puzzles.p2_delegated_conditions
|
||||||
|
chia.wallet.puzzles.p2_delegated_puzzle
|
||||||
|
chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle
|
||||||
|
chia.wallet.puzzles.p2_m_of_n_delegate_direct
|
||||||
|
chia.wallet.puzzles.p2_puzzle_hash
|
||||||
|
chia.wallet.puzzles.prefarm.spend_prefarm
|
||||||
|
chia.wallet.puzzles.puzzle_utils
|
||||||
|
chia.wallet.puzzles.singleton_top_layer
|
||||||
|
chia.wallet.puzzles.tails
|
||||||
|
chia.wallet.secret_key_store
|
||||||
|
chia.wallet.trading.trade_store
|
||||||
|
chia.wallet.transaction_record
|
||||||
|
chia.wallet.util.debug_spend_bundle
|
||||||
|
chia.wallet.util.new_peak_queue
|
||||||
|
chia.wallet.wallet
|
||||||
|
chia.wallet.wallet_coin_store
|
||||||
|
chia.wallet.wallet_interested_store
|
||||||
|
chia.wallet.wallet_node_api
|
||||||
|
chia.wallet.wallet_pool_store
|
||||||
|
chia.wallet.wallet_puzzle_store
|
||||||
|
chia.wallet.wallet_transaction_store
|
||||||
|
chia.wallet.wallet_user_store
|
||||||
|
installhelper
|
||||||
|
tests.blockchain.blockchain_test_utils
|
||||||
|
tests.blockchain.test_blockchain
|
||||||
|
tests.build-init-files
|
||||||
|
tests.clvm.coin_store
|
||||||
|
tests.clvm.test_chialisp_deserialization
|
||||||
|
tests.clvm.test_program
|
||||||
|
tests.clvm.test_puzzle_compression
|
||||||
|
tests.clvm.test_puzzles
|
||||||
|
tests.clvm.test_serialized_program
|
||||||
|
tests.clvm.test_singletons
|
||||||
|
tests.clvm.test_spend_sim
|
||||||
|
tests.conftest
|
||||||
|
tests.connection_utils
|
||||||
|
tests.core.cmds.test_keys
|
||||||
|
tests.core.consensus.test_pot_iterations
|
||||||
|
tests.core.custom_types.test_coin
|
||||||
|
tests.core.custom_types.test_spend_bundle
|
||||||
|
tests.core.daemon.test_daemon
|
||||||
|
tests.core.full_node.full_sync.test_full_sync
|
||||||
|
tests.core.full_node.stores.test_block_store
|
||||||
|
tests.core.full_node.stores.test_coin_store
|
||||||
|
tests.core.full_node.stores.test_full_node_store
|
||||||
|
tests.core.full_node.stores.test_hint_store
|
||||||
|
tests.core.full_node.stores.test_sync_store
|
||||||
|
tests.core.full_node.test_address_manager
|
||||||
|
tests.core.full_node.test_block_height_map
|
||||||
|
tests.core.full_node.test_conditions
|
||||||
|
tests.core.full_node.test_full_node
|
||||||
|
tests.core.full_node.test_node_load
|
||||||
|
tests.core.full_node.test_peer_store_resolver
|
||||||
|
tests.core.full_node.test_performance
|
||||||
|
tests.core.full_node.test_transactions
|
||||||
|
tests.core.make_block_generator
|
||||||
|
tests.core.mempool.test_mempool
|
||||||
|
tests.core.mempool.test_mempool_performance
|
||||||
|
tests.core.node_height
|
||||||
|
tests.core.server.test_dos
|
||||||
|
tests.core.server.test_rate_limits
|
||||||
|
tests.core.ssl.test_ssl
|
||||||
|
tests.core.test_cost_calculation
|
||||||
|
tests.core.test_crawler_rpc
|
||||||
|
tests.core.test_daemon_rpc
|
||||||
|
tests.core.test_db_conversion
|
||||||
|
tests.core.test_farmer_harvester_rpc
|
||||||
|
tests.core.test_filter
|
||||||
|
tests.core.test_full_node_rpc
|
||||||
|
tests.core.test_merkle_set
|
||||||
|
tests.core.test_setproctitle
|
||||||
|
tests.core.util.test_cached_bls
|
||||||
|
tests.core.util.test_config
|
||||||
|
tests.core.util.test_file_keyring_synchronization
|
||||||
|
tests.core.util.test_files
|
||||||
|
tests.core.util.test_keychain
|
||||||
|
tests.core.util.test_keyring_wrapper
|
||||||
|
tests.core.util.test_lru_cache
|
||||||
|
tests.core.util.test_significant_bits
|
||||||
|
tests.farmer_harvester.test_farmer_harvester
|
||||||
|
tests.generator.test_list_to_batches
|
||||||
|
tests.generator.test_scan
|
||||||
|
tests.plot_sync.test_plot_sync
|
||||||
|
tests.plot_sync.test_sync_simulated
|
||||||
|
tests.plotting.test_plot_manager
|
||||||
|
tests.pools.test_pool_cmdline
|
||||||
|
tests.pools.test_pool_config
|
||||||
|
tests.pools.test_pool_puzzles_lifecycle
|
||||||
|
tests.pools.test_wallet_pool_store
|
||||||
|
tests.simulation.test_simulation
|
||||||
|
tests.tools.test_full_sync
|
||||||
|
tests.tools.test_run_block
|
||||||
|
tests.util.benchmark_cost
|
||||||
|
tests.util.build_network_protocol_files
|
||||||
|
tests.util.db_connection
|
||||||
|
tests.util.generator_tools_testing
|
||||||
|
tests.util.key_tool
|
||||||
|
tests.util.test_full_block_utils
|
||||||
|
tests.util.test_lock_queue
|
||||||
|
tests.util.test_misc
|
||||||
|
tests.util.test_network
|
||||||
|
tests.wallet.cat_wallet.test_cat_lifecycle
|
||||||
|
tests.wallet.cat_wallet.test_cat_wallet
|
||||||
|
tests.wallet.cat_wallet.test_offer_lifecycle
|
||||||
|
tests.wallet.cat_wallet.test_trades
|
||||||
|
tests.wallet.did_wallet.test_did
|
||||||
|
tests.wallet.nft_wallet.test_nft_puzzles
|
||||||
|
tests.wallet.nft_wallet.test_nft_wallet
|
||||||
|
tests.wallet.rpc.test_wallet_rpc
|
||||||
|
tests.wallet.simple_sync.test_simple_sync_protocol
|
||||||
|
tests.wallet.sync.test_wallet_sync
|
||||||
|
tests.wallet.test_bech32m
|
||||||
|
tests.wallet.test_chialisp
|
||||||
|
tests.wallet.test_puzzle_store
|
||||||
|
tests.wallet.test_singleton
|
||||||
|
tests.wallet.test_singleton_lifecycle
|
||||||
|
tests.wallet.test_singleton_lifecycle_fast
|
||||||
|
tests.wallet.test_taproot
|
||||||
|
tests.wallet.test_wallet_blockchain
|
||||||
|
tests.wallet.test_wallet_interested_store
|
||||||
|
tests.wallet.test_wallet_key_val_store
|
||||||
|
tests.wallet.test_wallet_user_store
|
||||||
|
tests.weight_proof.test_weight_proof
|
||||||
|
tools.analyze-chain
|
||||||
|
tools.run_block
|
||||||
|
tools.test_full_sync
|
|
@ -0,0 +1,32 @@
|
||||||
|
[mypy]
|
||||||
|
files = benchmarks,build_scripts,chia,tests,tools,*.py
|
||||||
|
ignore_missing_imports = True
|
||||||
|
show_error_codes = True
|
||||||
|
warn_unused_ignores = True
|
||||||
|
|
||||||
|
disallow_any_generics = True
|
||||||
|
disallow_subclassing_any = True
|
||||||
|
disallow_untyped_calls = True
|
||||||
|
disallow_untyped_defs = True
|
||||||
|
disallow_incomplete_defs = True
|
||||||
|
check_untyped_defs = True
|
||||||
|
disallow_untyped_decorators = True
|
||||||
|
no_implicit_optional = True
|
||||||
|
warn_return_any = True
|
||||||
|
no_implicit_reexport = True
|
||||||
|
strict_equality = True
|
||||||
|
warn_redundant_casts = True
|
||||||
|
|
||||||
|
[mypy-chia-exclusions]
|
||||||
|
disable_error_code = annotation-unchecked
|
||||||
|
disallow_any_generics = False
|
||||||
|
disallow_subclassing_any = False
|
||||||
|
disallow_untyped_calls = False
|
||||||
|
disallow_untyped_defs = False
|
||||||
|
disallow_incomplete_defs = False
|
||||||
|
check_untyped_defs = False
|
||||||
|
disallow_untyped_decorators = False
|
||||||
|
no_implicit_optional = False
|
||||||
|
warn_return_any = False
|
||||||
|
no_implicit_reexport = False
|
||||||
|
strict_equality = False
|
Loading…
Reference in New Issue