mypy: Build `mypy.ini` config with exclusions dynamically (#15158)
* mypy: Build `mypy.ini` config with exclusions dynamically * Apply suggestions from code review * Revert suggested `check=True` because `mypy` is expected to fails here * Only `read_text` once * Fix exception message * Don't `touch` * More `joinpath` * Revert `newline="\n"`, its python 3.10 only See https://bugs.python.org/issue23706 * Apply suggestions * Fail if new issues get introduced * Print the newly introduces issues * Handle error codes * Add `write_file` * Use `sys.executable` Co-authored-by: Kyle Altendorf <sda@fstab.net> * `mypy_failures[:-1]` * Add an example comment * Improve failure check * Drop exclusions after rebase * Drop one more exclusion after another rebase --------- Co-authored-by: Kyle Altendorf <sda@fstab.net>
This commit is contained in:
dustinface
GitHub
parent
fab36d5b4e
commit
ec5f85dfc4
|
|
@ -74,4 +74,5 @@ jobs:
|
|||
|
||||
- env:
|
||||
CHIA_MANAGE_CLVM_CHECK_USE_CACHE: "false"
|
||||
CHIA_MANAGE_MYPY_CHECK_EXCLUSIONS: "true"
|
||||
run: pre-commit run --all-files --verbose
|
||||
|
|
|
|||
|
|
@ -62,6 +62,7 @@ jobs:
|
|||
- name: mypy
|
||||
command: |
|
||||
echo "MYPY VERSION IS: $(mypy --version)"
|
||||
python manage-mypy.py build-mypy-ini
|
||||
mypy
|
||||
exclude:
|
||||
- os:
|
||||
|
|
|
|||
|
|
@ -261,6 +261,7 @@ venv.bak/
|
|||
.mypy_cache/
|
||||
.dmypy.json
|
||||
dmypy.json
|
||||
mypy.ini
|
||||
|
||||
# Pyre type checker
|
||||
.pyre/
|
||||
|
|
|
|||
|
|
@ -44,6 +44,13 @@ repos:
|
|||
entry: ./activated.py python tools/chialispp.py .
|
||||
language: python
|
||||
pass_filenames: false
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: build mypy.ini
|
||||
name: build mypy.ini
|
||||
entry: ./activated.py python manage-mypy.py build-mypy-ini
|
||||
language: system
|
||||
pass_filenames: false
|
||||
- repo: local
|
||||
hooks:
|
||||
- id: mypy
|
||||
|
|
|
|||
|
|
@ -0,0 +1,97 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from subprocess import CalledProcessError, run
|
||||
from typing import List, cast
|
||||
|
||||
import click
|
||||
|
||||
file_path = Path(__file__)
|
||||
here = file_path.parent
|
||||
exclusion_file = here.joinpath("mypy-exclusions.txt")
|
||||
|
||||
|
||||
def write_file(path: Path, content: str) -> None:
|
||||
with path.open(mode="w", encoding="utf-8", newline="\n") as file:
|
||||
file.write(content.strip() + "\n")
|
||||
|
||||
|
||||
def get_mypy_failures() -> List[str]:
|
||||
# Get a list of all mypy failures when only running mypy with the template file `mypy.ini.template`
|
||||
command = [sys.executable, "activated.py", "mypy", "--config-file", "mypy.ini.template"]
|
||||
try:
|
||||
run(command, capture_output=True, check=True, encoding="utf-8")
|
||||
except CalledProcessError as e:
|
||||
if e.returncode == 1:
|
||||
return cast(List[str], e.stdout.splitlines())
|
||||
raise click.ClickException(f"Unexpected mypy failure:\n{e.stderr}") from e
|
||||
return []
|
||||
|
||||
|
||||
def split_mypy_failure(line: str) -> List[str]:
|
||||
return list(Path(line[: line.find(".py")]).parts)
|
||||
|
||||
|
||||
def build_exclusion_list(mypy_failures: List[str]) -> List[str]:
|
||||
# Create content for `mypy-exclusions.txt` from a list of mypy failures which look like:
|
||||
# # chia/cmds/wallet_funcs.py:1251: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] # noqa
|
||||
return sorted({".".join(split_mypy_failure(line)) for line in mypy_failures[:-1]})
|
||||
|
||||
|
||||
@click.group()
|
||||
def main() -> None:
|
||||
pass
|
||||
|
||||
|
||||
@main.command()
|
||||
@click.option("--check-exclusions/--no-check-exclusions", show_default=True, envvar="CHIA_MANAGE_MYPY_CHECK_EXCLUSIONS")
|
||||
def build_mypy_ini(check_exclusions: bool = False) -> None:
|
||||
if not exclusion_file.exists():
|
||||
raise click.ClickException(f"{exclusion_file.name} missing, run `{file_path.name} build-exclusions`")
|
||||
exclusion_file_content = exclusion_file.read_text(encoding="utf-8").splitlines()
|
||||
exclusion_lines = [line for line in exclusion_file_content if not line.startswith("#") and len(line.strip()) > 0]
|
||||
if check_exclusions:
|
||||
mypy_failures = get_mypy_failures()
|
||||
updated_exclusions = build_exclusion_list(mypy_failures)
|
||||
# Compare the old content with the new content and fail if some file without issues is excluded.
|
||||
updated_set = set(updated_exclusions)
|
||||
old_set = set(exclusion_lines)
|
||||
if updated_set != old_set:
|
||||
fixed = "\n".join(f" -> {entry}" for entry in sorted(old_set - updated_set))
|
||||
if len(fixed) > 0:
|
||||
raise click.ClickException(
|
||||
f"The following fixed files need to be dropped from {exclusion_file.name}:\n{fixed}"
|
||||
)
|
||||
new_exclusions = sorted(updated_set - old_set)
|
||||
new_failures = sorted(
|
||||
line.strip()
|
||||
for line in mypy_failures
|
||||
if any(exclusion.split(".") == split_mypy_failure(line) for exclusion in new_exclusions)
|
||||
)
|
||||
if len(new_failures) > 0:
|
||||
new_failures_string = "\n".join(new_failures)
|
||||
raise click.ClickException(f"The following new issues have been introduced:\n{new_failures_string}")
|
||||
|
||||
# Create the `mypy.ini` with all entries from `mypy-exclusions.txt`
|
||||
exclusion_section = f"[mypy-{','.join(exclusion_lines)}]"
|
||||
mypy_config_data = (
|
||||
here.joinpath("mypy.ini.template")
|
||||
.read_text(encoding="utf-8")
|
||||
.replace("[mypy-chia-exclusions]", exclusion_section)
|
||||
)
|
||||
write_file(here.joinpath("mypy.ini"), mypy_config_data)
|
||||
|
||||
|
||||
@main.command()
|
||||
def build_exclusions() -> None:
|
||||
updated_file_content = [
|
||||
f"# File created by: python {file_path.name} build-exclusions",
|
||||
*build_exclusion_list(get_mypy_failures()),
|
||||
]
|
||||
write_file(exclusion_file, "\n".join(updated_file_content))
|
||||
|
||||
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,201 @@
|
|||
# File created by: python manage-mypy.py build-exclusions
|
||||
chia.cmds.plots
|
||||
chia.cmds.plotters
|
||||
chia.cmds.start_funcs
|
||||
chia.cmds.wallet
|
||||
chia.cmds.wallet_funcs
|
||||
chia.daemon.server
|
||||
chia.data_layer.data_layer_wallet
|
||||
chia.farmer.farmer_api
|
||||
chia.introducer.introducer
|
||||
chia.introducer.introducer_api
|
||||
chia.plotters.bladebit
|
||||
chia.plotters.chiapos
|
||||
chia.plotters.madmax
|
||||
chia.plotters.plotters
|
||||
chia.plotters.plotters_util
|
||||
chia.plotting.manager
|
||||
chia.plotting.util
|
||||
chia.pools.pool_puzzles
|
||||
chia.pools.pool_wallet
|
||||
chia.pools.pool_wallet_info
|
||||
chia.rpc.harvester_rpc_api
|
||||
chia.rpc.harvester_rpc_client
|
||||
chia.rpc.rpc_client
|
||||
chia.rpc.util
|
||||
chia.rpc.wallet_rpc_api
|
||||
chia.rpc.wallet_rpc_client
|
||||
chia.seeder.crawl_store
|
||||
chia.seeder.crawler
|
||||
chia.seeder.crawler_api
|
||||
chia.seeder.dns_server
|
||||
chia.seeder.peer_record
|
||||
chia.seeder.start_crawler
|
||||
chia.server.start_harvester
|
||||
chia.simulator.block_tools
|
||||
chia.simulator.full_node_simulator
|
||||
chia.simulator.keyring
|
||||
chia.simulator.setup_services
|
||||
chia.simulator.start_simulator
|
||||
chia.simulator.time_out_assert
|
||||
chia.simulator.wallet_tools
|
||||
chia.ssl.create_ssl
|
||||
chia.timelord.iters_from_block
|
||||
chia.timelord.timelord
|
||||
chia.timelord.timelord_api
|
||||
chia.timelord.timelord_launcher
|
||||
chia.timelord.timelord_state
|
||||
chia.types.blockchain_format.program
|
||||
chia.util.block_cache
|
||||
chia.util.chia_logging
|
||||
chia.util.config
|
||||
chia.util.db_wrapper
|
||||
chia.util.hash
|
||||
chia.util.json_util
|
||||
chia.util.keychain
|
||||
chia.util.keyring_wrapper
|
||||
chia.util.log_exceptions
|
||||
chia.util.make_test_constants
|
||||
chia.util.merkle_set
|
||||
chia.util.partial_func
|
||||
chia.util.profiler
|
||||
chia.util.safe_cancel_task
|
||||
chia.util.service_groups
|
||||
chia.util.ssl_check
|
||||
chia.wallet.block_record
|
||||
chia.wallet.cat_wallet.cat_wallet
|
||||
chia.wallet.chialisp
|
||||
chia.wallet.did_wallet.did_wallet
|
||||
chia.wallet.did_wallet.did_wallet_puzzles
|
||||
chia.wallet.key_val_store
|
||||
chia.wallet.lineage_proof
|
||||
chia.wallet.nft_wallet.nft_puzzles
|
||||
chia.wallet.payment
|
||||
chia.wallet.puzzles.load_clvm
|
||||
chia.wallet.puzzles.p2_conditions
|
||||
chia.wallet.puzzles.p2_delegated_conditions
|
||||
chia.wallet.puzzles.p2_delegated_puzzle
|
||||
chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle
|
||||
chia.wallet.puzzles.p2_m_of_n_delegate_direct
|
||||
chia.wallet.puzzles.p2_puzzle_hash
|
||||
chia.wallet.puzzles.prefarm.spend_prefarm
|
||||
chia.wallet.puzzles.puzzle_utils
|
||||
chia.wallet.puzzles.singleton_top_layer
|
||||
chia.wallet.puzzles.tails
|
||||
chia.wallet.secret_key_store
|
||||
chia.wallet.trading.trade_store
|
||||
chia.wallet.transaction_record
|
||||
chia.wallet.util.debug_spend_bundle
|
||||
chia.wallet.util.new_peak_queue
|
||||
chia.wallet.wallet
|
||||
chia.wallet.wallet_coin_store
|
||||
chia.wallet.wallet_interested_store
|
||||
chia.wallet.wallet_node_api
|
||||
chia.wallet.wallet_pool_store
|
||||
chia.wallet.wallet_puzzle_store
|
||||
chia.wallet.wallet_transaction_store
|
||||
chia.wallet.wallet_user_store
|
||||
installhelper
|
||||
tests.blockchain.blockchain_test_utils
|
||||
tests.blockchain.test_blockchain
|
||||
tests.build-init-files
|
||||
tests.clvm.coin_store
|
||||
tests.clvm.test_chialisp_deserialization
|
||||
tests.clvm.test_program
|
||||
tests.clvm.test_puzzle_compression
|
||||
tests.clvm.test_puzzles
|
||||
tests.clvm.test_serialized_program
|
||||
tests.clvm.test_singletons
|
||||
tests.clvm.test_spend_sim
|
||||
tests.conftest
|
||||
tests.connection_utils
|
||||
tests.core.cmds.test_keys
|
||||
tests.core.consensus.test_pot_iterations
|
||||
tests.core.custom_types.test_coin
|
||||
tests.core.custom_types.test_spend_bundle
|
||||
tests.core.daemon.test_daemon
|
||||
tests.core.full_node.full_sync.test_full_sync
|
||||
tests.core.full_node.stores.test_block_store
|
||||
tests.core.full_node.stores.test_coin_store
|
||||
tests.core.full_node.stores.test_full_node_store
|
||||
tests.core.full_node.stores.test_hint_store
|
||||
tests.core.full_node.stores.test_sync_store
|
||||
tests.core.full_node.test_address_manager
|
||||
tests.core.full_node.test_block_height_map
|
||||
tests.core.full_node.test_conditions
|
||||
tests.core.full_node.test_full_node
|
||||
tests.core.full_node.test_node_load
|
||||
tests.core.full_node.test_peer_store_resolver
|
||||
tests.core.full_node.test_performance
|
||||
tests.core.full_node.test_transactions
|
||||
tests.core.make_block_generator
|
||||
tests.core.mempool.test_mempool
|
||||
tests.core.mempool.test_mempool_performance
|
||||
tests.core.node_height
|
||||
tests.core.server.test_dos
|
||||
tests.core.server.test_rate_limits
|
||||
tests.core.ssl.test_ssl
|
||||
tests.core.test_cost_calculation
|
||||
tests.core.test_crawler_rpc
|
||||
tests.core.test_daemon_rpc
|
||||
tests.core.test_db_conversion
|
||||
tests.core.test_farmer_harvester_rpc
|
||||
tests.core.test_filter
|
||||
tests.core.test_full_node_rpc
|
||||
tests.core.test_merkle_set
|
||||
tests.core.test_setproctitle
|
||||
tests.core.util.test_cached_bls
|
||||
tests.core.util.test_config
|
||||
tests.core.util.test_file_keyring_synchronization
|
||||
tests.core.util.test_files
|
||||
tests.core.util.test_keychain
|
||||
tests.core.util.test_keyring_wrapper
|
||||
tests.core.util.test_lru_cache
|
||||
tests.core.util.test_significant_bits
|
||||
tests.farmer_harvester.test_farmer_harvester
|
||||
tests.generator.test_list_to_batches
|
||||
tests.generator.test_scan
|
||||
tests.plot_sync.test_plot_sync
|
||||
tests.plot_sync.test_sync_simulated
|
||||
tests.plotting.test_plot_manager
|
||||
tests.pools.test_pool_cmdline
|
||||
tests.pools.test_pool_config
|
||||
tests.pools.test_pool_puzzles_lifecycle
|
||||
tests.pools.test_wallet_pool_store
|
||||
tests.simulation.test_simulation
|
||||
tests.tools.test_full_sync
|
||||
tests.tools.test_run_block
|
||||
tests.util.benchmark_cost
|
||||
tests.util.build_network_protocol_files
|
||||
tests.util.db_connection
|
||||
tests.util.generator_tools_testing
|
||||
tests.util.key_tool
|
||||
tests.util.test_full_block_utils
|
||||
tests.util.test_lock_queue
|
||||
tests.util.test_misc
|
||||
tests.util.test_network
|
||||
tests.wallet.cat_wallet.test_cat_lifecycle
|
||||
tests.wallet.cat_wallet.test_cat_wallet
|
||||
tests.wallet.cat_wallet.test_offer_lifecycle
|
||||
tests.wallet.cat_wallet.test_trades
|
||||
tests.wallet.did_wallet.test_did
|
||||
tests.wallet.nft_wallet.test_nft_puzzles
|
||||
tests.wallet.nft_wallet.test_nft_wallet
|
||||
tests.wallet.rpc.test_wallet_rpc
|
||||
tests.wallet.simple_sync.test_simple_sync_protocol
|
||||
tests.wallet.sync.test_wallet_sync
|
||||
tests.wallet.test_bech32m
|
||||
tests.wallet.test_chialisp
|
||||
tests.wallet.test_puzzle_store
|
||||
tests.wallet.test_singleton
|
||||
tests.wallet.test_singleton_lifecycle
|
||||
tests.wallet.test_singleton_lifecycle_fast
|
||||
tests.wallet.test_taproot
|
||||
tests.wallet.test_wallet_blockchain
|
||||
tests.wallet.test_wallet_interested_store
|
||||
tests.wallet.test_wallet_key_val_store
|
||||
tests.wallet.test_wallet_user_store
|
||||
tests.weight_proof.test_weight_proof
|
||||
tools.analyze-chain
|
||||
tools.run_block
|
||||
tools.test_full_sync
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
[mypy]
|
||||
files = benchmarks,build_scripts,chia,tests,tools,*.py
|
||||
ignore_missing_imports = True
|
||||
show_error_codes = True
|
||||
warn_unused_ignores = True
|
||||
|
||||
disallow_any_generics = True
|
||||
disallow_subclassing_any = True
|
||||
disallow_untyped_calls = True
|
||||
disallow_untyped_defs = True
|
||||
disallow_incomplete_defs = True
|
||||
check_untyped_defs = True
|
||||
disallow_untyped_decorators = True
|
||||
no_implicit_optional = True
|
||||
warn_return_any = True
|
||||
no_implicit_reexport = True
|
||||
strict_equality = True
|
||||
warn_redundant_casts = True
|
||||
|
||||
[mypy-chia-exclusions]
|
||||
disable_error_code = annotation-unchecked
|
||||
disallow_any_generics = False
|
||||
disallow_subclassing_any = False
|
||||
disallow_untyped_calls = False
|
||||
disallow_untyped_defs = False
|
||||
disallow_incomplete_defs = False
|
||||
check_untyped_defs = False
|
||||
disallow_untyped_decorators = False
|
||||
no_implicit_optional = False
|
||||
warn_return_any = False
|
||||
no_implicit_reexport = False
|
||||
strict_equality = False
|
||||
Loading…
Reference in New Issue