From ec5f85dfc46c2114f595643512fc3f727893977c Mon Sep 17 00:00:00 2001 From: dustinface <35775977+xdustinface@users.noreply.github.com> Date: Mon, 22 May 2023 23:50:13 +0200 Subject: [PATCH] mypy: Build `mypy.ini` config with exclusions dynamically (#15158) * mypy: Build `mypy.ini` config with exclusions dynamically * Apply suggestions from code review * Revert suggested `check=True` because `mypy` is expected to fails here * Only `read_text` once * Fix exception message * Don't `touch` * More `joinpath` * Revert `newline="\n"`, its python 3.10 only See https://bugs.python.org/issue23706 * Apply suggestions * Fail if new issues get introduced * Print the newly introduces issues * Handle error codes * Add `write_file` * Use `sys.executable` Co-authored-by: Kyle Altendorf * `mypy_failures[:-1]` * Add an example comment * Improve failure check * Drop exclusions after rebase * Drop one more exclusion after another rebase --------- Co-authored-by: Kyle Altendorf --- .github/workflows/pre-commit.yml | 1 + .github/workflows/upload-pypi-source.yml | 1 + .gitignore | 1 + .pre-commit-config.yaml | 7 + manage-mypy.py | 97 +++++++++++ mypy-exclusions.txt | 201 +++++++++++++++++++++++ mypy.ini | 33 ---- mypy.ini.template | 32 ++++ 8 files changed, 340 insertions(+), 33 deletions(-) create mode 100755 manage-mypy.py create mode 100644 mypy-exclusions.txt delete mode 100644 mypy.ini create mode 100644 mypy.ini.template diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e847826953..aa4b86fd68 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -74,4 +74,5 @@ jobs: - env: CHIA_MANAGE_CLVM_CHECK_USE_CACHE: "false" + CHIA_MANAGE_MYPY_CHECK_EXCLUSIONS: "true" run: pre-commit run --all-files --verbose diff --git a/.github/workflows/upload-pypi-source.yml b/.github/workflows/upload-pypi-source.yml index b2f8c1dfd8..1233b33d29 100644 --- a/.github/workflows/upload-pypi-source.yml +++ b/.github/workflows/upload-pypi-source.yml @@ -62,6 +62,7 @@ jobs: - name: mypy command: | echo "MYPY VERSION IS: $(mypy --version)" + python manage-mypy.py build-mypy-ini mypy exclude: - os: diff --git a/.gitignore b/.gitignore index f6de6cf092..083b948e6c 100644 --- a/.gitignore +++ b/.gitignore @@ -261,6 +261,7 @@ venv.bak/ .mypy_cache/ .dmypy.json dmypy.json +mypy.ini # Pyre type checker .pyre/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f457e38941..e051394b72 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -44,6 +44,13 @@ repos: entry: ./activated.py python tools/chialispp.py . language: python pass_filenames: false +- repo: local + hooks: + - id: build mypy.ini + name: build mypy.ini + entry: ./activated.py python manage-mypy.py build-mypy-ini + language: system + pass_filenames: false - repo: local hooks: - id: mypy diff --git a/manage-mypy.py b/manage-mypy.py new file mode 100755 index 0000000000..fb2cd07dbd --- /dev/null +++ b/manage-mypy.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import sys +from pathlib import Path +from subprocess import CalledProcessError, run +from typing import List, cast + +import click + +file_path = Path(__file__) +here = file_path.parent +exclusion_file = here.joinpath("mypy-exclusions.txt") + + +def write_file(path: Path, content: str) -> None: + with path.open(mode="w", encoding="utf-8", newline="\n") as file: + file.write(content.strip() + "\n") + + +def get_mypy_failures() -> List[str]: + # Get a list of all mypy failures when only running mypy with the template file `mypy.ini.template` + command = [sys.executable, "activated.py", "mypy", "--config-file", "mypy.ini.template"] + try: + run(command, capture_output=True, check=True, encoding="utf-8") + except CalledProcessError as e: + if e.returncode == 1: + return cast(List[str], e.stdout.splitlines()) + raise click.ClickException(f"Unexpected mypy failure:\n{e.stderr}") from e + return [] + + +def split_mypy_failure(line: str) -> List[str]: + return list(Path(line[: line.find(".py")]).parts) + + +def build_exclusion_list(mypy_failures: List[str]) -> List[str]: + # Create content for `mypy-exclusions.txt` from a list of mypy failures which look like: + # # chia/cmds/wallet_funcs.py:1251: error: Incompatible types in assignment (expression has type "str", variable has type "int") [assignment] # noqa + return sorted({".".join(split_mypy_failure(line)) for line in mypy_failures[:-1]}) + + +@click.group() +def main() -> None: + pass + + +@main.command() +@click.option("--check-exclusions/--no-check-exclusions", show_default=True, envvar="CHIA_MANAGE_MYPY_CHECK_EXCLUSIONS") +def build_mypy_ini(check_exclusions: bool = False) -> None: + if not exclusion_file.exists(): + raise click.ClickException(f"{exclusion_file.name} missing, run `{file_path.name} build-exclusions`") + exclusion_file_content = exclusion_file.read_text(encoding="utf-8").splitlines() + exclusion_lines = [line for line in exclusion_file_content if not line.startswith("#") and len(line.strip()) > 0] + if check_exclusions: + mypy_failures = get_mypy_failures() + updated_exclusions = build_exclusion_list(mypy_failures) + # Compare the old content with the new content and fail if some file without issues is excluded. + updated_set = set(updated_exclusions) + old_set = set(exclusion_lines) + if updated_set != old_set: + fixed = "\n".join(f" -> {entry}" for entry in sorted(old_set - updated_set)) + if len(fixed) > 0: + raise click.ClickException( + f"The following fixed files need to be dropped from {exclusion_file.name}:\n{fixed}" + ) + new_exclusions = sorted(updated_set - old_set) + new_failures = sorted( + line.strip() + for line in mypy_failures + if any(exclusion.split(".") == split_mypy_failure(line) for exclusion in new_exclusions) + ) + if len(new_failures) > 0: + new_failures_string = "\n".join(new_failures) + raise click.ClickException(f"The following new issues have been introduced:\n{new_failures_string}") + + # Create the `mypy.ini` with all entries from `mypy-exclusions.txt` + exclusion_section = f"[mypy-{','.join(exclusion_lines)}]" + mypy_config_data = ( + here.joinpath("mypy.ini.template") + .read_text(encoding="utf-8") + .replace("[mypy-chia-exclusions]", exclusion_section) + ) + write_file(here.joinpath("mypy.ini"), mypy_config_data) + + +@main.command() +def build_exclusions() -> None: + updated_file_content = [ + f"# File created by: python {file_path.name} build-exclusions", + *build_exclusion_list(get_mypy_failures()), + ] + write_file(exclusion_file, "\n".join(updated_file_content)) + + +sys.exit(main()) diff --git a/mypy-exclusions.txt b/mypy-exclusions.txt new file mode 100644 index 0000000000..60855e27c0 --- /dev/null +++ b/mypy-exclusions.txt @@ -0,0 +1,201 @@ +# File created by: python manage-mypy.py build-exclusions +chia.cmds.plots +chia.cmds.plotters +chia.cmds.start_funcs +chia.cmds.wallet +chia.cmds.wallet_funcs +chia.daemon.server +chia.data_layer.data_layer_wallet +chia.farmer.farmer_api +chia.introducer.introducer +chia.introducer.introducer_api +chia.plotters.bladebit +chia.plotters.chiapos +chia.plotters.madmax +chia.plotters.plotters +chia.plotters.plotters_util +chia.plotting.manager +chia.plotting.util +chia.pools.pool_puzzles +chia.pools.pool_wallet +chia.pools.pool_wallet_info +chia.rpc.harvester_rpc_api +chia.rpc.harvester_rpc_client +chia.rpc.rpc_client +chia.rpc.util +chia.rpc.wallet_rpc_api +chia.rpc.wallet_rpc_client +chia.seeder.crawl_store +chia.seeder.crawler +chia.seeder.crawler_api +chia.seeder.dns_server +chia.seeder.peer_record +chia.seeder.start_crawler +chia.server.start_harvester +chia.simulator.block_tools +chia.simulator.full_node_simulator +chia.simulator.keyring +chia.simulator.setup_services +chia.simulator.start_simulator +chia.simulator.time_out_assert +chia.simulator.wallet_tools +chia.ssl.create_ssl +chia.timelord.iters_from_block +chia.timelord.timelord +chia.timelord.timelord_api +chia.timelord.timelord_launcher +chia.timelord.timelord_state +chia.types.blockchain_format.program +chia.util.block_cache +chia.util.chia_logging +chia.util.config +chia.util.db_wrapper +chia.util.hash +chia.util.json_util +chia.util.keychain +chia.util.keyring_wrapper +chia.util.log_exceptions +chia.util.make_test_constants +chia.util.merkle_set +chia.util.partial_func +chia.util.profiler +chia.util.safe_cancel_task +chia.util.service_groups +chia.util.ssl_check +chia.wallet.block_record +chia.wallet.cat_wallet.cat_wallet +chia.wallet.chialisp +chia.wallet.did_wallet.did_wallet +chia.wallet.did_wallet.did_wallet_puzzles +chia.wallet.key_val_store +chia.wallet.lineage_proof +chia.wallet.nft_wallet.nft_puzzles +chia.wallet.payment +chia.wallet.puzzles.load_clvm +chia.wallet.puzzles.p2_conditions +chia.wallet.puzzles.p2_delegated_conditions +chia.wallet.puzzles.p2_delegated_puzzle +chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle +chia.wallet.puzzles.p2_m_of_n_delegate_direct +chia.wallet.puzzles.p2_puzzle_hash +chia.wallet.puzzles.prefarm.spend_prefarm +chia.wallet.puzzles.puzzle_utils +chia.wallet.puzzles.singleton_top_layer +chia.wallet.puzzles.tails +chia.wallet.secret_key_store +chia.wallet.trading.trade_store +chia.wallet.transaction_record +chia.wallet.util.debug_spend_bundle +chia.wallet.util.new_peak_queue +chia.wallet.wallet +chia.wallet.wallet_coin_store +chia.wallet.wallet_interested_store +chia.wallet.wallet_node_api +chia.wallet.wallet_pool_store +chia.wallet.wallet_puzzle_store +chia.wallet.wallet_transaction_store +chia.wallet.wallet_user_store +installhelper +tests.blockchain.blockchain_test_utils +tests.blockchain.test_blockchain +tests.build-init-files +tests.clvm.coin_store +tests.clvm.test_chialisp_deserialization +tests.clvm.test_program +tests.clvm.test_puzzle_compression +tests.clvm.test_puzzles +tests.clvm.test_serialized_program +tests.clvm.test_singletons +tests.clvm.test_spend_sim +tests.conftest +tests.connection_utils +tests.core.cmds.test_keys +tests.core.consensus.test_pot_iterations +tests.core.custom_types.test_coin +tests.core.custom_types.test_spend_bundle +tests.core.daemon.test_daemon +tests.core.full_node.full_sync.test_full_sync +tests.core.full_node.stores.test_block_store +tests.core.full_node.stores.test_coin_store +tests.core.full_node.stores.test_full_node_store +tests.core.full_node.stores.test_hint_store +tests.core.full_node.stores.test_sync_store +tests.core.full_node.test_address_manager +tests.core.full_node.test_block_height_map +tests.core.full_node.test_conditions +tests.core.full_node.test_full_node +tests.core.full_node.test_node_load +tests.core.full_node.test_peer_store_resolver +tests.core.full_node.test_performance +tests.core.full_node.test_transactions +tests.core.make_block_generator +tests.core.mempool.test_mempool +tests.core.mempool.test_mempool_performance +tests.core.node_height +tests.core.server.test_dos +tests.core.server.test_rate_limits +tests.core.ssl.test_ssl +tests.core.test_cost_calculation +tests.core.test_crawler_rpc +tests.core.test_daemon_rpc +tests.core.test_db_conversion +tests.core.test_farmer_harvester_rpc +tests.core.test_filter +tests.core.test_full_node_rpc +tests.core.test_merkle_set +tests.core.test_setproctitle +tests.core.util.test_cached_bls +tests.core.util.test_config +tests.core.util.test_file_keyring_synchronization +tests.core.util.test_files +tests.core.util.test_keychain +tests.core.util.test_keyring_wrapper +tests.core.util.test_lru_cache +tests.core.util.test_significant_bits +tests.farmer_harvester.test_farmer_harvester +tests.generator.test_list_to_batches +tests.generator.test_scan +tests.plot_sync.test_plot_sync +tests.plot_sync.test_sync_simulated +tests.plotting.test_plot_manager +tests.pools.test_pool_cmdline +tests.pools.test_pool_config +tests.pools.test_pool_puzzles_lifecycle +tests.pools.test_wallet_pool_store +tests.simulation.test_simulation +tests.tools.test_full_sync +tests.tools.test_run_block +tests.util.benchmark_cost +tests.util.build_network_protocol_files +tests.util.db_connection +tests.util.generator_tools_testing +tests.util.key_tool +tests.util.test_full_block_utils +tests.util.test_lock_queue +tests.util.test_misc +tests.util.test_network +tests.wallet.cat_wallet.test_cat_lifecycle +tests.wallet.cat_wallet.test_cat_wallet +tests.wallet.cat_wallet.test_offer_lifecycle +tests.wallet.cat_wallet.test_trades +tests.wallet.did_wallet.test_did +tests.wallet.nft_wallet.test_nft_puzzles +tests.wallet.nft_wallet.test_nft_wallet +tests.wallet.rpc.test_wallet_rpc +tests.wallet.simple_sync.test_simple_sync_protocol +tests.wallet.sync.test_wallet_sync +tests.wallet.test_bech32m +tests.wallet.test_chialisp +tests.wallet.test_puzzle_store +tests.wallet.test_singleton +tests.wallet.test_singleton_lifecycle +tests.wallet.test_singleton_lifecycle_fast +tests.wallet.test_taproot +tests.wallet.test_wallet_blockchain +tests.wallet.test_wallet_interested_store +tests.wallet.test_wallet_key_val_store +tests.wallet.test_wallet_user_store +tests.weight_proof.test_weight_proof +tools.analyze-chain +tools.run_block +tools.test_full_sync diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index 70c64912b4..0000000000 --- a/mypy.ini +++ /dev/null @@ -1,33 +0,0 @@ -[mypy] -files = benchmarks,build_scripts,chia,tests,tools,*.py -ignore_missing_imports = True -show_error_codes = True -warn_unused_ignores = True - -disallow_any_generics = True -disallow_subclassing_any = True -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_incomplete_defs = True -check_untyped_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_return_any = True -no_implicit_reexport = True -strict_equality = True -warn_redundant_casts = True - -# list created by: venv/bin/mypy | sed -n 's/.py:.*//p' | sort | uniq | tr '/' '.' | tr '\n' ',' -[mypy-chia.cmds.passphrase,chia.cmds.passphrase_funcs,chia.cmds.plots,chia.cmds.plotters,chia.cmds.show,chia.cmds.start_funcs,chia.cmds.wallet,chia.cmds.wallet_funcs,chia.daemon.server,chia.farmer.farmer_api,chia.introducer.introducer,chia.introducer.introducer_api,chia.plotters.bladebit,chia.plotters.chiapos,chia.plotters.madmax,chia.plotters.plotters,chia.plotters.plotters_util,chia.plotting.create_plots,chia.plotting.manager,chia.plotting.util,chia.pools.pool_puzzles,chia.pools.pool_wallet,chia.pools.pool_wallet_info,chia.rpc.full_node_rpc_client,chia.rpc.harvester_rpc_api,chia.rpc.harvester_rpc_client,chia.rpc.rpc_client,chia.rpc.timelord_rpc_api,chia.rpc.util,chia.rpc.wallet_rpc_api,chia.rpc.wallet_rpc_client,chia.seeder.crawler,chia.seeder.crawler_api,chia.seeder.crawl_store,chia.seeder.dns_server,chia.seeder.peer_record,chia.seeder.start_crawler,chia.simulator.full_node_simulator,chia.simulator.start_simulator,chia.ssl.create_ssl,chia.timelord.iters_from_block,chia.timelord.timelord,chia.timelord.timelord_api,chia.timelord.timelord_launcher,chia.timelord.timelord_state,chia.types.blockchain_format.program,chia.util.block_cache,chia.util.check_fork_next_block,chia.util.chia_logging,chia.util.config,chia.util.db_wrapper,chia.util.hash,chia.util.json_util,chia.util.keychain,chia.util.keyring_wrapper,chia.util.log_exceptions,chia.util.make_test_constants,chia.util.merkle_set,chia.util.network,chia.util.partial_func,chia.util.profiler,chia.util.safe_cancel_task,chia.util.service_groups,chia.util.ssl_check,chia.wallet.block_record,chia.wallet.chialisp,chia.wallet.did_wallet.did_wallet,chia.wallet.did_wallet.did_wallet_puzzles,chia.wallet.key_val_store,chia.wallet.lineage_proof,chia.wallet.payment,chia.wallet.puzzles.load_clvm,chia.wallet.puzzles.p2_conditions,chia.wallet.puzzles.p2_delegated_conditions,chia.wallet.puzzles.p2_delegated_puzzle,chia.wallet.puzzles.p2_delegated_puzzle_or_hidden_puzzle,chia.wallet.puzzles.p2_m_of_n_delegate_direct,chia.wallet.puzzles.p2_puzzle_hash,chia.wallet.puzzles.prefarm.spend_prefarm,chia.wallet.puzzles.puzzle_utils,chia.wallet.puzzles.singleton_top_layer,chia.wallet.puzzles.tails,chia.wallet.secret_key_store,chia.wallet.trade_record,chia.wallet.trading.trade_store,chia.wallet.transaction_record,chia.wallet.util.debug_spend_bundle,chia.wallet.util.new_peak_queue,chia.wallet.wallet,chia.wallet.wallet_coin_store,chia.wallet.wallet_interested_store,chia.wallet.wallet_node_api,chia.wallet.wallet_pool_store,chia.wallet.wallet_puzzle_store,chia.wallet.wallet_sync_store,chia.wallet.wallet_transaction_store,chia.wallet.wallet_user_store,installhelper,tests.blockchain.blockchain_test_utils,tests.blockchain.test_blockchain,chia.simulator.block_tools,tests.build-init-files,tests.build-workflows,tests.clvm.coin_store,tests.clvm.test_chialisp_deserialization,tests.clvm.test_clvm_compilation,tests.clvm.test_program,tests.clvm.test_puzzle_compression,tests.clvm.test_puzzles,tests.clvm.test_serialized_program,tests.clvm.test_singletons,tests.clvm.test_spend_sim,tests.conftest,tests.connection_utils,tests.core.cmds.test_keys,tests.core.consensus.test_pot_iterations,tests.core.custom_types.test_coin,tests.core.custom_types.test_spend_bundle,tests.core.daemon.test_daemon,tests.core.full_node.full_sync.test_full_sync,tests.core.full_node.stores.test_block_store,tests.core.full_node.stores.test_coin_store,tests.core.full_node.stores.test_full_node_store,tests.core.full_node.stores.test_hint_store,tests.core.full_node.stores.test_sync_store,tests.core.full_node.test_address_manager,tests.core.full_node.test_block_height_map,tests.core.full_node.test_conditions,tests.core.full_node.test_full_node,tests.core.mempool.test_mempool,tests.core.mempool.test_mempool_performance,tests.core.full_node.test_node_load,tests.core.full_node.test_peer_store_resolver,tests.core.full_node.test_performance,tests.core.full_node.test_transactions,tests.core.make_block_generator,tests.core.node_height,tests.core.server.test_dos,tests.core.server.test_rate_limits,tests.core.ssl.test_ssl,tests.core.test_cost_calculation,tests.core.test_crawler_rpc,tests.core.test_daemon_rpc,tests.core.test_db_conversion,tests.core.test_db_validation,tests.core.test_farmer_harvester_rpc,tests.core.test_filter,tests.core.test_full_node_rpc,tests.core.test_merkle_set,tests.core.test_setproctitle,tests.core.util.test_cached_bls,tests.core.util.test_config,tests.core.util.test_file_keyring_synchronization,tests.core.util.test_files,tests.core.util.test_keychain,tests.core.util.test_keyring_wrapper,tests.core.util.test_lru_cache,tests.core.util.test_significant_bits,tests.farmer_harvester.test_farmer_harvester,tests.generator.test_list_to_batches,tests.generator.test_scan,tests.plotting.test_plot_manager,tests.pools.test_pool_cmdline,tests.pools.test_pool_config,tests.pools.test_pool_puzzles_lifecycle,tests.pools.test_wallet_pool_store,tests.simulation.test_simulation,chia.simulator.time_out_assert,tests.tools.test_full_sync,tests.tools.test_run_block,tests.util.benchmark_cost,tests.util.build_network_protocol_files,tests.util.db_connection,tests.util.generator_tools_testing,chia.simulator.keyring,tests.util.key_tool,tests.util.test_full_block_utils,tests.util.test_lock_queue,tests.util.test_misc,tests.util.test_network,tests.util.test_network_protocol_files,tests.wallet.cat_wallet.test_cat_lifecycle,tests.wallet.cat_wallet.test_cat_wallet,tests.wallet.cat_wallet.test_offer_lifecycle,tests.wallet.cat_wallet.test_trades,tests.wallet.did_wallet.test_did,tests.wallet.did_wallet.test_did_rpc,tests.wallet.did_wallet.test_nft_rpc,tests.wallet.did_wallet.test_nft_wallet,tests.wallet.rpc.test_wallet_rpc,tests.wallet.simple_sync.test_simple_sync_protocol,tests.wallet.sync.test_wallet_sync,tests.wallet.test_bech32m,tests.wallet.test_chialisp,tests.wallet.test_puzzle_store,tests.wallet.test_singleton,tests.wallet.test_singleton_lifecycle,tests.wallet.test_singleton_lifecycle_fast,tests.wallet.test_taproot,tests.wallet.test_wallet_blockchain,tests.wallet.test_wallet_interested_store,tests.wallet.test_wallet_key_val_store,tests.wallet.test_wallet_user_store,chia.simulator.wallet_tools,tests.weight_proof.test_weight_proof,tools.analyze-chain,tools.run_block,tools.test_full_sync,tests.wallet.nft_wallet.test_nft_wallet,chia.wallet.nft_wallet.nft_puzzles,tests.wallet.nft_wallet.test_nft_puzzles] -disable_error_code = annotation-unchecked -disallow_any_generics = False -disallow_subclassing_any = False -disallow_untyped_calls = False -disallow_untyped_defs = False -disallow_incomplete_defs = False -check_untyped_defs = False -disallow_untyped_decorators = False -no_implicit_optional = False -warn_return_any = False -no_implicit_reexport = False -strict_equality = False diff --git a/mypy.ini.template b/mypy.ini.template new file mode 100644 index 0000000000..69f04397cc --- /dev/null +++ b/mypy.ini.template @@ -0,0 +1,32 @@ +[mypy] +files = benchmarks,build_scripts,chia,tests,tools,*.py +ignore_missing_imports = True +show_error_codes = True +warn_unused_ignores = True + +disallow_any_generics = True +disallow_subclassing_any = True +disallow_untyped_calls = True +disallow_untyped_defs = True +disallow_incomplete_defs = True +check_untyped_defs = True +disallow_untyped_decorators = True +no_implicit_optional = True +warn_return_any = True +no_implicit_reexport = True +strict_equality = True +warn_redundant_casts = True + +[mypy-chia-exclusions] +disable_error_code = annotation-unchecked +disallow_any_generics = False +disallow_subclassing_any = False +disallow_untyped_calls = False +disallow_untyped_defs = False +disallow_incomplete_defs = False +check_untyped_defs = False +disallow_untyped_decorators = False +no_implicit_optional = False +warn_return_any = False +no_implicit_reexport = False +strict_equality = False