diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml index a65136289..39282a3cc 100644 --- a/.azure-pipelines/steps/run-tests-windows.yml +++ b/.azure-pipelines/steps/run-tests-windows.yml @@ -43,7 +43,7 @@ steps: # https://bugs.python.org/issue18199 $env:TEMP = "R:\Temp" - tox -e py -- -m integration -n auto --duration=5 --junit-xml=junit/integration-test.xml + tox -e py -- -m integration -n auto --durations=5 --junit-xml=junit/integration-test.xml displayName: Tox run integration tests - task: PublishTestResults@2 diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index 11ea22727..5b9a9c50c 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -11,10 +11,10 @@ steps: displayName: Tox run unit tests # Run integration tests in two groups so we will fail faster if there is a failure in the first group -- script: tox -e py -- -m integration -n auto --duration=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml +- script: tox -e py -- -m integration -n auto --durations=5 -k "not test_install" --junit-xml=junit/integration-test-group0.xml displayName: Tox run Group 0 integration tests -- script: tox -e py -- -m integration -n auto --duration=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml +- script: tox -e py -- -m integration -n auto --durations=5 -k "test_install" --junit-xml=junit/integration-test-group1.xml displayName: Tox run Group 1 integration tests - task: PublishTestResults@2 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 3babf35bd..8e5c268c1 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,7 +1,7 @@ # Ref: https://help.github.com/en/github/building-a-strong-community/configuring-issue-templates-for-your-repository#configuring-the-template-chooser blank_issues_enabled: true # default contact_links: -- name: 🤷💻🤦 Discourse +- name: 💬 Discourse url: https://discuss.python.org/c/packaging about: | Please ask typical Q&A here: general ideas for Python packaging, @@ -9,6 +9,3 @@ contact_links: - name: '💬 IRC: #pypa @ Freenode' url: https://webchat.freenode.net/#pypa about: Chat with devs -- name: 📝 PyPA Code of Conduct - url: https://www.pypa.io/en/latest/code-of-conduct/ - about: ❤ Be nice to other members of the community. ☮ Behave. diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04c72d8e3..13b3abc62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ exclude: 'src/pip/_vendor/' repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.2.0 hooks: - id: check-builtin-literals - id: check-added-large-files @@ -16,8 +16,44 @@ repos: - id: trailing-whitespace exclude: .patch +- repo: https://github.com/psf/black + rev: 20.8b1 + hooks: + - id: black + exclude: | + (?x) + ^docs/| + ^src/pip/_internal/cli| + ^src/pip/_internal/commands| + ^src/pip/_internal/distributions| + ^src/pip/_internal/index| + ^src/pip/_internal/models| + ^src/pip/_internal/network| + ^src/pip/_internal/operations| + ^src/pip/_internal/req| + ^src/pip/_internal/resolution| + ^src/pip/_internal/utils| + ^src/pip/_internal/vcs| + ^src/pip/_internal/\w+\.py$| + ^src/pip/__main__.py$| + ^tools/| + # Tests + ^tests/conftest.py| + ^tests/yaml| + ^tests/lib| + ^tests/data| + ^tests/unit| + ^tests/functional/(?!test_install)| + ^tests/functional/test_install| + # Files in the root of the repository + ^setup.py| + ^noxfile.py| + # A blank ignore, to avoid merge conflicts later. + ^$ + + - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.1 + rev: 3.8.3 hooks: - id: flake8 additional_dependencies: [ @@ -27,7 +63,7 @@ repos: exclude: tests/data - repo: https://github.com/timothycrosley/isort - rev: 4.3.21 + rev: 5.5.3 hooks: - id: isort files: \.py$ @@ -44,18 +80,25 @@ repos: args: ["--pretty", "-2"] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.5.1 + rev: v1.6.0 hooks: - id: python-no-log-warn - id: python-no-eval - id: rst-backticks - # Validate existing ReST files and NEWS fragments. - files: .*\.rst$|^news/.* + files: .*\.rst$ types: [file] - # The errors flagged in NEWS.rst are old. - exclude: NEWS.rst + exclude: NEWS.rst # The errors flagged in NEWS.rst are old. + +- repo: local + hooks: + - id: news-fragment-filenames + name: NEWS fragment + language: fail + entry: NEWS fragment files must be named *.(process|removal|feature|bugfix|vendor|doc|trivial).rst + exclude: ^news/(.gitignore|.*\.(process|removal|feature|bugfix|vendor|doc|trivial).rst) + files: ^news/ - repo: https://github.com/mgedmin/check-manifest - rev: '0.42' + rev: '0.43' hooks: - id: check-manifest diff --git a/LICENSE.txt b/LICENSE.txt index 737fec5c5..75eb0fd80 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) +Copyright (c) 2008-2020 The pip developers (see AUTHORS.txt file) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/MANIFEST.in b/MANIFEST.in index aa6a1d0e7..24d455378 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -22,7 +22,7 @@ exclude noxfile.py recursive-include src/pip/_vendor *.pem recursive-include src/pip/_vendor py.typed -recursive-include docs Makefile *.rst *.py *.bat +recursive-include docs *.css *.rst *.py exclude src/pip/_vendor/six exclude src/pip/_vendor/six/moves diff --git a/README.rst b/README.rst index 4f0f210f0..395b642d6 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Code of Conduct --------------- Everyone interacting in the pip project's codebases, issue trackers, chat -rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _package installer: https://packaging.python.org/guides/tool-recommendations/ .. _Python Package Index: https://pypi.org @@ -54,4 +54,4 @@ rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. .. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/ .. _User IRC: https://webchat.freenode.net/?channels=%23pypa .. _Development IRC: https://webchat.freenode.net/?channels=%23pypa-dev -.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/docs/docs_feedback_sphinxext.py b/docs/docs_feedback_sphinxext.py new file mode 100644 index 000000000..90f2ddd74 --- /dev/null +++ b/docs/docs_feedback_sphinxext.py @@ -0,0 +1,165 @@ +"""A sphinx extension for collecting per doc feedback.""" + +from __future__ import annotations + +from itertools import chain +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import Dict, List, Union + + from sphinx.application import Sphinx + + +DEFAULT_DOC_LINES_THRESHOLD = 250 +RST_INDENT = 4 +EMAIL_INDENT = 6 + + +def _modify_rst_document_source_on_read( + app: Sphinx, + docname: str, + source: List[str], +) -> None: + """Add info block to top and bottom of each document source. + + This function modifies RST source in-place by adding an admonition + block at the top and the bottom of each document right after it's + been read from disk preserving :orphan: at top, if present. + """ + admonition_type = app.config.docs_feedback_admonition_type + big_doc_lines = app.config.docs_feedback_big_doc_lines + escaped_email = app.config.docs_feedback_email.replace(' ', r'\ ') + excluded_documents = set(app.config.docs_feedback_excluded_documents) + questions_list = app.config.docs_feedback_questions_list + + valid_admonitions = { + 'attention', 'caution', 'danger', 'error', 'hint', + 'important', 'note', 'tip', 'warning', 'admonition', + } + + if admonition_type not in valid_admonitions: + raise ValueError( + 'Expected `docs_feedback_admonition_type` to be one of ' + f'{valid_admonitions} but got {admonition_type}.' + ) + + if not questions_list: + raise ValueError( + 'Expected `docs_feedback_questions_list` to list questions ' + 'but got none.' + ) + + if docname in excluded_documents: + # NOTE: Completely ignore any document + # NOTE: listed in 'docs_feedback_excluded_documents'. + return + + is_doc_big = source[0].count('\n') >= big_doc_lines + + questions_list_rst = '\n'.join( + f'{" " * RST_INDENT}{number!s}. {question}' + for number, question in enumerate(questions_list, 1) + ) + questions_list_urlencoded = ( + '\n'.join( + f'\n{" " * RST_INDENT}{number!s}. {question} ' + for number, question in enumerate( + chain( + (f'Document: {docname}. Page URL: https://', ), + questions_list, + ), + ) + ). + rstrip('\r\n\t '). + replace('\r', '%0D'). + replace('\n', '%0A'). + replace(' ', '%20') + ) + + admonition_msg = rf""" + **Did this article help?** + + We are currently doing research to improve pip's documentation + and would love your feedback. + Please `email us`_ and let us know{{let_us_know_ending}} + +{{questions_list_rst}} + + .. _email us: + mailto:{escaped_email}\ + ?subject=[Doc:\ {docname}]\ Pip\ docs\ feedback\ \ + (URL\:\ https\://)\ + &body={questions_list_urlencoded} + """ + let_us_know_ending = ':' + + info_block_bottom = ( + f'.. {admonition_type}::\n\t\t{admonition_msg.format_map(locals())}\n' + ) + + questions_list_rst = '' + let_us_know_ending = ( + ' why you came to this page and what on it helped ' + 'you and what did not. ' + '(:issue:`Read more about this research <8517>`)' + ) + info_block_top = '' if is_doc_big else ( + f'.. {admonition_type}::\n\t\t{admonition_msg.format_map(locals())}\n' + ) + + orphan_mark = ':orphan:' + is_orphan = orphan_mark in source[0] + if is_orphan: + source[0].replace(orphan_mark, '') + else: + orphan_mark = '' + + source[0] = '\n\n'.join(( + orphan_mark, info_block_top, source[0], info_block_bottom, + )) + + +def setup(app: Sphinx) -> Dict[str, Union[bool, str]]: + """Initialize the Sphinx extension. + + This function adds a callback for modifying the document sources + in-place on read. + + It also declares the extension settings changable via :file:`conf.py`. + """ + rebuild_trigger = 'html' # rebuild full html on settings change + app.add_config_value( + 'docs_feedback_admonition_type', + default='important', + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_big_doc_lines', + default=DEFAULT_DOC_LINES_THRESHOLD, + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_email', + default='Docs UX Team ', + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_excluded_documents', + default=set(), + rebuild=rebuild_trigger, + ) + app.add_config_value( + 'docs_feedback_questions_list', + default=(), + rebuild=rebuild_trigger, + ) + + app.add_css_file('important-admonition.css') + app.connect('source-read', _modify_rst_document_source_on_read) + + return { + 'parallel_read_safe': True, + 'parallel_write_safe': True, + 'version': 'builtin', + } diff --git a/docs/html/_static/important-admonition.css b/docs/html/_static/important-admonition.css new file mode 100644 index 000000000..a73ae2e4d --- /dev/null +++ b/docs/html/_static/important-admonition.css @@ -0,0 +1,8 @@ +.admonition.important { + background-color: rgb(219, 250, 244); + border: 1px solid rgb(26, 188, 156); +} + +.admonition.important>.admonition-title { + color: rgb(26, 188, 156); +} diff --git a/docs/html/conf.py b/docs/html/conf.py index bd44b9bff..444d15a81 100644 --- a/docs/html/conf.py +++ b/docs/html/conf.py @@ -30,7 +30,16 @@ sys.path.insert(0, docs_dir) # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones. # extensions = ['sphinx.ext.autodoc'] -extensions = ['sphinx.ext.extlinks', 'pip_sphinxext', 'sphinx.ext.intersphinx'] +extensions = [ + # native: + 'sphinx.ext.extlinks', + 'sphinx.ext.intersphinx', + # third-party: + 'sphinx_tabs.tabs', + # in-tree: + 'docs_feedback_sphinxext', + 'pip_sphinxext', +] # intersphinx intersphinx_cache_limit = 0 @@ -130,6 +139,9 @@ extlinks = { 'pypi': ('https://pypi.org/project/%s/', ''), } +# Turn off sphinx build warnings because of sphinx tabs during man pages build +sphinx_tabs_nowarn = True + # -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with @@ -143,7 +155,9 @@ html_theme_options = { 'collapsiblesidebar': True, 'externalrefs': True, 'navigation_depth': 3, - 'issues_url': 'https://github.com/pypa/pip/issues' + 'issues_url': 'https://github.com/pypa/pip/issues', + 'codebgcolor': '#eeffcc', + 'codetextcolor': '#333333', } # Add any paths that contain custom themes here, relative to this directory. @@ -167,7 +181,7 @@ html_theme_options = { # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = [] +html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. @@ -298,3 +312,19 @@ for fname in raw_subcommands: ) man_pages.append((fname_base, outname, description, u'pip developers', 1)) + +# -- Options for docs_feedback_sphinxext -------------------------------------- + +# NOTE: Must be one of 'attention', 'caution', 'danger', 'error', 'hint', +# NOTE: 'important', 'note', 'tip', 'warning' or 'admonition'. +docs_feedback_admonition_type = 'important' +docs_feedback_big_doc_lines = 50 # bigger docs will have a banner on top +docs_feedback_email = 'Docs UX Team ' +docs_feedback_excluded_documents = { # these won't have any banners + 'news', +} +docs_feedback_questions_list = ( + 'What problem were you trying to solve when you came to this page?', + 'What content was useful?', + 'What content was not useful?', +) diff --git a/docs/html/development/architecture/configuration-files.rst b/docs/html/development/architecture/configuration-files.rst index ce0ef40ee..2f96ea5ca 100644 --- a/docs/html/development/architecture/configuration-files.rst +++ b/docs/html/development/architecture/configuration-files.rst @@ -109,6 +109,7 @@ manipulated. In addition to the methods discussed in the previous section, the methods used would be: .. py:class:: Configuration + :noindex: .. py:method:: get_file_to_edit() diff --git a/docs/html/development/contributing.rst b/docs/html/development/contributing.rst index 15690dae4..3d2d5ead0 100644 --- a/docs/html/development/contributing.rst +++ b/docs/html/development/contributing.rst @@ -70,15 +70,17 @@ such, but it is preferred to have a dedicated issue (for example, in case the PR ends up rejected due to code quality reasons). Once you have an issue or pull request, you take the number and you create a -file inside of the ``news/`` directory named after that issue number with an -extension of ``removal``, ``feature``, ``bugfix``, or ``doc``. Thus if your -issue or PR number is ``1234`` and this change is fixing a bug, then you would -create a file ``news/1234.bugfix``. PRs can span multiple categories by creating -multiple files (for instance, if you added a feature and deprecated/removed the -old feature at the same time, you would create ``news/NNNN.feature`` and -``news/NNNN.removal``). Likewise if a PR touches multiple issues/PRs you may -create a file for each of them with the exact same contents and Towncrier will -deduplicate them. +file inside of the ``news/`` directory, named after that issue number with a +"type" of ``removal``, ``feature``, ``bugfix``, or ``doc`` associated with it. + +If your issue or PR number is ``1234`` and this change is fixing a bug, +then you would create a file ``news/1234.bugfix.rst``. PRs can span multiple +categories by creating multiple files (for instance, if you added a feature and +deprecated/removed the old feature at the same time, you would create +``news/NNNN.feature.rst`` and ``news/NNNN.removal.rst``). + +If a PR touches multiple issues/PRs, you may create a file for each of them +with the exact same contents and Towncrier will deduplicate them. Contents of a NEWS entry ------------------------ diff --git a/docs/html/development/getting-started.rst b/docs/html/development/getting-started.rst index 326543202..1bc4a5516 100644 --- a/docs/html/development/getting-started.rst +++ b/docs/html/development/getting-started.rst @@ -38,13 +38,25 @@ To run the pip executable from your source tree during development, install pip locally using editable installation (inside a virtualenv). You can then invoke your local source tree pip normally. -.. code-block:: console +.. tabs:: - $ virtualenv venv # You can also use "python -m venv venv" from python3.3+ - $ source venv/bin/activate - $ python -m pip install -e . - $ python -m pip --version + .. group-tab:: Unix/macOS + .. code-block:: shell + + virtualenv venv # You can also use "python -m venv venv" from python3.3+ + source venv/bin/activate + python -m pip install -e . + python -m pip --version + + .. group-tab:: Windows + + .. code-block:: shell + + virtualenv venv # You can also use "py -m venv venv" from python3.3+ + venv\Scripts\activate + py -m pip install -e . + py -m pip --version Running Tests ============= diff --git a/docs/html/index.rst b/docs/html/index.rst index 0f4f4b7dc..64fc34c9d 100644 --- a/docs/html/index.rst +++ b/docs/html/index.rst @@ -36,7 +36,7 @@ Code of Conduct =============== Everyone interacting in the pip project's codebases, issue trackers, chat -rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. +rooms, and mailing lists is expected to follow the `PSF Code of Conduct`_. .. _package installer: https://packaging.python.org/guides/tool-recommendations/ .. _Python Package Index: https://pypi.org @@ -51,4 +51,4 @@ rooms, and mailing lists is expected to follow the `PyPA Code of Conduct`_. .. _Development mailing list: https://mail.python.org/mailman3/lists/distutils-sig.python.org/ .. _User IRC: https://webchat.freenode.net/?channels=%23pypa .. _Development IRC: https://webchat.freenode.net/?channels=%23pypa-dev -.. _PyPA Code of Conduct: https://www.pypa.io/en/latest/code-of-conduct/ +.. _PSF Code of Conduct: https://github.com/pypa/.github/blob/main/CODE_OF_CONDUCT.md diff --git a/docs/html/installing.rst b/docs/html/installing.rst index 0a263ac41..5379c1da0 100644 --- a/docs/html/installing.rst +++ b/docs/html/installing.rst @@ -26,9 +26,21 @@ this link: `get-pip.py curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py Then run the following command in the folder where you -have downloaded ``get-pip.py``:: +have downloaded ``get-pip.py``: - python get-pip.py +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python get-pip.py + + .. group-tab:: Windows + + .. code-block:: shell + + py get-pip.py .. warning:: @@ -67,23 +79,70 @@ get-pip.py options install Options>` and the :ref:`general options `. Below are some examples: -Install from local copies of pip and setuptools:: +Install from local copies of pip and setuptools: - python get-pip.py --no-index --find-links=/local/copies +.. tabs:: -Install to the user site [3]_:: + .. group-tab:: Unix/macOS - python get-pip.py --user + .. code-block:: shell -Install behind a proxy:: + python get-pip.py --no-index --find-links=/local/copies - python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + .. group-tab:: Windows + + .. code-block:: shell + + py get-pip.py --no-index --find-links=/local/copies + +Install to the user site [3]_: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python get-pip.py --user + + .. group-tab:: Windows + + .. code-block:: shell + + py get-pip.py --user + +Install behind a proxy: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python get-pip.py --proxy="http://[user:passwd@]proxy.server:port" + + .. group-tab:: Windows + + .. code-block:: shell + + py get-pip.py --proxy="http://[user:passwd@]proxy.server:port" ``get-pip.py`` can also be used to install a specified combination of ``pip``, -``setuptools``, and ``wheel`` using the same requirements syntax as pip:: +``setuptools``, and ``wheel`` using the same requirements syntax as pip: - python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 +.. tabs:: + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 + + .. group-tab:: Windows + + .. code-block:: shell + + py get-pip.py pip==9.0.2 wheel==0.30.0 setuptools==28.8.0 Using Linux Package Managers ============================ @@ -97,14 +156,19 @@ the `Python Packaging User Guide Upgrading pip ============= -On Linux or macOS:: +.. tabs:: - pip install -U pip + .. group-tab:: Unix/macOS + .. code-block:: shell -On Windows [4]_:: + python -m pip install -U pip - python -m pip install -U pip + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -U pip .. _compatibility-requirements: @@ -134,5 +198,3 @@ pip works on Unix/Linux, macOS, and Windows. ``--user`` installs for pip itself, should not be considered to be fully tested or endorsed. For discussion, see `Issue 1668 `_. - -.. [4] https://github.com/pypa/pip/issues/1299 diff --git a/docs/html/quickstart.rst b/docs/html/quickstart.rst index c2250399c..9591e1127 100644 --- a/docs/html/quickstart.rst +++ b/docs/html/quickstart.rst @@ -6,62 +6,145 @@ First, :doc:`install pip `. Install a package from `PyPI`_: -:: +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install SomePackage + [...] + Successfully installed SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install SomePackage + [...] + Successfully installed SomePackage - $ pip install SomePackage - [...] - Successfully installed SomePackage Install a package that's already been downloaded from `PyPI`_ or obtained from elsewhere. This is useful if the target machine does not have a network connection: -:: +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install SomePackage-1.0-py2.py3-none-any.whl + [...] + Successfully installed SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install SomePackage-1.0-py2.py3-none-any.whl + [...] + Successfully installed SomePackage - $ pip install SomePackage-1.0-py2.py3-none-any.whl - [...] - Successfully installed SomePackage Show what files were installed: -:: +.. tabs:: - $ pip show --files SomePackage - Name: SomePackage - Version: 1.0 - Location: /my/env/lib/pythonx.x/site-packages - Files: - ../somepackage/__init__.py - [...] + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip show --files SomePackage + Name: SomePackage + Version: 1.0 + Location: /my/env/lib/pythonx.x/site-packages + Files: + ../somepackage/__init__.py + [...] + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip show --files SomePackage + Name: SomePackage + Version: 1.0 + Location: /my/env/lib/pythonx.x/site-packages + Files: + ../somepackage/__init__.py + [...] List what packages are outdated: -:: +.. tabs:: - $ pip list --outdated - SomePackage (Current: 1.0 Latest: 2.0) + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --outdated + SomePackage (Current: 1.0 Latest: 2.0) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --outdated + SomePackage (Current: 1.0 Latest: 2.0) Upgrade a package: -:: +.. tabs:: - $ pip install --upgrade SomePackage - [...] - Found existing installation: SomePackage 1.0 - Uninstalling SomePackage: - Successfully uninstalled SomePackage - Running setup.py install for SomePackage - Successfully installed SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install --upgrade SomePackage + [...] + Found existing installation: SomePackage 1.0 + Uninstalling SomePackage: + Successfully uninstalled SomePackage + Running setup.py install for SomePackage + Successfully installed SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --upgrade SomePackage + [...] + Found existing installation: SomePackage 1.0 + Uninstalling SomePackage: + Successfully uninstalled SomePackage + Running setup.py install for SomePackage + Successfully installed SomePackage Uninstall a package: -:: +.. tabs:: - $ pip uninstall SomePackage - Uninstalling SomePackage: - /my/env/lib/pythonx.x/site-packages/somepackage - Proceed (y/n)? y - Successfully uninstalled SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip uninstall SomePackage + Uninstalling SomePackage: + /my/env/lib/pythonx.x/site-packages/somepackage + Proceed (y/n)? y + Successfully uninstalled SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip uninstall SomePackage + Uninstalling SomePackage: + /my/env/lib/pythonx.x/site-packages/somepackage + Proceed (y/n)? y + Successfully uninstalled SomePackage .. _PyPI: https://pypi.org/ diff --git a/docs/html/reference/pip.rst b/docs/html/reference/pip.rst index 9c218f355..9fd42c676 100644 --- a/docs/html/reference/pip.rst +++ b/docs/html/reference/pip.rst @@ -7,10 +7,19 @@ pip Usage ***** -:: +.. tabs:: - pip [options] + .. group-tab:: Unix/macOS + .. code-block:: shell + + python -m pip [options] + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip [options] Description *********** @@ -221,9 +230,21 @@ Build Options The ``--global-option`` and ``--build-option`` arguments to the ``pip install`` and ``pip wheel`` inject additional arguments into the ``setup.py`` command (``--build-option`` is only available in ``pip wheel``). These arguments are -included in the command as follows:: +included in the command as follows: - python setup.py BUILD COMMAND +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + python setup.py BUILD COMMAND + + .. group-tab:: Windows + + .. code-block:: shell + + py setup.py BUILD COMMAND The options are passed unmodified, and presently offer direct access to the distutils command line. Use of ``--global-option`` and ``--build-option`` diff --git a/docs/html/reference/pip_cache.rst b/docs/html/reference/pip_cache.rst index 8ad99f65c..35e0dfcad 100644 --- a/docs/html/reference/pip_cache.rst +++ b/docs/html/reference/pip_cache.rst @@ -9,7 +9,15 @@ pip cache Usage ***** -.. pip-command-usage:: cache +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: cache "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: cache "py -m pip" Description *********** diff --git a/docs/html/reference/pip_check.rst b/docs/html/reference/pip_check.rst index a12d5b3ec..d3bb457e1 100644 --- a/docs/html/reference/pip_check.rst +++ b/docs/html/reference/pip_check.rst @@ -10,7 +10,15 @@ pip check Usage ===== -.. pip-command-usage:: check +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: check "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: check "py -m pip" Description @@ -24,27 +32,66 @@ Examples #. If all dependencies are compatible: - :: + .. tabs:: - $ pip check - No broken requirements found. - $ echo $? - 0 + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip check + No broken requirements found. + $ echo $? + 0 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip check + No broken requirements found. + C:\> echo %errorlevel% + 0 #. If a package is missing: - :: + .. tabs:: - $ pip check - pyramid 1.5.2 requires WebOb, which is not installed. - $ echo $? - 1 + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip check + pyramid 1.5.2 requires WebOb, which is not installed. + $ echo $? + 1 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip check + pyramid 1.5.2 requires WebOb, which is not installed. + C:\> echo %errorlevel% + 1 #. If a package has the wrong version: - :: + .. tabs:: - $ pip check - pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. - $ echo $? - 1 + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip check + pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. + $ echo $? + 1 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip check + pyramid 1.5.2 has requirement WebOb>=1.3.1, but you have WebOb 0.8. + C:\> echo %errorlevel% + 1 diff --git a/docs/html/reference/pip_config.rst b/docs/html/reference/pip_config.rst index 70d9406c5..d9bf0afc8 100644 --- a/docs/html/reference/pip_config.rst +++ b/docs/html/reference/pip_config.rst @@ -11,7 +11,15 @@ pip config Usage ===== -.. pip-command-usage:: config +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: config "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: config "py -m pip" Description diff --git a/docs/html/reference/pip_debug.rst b/docs/html/reference/pip_debug.rst index da147bcf2..2ef98228a 100644 --- a/docs/html/reference/pip_debug.rst +++ b/docs/html/reference/pip_debug.rst @@ -10,7 +10,15 @@ pip debug Usage ===== -.. pip-command-usage:: debug +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: debug "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: debug "py -m pip" .. warning:: diff --git a/docs/html/reference/pip_download.rst b/docs/html/reference/pip_download.rst index b74b1d240..7983bb95b 100644 --- a/docs/html/reference/pip_download.rst +++ b/docs/html/reference/pip_download.rst @@ -11,7 +11,15 @@ pip download Usage ===== -.. pip-command-usage:: download +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: download "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: download "py -m pip" Description @@ -56,64 +64,148 @@ Examples #. Download a package and all of its dependencies - :: + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download SomePackage + python -m pip download -d . SomePackage # equivalent to above + python -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip download SomePackage + py -m pip download -d . SomePackage # equivalent to above + py -m pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage - $ pip download SomePackage - $ pip download -d . SomePackage # equivalent to above - $ pip download --no-index --find-links=/tmp/wheelhouse -d /tmp/otherwheelhouse SomePackage #. Download a package and all of its dependencies with OSX specific interpreter constraints. - This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, - this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, - etc. - It will also match deps with platform ``any``. Also force the interpreter version to ``27`` - (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). + This forces OSX 10.10 or lower compatibility. Since OSX deps are forward compatible, + this will also match ``macosx-10_9_x86_64``, ``macosx-10_8_x86_64``, ``macosx-10_8_intel``, + etc. + It will also match deps with platform ``any``. Also force the interpreter version to ``27`` + (or more generic, i.e. ``2``) and implementation to ``cp`` (or more generic, i.e. ``py``). - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform macosx-10_10_x86_64 \ - --python-version 27 \ - --implementation cp \ - SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download \ + --only-binary=:all: \ + --platform macosx-10_10_x86_64 \ + --python-version 27 \ + --implementation cp \ + SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip download ^ + --only-binary=:all: ^ + --platform macosx-10_10_x86_64 ^ + --python-version 27 ^ + --implementation cp ^ + SomePackage #. Download a package and its dependencies with linux specific constraints. - Force the interpreter to be any minor version of py3k, and only accept - ``cp34m`` or ``none`` as the abi. + Force the interpreter to be any minor version of py3k, and only accept + ``cp34m`` or ``none`` as the abi. - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 3 \ - --implementation cp \ - --abi cp34m \ - SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 3 \ + --implementation cp \ + --abi cp34m \ + SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 3 ^ + --implementation cp ^ + --abi cp34m ^ + SomePackage #. Force platform, implementation, and abi agnostic deps. - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform any \ - --python-version 3 \ - --implementation py \ - --abi none \ - SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download \ + --only-binary=:all: \ + --platform any \ + --python-version 3 \ + --implementation py \ + --abi none \ + SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip download ^ + --only-binary=:all: ^ + --platform any ^ + --python-version 3 ^ + --implementation py ^ + --abi none ^ + SomePackage #. Even when overconstrained, this will still correctly fetch the pip universal wheel. - :: + .. tabs:: - $ pip download \ - --only-binary=:all: \ - --platform linux_x86_64 \ - --python-version 33 \ - --implementation cp \ - --abi cp34m \ - pip>=8 - $ ls pip-8.1.1-py2.py3-none-any.whl - pip-8.1.1-py2.py3-none-any.whl + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip download \ + --only-binary=:all: \ + --platform linux_x86_64 \ + --python-version 33 \ + --implementation cp \ + --abi cp34m \ + pip>=8 + + .. code-block:: console + + $ ls pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip download ^ + --only-binary=:all: ^ + --platform linux_x86_64 ^ + --python-version 33 ^ + --implementation cp ^ + --abi cp34m ^ + pip>=8 + + .. code-block:: console + + C:\> dir pip-8.1.1-py2.py3-none-any.whl + pip-8.1.1-py2.py3-none-any.whl diff --git a/docs/html/reference/pip_freeze.rst b/docs/html/reference/pip_freeze.rst index 31efd571b..d4ed00bfb 100644 --- a/docs/html/reference/pip_freeze.rst +++ b/docs/html/reference/pip_freeze.rst @@ -11,7 +11,15 @@ pip freeze Usage ===== -.. pip-command-usage:: freeze +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: freeze "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: freeze "py -m pip" Description @@ -31,19 +39,45 @@ Examples #. Generate output suitable for a requirements file. - :: + .. tabs:: - $ pip freeze - docutils==0.11 - Jinja2==2.7.2 - MarkupSafe==0.19 - Pygments==1.6 - Sphinx==1.2.2 + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip freeze + docutils==0.11 + Jinja2==2.7.2 + MarkupSafe==0.19 + Pygments==1.6 + Sphinx==1.2.2 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip freeze + docutils==0.11 + Jinja2==2.7.2 + MarkupSafe==0.19 + Pygments==1.6 + Sphinx==1.2.2 #. Generate a requirements file and then install from it in another environment. - :: + .. tabs:: - $ env1/bin/pip freeze > requirements.txt - $ env2/bin/pip install -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + env1/bin/python -m pip freeze > requirements.txt + env2/bin/python -m pip install -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + env1\bin\python -m pip freeze > requirements.txt + env2\bin\python -m pip install -r requirements.txt diff --git a/docs/html/reference/pip_hash.rst b/docs/html/reference/pip_hash.rst index dbf1f3e94..71e1cf4be 100644 --- a/docs/html/reference/pip_hash.rst +++ b/docs/html/reference/pip_hash.rst @@ -10,7 +10,15 @@ pip hash Usage ===== -.. pip-command-usage:: hash +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: hash "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: hash "py -m pip" Description @@ -39,13 +47,32 @@ Options Example ======= -Compute the hash of a downloaded archive:: +Compute the hash of a downloaded archive: - $ pip download SomePackage - Collecting SomePackage - Downloading SomePackage-2.2.tar.gz - Saved ./pip_downloads/SomePackage-2.2.tar.gz - Successfully downloaded SomePackage - $ pip hash ./pip_downloads/SomePackage-2.2.tar.gz - ./pip_downloads/SomePackage-2.2.tar.gz: - --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + $ python -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip download SomePackage + Collecting SomePackage + Downloading SomePackage-2.2.tar.gz + Saved ./pip_downloads/SomePackage-2.2.tar.gz + Successfully downloaded SomePackage + C:\> py -m pip hash ./pip_downloads/SomePackage-2.2.tar.gz + ./pip_downloads/SomePackage-2.2.tar.gz: + --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 diff --git a/docs/html/reference/pip_install.rst b/docs/html/reference/pip_install.rst index fd962cd36..cb97c8ee0 100644 --- a/docs/html/reference/pip_install.rst +++ b/docs/html/reference/pip_install.rst @@ -10,7 +10,16 @@ pip install Usage ===== -.. pip-command-usage:: install +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: install "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: install "py -m pip" + Description @@ -89,15 +98,33 @@ implementation (which might possibly change later) has it such that the first encountered member of the cycle is installed last. For instance, if quux depends on foo which depends on bar which depends on baz, -which depends on foo:: +which depends on foo: - pip install quux - ... - Installing collected packages baz, bar, foo, quux +.. tabs:: - pip install bar - ... - Installing collected packages foo, baz, bar + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install quux + ... + Installing collected packages baz, bar, foo, quux + + $ python -m pip install bar + ... + Installing collected packages foo, baz, bar + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install quux + ... + Installing collected packages baz, bar, foo, quux + + C:\> py -m pip install bar + ... + Installing collected packages foo, baz, bar Prior to v6.1.0, pip made no commitments about install order. @@ -387,9 +414,21 @@ If your repository layout is:: └── some_file some_other_file -Then, to install from this repository, the syntax would be:: +Then, to install from this repository, the syntax would be: - $ pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -e "vcs+protocol://repo_url/#egg=pkg&subdirectory=pkg_dir" Git @@ -636,17 +675,38 @@ against any requirement not only checks that hash but also activates a global .. _`--require-hashes`: Hash-checking mode can be forced on with the ``--require-hashes`` command-line -option:: +option: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install --require-hashes -r requirements.txt + ... + Hashes are required in --require-hashes mode (implicitly on when a hash is + specified for any package). These requirements were missing hashes, + leaving them open to tampering. These are the hashes the downloaded + archives actually had. You can add lines like these to your requirements + files to prevent tampering. + pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa + more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --require-hashes -r requirements.txt + ... + Hashes are required in --require-hashes mode (implicitly on when a hash is + specified for any package). These requirements were missing hashes, + leaving them open to tampering. These are the hashes the downloaded + archives actually had. You can add lines like these to your requirements + files to prevent tampering. + pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa + more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 - $ pip install --require-hashes -r requirements.txt - ... - Hashes are required in --require-hashes mode (implicitly on when a hash is - specified for any package). These requirements were missing hashes, - leaving them open to tampering. These are the hashes the downloaded - archives actually had. You can add lines like these to your requirements - files to prevent tampering. - pyelasticsearch==1.0 --hash=sha256:44ddfb1225054d7d6b1d02e9338e7d4809be94edbe9929a2ec0807d38df993fa - more-itertools==2.2 --hash=sha256:93e62e05c7ad3da1a233def6731e8285156701e3419a5fe279017c429ec67ce0 This can be useful in deploy scripts, to ensure that the author of the requirements file provided hashes. It is also a convenient way to bootstrap @@ -692,14 +752,38 @@ Hash-checking mode also works with :ref:`pip download` and :ref:`pip wheel`. A as your project evolves. To be safe, install your project using pip and :ref:`--no-deps `. - Instead of ``python setup.py develop``, use... :: + Instead of ``python setup.py develop``, use... - pip install --no-deps -e . + .. tabs:: - Instead of ``python setup.py install``, use... :: + .. group-tab:: Unix/macOS - pip install --no-deps . + .. code-block:: shell + python -m pip install --no-deps -e . + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --no-deps -e . + + + Instead of ``python setup.py install``, use... + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --no-deps . + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --no-deps . Hashes from PyPI ^^^^^^^^^^^^^^^^ @@ -717,9 +801,22 @@ Local project installs ---------------------- pip supports installing local project in both regular mode and editable mode. -You can install local projects by specifying the project path to pip:: +You can install local projects by specifying the project path to pip: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install path/to/SomeProject + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install path/to/SomeProject -$ pip install path/to/SomeProject During regular installation, pip will copy the entire project directory to a temporary location and install from there. The exception is that pip will @@ -736,10 +833,24 @@ being copied. `_ installs. -You can install local projects or VCS projects in "editable" mode:: +You can install local projects or VCS projects in "editable" mode: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -e path/to/SomeProject + python -m pip install -e git+http://repo/my_project.git#egg=SomeProject + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -e path/to/SomeProject + py -m pip install -e git+http://repo/my_project.git#egg=SomeProject -$ pip install -e path/to/SomeProject -$ pip install -e git+http://repo/my_project.git#egg=SomeProject (See the :ref:`VCS Support` section above for more information on VCS-related syntax.) @@ -846,113 +957,292 @@ Examples #. Install ``SomePackage`` and its dependencies from `PyPI`_ using :ref:`Requirement Specifiers` - :: + .. tabs:: - $ pip install SomePackage # latest version - $ pip install SomePackage==1.0.4 # specific version - $ pip install 'SomePackage>=1.0.4' # minimum version + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomePackage # latest version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomePackage # latest version + py -m pip install SomePackage==1.0.4 # specific version + py -m pip install 'SomePackage>=1.0.4' # minimum version #. Install a list of requirements specified in a file. See the :ref:`Requirements files `. - :: + .. tabs:: - $ pip install -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -r requirements.txt #. Upgrade an already installed ``SomePackage`` to the latest from PyPI. - :: + .. tabs:: - $ pip install --upgrade SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --upgrade SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --upgrade SomePackage #. Install a local project in "editable" mode. See the section on :ref:`Editable Installs `. - :: + .. tabs:: - $ pip install -e . # project in current directory - $ pip install -e path/to/project # project in another directory + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -e . # project in current directory + python -m pip install -e path/to/project # project in another directory + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -e . # project in current directory + py -m pip install -e path/to/project # project in another directory #. Install a project from VCS - :: + .. tabs:: - $ pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomeProject@git+https://git.repo/some_pkg.git@1.3.1 #. Install a project from VCS in "editable" mode. See the sections on :ref:`VCS Support ` and :ref:`Editable Installs `. - :: + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + python -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + python -m python -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + python -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + python -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git + py -m pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial + py -m pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn + py -m pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch + py -m pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory - $ pip install -e git+https://git.repo/some_pkg.git#egg=SomePackage # from git - $ pip install -e hg+https://hg.repo/some_pkg.git#egg=SomePackage # from mercurial - $ pip install -e svn+svn://svn.repo/some_pkg/trunk/#egg=SomePackage # from svn - $ pip install -e git+https://git.repo/some_pkg.git@feature#egg=SomePackage # from 'feature' branch - $ pip install -e "git+https://git.repo/some_repo.git#egg=subdir&subdirectory=subdir_path" # install a python package from a repo subdirectory #. Install a package with `setuptools extras`_. - :: + .. tabs:: - $ pip install SomePackage[PDF] - $ pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" - $ pip install .[PDF] # project in current directory - $ pip install SomePackage[PDF]==3.0 - $ pip install SomePackage[PDF,EPUB] # multiple extras + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomePackage[PDF] + python -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + python -m pip install .[PDF] # project in current directory + python -m pip install SomePackage[PDF]==3.0 + python -m pip install SomePackage[PDF,EPUB] # multiple extras + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomePackage[PDF] + py -m pip install "SomePackage[PDF] @ git+https://git.repo/SomePackage@master#subdirectory=subdir_path" + py -m pip install .[PDF] # project in current directory + py -m pip install SomePackage[PDF]==3.0 + py -m pip install SomePackage[PDF,EPUB] # multiple extras #. Install a particular source archive file. - :: + .. tabs:: - $ pip install ./downloads/SomePackage-1.0.4.tar.gz - $ pip install http://my.package.repo/SomePackage-1.0.4.zip + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install ./downloads/SomePackage-1.0.4.tar.gz + python -m pip install http://my.package.repo/SomePackage-1.0.4.zip + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install ./downloads/SomePackage-1.0.4.tar.gz + py -m pip install http://my.package.repo/SomePackage-1.0.4.zip #. Install a particular source archive file following :pep:`440` direct references. - :: + .. tabs:: - $ pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl - $ pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" - $ pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + python -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + python -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomeProject@http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl + py -m pip install "SomeProject @ http://my.package.repo/SomeProject-1.2.3-py33-none-any.whl" + py -m pip install SomeProject@http://my.package.repo/1.2.3.tar.gz #. Install from alternative package repositories. - Install from a different index, and not `PyPI`_ :: + Install from a different index, and not `PyPI`_ - $ pip install --index-url http://my.package.repo/simple/ SomePackage + .. tabs:: - Search an additional index during install, in addition to `PyPI`_ :: + .. group-tab:: Unix/macOS - $ pip install --extra-index-url http://my.package.repo/simple SomePackage + .. code-block:: shell - Install from a local flat directory containing archives (and don't scan indexes):: + python -m pip install --index-url http://my.package.repo/simple/ SomePackage - $ pip install --no-index --find-links=file:///local/dir/ SomePackage - $ pip install --no-index --find-links=/local/dir/ SomePackage - $ pip install --no-index --find-links=relative/dir/ SomePackage + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --index-url http://my.package.repo/simple/ SomePackage + + + Search an additional index during install, in addition to `PyPI`_ + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --extra-index-url http://my.package.repo/simple SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --extra-index-url http://my.package.repo/simple SomePackage + + + Install from a local flat directory containing archives (and don't scan indexes): + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --no-index --find-links=file:///local/dir/ SomePackage + python -m pip install --no-index --find-links=/local/dir/ SomePackage + python -m pip install --no-index --find-links=relative/dir/ SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --no-index --find-links=file:///local/dir/ SomePackage + py -m pip install --no-index --find-links=/local/dir/ SomePackage + py -m pip install --no-index --find-links=relative/dir/ SomePackage #. Find pre-release and development versions, in addition to stable versions. By default, pip only finds stable versions. - :: + .. tabs:: - $ pip install --pre SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --pre SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --pre SomePackage #. Install packages from source. - Do not use any binary packages:: + Do not use any binary packages - $ pip install SomePackage1 SomePackage2 --no-binary :all: + .. tabs:: - Specify ``SomePackage1`` to be installed from source:: + .. group-tab:: Unix/macOS - $ pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + .. code-block:: shell + + python -m pip install SomePackage1 SomePackage2 --no-binary :all: + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomePackage1 SomePackage2 --no-binary :all: + + Specify ``SomePackage1`` to be installed from source: + + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomePackage1 SomePackage2 --no-binary SomePackage1 ---- diff --git a/docs/html/reference/pip_list.rst b/docs/html/reference/pip_list.rst index 15d0920a7..1489ed751 100644 --- a/docs/html/reference/pip_list.rst +++ b/docs/html/reference/pip_list.rst @@ -10,7 +10,15 @@ pip list Usage ===== -.. pip-command-usage:: list +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: list "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: list "py -m pip" Description @@ -32,75 +40,182 @@ Examples #. List installed packages. - :: + .. tabs:: - $ pip list - docutils (0.10) - Jinja2 (2.7.2) - MarkupSafe (0.18) - Pygments (1.6) - Sphinx (1.2.1) + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list + docutils (0.10) + Jinja2 (2.7.2) + MarkupSafe (0.18) + Pygments (1.6) + Sphinx (1.2.1) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list + docutils (0.10) + Jinja2 (2.7.2) + MarkupSafe (0.18) + Pygments (1.6) + Sphinx (1.2.1) #. List outdated packages (excluding editables), and the latest version available. - :: + .. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --outdated + docutils (Current: 0.10 Latest: 0.11) + Sphinx (Current: 1.2.1 Latest: 1.2.2) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --outdated + docutils (Current: 0.10 Latest: 0.11) + Sphinx (Current: 1.2.1 Latest: 1.2.2) - $ pip list --outdated - docutils (Current: 0.10 Latest: 0.11) - Sphinx (Current: 1.2.1 Latest: 1.2.2) #. List installed packages with column formatting. - :: + .. tabs:: - $ pip list --format columns - Package Version - ------- ------- - docopt 0.6.2 - idlex 1.13 - jedi 0.9.0 + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --format columns + Package Version + ------- ------- + docopt 0.6.2 + idlex 1.13 + jedi 0.9.0 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --format columns + Package Version + ------- ------- + docopt 0.6.2 + idlex 1.13 + jedi 0.9.0 #. List outdated packages with column formatting. - :: + .. tabs:: - $ pip list -o --format columns - Package Version Latest Type - ---------- ------- ------ ----- - retry 0.8.1 0.9.1 wheel - setuptools 20.6.7 21.0.0 wheel + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list -o --format columns + Package Version Latest Type + ---------- ------- ------ ----- + retry 0.8.1 0.9.1 wheel + setuptools 20.6.7 21.0.0 wheel + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list -o --format columns + Package Version Latest Type + ---------- ------- ------ ----- + retry 0.8.1 0.9.1 wheel + setuptools 20.6.7 21.0.0 wheel #. List packages that are not dependencies of other packages. Can be combined with other options. - :: + .. tabs:: - $ pip list --outdated --not-required - docutils (Current: 0.10 Latest: 0.11) + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --outdated --not-required + docutils (Current: 0.10 Latest: 0.11) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --outdated --not-required + docutils (Current: 0.10 Latest: 0.11) #. Use legacy formatting - :: + .. tabs:: - $ pip list --format=legacy - colorama (0.3.7) - docopt (0.6.2) - idlex (1.13) - jedi (0.9.0) + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --format=legacy + colorama (0.3.7) + docopt (0.6.2) + idlex (1.13) + jedi (0.9.0) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --format=legacy + colorama (0.3.7) + docopt (0.6.2) + idlex (1.13) + jedi (0.9.0) #. Use json formatting - :: + .. tabs:: - $ pip list --format=json - [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --format=json + [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --format=json + [{'name': 'colorama', 'version': '0.3.7'}, {'name': 'docopt', 'version': '0.6.2'}, ... #. Use freeze formatting - :: + .. tabs:: - $ pip list --format=freeze - colorama==0.3.7 - docopt==0.6.2 - idlex==1.13 - jedi==0.9.0 + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list --format=freeze + colorama==0.3.7 + docopt==0.6.2 + idlex==1.13 + jedi==0.9.0 + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --format=freeze + colorama==0.3.7 + docopt==0.6.2 + idlex==1.13 + jedi==0.9.0 diff --git a/docs/html/reference/pip_search.rst b/docs/html/reference/pip_search.rst index db1bd2be8..fba629593 100644 --- a/docs/html/reference/pip_search.rst +++ b/docs/html/reference/pip_search.rst @@ -10,7 +10,15 @@ pip search Usage ===== -.. pip-command-usage:: search +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: search "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: search "py -m pip" Description @@ -30,8 +38,20 @@ Examples #. Search for "peppercorn" - :: + .. tabs:: - $ pip search peppercorn - pepperedform - Helpers for using peppercorn with formprocess. - peppercorn - A library for converting a token stream into [...] + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip search peppercorn + pepperedform - Helpers for using peppercorn with formprocess. + peppercorn - A library for converting a token stream into [...] + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip search peppercorn + pepperedform - Helpers for using peppercorn with formprocess. + peppercorn - A library for converting a token stream into [...] diff --git a/docs/html/reference/pip_show.rst b/docs/html/reference/pip_show.rst index e9568b6b0..6bd3718b9 100644 --- a/docs/html/reference/pip_show.rst +++ b/docs/html/reference/pip_show.rst @@ -10,7 +10,15 @@ pip show Usage ===== -.. pip-command-usage:: show +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: show "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: show "py -m pip" Description @@ -30,58 +38,124 @@ Examples #. Show information about a package: - :: + .. tabs:: - $ pip show sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip show sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip show sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six #. Show all information about a package - :: + .. tabs:: - $ pip show --verbose sphinx - Name: Sphinx - Version: 1.4.5 - Summary: Python documentation generator - Home-page: http://sphinx-doc.org/ - Author: Georg Brandl - Author-email: georg@python.org - License: BSD - Location: /my/env/lib/python2.7/site-packages - Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six - Metadata-Version: 2.0 - Installer: - Classifiers: - Development Status :: 5 - Production/Stable - Environment :: Console - Environment :: Web Environment - Intended Audience :: Developers - Intended Audience :: Education - License :: OSI Approved :: BSD License - Operating System :: OS Independent - Programming Language :: Python - Programming Language :: Python :: 2 - Programming Language :: Python :: 3 - Framework :: Sphinx - Framework :: Sphinx :: Extension - Framework :: Sphinx :: Theme - Topic :: Documentation - Topic :: Documentation :: Sphinx - Topic :: Text Processing - Topic :: Utilities - Entry-points: - [console_scripts] - sphinx-apidoc = sphinx.apidoc:main - sphinx-autogen = sphinx.ext.autosummary.generate:main - sphinx-build = sphinx:main - sphinx-quickstart = sphinx.quickstart:main - [distutils.commands] - build_sphinx = sphinx.setup_command:BuildDoc + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip show --verbose sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + Metadata-Version: 2.0 + Installer: + Classifiers: + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: Education + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Framework :: Sphinx + Framework :: Sphinx :: Extension + Framework :: Sphinx :: Theme + Topic :: Documentation + Topic :: Documentation :: Sphinx + Topic :: Text Processing + Topic :: Utilities + Entry-points: + [console_scripts] + sphinx-apidoc = sphinx.apidoc:main + sphinx-autogen = sphinx.ext.autosummary.generate:main + sphinx-build = sphinx:main + sphinx-quickstart = sphinx.quickstart:main + [distutils.commands] + build_sphinx = sphinx.setup_command:BuildDoc + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip show --verbose sphinx + Name: Sphinx + Version: 1.4.5 + Summary: Python documentation generator + Home-page: http://sphinx-doc.org/ + Author: Georg Brandl + Author-email: georg@python.org + License: BSD + Location: /my/env/lib/python2.7/site-packages + Requires: docutils, snowballstemmer, alabaster, Pygments, imagesize, Jinja2, babel, six + Metadata-Version: 2.0 + Installer: + Classifiers: + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Web Environment + Intended Audience :: Developers + Intended Audience :: Education + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: 2 + Programming Language :: Python :: 3 + Framework :: Sphinx + Framework :: Sphinx :: Extension + Framework :: Sphinx :: Theme + Topic :: Documentation + Topic :: Documentation :: Sphinx + Topic :: Text Processing + Topic :: Utilities + Entry-points: + [console_scripts] + sphinx-apidoc = sphinx.apidoc:main + sphinx-autogen = sphinx.ext.autosummary.generate:main + sphinx-build = sphinx:main + sphinx-quickstart = sphinx.quickstart:main + [distutils.commands] + build_sphinx = sphinx.setup_command:BuildDoc diff --git a/docs/html/reference/pip_uninstall.rst b/docs/html/reference/pip_uninstall.rst index 67d752d6b..8b31c5673 100644 --- a/docs/html/reference/pip_uninstall.rst +++ b/docs/html/reference/pip_uninstall.rst @@ -10,7 +10,15 @@ pip uninstall Usage ===== -.. pip-command-usage:: uninstall +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: uninstall "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: uninstall "py -m pip" Description @@ -30,11 +38,26 @@ Examples #. Uninstall a package. - :: + .. tabs:: - $ pip uninstall simplejson - Uninstalling simplejson: - /home/me/env/lib/python2.7/site-packages/simplejson - /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info - Proceed (y/n)? y - Successfully uninstalled simplejson + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip uninstall simplejson + Uninstalling simplejson: + /home/me/env/lib/python2.7/site-packages/simplejson + /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info + Proceed (y/n)? y + Successfully uninstalled simplejson + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip uninstall simplejson + Uninstalling simplejson: + /home/me/env/lib/python2.7/site-packages/simplejson + /home/me/env/lib/python2.7/site-packages/simplejson-2.2.1-py2.7.egg-info + Proceed (y/n)? y + Successfully uninstalled simplejson diff --git a/docs/html/reference/pip_wheel.rst b/docs/html/reference/pip_wheel.rst index dc32dda46..c1bdf37f8 100644 --- a/docs/html/reference/pip_wheel.rst +++ b/docs/html/reference/pip_wheel.rst @@ -11,7 +11,15 @@ pip wheel Usage ===== -.. pip-command-usage:: wheel +.. tabs:: + + .. group-tab:: Unix/macOS + + .. pip-command-usage:: wheel "python -m pip" + + .. group-tab:: Windows + + .. pip-command-usage:: wheel "py -m pip" Description @@ -24,9 +32,22 @@ Build System Interface ---------------------- In order for pip to build a wheel, ``setup.py`` must implement the -``bdist_wheel`` command with the following syntax:: +``bdist_wheel`` command with the following syntax: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python setup.py bdist_wheel -d TARGET + + .. group-tab:: Windows + + .. code-block:: shell + + py setup.py bdist_wheel -d TARGET - python setup.py bdist_wheel -d TARGET This command must create a wheel compatible with the invoking Python interpreter, and save that wheel in the directory TARGET. @@ -39,9 +60,22 @@ Customising the build It is possible using ``--global-option`` to include additional build commands with their arguments in the ``setup.py`` command. This is currently the only way to influence the building of C extensions from the command line. For -example:: +example: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip wheel --global-option bdist_ext --global-option -DFOO wheel + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip wheel --global-option bdist_ext --global-option -DFOO wheel - pip wheel --global-option bdist_ext --global-option -DFOO wheel will result in a build command of @@ -69,13 +103,34 @@ Examples #. Build wheels for a requirement (and all its dependencies), and then install - :: + .. tabs:: - $ pip wheel --wheel-dir=/tmp/wheelhouse SomePackage - $ pip install --no-index --find-links=/tmp/wheelhouse SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage + python -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip wheel --wheel-dir=/tmp/wheelhouse SomePackage + py -m pip install --no-index --find-links=/tmp/wheelhouse SomePackage #. Build a wheel for a package from source - :: + .. tabs:: - $ pip wheel --no-binary SomePackage SomePackage + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip wheel --no-binary SomePackage SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip wheel --no-binary SomePackage SomePackage diff --git a/docs/html/user_guide.rst b/docs/html/user_guide.rst index 77ffcecdd..a2d13c433 100644 --- a/docs/html/user_guide.rst +++ b/docs/html/user_guide.rst @@ -9,23 +9,28 @@ Running pip =========== pip is a command line program. When you install pip, a ``pip`` command is added -to your system, which can be run from the command prompt as follows:: +to your system, which can be run from the command prompt as follows: - $ pip +.. tabs:: -If you cannot run the ``pip`` command directly (possibly because the location -where it was installed isn't on your operating system's ``PATH``) then you can -run pip via the Python interpreter:: + .. group-tab:: Unix/macOS - $ python -m pip + .. code-block:: shell -On Windows, the ``py`` launcher can be used:: + python -m pip - $ py -m pip + ``python -m pip`` executes pip using the Python interpreter you + specified as python. So ``/usr/bin/python3.7 -m pip`` means + you are executing pip for your interpreter located at /usr/bin/python3.7. -Even though pip is available from your Python installation as an importable -module, via ``import pip``, it is *not supported* to use pip in this way. For -more details, see :ref:`Using pip from your program`. + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip + + ``py -m pip`` executes pip using the latest Python interpreter you + have installed. For more details, read the `Python Windows launcher`_ docs. Installing Packages @@ -36,12 +41,25 @@ directly from distribution files. The most common scenario is to install from `PyPI`_ using :ref:`Requirement -Specifiers` :: +Specifiers` - $ pip install SomePackage # latest version - $ pip install SomePackage==1.0.4 # specific version - $ pip install 'SomePackage>=1.0.4' # minimum version +.. tabs:: + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomePackage # latest version + python -m pip install SomePackage==1.0.4 # specific version + python -m pip install 'SomePackage>=1.0.4' # minimum version + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomePackage # latest version + py -m pip install SomePackage==1.0.4 # specific version + py -m pip install 'SomePackage>=1.0.4' # minimum version For more information and examples, see the :ref:`pip install` reference. @@ -77,6 +95,47 @@ as the "username" and do not provide a password, for example - ``https://0123456789abcdef@pypi.company.com`` +netrc Support +------------- + +If no credentials are part of the URL, pip will attempt to get authentication credentials +for the URL’s hostname from the user’s .netrc file. This behaviour comes from the underlying +use of `requests`_ which in turn delegates it to the `Python standard library`_. + +The .netrc file contains login and initialization information used by the auto-login process. +It resides in the user's home directory. The .netrc file format is simple. You specify lines +with a machine name and follow that with lines for the login and password that are +associated with that machine. Machine name is the hostname in your URL. + +An example .netrc for the host example.com with a user named 'daniel', using the password +'qwerty' would look like: + +.. code-block:: shell + + machine example.com + login daniel + password qwerty + +As mentioned in the `standard library docs `_, +only ASCII characters are allowed. Whitespace and non-printable characters are not allowed in passwords. + + +Keyring Support +--------------- + +pip also supports credentials stored in your keyring using the `keyring`_ +library. Note that ``keyring`` will need to be installed separately, as pip +does not come with it included. + +.. code-block:: shell + + pip install keyring + echo your-password | keyring set pypi.company.com your-username + pip install your-package --extra-index-url https://pypi.company.com/ + +.. _keyring: https://pypi.org/project/keyring/ + + Using a Proxy Server ==================== @@ -101,10 +160,21 @@ Requirements Files ================== "Requirements files" are files containing a list of items to be -installed using :ref:`pip install` like so:: +installed using :ref:`pip install` like so: - pip install -r requirements.txt +.. tabs:: + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -r requirements.txt Details on the format of the files are here: :ref:`Requirements File Format`. @@ -119,10 +189,21 @@ In practice, there are 4 common uses of Requirements files: this case, your requirement file contains a pinned version of everything that was installed when ``pip freeze`` was run. - :: +.. tabs:: - pip freeze > requirements.txt - pip install -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip freeze > requirements.txt + python -m pip install -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip freeze > requirements.txt + py -m pip install -r requirements.txt 2. Requirements files are used to force pip to properly resolve dependencies. As it is now, pip `doesn't have true dependency resolution @@ -187,9 +268,21 @@ contents is nearly identical to :ref:`Requirements Files`. There is one key difference: Including a package in a constraints file does not trigger installation of the package. -Use a constraints file like so:: +Use a constraints file like so: - pip install -c constraints.txt +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install -c constraints.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install -c constraints.txt Constraints files are used for exactly the same reason as requirements files when you don't know exactly what things you want to install. For instance, say @@ -227,9 +320,19 @@ archives. To install directly from a wheel archive: -:: +.. tabs:: - pip install SomePackage-1.0-py2.py3-none-any.whl + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install SomePackage-1.0-py2.py3-none-any.whl + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install SomePackage-1.0-py2.py3-none-any.whl For the cases where wheels are not available, pip offers :ref:`pip wheel` as a @@ -242,17 +345,38 @@ convenience, to build wheels for all your requirements and dependencies. To build wheels for your requirements and all their dependencies to a local directory: -:: +.. tabs:: - pip install wheel - pip wheel --wheel-dir=/local/wheels -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install wheel + python -m pip wheel --wheel-dir=/local/wheels -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install wheel + py -m pip wheel --wheel-dir=/local/wheels -r requirements.txt And *then* to install those requirements just using your local directory of wheels (and not from PyPI): -:: +.. tabs:: - pip install --no-index --find-links=/local/wheels -r requirements.txt + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --no-index --find-links=/local/wheels -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --no-index --find-links=/local/wheels -r requirements.txt Uninstalling Packages @@ -260,9 +384,20 @@ Uninstalling Packages pip is able to uninstall most packages like so: -:: +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip uninstall SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip uninstall SomePackage - $ pip uninstall SomePackage pip also performs an automatic uninstall of an old version of a package before upgrading to a newer version. @@ -275,33 +410,74 @@ Listing Packages To list installed packages: -:: +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip list + docutils (0.9.1) + Jinja2 (2.6) + Pygments (1.5) + Sphinx (1.1.2) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list + docutils (0.9.1) + Jinja2 (2.6) + Pygments (1.5) + Sphinx (1.1.2) - $ pip list - docutils (0.9.1) - Jinja2 (2.6) - Pygments (1.5) - Sphinx (1.1.2) To list outdated packages, and show the latest version available: -:: +.. tabs:: - $ pip list --outdated - docutils (Current: 0.9.1 Latest: 0.10) - Sphinx (Current: 1.1.2 Latest: 1.1.3) + .. group-tab:: Unix/macOS + .. code-block:: console + + $ python -m pip list --outdated + docutils (Current: 0.9.1 Latest: 0.10) + Sphinx (Current: 1.1.2 Latest: 1.1.3) + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip list --outdated + docutils (Current: 0.9.1 Latest: 0.10) + Sphinx (Current: 1.1.2 Latest: 1.1.3) To show details about an installed package: -:: +.. tabs:: - $ pip show sphinx - --- - Name: Sphinx - Version: 1.1.3 - Location: /my/env/lib/pythonx.x/site-packages - Requires: Pygments, Jinja2, docutils + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip show sphinx + --- + Name: Sphinx + Version: 1.1.3 + Location: /my/env/lib/pythonx.x/site-packages + Requires: Pygments, Jinja2, docutils + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip show sphinx + --- + Name: Sphinx + Version: 1.1.3 + Location: /my/env/lib/pythonx.x/site-packages + Requires: Pygments, Jinja2, docutils For more information and examples, see the :ref:`pip list` and :ref:`pip show` @@ -312,9 +488,21 @@ Searching for Packages ====================== pip can search `PyPI`_ for packages using the ``pip search`` -command:: +command: - $ pip search "query" +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip search "query" + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip search "query" The query will be used to search the names and summaries of all packages. @@ -436,6 +624,15 @@ and ``--no-cache-dir``, falsy values have to be used: no-compile = no no-warn-script-location = false +For options which can be repeated like ``--verbose`` and ``--quiet``, +a non-negative integer can be used to represent the level to be specified: + +.. code-block:: ini + + [global] + quiet = 0 + verbose = 2 + It is possible to append values to a section within a configuration file such as the pip.ini file. This is applicable to appending options like ``--find-links`` or ``--trusted-host``, which can be written on multiple lines: @@ -465,22 +662,81 @@ pip's command line options can be set with environment variables using the format ``PIP_`` . Dashes (``-``) have to be replaced with underscores (``_``). -For example, to set the default timeout:: +For example, to set the default timeout: - export PIP_DEFAULT_TIMEOUT=60 +.. tabs:: -This is the same as passing the option to pip directly:: + .. group-tab:: Unix/macOS - pip --default-timeout=60 [...] + .. code-block:: shell + + export PIP_DEFAULT_TIMEOUT=60 + + .. group-tab:: Windows + + .. code-block:: shell + + set PIP_DEFAULT_TIMEOUT=60 + +This is the same as passing the option to pip directly: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip --default-timeout=60 [...] + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip --default-timeout=60 [...] For command line options which can be repeated, use a space to separate -multiple values. For example:: +multiple values. For example: - export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + export PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + + .. group-tab:: Windows + + .. code-block:: shell + + set PIP_FIND_LINKS="http://mirror1.example.com http://mirror2.example.com" + + +is the same as calling: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + + +Options that do not take a value, but can be repeated (such as ``--verbose``) +can be specified using the number of repetitions, so:: + + export PIP_VERBOSE=3 is the same as calling:: - pip install --find-links=http://mirror1.example.com --find-links=http://mirror2.example.com + pip install -vvv .. note:: @@ -514,15 +770,15 @@ pip comes with support for command line completion in bash, zsh and fish. To setup for bash:: - $ pip completion --bash >> ~/.profile + python -m pip completion --bash >> ~/.profile To setup for zsh:: - $ pip completion --zsh >> ~/.zprofile + python -m pip completion --zsh >> ~/.zprofile To setup for fish:: -$ pip completion --fish > ~/.config/fish/completions/pip.fish + python -m pip completion --fish > ~/.config/fish/completions/pip.fish Alternatively, you can use the result of the ``completion`` command directly with the eval function of your shell, e.g. by adding the following to your @@ -541,24 +797,59 @@ Installing from local packages In some cases, you may want to install from local packages only, with no traffic to PyPI. -First, download the archives that fulfill your requirements:: +First, download the archives that fulfill your requirements: -$ pip download --destination-directory DIR -r requirements.txt +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip download --destination-directory DIR -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip download --destination-directory DIR -r requirements.txt Note that ``pip download`` will look in your wheel cache first, before trying to download from PyPI. If you've never installed your requirements before, you won't have a wheel cache for those items. In that case, if some of your requirements don't come as wheels from PyPI, and you want wheels, then run -this instead:: +this instead: -$ pip wheel --wheel-dir DIR -r requirements.txt +.. tabs:: + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip wheel --wheel-dir DIR -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip wheel --wheel-dir DIR -r requirements.txt Then, to install from local only, you'll be using :ref:`--find-links -` and :ref:`--no-index ` like so:: +` and :ref:`--no-index ` like so: -$ pip install --no-index --find-links=DIR -r requirements.txt +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --no-index --find-links=DIR -r requirements.txt + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --no-index --find-links=DIR -r requirements.txt "Only if needed" Recursive Upgrade @@ -577,10 +868,24 @@ The default strategy is ``only-if-needed``. This was changed in pip 10.0 due to the breaking nature of ``eager`` when upgrading conflicting dependencies. As an historic note, an earlier "fix" for getting the ``only-if-needed`` -behaviour was:: +behaviour was: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install --upgrade --no-deps SomePackage + python -m pip install SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install --upgrade --no-deps SomePackage + py -m pip install SomePackage - pip install --upgrade --no-deps SomePackage - pip install SomePackage A proposal for an ``upgrade-all`` command is being considered as a safer alternative to the behaviour of eager upgrading. @@ -603,11 +908,23 @@ Moreover, the "user scheme" can be customized by setting the ``site.USER_BASE``. To install "SomePackage" into an environment with site.USER_BASE customized to -'/myappenv', do the following:: +'/myappenv', do the following: - export PYTHONUSERBASE=/myappenv - pip install --user SomePackage +.. tabs:: + .. group-tab:: Unix/macOS + + .. code-block:: shell + + export PYTHONUSERBASE=/myappenv + python -m pip install --user SomePackage + + .. group-tab:: Windows + + .. code-block:: shell + + set PYTHONUSERBASE=c:/myappenv + py -m pip install --user SomePackage ``pip install --user`` follows four rules: @@ -630,54 +947,125 @@ To install "SomePackage" into an environment with site.USER_BASE customized to To make the rules clearer, here are some examples: -From within a ``--no-site-packages`` virtualenv (i.e. the default kind):: +From within a ``--no-site-packages`` virtualenv (i.e. the default kind): - $ pip install --user SomePackage - Can not perform a '--user' install. User site-packages are not visible in this virtualenv. +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install --user SomePackage + Can not perform a '--user' install. User site-packages are not visible in this virtualenv. + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --user SomePackage + Can not perform a '--user' install. User site-packages are not visible in this virtualenv. From within a ``--system-site-packages`` virtualenv where ``SomePackage==0.3`` -is already installed in the virtualenv:: +is already installed in the virtualenv: - $ pip install --user SomePackage==0.4 - Will not install to the user site because it will lack sys.path precedence +.. tabs:: + .. group-tab:: Unix/macOS -From within a real python, where ``SomePackage`` is *not* installed globally:: + .. code-block:: console - $ pip install --user SomePackage - [...] - Successfully installed SomePackage + $ python -m pip install --user SomePackage==0.4 + Will not install to the user site because it will lack sys.path precedence + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --user SomePackage==0.4 + Will not install to the user site because it will lack sys.path precedence + +From within a real python, where ``SomePackage`` is *not* installed globally: + +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: console + + $ python -m pip install --user SomePackage + [...] + Successfully installed SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --user SomePackage + [...] + Successfully installed SomePackage From within a real python, where ``SomePackage`` *is* installed globally, but -is *not* the latest version:: +is *not* the latest version: - $ pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) +.. tabs:: - $ pip install --user --upgrade SomePackage - [...] - Successfully installed SomePackage + .. group-tab:: Unix/macOS + .. code-block:: console + + $ python -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + $ python -m pip install --user --upgrade SomePackage + [...] + Successfully installed SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + C:\> py -m pip install --user --upgrade SomePackage + [...] + Successfully installed SomePackage From within a real python, where ``SomePackage`` *is* installed globally, and -is the latest version:: +is the latest version: - $ pip install --user SomePackage - [...] - Requirement already satisfied (use --upgrade to upgrade) +.. tabs:: - $ pip install --user --upgrade SomePackage - [...] - Requirement already up-to-date: SomePackage + .. group-tab:: Unix/macOS - # force the install - $ pip install --user --ignore-installed SomePackage - [...] - Successfully installed SomePackage + .. code-block:: console + $ python -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + $ python -m pip install --user --upgrade SomePackage + [...] + Requirement already up-to-date: SomePackage + # force the install + $ python -m pip install --user --ignore-installed SomePackage + [...] + Successfully installed SomePackage + + .. group-tab:: Windows + + .. code-block:: console + + C:\> py -m pip install --user SomePackage + [...] + Requirement already satisfied (use --upgrade to upgrade) + C:\> py -m pip install --user --upgrade SomePackage + [...] + Requirement already up-to-date: SomePackage + # force the install + C:\> py -m pip install --user --ignore-installed SomePackage + [...] + Successfully installed SomePackage .. _`Repeatability`: @@ -742,7 +1130,7 @@ index servers are unavailable and avoids time-consuming recompilation. Create an archive like this:: $ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX) - $ pip wheel -r requirements.txt --wheel-dir=$tempdir + $ python -m pip wheel -r requirements.txt --wheel-dir=$tempdir $ cwd=`pwd` $ (cd "$tempdir"; tar -cjvf "$cwd/bundled.tar.bz2" *) @@ -750,10 +1138,10 @@ You can then install from the archive like this:: $ tempdir=$(mktemp -d /tmp/wheelhouse-XXXXX) $ (cd $tempdir; tar -xvf /path/to/bundled.tar.bz2) - $ pip install --force-reinstall --ignore-installed --upgrade --no-index --no-deps $tempdir/* + $ python -m pip install --force-reinstall --ignore-installed --upgrade --no-index --no-deps $tempdir/* Note that compiled packages are typically OS- and architecture-specific, so -these archives are not necessarily portable across machines. +these archives are not necessarily portable across macOShines. Hash-checking mode can be used along with this method to ensure that future archives are built with identical packages. @@ -766,8 +1154,6 @@ archives are built with identical packages. to use such a package, see :ref:`Controlling setup_requires`. -.. _`Using pip from your program`: - Fixing conflicting dependencies =============================== @@ -785,10 +1171,22 @@ Understanding your error message When you get a ``ResolutionImpossible`` error, you might see something like this: -.. code-block:: console +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install package_coffee==0.44.1 package_tea==4.3.0 + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install package_coffee==0.44.1 package_tea==4.3.0 + +:: - $ pip install package_coffee==0.44.1 package_tea==4.3.0 - ... Due to conflicting dependencies pip cannot install package_coffee and package_tea: - package_coffee depends on package_water<3.0.0,>=2.4.2 @@ -879,7 +1277,7 @@ the same version of ``package_water``, you might consider: (e.g. ``pip install "package_coffee>0.44.*" "package_tea>4.0.0"``) - Asking pip to install *any* version of ``package_coffee`` and ``package_tea`` by removing the version specifiers altogether (e.g. - ``pip install package_coffee package_tea``) + ``python -m pip install package_coffee package_tea``) In the second case, pip will automatically find a version of both ``package_coffee`` and ``package_tea`` that depend on the same version of @@ -889,9 +1287,21 @@ In the second case, pip will automatically find a version of both - ``package_tea 4.3.0`` which *also* depends on ``package_water 2.6.1`` If you want to prioritize one package over another, you can add version -specifiers to *only* the more important package:: +specifiers to *only* the more important package: - pip install package_coffee==0.44.1b0 package_tea +.. tabs:: + + .. group-tab:: Unix/macOS + + .. code-block:: shell + + python -m pip install package_coffee==0.44.1b0 package_tea + + .. group-tab:: Windows + + .. code-block:: shell + + py -m pip install package_coffee==0.44.1b0 package_tea This will result in: @@ -957,6 +1367,8 @@ issue tracker`_ if you believe that your problem has exposed a bug in pip. .. _"How do I ask a good question?": https://stackoverflow.com/help/how-to-ask .. _pip issue tracker: https://github.com/pypa/pip/issues +.. _`Using pip from your program`: + Using pip from your program =========================== @@ -1046,7 +1458,7 @@ The big change in this release is to the pip dependency resolver within pip. Computers need to know the right order to install pieces of software -("to install `x`, you need to install `y` first"). So, when Python +("to install ``x``, you need to install ``y`` first"). So, when Python programmers share software as packages, they have to precisely describe those installation prerequisites, and pip needs to navigate tricky situations where it's getting conflicting instructions. This new @@ -1271,3 +1683,6 @@ announcements on the `low-traffic packaging announcements list`_ and .. _low-traffic packaging announcements list: https://mail.python.org/mailman3/lists/pypi-announce.python.org/ .. _our survey on upgrades that create conflicts: https://docs.google.com/forms/d/e/1FAIpQLSeBkbhuIlSofXqCyhi3kGkLmtrpPOEBwr6iJA6SzHdxWKfqdA/viewform .. _the official Python blog: https://blog.python.org/ +.. _requests: https://requests.readthedocs.io/en/master/user/authentication/#netrc-authentication +.. _Python standard library: https://docs.python.org/3/library/netrc.html +.. _Python Windows launcher: https://docs.python.org/3/using/windows.html#launcher diff --git a/docs/pip_sphinxext.py b/docs/pip_sphinxext.py index 6cc7a2c82..9386d71e7 100644 --- a/docs/pip_sphinxext.py +++ b/docs/pip_sphinxext.py @@ -15,11 +15,17 @@ from pip._internal.req.req_file import SUPPORTED_OPTIONS class PipCommandUsage(rst.Directive): required_arguments = 1 + optional_arguments = 3 def run(self): cmd = create_command(self.arguments[0]) + cmd_prefix = 'python -m pip' + if len(self.arguments) > 1: + cmd_prefix = " ".join(self.arguments[1:]) + cmd_prefix = cmd_prefix.strip('"') + cmd_prefix = cmd_prefix.strip("'") usage = dedent( - cmd.usage.replace('%prog', 'pip {}'.format(cmd.name)) + cmd.usage.replace('%prog', '{} {}'.format(cmd_prefix, cmd.name)) ).strip() node = nodes.literal_block(usage, usage) return [node] diff --git a/news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial.rst b/news/093f7456-cc25-4df9-9518-4732b1e07fe5.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial.rst b/news/0e494986-202e-4275-b7ec-d6f046c0aa05.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/24715eb2-c118-473b-9d99-4f6ce8bbfa83.trivial.rst b/news/24715eb2-c118-473b-9d99-4f6ce8bbfa83.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial.rst b/news/2a1e2773-eae6-43a4-b075-55f49e713fb1.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial.rst b/news/4c82ffb4-cde3-4dd5-8f37-6f4ef53e028b.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial.rst b/news/50cf024d-0a74-44c8-b3e9-483dd826fff2.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial.rst b/news/559bb022-21ae-498c-a2ce-2c354d880f5e.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial.rst b/news/5e8c60c2-d540-4a25-af03-100d848acbc0.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial.rst b/news/629892ca-55da-4ca9-9cff-c15373e97ad1.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial.rst b/news/6f8eff8d-9886-4e00-b431-5c809500e6bf.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/7231.doc.rst b/news/7231.doc.rst new file mode 100644 index 000000000..bef9bf3e6 --- /dev/null +++ b/news/7231.doc.rst @@ -0,0 +1 @@ +Add documentation for '.netrc' support. diff --git a/news/7311.doc.rst b/news/7311.doc.rst new file mode 100644 index 000000000..6ed2c4204 --- /dev/null +++ b/news/7311.doc.rst @@ -0,0 +1 @@ +Add OS tabs for OS-specific commands. diff --git a/news/8023.feature.rst b/news/8023.feature.rst new file mode 100644 index 000000000..c886e9a66 --- /dev/null +++ b/news/8023.feature.rst @@ -0,0 +1,2 @@ +New resolver: Avoid accessing indexes when the installed candidate is preferred +and considered good enough. diff --git a/news/8103.bugfix.rst b/news/8103.bugfix.rst new file mode 100644 index 000000000..55e4d6571 --- /dev/null +++ b/news/8103.bugfix.rst @@ -0,0 +1,2 @@ +Propagate ``--extra-index-url`` from requirements file properly to session auth, +so that keyring auth will work as expected. diff --git a/news/8181.removal.rst b/news/8181.removal.rst new file mode 100644 index 000000000..ae6bbe9f8 --- /dev/null +++ b/news/8181.removal.rst @@ -0,0 +1 @@ +Deprecate support for Python 3.5 diff --git a/news/8355.feature.rst b/news/8355.feature.rst new file mode 100644 index 000000000..3a7fb5375 --- /dev/null +++ b/news/8355.feature.rst @@ -0,0 +1 @@ +Add option ``--format`` to subcommand ``list`` of ``pip cache``, with ``abspath`` choice to output the full path of a wheel file. diff --git a/news/8417.removal.rst b/news/8417.removal.rst new file mode 100644 index 000000000..8f280b535 --- /dev/null +++ b/news/8417.removal.rst @@ -0,0 +1 @@ +Document that certain removals can be fast tracked. diff --git a/news/8578.bugfix.rst b/news/8578.bugfix.rst new file mode 100644 index 000000000..3df7ed078 --- /dev/null +++ b/news/8578.bugfix.rst @@ -0,0 +1,4 @@ +Allow specifying verbosity and quiet level via configuration files +and environment variables. Previously these options were treated as +boolean values when read from there while through CLI the level can be +specified. diff --git a/news/8636.doc.rst b/news/8636.doc.rst new file mode 100644 index 000000000..081cf1c7e --- /dev/null +++ b/news/8636.doc.rst @@ -0,0 +1 @@ +Add note and example on keyring support for index basic-auth diff --git a/news/8676.feature.rst b/news/8676.feature.rst new file mode 100644 index 000000000..f8da963f6 --- /dev/null +++ b/news/8676.feature.rst @@ -0,0 +1,2 @@ +Improve error message friendliness when an environment has packages with +corrupted metadata. diff --git a/news/8696.bugfix.rst b/news/8696.bugfix.rst new file mode 100644 index 000000000..989d2d029 --- /dev/null +++ b/news/8696.bugfix.rst @@ -0,0 +1,3 @@ +List downloaded distributions before exiting ``pip download`` +when using the new resolver to make the behavior the same as +that on the legacy resolver. diff --git a/news/8752.feature.rst b/news/8752.feature.rst new file mode 100644 index 000000000..d2560da18 --- /dev/null +++ b/news/8752.feature.rst @@ -0,0 +1,3 @@ +Make the ``setup.py install`` deprecation warning less noisy. We warn only +when ``setup.py install`` succeeded and ``setup.py bdist_wheel`` failed, as +situations where both fails are most probably irrelevant to this deprecation. diff --git a/news/8758.bugfix.rst b/news/8758.bugfix.rst new file mode 100644 index 000000000..9f44b7e47 --- /dev/null +++ b/news/8758.bugfix.rst @@ -0,0 +1,2 @@ +New resolver: Correctly respect ``Requires-Python`` metadata to reject +incompatible packages in ``--no-deps`` mode. diff --git a/news/8781.trivial.rst b/news/8781.trivial.rst new file mode 100644 index 000000000..e6044f52f --- /dev/null +++ b/news/8781.trivial.rst @@ -0,0 +1 @@ +Fix a broken slug anchor in user guide. diff --git a/news/8783.doc.rst b/news/8783.doc.rst new file mode 100644 index 000000000..6d2bb8762 --- /dev/null +++ b/news/8783.doc.rst @@ -0,0 +1 @@ +Added initial UX feedback widgets to docs. diff --git a/news/8792.bugfix.rst b/news/8792.bugfix.rst new file mode 100644 index 000000000..e83bdb09c --- /dev/null +++ b/news/8792.bugfix.rst @@ -0,0 +1,2 @@ +New resolver: Pick up hash declarations in constraints files and use them to +filter available distributions. diff --git a/news/8804.feature.rst b/news/8804.feature.rst new file mode 100644 index 000000000..a29333342 --- /dev/null +++ b/news/8804.feature.rst @@ -0,0 +1,3 @@ +Check the download directory for existing wheels to possibly avoid +fetching metadata when the ``fast-deps`` feature is used with +``pip wheel`` and ``pip download``. diff --git a/news/8807.doc.rst b/news/8807.doc.rst new file mode 100644 index 000000000..6ef1a123a --- /dev/null +++ b/news/8807.doc.rst @@ -0,0 +1 @@ +Add ux documentation diff --git a/news/8815.feature.rst b/news/8815.feature.rst new file mode 100644 index 000000000..7d9149d69 --- /dev/null +++ b/news/8815.feature.rst @@ -0,0 +1,2 @@ +When installing a git URL that refers to a commit that is not available locally +after git clone, attempt to fetch it from the remote. diff --git a/news/8827.bugfix.rst b/news/8827.bugfix.rst new file mode 100644 index 000000000..608cd3d5c --- /dev/null +++ b/news/8827.bugfix.rst @@ -0,0 +1,2 @@ +Avoid polluting the destination directory by resolution artifacts +when the new resolver is used for ``pip download`` or ``pip wheel``. diff --git a/news/8839.bugfix.rst b/news/8839.bugfix.rst new file mode 100644 index 000000000..987b801e9 --- /dev/null +++ b/news/8839.bugfix.rst @@ -0,0 +1,3 @@ +New resolver: If a package appears multiple times in user specification with +different ``--hash`` options, only hashes that present in all specifications +should be allowed. diff --git a/news/8848.doc.rst b/news/8848.doc.rst new file mode 100644 index 000000000..6d2bb8762 --- /dev/null +++ b/news/8848.doc.rst @@ -0,0 +1 @@ +Added initial UX feedback widgets to docs. diff --git a/news/8861.bugfix.rst b/news/8861.bugfix.rst new file mode 100644 index 000000000..d623419fa --- /dev/null +++ b/news/8861.bugfix.rst @@ -0,0 +1 @@ +Tweak the output during dependency resolution in the new resolver. diff --git a/news/8892.feature.rst b/news/8892.feature.rst new file mode 100644 index 000000000..96c99bf8c --- /dev/null +++ b/news/8892.feature.rst @@ -0,0 +1 @@ +Include http subdirectory in ``pip cache info`` and ``pip cache purge`` commands. diff --git a/news/8905.feature.rst b/news/8905.feature.rst new file mode 100644 index 000000000..5d27d40c2 --- /dev/null +++ b/news/8905.feature.rst @@ -0,0 +1,3 @@ +Cache package listings on index packages so they are guarenteed to stay stable +during a pip command session. This also improves performance when a index page +is accessed multiple times during the command session. diff --git a/news/8924.feature.rst b/news/8924.feature.rst new file mode 100644 index 000000000..c607aa0d0 --- /dev/null +++ b/news/8924.feature.rst @@ -0,0 +1,2 @@ +New resolver: Tweak resolution logic to improve user experience when +user-supplied requirements conflict. diff --git a/news/8927.removal.rst b/news/8927.removal.rst new file mode 100644 index 000000000..0032fa5f2 --- /dev/null +++ b/news/8927.removal.rst @@ -0,0 +1 @@ +Document that Python versions are generally supported until PyPI usage falls below 5%. diff --git a/news/8963.bugfix.rst b/news/8963.bugfix.rst new file mode 100644 index 000000000..62c01b464 --- /dev/null +++ b/news/8963.bugfix.rst @@ -0,0 +1,2 @@ +Correctly search for installed distributions in new resolver logic in order +to not miss packages (virtualenv packages from system-wide-packages for example) diff --git a/news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial.rst b/news/946beace-6164-4d1a-a05d-e9bebf43ccd0.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial.rst b/news/9f8da1d9-dd18-47e9-b334-5eb862054409.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial.rst b/news/a2fa2e68-01bf-11eb-a0b1-4fe8cb1f9dcf.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial.rst b/news/a3a2b1b7-744e-4533-b3ff-6e7a1843d573.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial.rst b/news/bc7f9ea0-030d-11eb-92cb-6b2b625d02fc.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial.rst b/news/c6182139-edb4-4bf6-bc3f-2d37cb5759ad.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/d53425e4-767e-4d73-bce5-88644b781855.trivial.rst b/news/d53425e4-767e-4d73-bce5-88644b781855.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial.rst b/news/d90a40c1-15b7-46b9-9162-335bb346b53f.trivial.rst new file mode 100644 index 000000000..e69de29bb diff --git a/setup.cfg b/setup.cfg index 45fd58a3e..5f4fd4036 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,4 +1,5 @@ [isort] +profile = black skip = ./build, .nox, @@ -6,16 +7,11 @@ skip = .scratch, _vendor, data -multi_line_output = 3 known_third_party = pip._vendor -known_first_party = - pip - tests -default_section = THIRDPARTY -include_trailing_comma = true [flake8] +max-line-length = 88 exclude = ./build, .nox, @@ -24,10 +20,10 @@ exclude = _vendor, data enable-extensions = G -ignore = +extend-ignore = G200, G202, - # pycodestyle checks ignored in the default configuration - E121, E123, E126, E133, E226, E241, E242, E704, W503, W504, W505, + # black adds spaces around ':' + E203, per-file-ignores = # G: The plugin logging-format treats every .log and .error as logging. noxfile.py: G diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 264c2cab8..a778e9948 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -2,7 +2,7 @@ import pip._internal.utils.inject_securetransport # noqa from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List + from typing import List, Optional def main(args=None): diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 28d1ad689..a08e63cd0 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -19,7 +19,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from types import TracebackType - from typing import Tuple, Set, Iterable, Optional, List, Type + from typing import Iterable, List, Optional, Set, Tuple, Type + from pip._internal.index.package_finder import PackageFinder logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 07db948b9..def8dd64a 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -17,7 +17,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.urls import path_to_url if MYPY_CHECK_RUNNING: - from typing import Optional, Set, List, Any, Dict + from typing import Any, Dict, List, Optional, Set from pip._vendor.packaging.tags import Tag diff --git a/src/pip/_internal/cli/base_command.py b/src/pip/_internal/cli/base_command.py index 197400a72..e4b07e0ce 100644 --- a/src/pip/_internal/cli/base_command.py +++ b/src/pip/_internal/cli/base_command.py @@ -12,10 +12,7 @@ import traceback from pip._internal.cli import cmdoptions from pip._internal.cli.command_context import CommandContextMixIn -from pip._internal.cli.parser import ( - ConfigOptionParser, - UpdatingDefaultsHelpFormatter, -) +from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.cli.status_codes import ( ERROR, PREVIOUS_BUILD_DIR_ERROR, @@ -35,19 +32,16 @@ from pip._internal.utils.deprecation import deprecated from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging from pip._internal.utils.misc import get_prog, normalize_path -from pip._internal.utils.temp_dir import ( - global_tempdir_manager, - tempdir_registry, -) +from pip._internal.utils.temp_dir import global_tempdir_manager, tempdir_registry from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.virtualenv import running_under_virtualenv if MYPY_CHECK_RUNNING: - from typing import List, Optional, Tuple, Any from optparse import Values + from typing import Any, List, Optional, Tuple from pip._internal.utils.temp_dir import ( - TempDirectoryTypeRegistry as TempDirRegistry + TempDirectoryTypeRegistry as TempDirRegistry, ) __all__ = ['Command'] diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index ed42c5f5a..2f640b2cb 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -30,8 +30,9 @@ from pip._internal.utils.hashes import STRONG_HASHES from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Callable, Dict, Optional, Tuple from optparse import OptionParser, Values + from typing import Any, Callable, Dict, Optional, Tuple + from pip._internal.cli.parser import ConfigOptionParser @@ -187,7 +188,7 @@ no_color = partial( dest='no_color', action='store_true', default=False, - help="Suppress colored output", + help="Suppress colored output.", ) # type: Callable[..., Option] version = partial( diff --git a/src/pip/_internal/cli/command_context.py b/src/pip/_internal/cli/command_context.py index d1a64a776..669c77774 100644 --- a/src/pip/_internal/cli/command_context.py +++ b/src/pip/_internal/cli/command_context.py @@ -5,7 +5,7 @@ from pip._vendor.contextlib2 import ExitStack from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterator, ContextManager, TypeVar + from typing import ContextManager, Iterator, TypeVar _T = TypeVar('_T', covariant=True) diff --git a/src/pip/_internal/cli/main_parser.py b/src/pip/_internal/cli/main_parser.py index 08c82c1f7..ba3cf68aa 100644 --- a/src/pip/_internal/cli/main_parser.py +++ b/src/pip/_internal/cli/main_parser.py @@ -5,17 +5,14 @@ import os import sys from pip._internal.cli import cmdoptions -from pip._internal.cli.parser import ( - ConfigOptionParser, - UpdatingDefaultsHelpFormatter, -) +from pip._internal.cli.parser import ConfigOptionParser, UpdatingDefaultsHelpFormatter from pip._internal.commands import commands_dict, get_similar_commands from pip._internal.exceptions import CommandError from pip._internal.utils.misc import get_pip_version, get_prog from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Tuple, List + from typing import List, Tuple __all__ = ["create_main_parser", "parse_command"] diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py index 04e00b721..b6b78318a 100644 --- a/src/pip/_internal/cli/parser.py +++ b/src/pip/_internal/cli/parser.py @@ -11,6 +11,7 @@ import sys import textwrap from distutils.util import strtobool +from pip._vendor.contextlib2 import suppress from pip._vendor.six import string_types from pip._internal.cli.status_codes import UNKNOWN_ERROR @@ -197,15 +198,27 @@ class ConfigOptionParser(CustomOptionParser): if option is None: continue - if option.action in ('store_true', 'store_false', 'count'): + if option.action in ('store_true', 'store_false'): try: val = strtobool(val) except ValueError: - error_msg = invalid_config_error_message( - option.action, key, val + self.error( + '{} is not a valid value for {} option, ' # noqa + 'please specify a boolean value like yes/no, ' + 'true/false or 1/0 instead.'.format(val, key) + ) + elif option.action == 'count': + with suppress(ValueError): + val = strtobool(val) + with suppress(ValueError): + val = int(val) + if not isinstance(val, int) or val < 0: + self.error( + '{} is not a valid value for {} option, ' # noqa + 'please instead specify either a non-negative integer ' + 'or a boolean value like yes/no or false/true ' + 'which is equivalent to 1/0.'.format(val, key) ) - self.error(error_msg) - elif option.action == 'append': val = val.split() val = [self.check_default(option, key, v) for v in val] @@ -251,16 +264,3 @@ class ConfigOptionParser(CustomOptionParser): def error(self, msg): self.print_usage(sys.stderr) self.exit(UNKNOWN_ERROR, "{}\n".format(msg)) - - -def invalid_config_error_message(action, key, val): - """Returns a better error message when invalid configuration option - is provided.""" - if action in ('store_true', 'store_false'): - return ("{0} is not a valid value for {1} option, " - "please specify a boolean value like yes/no, " - "true/false or 1/0 instead.").format(val, key) - - return ("{0} is not a valid value for {1} option, " - "please specify a numerical value like 1/0 " - "instead.").format(val, key) diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 78b5ce6a1..03cc52f69 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -16,7 +16,6 @@ from pip._internal.exceptions import CommandError, PreviousBuildDirError from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.selection_prefs import SelectionPreferences -from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req.constructors import ( @@ -39,10 +38,7 @@ if MYPY_CHECK_RUNNING: from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolution.base import BaseResolver - from pip._internal.utils.temp_dir import ( - TempDirectory, - TempDirectoryTypeRegistry, - ) + from pip._internal.utils.temp_dir import TempDirectory, TempDirectoryTypeRegistry logger = logging.getLogger(__name__) @@ -207,28 +203,39 @@ class RequirementCommand(IndexGroupCommand): finder, # type: PackageFinder use_user_site, # type: bool download_dir=None, # type: str - wheel_download_dir=None, # type: str ): # type: (...) -> RequirementPreparer """ Create a RequirementPreparer instance for the given parameters. """ - downloader = Downloader(session, progress_bar=options.progress_bar) - temp_build_dir_path = temp_build_dir.path assert temp_build_dir_path is not None + if '2020-resolver' in options.features_enabled: + lazy_wheel = 'fast-deps' in options.features_enabled + if lazy_wheel: + logger.warning( + 'pip is using lazily downloaded wheels using HTTP ' + 'range requests to obtain dependency information. ' + 'This experimental feature is enabled through ' + '--use-feature=fast-deps and it is not ready for ' + 'production.' + ) + else: + lazy_wheel = False + return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, download_dir=download_dir, - wheel_download_dir=wheel_download_dir, build_isolation=options.build_isolation, req_tracker=req_tracker, - downloader=downloader, + session=session, + progress_bar=options.progress_bar, finder=finder, require_hashes=options.require_hashes, use_user_site=use_user_site, + lazy_wheel=lazy_wheel, ) @staticmethod @@ -259,6 +266,7 @@ class RequirementCommand(IndexGroupCommand): # "Resolver" class being redefined. if '2020-resolver' in options.features_enabled: import pip._internal.resolution.resolvelib.resolver + return pip._internal.resolution.resolvelib.resolver.Resolver( preparer=preparer, finder=finder, @@ -271,7 +279,6 @@ class RequirementCommand(IndexGroupCommand): force_reinstall=force_reinstall, upgrade_strategy=upgrade_strategy, py_version_info=py_version_info, - lazy_wheel='fast-deps' in options.features_enabled, ) import pip._internal.resolution.legacy.resolver return pip._internal.resolution.legacy.resolver.Resolver( diff --git a/src/pip/_internal/cli/spinners.py b/src/pip/_internal/cli/spinners.py index c6c4c5cd1..65c3c23d7 100644 --- a/src/pip/_internal/cli/spinners.py +++ b/src/pip/_internal/cli/spinners.py @@ -13,7 +13,7 @@ from pip._internal.utils.logging import get_indentation from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterator, IO + from typing import IO, Iterator logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index 6825fa6e2..4f0c4ba3a 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -18,6 +18,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Any + from pip._internal.cli.base_command import Command diff --git a/src/pip/_internal/commands/cache.py b/src/pip/_internal/commands/cache.py index 747277f6e..ec21be68f 100644 --- a/src/pip/_internal/commands/cache.py +++ b/src/pip/_internal/commands/cache.py @@ -37,11 +37,25 @@ class CacheCommand(Command): usage = """ %prog dir %prog info - %prog list [] + %prog list [] [--format=[human, abspath]] %prog remove %prog purge """ + def add_options(self): + # type: () -> None + + self.cmd_opts.add_option( + '--format', + action='store', + dest='list_format', + default="human", + choices=('human', 'abspath'), + help="Select the output format among: human (default) or abspath" + ) + + self.parser.insert_option_group(0, self.cmd_opts) + def run(self, options, args): # type: (Values, List[Any]) -> int handlers = { @@ -88,19 +102,30 @@ class CacheCommand(Command): if args: raise CommandError('Too many arguments') + num_http_files = len(self._find_http_files(options)) num_packages = len(self._find_wheels(options, '*')) - cache_location = self._wheels_cache_dir(options) - cache_size = filesystem.format_directory_size(cache_location) + http_cache_location = self._cache_dir(options, 'http') + wheels_cache_location = self._cache_dir(options, 'wheels') + http_cache_size = filesystem.format_directory_size(http_cache_location) + wheels_cache_size = filesystem.format_directory_size( + wheels_cache_location + ) message = textwrap.dedent(""" - Location: {location} - Size: {size} + Package index page cache location: {http_cache_location} + Package index page cache size: {http_cache_size} + Number of HTTP files: {num_http_files} + Wheels location: {wheels_cache_location} + Wheels size: {wheels_cache_size} Number of wheels: {package_count} """).format( - location=cache_location, + http_cache_location=http_cache_location, + http_cache_size=http_cache_size, + num_http_files=num_http_files, + wheels_cache_location=wheels_cache_location, package_count=num_packages, - size=cache_size, + wheels_cache_size=wheels_cache_size, ).strip() logger.info(message) @@ -116,7 +141,13 @@ class CacheCommand(Command): pattern = '*' files = self._find_wheels(options, pattern) + if options.list_format == 'human': + self.format_for_human(files) + else: + self.format_for_abspath(files) + def format_for_human(self, files): + # type: (List[str]) -> None if not files: logger.info('Nothing cached.') return @@ -129,6 +160,17 @@ class CacheCommand(Command): logger.info('Cache contents:\n') logger.info('\n'.join(sorted(results))) + def format_for_abspath(self, files): + # type: (List[str]) -> None + if not files: + return + + results = [] + for filename in files: + results.append(filename) + + logger.info('\n'.join(sorted(results))) + def remove_cache_items(self, options, args): # type: (Values, List[Any]) -> None if len(args) > 1: @@ -138,6 +180,11 @@ class CacheCommand(Command): raise CommandError('Please provide a pattern') files = self._find_wheels(options, args[0]) + + # Only fetch http files if no specific pattern given + if args[0] == '*': + files += self._find_http_files(options) + if not files: raise CommandError('No matching packages') @@ -153,13 +200,18 @@ class CacheCommand(Command): return self.remove_cache_items(options, ['*']) - def _wheels_cache_dir(self, options): - # type: (Values) -> str - return os.path.join(options.cache_dir, 'wheels') + def _cache_dir(self, options, subdir): + # type: (Values, str) -> str + return os.path.join(options.cache_dir, subdir) + + def _find_http_files(self, options): + # type: (Values) -> List[str] + http_dir = self._cache_dir(options, 'http') + return filesystem.find_files(http_dir, '*') def _find_wheels(self, options, pattern): # type: (Values, str) -> List[str] - wheel_dir = self._wheels_cache_dir(options) + wheel_dir = self._cache_dir(options, 'wheels') # The wheel filename format, as specified in PEP 427, is: # {distribution}-{version}(-{build})?-{python}-{abi}-{platform}.whl diff --git a/src/pip/_internal/commands/check.py b/src/pip/_internal/commands/check.py index b557ca641..e066bb63c 100644 --- a/src/pip/_internal/commands/check.py +++ b/src/pip/_internal/commands/check.py @@ -12,8 +12,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING logger = logging.getLogger(__name__) if MYPY_CHECK_RUNNING: - from typing import List, Any from optparse import Values + from typing import Any, List class CheckCommand(Command): diff --git a/src/pip/_internal/commands/completion.py b/src/pip/_internal/commands/completion.py index 9b99f51f0..b19f1ed1a 100644 --- a/src/pip/_internal/commands/completion.py +++ b/src/pip/_internal/commands/completion.py @@ -9,8 +9,8 @@ from pip._internal.utils.misc import get_prog from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List from optparse import Values + from typing import List BASE_COMPLETION = """ # pip {shell} completion start{script}# pip {shell} completion end diff --git a/src/pip/_internal/commands/configuration.py b/src/pip/_internal/commands/configuration.py index f9b3ab79d..1ab90b47b 100644 --- a/src/pip/_internal/commands/configuration.py +++ b/src/pip/_internal/commands/configuration.py @@ -4,19 +4,15 @@ import subprocess from pip._internal.cli.base_command import Command from pip._internal.cli.status_codes import ERROR, SUCCESS -from pip._internal.configuration import ( - Configuration, - get_configuration_files, - kinds, -) +from pip._internal.configuration import Configuration, get_configuration_files, kinds from pip._internal.exceptions import PipError from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import get_prog, write_output from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Any, Optional from optparse import Values + from typing import Any, List, Optional from pip._internal.configuration import Kind diff --git a/src/pip/_internal/commands/debug.py b/src/pip/_internal/commands/debug.py index ff369d7d9..1b65c4306 100644 --- a/src/pip/_internal/commands/debug.py +++ b/src/pip/_internal/commands/debug.py @@ -19,9 +19,10 @@ from pip._internal.utils.misc import get_pip_version from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from types import ModuleType - from typing import List, Optional, Dict from optparse import Values + from types import ModuleType + from typing import Dict, List, Optional + from pip._internal.configuration import Configuration logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 46e837126..2f151e049 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -87,7 +87,6 @@ class DownloadCommand(RequirementCommand): cmdoptions.check_dist_restriction(options) options.download_dir = normalize_path(options.download_dir) - ensure_dir(options.download_dir) session = self.get_default_session(options) @@ -134,10 +133,13 @@ class DownloadCommand(RequirementCommand): reqs, check_supported_wheels=True ) - downloaded = ' '.join([req.name # type: ignore - for req in requirement_set.requirements.values() - if req.successfully_downloaded]) + downloaded = [] # type: List[str] + for req in requirement_set.requirements.values(): + if not req.editable and req.satisfied_by is None: + assert req.name is not None + preparer.save_linked_requirement(req) + downloaded.append(req.name) if downloaded: - write_output('Successfully downloaded %s', downloaded) + write_output('Successfully downloaded %s', ' '.join(downloaded)) return SUCCESS diff --git a/src/pip/_internal/commands/help.py b/src/pip/_internal/commands/help.py index a2edc2989..2ab2b6d8f 100644 --- a/src/pip/_internal/commands/help.py +++ b/src/pip/_internal/commands/help.py @@ -6,8 +6,8 @@ from pip._internal.exceptions import CommandError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List from optparse import Values + from typing import List class HelpCommand(Command): @@ -20,7 +20,9 @@ class HelpCommand(Command): def run(self, options, args): # type: (Values, List[str]) -> int from pip._internal.commands import ( - commands_dict, create_command, get_similar_commands, + commands_dict, + create_command, + get_similar_commands, ) try: diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index 704e2d656..e41660070 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -21,7 +21,6 @@ from pip._internal.locations import distutils_scheme from pip._internal.operations.check import check_install_conflicts from pip._internal.req import install_given_reqs from pip._internal.req.req_tracker import get_requirement_tracker -from pip._internal.utils.datetime import today_is_later_than from pip._internal.utils.distutils_args import parse_distutils_args from pip._internal.utils.filesystem import test_writable_dir from pip._internal.utils.misc import ( @@ -543,19 +542,6 @@ class InstallCommand(RequirementCommand): "your packages with the new resolver before it becomes the " "default.\n" ) - elif not today_is_later_than(year=2020, month=7, day=31): - # NOTE: trailing newlines here are intentional - parts.append( - "Pip will install or upgrade your package(s) and its " - "dependencies without taking into account other packages you " - "already have installed. This may cause an uncaught " - "dependency conflict.\n" - ) - form_link = "https://forms.gle/cWKMoDs8sUVE29hz9" - parts.append( - "If you would like pip to take your other packages into " - "account, please tell us here: {}\n".format(form_link) - ) # NOTE: There is some duplication here, with commands/check.py for project_name in missing: diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 20e9bff2b..a6dfa5fd5 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -24,10 +24,11 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from optparse import Values - from typing import List, Set, Tuple, Iterator + from typing import Iterator, List, Set, Tuple + + from pip._vendor.pkg_resources import Distribution from pip._internal.network.session import PipSession - from pip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) @@ -201,7 +202,6 @@ class ListCommand(IndexGroupCommand): def latest_info(dist): # type: (Distribution) -> Distribution - typ = 'unknown' all_candidates = finder.find_all_candidates(dist.key) if not options.pre: # Remove prereleases diff --git a/src/pip/_internal/commands/search.py b/src/pip/_internal/commands/search.py index ff0947202..146d653e5 100644 --- a/src/pip/_internal/commands/search.py +++ b/src/pip/_internal/commands/search.py @@ -7,6 +7,7 @@ from collections import OrderedDict from pip._vendor import pkg_resources from pip._vendor.packaging.version import parse as parse_version + # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is # why we ignore the type on this import from pip._vendor.six.moves import xmlrpc_client # type: ignore @@ -24,7 +25,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from optparse import Values - from typing import List, Dict, Optional + from typing import Dict, List, Optional + from typing_extensions import TypedDict TransformedHit = TypedDict( 'TransformedHit', diff --git a/src/pip/_internal/commands/show.py b/src/pip/_internal/commands/show.py index 3892c5959..b0b3f3abd 100644 --- a/src/pip/_internal/commands/show.py +++ b/src/pip/_internal/commands/show.py @@ -14,7 +14,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from optparse import Values - from typing import List, Dict, Iterator + from typing import Dict, Iterator, List logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 0f718566b..8f5783c35 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -21,6 +21,7 @@ if MYPY_CHECK_RUNNING: from optparse import Values from typing import List + from pip._internal.req.req_install import InstallRequirement logger = logging.getLogger(__name__) @@ -137,7 +138,7 @@ class WheelCommand(RequirementCommand): req_tracker=req_tracker, session=session, finder=finder, - wheel_download_dir=options.wheel_dir, + download_dir=options.wheel_dir, use_user_site=False, ) @@ -156,10 +157,12 @@ class WheelCommand(RequirementCommand): reqs, check_supported_wheels=True ) - reqs_to_build = [ - r for r in requirement_set.requirements.values() - if should_build_for_wheel_command(r) - ] + reqs_to_build = [] # type: List[InstallRequirement] + for req in requirement_set.requirements.values(): + if req.is_wheel: + preparer.save_linked_requirement(req) + elif should_build_for_wheel_command(req): + reqs_to_build.append(req) # build wheels build_successes, build_failures = build( diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py index e49a5f4f5..23614fd2b 100644 --- a/src/pip/_internal/configuration.py +++ b/src/pip/_internal/configuration.py @@ -28,13 +28,25 @@ from pip._internal.utils.misc import ensure_dir, enum from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, NewType, Optional, Tuple - ) + from typing import Any, Dict, Iterable, List, NewType, Optional, Tuple RawConfigParser = configparser.RawConfigParser # Shorthand Kind = NewType("Kind", str) +CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' +ENV_NAMES_IGNORED = "version", "help" + +# The kinds of configurations there are. +kinds = enum( + USER="user", # User Specific + GLOBAL="global", # System Wide + SITE="site", # [Virtual] Environment Specific + ENV="env", # from PIP_CONFIG_FILE + ENV_VAR="env-var", # from Environment Variables +) +OVERRIDE_ORDER = kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR +VALID_LOAD_ONLY = kinds.USER, kinds.GLOBAL, kinds.SITE + logger = logging.getLogger(__name__) @@ -60,19 +72,6 @@ def _disassemble_key(name): return name.split(".", 1) -# The kinds of configurations there are. -kinds = enum( - USER="user", # User Specific - GLOBAL="global", # System Wide - SITE="site", # [Virtual] Environment Specific - ENV="env", # from PIP_CONFIG_FILE - ENV_VAR="env-var", # from Environment Variables -) - - -CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' - - def get_configuration_files(): # type: () -> Dict[Kind, List[str]] global_config_files = [ @@ -114,29 +113,21 @@ class Configuration(object): # type: (bool, Optional[Kind]) -> None super(Configuration, self).__init__() - _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None] - if load_only not in _valid_load_only: + if load_only is not None and load_only not in VALID_LOAD_ONLY: raise ConfigurationError( "Got invalid value for load_only - should be one of {}".format( - ", ".join(map(repr, _valid_load_only[:-1])) + ", ".join(map(repr, VALID_LOAD_ONLY)) ) ) self.isolated = isolated self.load_only = load_only - # The order here determines the override order. - self._override_order = [ - kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR - ] - - self._ignore_env_names = ["version", "help"] - # Because we keep track of where we got the data from self._parsers = { - variant: [] for variant in self._override_order + variant: [] for variant in OVERRIDE_ORDER } # type: Dict[Kind, List[Tuple[str, RawConfigParser]]] self._config = { - variant: {} for variant in self._override_order + variant: {} for variant in OVERRIDE_ORDER } # type: Dict[Kind, Dict[str, Any]] self._modified_parsers = [] # type: List[Tuple[str, RawConfigParser]] @@ -257,7 +248,7 @@ class Configuration(object): # are not needed here. retval = {} - for variant in self._override_order: + for variant in OVERRIDE_ORDER: retval.update(self._config[variant]) return retval @@ -348,12 +339,10 @@ class Configuration(object): # type: () -> Iterable[Tuple[str, str]] """Returns a generator with all environmental vars with prefix PIP_""" for key, val in os.environ.items(): - should_be_yielded = ( - key.startswith("PIP_") and - key[4:].lower() not in self._ignore_env_names - ) - if should_be_yielded: - yield key[4:].lower(), val + if key.startswith("PIP_"): + name = key[4:].lower() + if name not in ENV_NAMES_IGNORED: + yield name, val # XXX: This is patched in the tests. def iter_config_files(self): diff --git a/src/pip/_internal/distributions/base.py b/src/pip/_internal/distributions/base.py index b836b98d1..3a789f804 100644 --- a/src/pip/_internal/distributions/base.py +++ b/src/pip/_internal/distributions/base.py @@ -8,8 +8,9 @@ if MYPY_CHECK_RUNNING: from typing import Optional from pip._vendor.pkg_resources import Distribution - from pip._internal.req import InstallRequirement + from pip._internal.index.package_finder import PackageFinder + from pip._internal.req import InstallRequirement @add_metaclass(abc.ABCMeta) diff --git a/src/pip/_internal/distributions/installed.py b/src/pip/_internal/distributions/installed.py index 0d15bf424..a813b211f 100644 --- a/src/pip/_internal/distributions/installed.py +++ b/src/pip/_internal/distributions/installed.py @@ -5,6 +5,7 @@ if MYPY_CHECK_RUNNING: from typing import Optional from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/distributions/sdist.py b/src/pip/_internal/distributions/sdist.py index be3d7d97a..06b9df09c 100644 --- a/src/pip/_internal/distributions/sdist.py +++ b/src/pip/_internal/distributions/sdist.py @@ -10,6 +10,7 @@ if MYPY_CHECK_RUNNING: from typing import Set, Tuple from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/distributions/wheel.py b/src/pip/_internal/distributions/wheel.py index bf3482b15..2adc22862 100644 --- a/src/pip/_internal/distributions/wheel.py +++ b/src/pip/_internal/distributions/wheel.py @@ -6,6 +6,7 @@ from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel if MYPY_CHECK_RUNNING: from pip._vendor.pkg_resources import Distribution + from pip._internal.index.package_finder import PackageFinder diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py index 3f26215d6..62bde1eed 100644 --- a/src/pip/_internal/exceptions.py +++ b/src/pip/_internal/exceptions.py @@ -9,10 +9,10 @@ from pip._vendor.six import iteritems from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Optional, List, Dict, Text + from typing import Any, Dict, List, Optional, Text from pip._vendor.pkg_resources import Distribution - from pip._vendor.requests.models import Response, Request + from pip._vendor.requests.models import Request, Response from pip._vendor.six import PY3 from pip._vendor.six.moves import configparser diff --git a/src/pip/_internal/index/collector.py b/src/pip/_internal/index/collector.py index ef2100f89..7b9abbf69 100644 --- a/src/pip/_internal/index/collector.py +++ b/src/pip/_internal/index/collector.py @@ -29,6 +29,7 @@ from pip._internal.utils.urls import path_to_url, url_to_path from pip._internal.vcs import is_url, vcs if MYPY_CHECK_RUNNING: + import xml.etree.ElementTree from optparse import Values from typing import ( Callable, @@ -40,7 +41,6 @@ if MYPY_CHECK_RUNNING: Tuple, Union, ) - import xml.etree.ElementTree from pip._vendor.requests import Response diff --git a/src/pip/_internal/index/package_finder.py b/src/pip/_internal/index/package_finder.py index 8ceccee6c..b361e194d 100644 --- a/src/pip/_internal/index/package_finder.py +++ b/src/pip/_internal/index/package_finder.py @@ -35,9 +35,7 @@ from pip._internal.utils.unpacking import SUPPORTED_EXTENSIONS from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: - from typing import ( - FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union, - ) + from typing import FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union from pip._vendor.packaging.tags import Tag from pip._vendor.packaging.version import _BaseVersion diff --git a/src/pip/_internal/locations.py b/src/pip/_internal/locations.py index 0c1235488..35a4512b4 100644 --- a/src/pip/_internal/locations.py +++ b/src/pip/_internal/locations.py @@ -22,9 +22,8 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast from pip._internal.utils.virtualenv import running_under_virtualenv if MYPY_CHECK_RUNNING: - from typing import Dict, List, Optional, Union - from distutils.cmd import Command as DistutilsCommand + from typing import Dict, List, Optional, Union # Application Directories diff --git a/src/pip/_internal/main.py b/src/pip/_internal/main.py index 3208d5b88..1c99c49a1 100644 --- a/src/pip/_internal/main.py +++ b/src/pip/_internal/main.py @@ -1,7 +1,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List + from typing import List, Optional def main(args=None): diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py index 9149e0fc6..0d89a8c07 100644 --- a/src/pip/_internal/models/candidate.py +++ b/src/pip/_internal/models/candidate.py @@ -5,6 +5,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from pip._vendor.packaging.version import _BaseVersion + from pip._internal.models.link import Link diff --git a/src/pip/_internal/models/direct_url.py b/src/pip/_internal/models/direct_url.py index 87bd9fe4b..99aa68d12 100644 --- a/src/pip/_internal/models/direct_url.py +++ b/src/pip/_internal/models/direct_url.py @@ -8,9 +8,7 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, Optional, Type, TypeVar, Union - ) + from typing import Any, Dict, Iterable, Optional, Type, TypeVar, Union T = TypeVar("T") diff --git a/src/pip/_internal/models/format_control.py b/src/pip/_internal/models/format_control.py index c6275e721..adcf61e28 100644 --- a/src/pip/_internal/models/format_control.py +++ b/src/pip/_internal/models/format_control.py @@ -4,7 +4,7 @@ from pip._internal.exceptions import CommandError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Set, FrozenSet + from typing import FrozenSet, Optional, Set class FormatControl(object): diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py index c0d278ade..29ef402be 100644 --- a/src/pip/_internal/models/link.py +++ b/src/pip/_internal/models/link.py @@ -16,6 +16,7 @@ from pip._internal.utils.urls import path_to_url, url_to_path if MYPY_CHECK_RUNNING: from typing import Optional, Text, Tuple, Union + from pip._internal.index.collector import HTMLPage from pip._internal.utils.hashes import Hashes diff --git a/src/pip/_internal/models/selection_prefs.py b/src/pip/_internal/models/selection_prefs.py index 5db3ca91c..83110dd8f 100644 --- a/src/pip/_internal/models/selection_prefs.py +++ b/src/pip/_internal/models/selection_prefs.py @@ -2,6 +2,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Optional + from pip._internal.models.format_control import FormatControl diff --git a/src/pip/_internal/models/target_python.py b/src/pip/_internal/models/target_python.py index 6d1ca7964..ad7e506a6 100644 --- a/src/pip/_internal/models/target_python.py +++ b/src/pip/_internal/models/target_python.py @@ -1,9 +1,6 @@ import sys -from pip._internal.utils.compatibility_tags import ( - get_supported, - version_info_to_nodot, -) +from pip._internal.utils.compatibility_tags import get_supported, version_info_to_nodot from pip._internal.utils.misc import normalize_version_info from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index c49deaaf1..357811a16 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -20,12 +20,12 @@ from pip._internal.utils.misc import ( from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Dict, Optional, Tuple, List, Any + from typing import Any, Dict, List, Optional, Tuple + + from pip._vendor.requests.models import Request, Response from pip._internal.vcs.versioncontrol import AuthInfo - from pip._vendor.requests.models import Response, Request - Credentials = Tuple[str, str, str] logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index a0d55b5e9..d2a1b7313 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -13,7 +13,7 @@ from pip._internal.utils.misc import ensure_dir from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Iterator + from typing import Iterator, Optional def is_from_cache(response): diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 44f9985a3..76896e899 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -11,20 +11,12 @@ from pip._internal.cli.progress_bars import DownloadProgressProvider from pip._internal.exceptions import NetworkConnectionError from pip._internal.models.index import PyPI from pip._internal.network.cache import is_from_cache -from pip._internal.network.utils import ( - HEADERS, - raise_for_status, - response_chunks, -) -from pip._internal.utils.misc import ( - format_size, - redact_auth_from_url, - splitext, -) +from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks +from pip._internal.utils.misc import format_size, redact_auth_from_url, splitext from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Iterable, Optional + from typing import Iterable, Optional, Tuple from pip._vendor.requests.models import Response @@ -141,19 +133,6 @@ def _http_get_download(session, link): return resp -class Download(object): - def __init__( - self, - response, # type: Response - filename, # type: str - chunks, # type: Iterable[bytes] - ): - # type: (...) -> None - self.response = response - self.filename = filename - self.chunks = chunks - - class Downloader(object): def __init__( self, @@ -164,8 +143,9 @@ class Downloader(object): self._session = session self._progress_bar = progress_bar - def __call__(self, link): - # type: (Link) -> Download + def __call__(self, link, location): + # type: (Link, str) -> Tuple[str, str] + """Download the file given by link into location.""" try: resp = _http_get_download(self._session, link) except NetworkConnectionError as e: @@ -175,8 +155,48 @@ class Downloader(object): ) raise - return Download( - resp, - _get_http_response_filename(resp, link), - _prepare_download(resp, link, self._progress_bar), - ) + filename = _get_http_response_filename(resp, link) + filepath = os.path.join(location, filename) + + chunks = _prepare_download(resp, link, self._progress_bar) + with open(filepath, 'wb') as content_file: + for chunk in chunks: + content_file.write(chunk) + content_type = resp.headers.get('Content-Type', '') + return filepath, content_type + + +class BatchDownloader(object): + + def __init__( + self, + session, # type: PipSession + progress_bar, # type: str + ): + # type: (...) -> None + self._session = session + self._progress_bar = progress_bar + + def __call__(self, links, location): + # type: (Iterable[Link], str) -> Iterable[Tuple[str, Tuple[str, str]]] + """Download the files given by links into location.""" + for link in links: + try: + resp = _http_get_download(self._session, link) + except NetworkConnectionError as e: + assert e.response is not None + logger.critical( + "HTTP error %s while getting %s", + e.response.status_code, link, + ) + raise + + filename = _get_http_response_filename(resp, link) + filepath = os.path.join(location, filename) + + chunks = _prepare_download(resp, link, self._progress_bar) + with open(filepath, 'wb') as content_file: + for chunk in chunks: + content_file.write(chunk) + content_type = resp.headers.get('Content-Type', '') + yield link.url, (filepath, content_type) diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index a0f9e151d..608475aba 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -10,11 +10,7 @@ from zipfile import BadZipfile, ZipFile from pip._vendor.requests.models import CONTENT_CHUNK_SIZE from pip._vendor.six.moves import range -from pip._internal.network.utils import ( - HEADERS, - raise_for_status, - response_chunks, -) +from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.wheel import pkg_resources_distribution_for_wheel diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index 39a4a546e..454945d9a 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -25,6 +25,7 @@ from pip._vendor.urllib3.exceptions import InsecureRequestWarning from pip import __version__ from pip._internal.network.auth import MultiDomainBasicAuth from pip._internal.network.cache import SafeFileCache + # Import ssl from compat so the initial import occurs in only one place. from pip._internal.utils.compat import has_tls, ipaddress from pip._internal.utils.glibc import libc_ver @@ -37,9 +38,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.urls import url_to_path if MYPY_CHECK_RUNNING: - from typing import ( - Iterator, List, Optional, Tuple, Union, - ) + from typing import Iterator, List, Optional, Tuple, Union from pip._internal.models.link import Link @@ -305,6 +304,14 @@ class PipSession(requests.Session): for host in trusted_hosts: self.add_trusted_host(host, suppress_logging=True) + def update_index_urls(self, new_index_urls): + # type: (List[str]) -> None + """ + :param new_index_urls: New index urls to update the authentication + handler with. + """ + self.auth.index_urls = new_index_urls + def add_trusted_host(self, host, source=None, suppress_logging=False): # type: (str, Optional[str], bool) -> None """ diff --git a/src/pip/_internal/network/xmlrpc.py b/src/pip/_internal/network/xmlrpc.py index e61126241..504018f28 100644 --- a/src/pip/_internal/network/xmlrpc.py +++ b/src/pip/_internal/network/xmlrpc.py @@ -14,6 +14,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict + from pip._internal.network.session import PipSession diff --git a/src/pip/_internal/operations/build/metadata.py b/src/pip/_internal/operations/build/metadata.py index cf52f8d8f..5709962b0 100644 --- a/src/pip/_internal/operations/build/metadata.py +++ b/src/pip/_internal/operations/build/metadata.py @@ -8,9 +8,10 @@ from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from pip._internal.build_env import BuildEnvironment from pip._vendor.pep517.wrappers import Pep517HookCaller + from pip._internal.build_env import BuildEnvironment + def generate_metadata(build_env, backend): # type: (BuildEnvironment, Pep517HookCaller) -> str diff --git a/src/pip/_internal/operations/build/wheel.py b/src/pip/_internal/operations/build/wheel.py index 0c28c4989..d16ee0966 100644 --- a/src/pip/_internal/operations/build/wheel.py +++ b/src/pip/_internal/operations/build/wheel.py @@ -6,6 +6,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import List, Optional + from pip._vendor.pep517.wrappers import Pep517HookCaller logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/operations/build/wheel_legacy.py b/src/pip/_internal/operations/build/wheel_legacy.py index 37dc876ac..9da365e4d 100644 --- a/src/pip/_internal/operations/build/wheel_legacy.py +++ b/src/pip/_internal/operations/build/wheel_legacy.py @@ -2,9 +2,7 @@ import logging import os.path from pip._internal.cli.spinners import open_spinner -from pip._internal.utils.setuptools_build import ( - make_setuptools_bdist_wheel_args, -) +from pip._internal.utils.setuptools_build import make_setuptools_bdist_wheel_args from pip._internal.utils.subprocess import ( LOG_DIVIDER, call_subprocess, diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index 0d5963295..5dee6bcb4 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -7,19 +7,16 @@ from collections import namedtuple from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.pkg_resources import RequirementParseError -from pip._internal.distributions import ( - make_distribution_for_install_requirement, -) +from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.utils.misc import get_installed_distributions from pip._internal.utils.typing import MYPY_CHECK_RUNNING logger = logging.getLogger(__name__) if MYPY_CHECK_RUNNING: + from typing import Any, Callable, Dict, List, Optional, Set, Tuple + from pip._internal.req.req_install import InstallRequirement - from typing import ( - Any, Callable, Dict, Optional, Set, Tuple, List - ) # Shorthands PackageSet = Dict[str, 'PackageDetails'] diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index ddb9cb232..d4f790cd4 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -18,20 +18,25 @@ from pip._internal.utils.direct_url_helpers import ( direct_url_as_pep440_direct_reference, dist_get_direct_url, ) -from pip._internal.utils.misc import ( - dist_is_editable, - get_installed_distributions, -) +from pip._internal.utils.misc import dist_is_editable, get_installed_distributions from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import ( - Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union + Container, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, + Union, ) + + from pip._vendor.pkg_resources import Distribution, Requirement + from pip._internal.cache import WheelCache - from pip._vendor.pkg_resources import ( - Distribution, Requirement - ) RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] @@ -183,7 +188,7 @@ def get_requirement_info(dist): location = os.path.normcase(os.path.abspath(dist.location)) - from pip._internal.vcs import vcs, RemoteNotFoundError + from pip._internal.vcs import RemoteNotFoundError, vcs vcs_backend = vcs.get_backend_for_dir(location) if vcs_backend is None: diff --git a/src/pip/_internal/operations/install/wheel.py b/src/pip/_internal/operations/install/wheel.py index e91b1b8d5..8b67ebb94 100644 --- a/src/pip/_internal/operations/install/wheel.py +++ b/src/pip/_internal/operations/install/wheel.py @@ -21,14 +21,7 @@ from zipfile import ZipFile from pip._vendor import pkg_resources from pip._vendor.distlib.scripts import ScriptMaker from pip._vendor.distlib.util import get_export_entry -from pip._vendor.six import ( - PY2, - ensure_str, - ensure_text, - itervalues, - reraise, - text_type, -) +from pip._vendor.six import PY2, ensure_str, ensure_text, itervalues, reraise, text_type from pip._vendor.six.moves import filterfalse, map from pip._internal.exceptions import InstallationError @@ -36,12 +29,7 @@ from pip._internal.locations import get_major_minor_version from pip._internal.models.direct_url import DIRECT_URL_METADATA_NAME, DirectUrl from pip._internal.models.scheme import SCHEME_KEYS from pip._internal.utils.filesystem import adjacent_tmp_file, replace -from pip._internal.utils.misc import ( - captured_stdout, - ensure_dir, - hash_file, - partition, -) +from pip._internal.utils.misc import captured_stdout, ensure_dir, hash_file, partition from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.unpacking import ( current_umask, @@ -49,10 +37,7 @@ from pip._internal.utils.unpacking import ( set_extracted_file_to_default_mode_plus_executable, zip_item_is_executable, ) -from pip._internal.utils.wheel import ( - parse_wheel, - pkg_resources_distribution_for_wheel, -) +from pip._internal.utils.wheel import parse_wheel, pkg_resources_distribution_for_wheel # Use the custom cast function at runtime to make cast work, # and import typing.cast when performing pre-commit and type @@ -62,10 +47,10 @@ if not MYPY_CHECK_RUNNING: else: from email.message import Message from typing import ( + IO, Any, Callable, Dict, - IO, Iterable, Iterator, List, diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index a5455fcc8..de017504a 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -9,11 +9,10 @@ import mimetypes import os import shutil +from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.six import PY2 -from pip._internal.distributions import ( - make_distribution_for_install_requirement, -) +from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( DirectoryUrlHashUnsupported, @@ -24,31 +23,30 @@ from pip._internal.exceptions import ( PreviousBuildDirError, VcsHashUnsupported, ) +from pip._internal.models.wheel import Wheel +from pip._internal.network.download import BatchDownloader, Downloader +from pip._internal.network.lazy_wheel import ( + HTTPRangeRequestUnsupported, + dist_from_wheel_url, +) from pip._internal.utils.filesystem import copy2_fixed from pip._internal.utils.hashes import MissingHashes from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import ( - display_path, - hide_url, - path_to_display, - rmtree, -) +from pip._internal.utils.misc import display_path, hide_url, path_to_display, rmtree from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.utils.unpacking import unpack_file from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Callable, List, Optional, Tuple, - ) + from typing import Callable, Dict, Iterable, List, Optional, Tuple from mypy_extensions import TypedDict + from pip._vendor.pkg_resources import Distribution - from pip._internal.distributions import AbstractDistribution from pip._internal.index.package_finder import PackageFinder from pip._internal.models.link import Link - from pip._internal.network.download import Downloader + from pip._internal.network.session import PipSession from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_tracker import RequirementTracker from pip._internal.utils.hashes import Hashes @@ -78,18 +76,17 @@ logger = logging.getLogger(__name__) def _get_prepared_distribution( - req, # type: InstallRequirement - req_tracker, # type: RequirementTracker - finder, # type: PackageFinder - build_isolation # type: bool + req, # type: InstallRequirement + req_tracker, # type: RequirementTracker + finder, # type: PackageFinder + build_isolation, # type: bool ): - # type: (...) -> AbstractDistribution - """Prepare a distribution for installation. - """ + # type: (...) -> Distribution + """Prepare a distribution for installation.""" abstract_dist = make_distribution_for_install_requirement(req) with req_tracker.track(req): abstract_dist.prepare_distribution_metadata(finder, build_isolation) - return abstract_dist + return abstract_dist.get_pkg_resources_distribution() def unpack_vcs_link(link, location): @@ -100,15 +97,19 @@ def unpack_vcs_link(link, location): class File(object): + def __init__(self, path, content_type): - # type: (str, str) -> None + # type: (str, Optional[str]) -> None self.path = path - self.content_type = content_type + if content_type is None: + self.content_type = mimetypes.guess_type(path)[0] + else: + self.content_type = content_type def get_http_url( link, # type: Link - downloader, # type: Downloader + download, # type: Downloader download_dir=None, # type: Optional[str] hashes=None, # type: Optional[Hashes] ): @@ -123,12 +124,12 @@ def get_http_url( if already_downloaded_path: from_path = already_downloaded_path - content_type = mimetypes.guess_type(from_path)[0] + content_type = None else: # let's download to a tmp dir - from_path, content_type = _download_http_url( - link, downloader, temp_dir.path, hashes - ) + from_path, content_type = download(link, temp_dir.path) + if hashes: + hashes.check_against_path(from_path) return File(from_path, content_type) @@ -213,16 +214,13 @@ def get_file_url( # one; no internet-sourced hash will be in `hashes`. if hashes: hashes.check_against_path(from_path) - - content_type = mimetypes.guess_type(from_path)[0] - - return File(from_path, content_type) + return File(from_path, None) def unpack_url( link, # type: Link location, # type: str - downloader, # type: Downloader + download, # type: Downloader download_dir=None, # type: Optional[str] hashes=None, # type: Optional[Hashes] ): @@ -254,7 +252,7 @@ def unpack_url( else: file = get_http_url( link, - downloader, + download, download_dir, hashes=hashes, ) @@ -267,27 +265,6 @@ def unpack_url( return file -def _download_http_url( - link, # type: Link - downloader, # type: Downloader - temp_dir, # type: str - hashes, # type: Optional[Hashes] -): - # type: (...) -> Tuple[str, str] - """Download link url into temp_dir using provided session""" - download = downloader(link) - - file_path = os.path.join(temp_dir, download.filename) - with open(file_path, 'wb') as content_file: - for chunk in download.chunks: - content_file.write(chunk) - - if hashes: - hashes.check_against_path(file_path) - - return file_path, download.response.headers.get('content-type', '') - - def _check_download_dir(link, download_dir, hashes): # type: (Link, str, Optional[Hashes]) -> Optional[str] """ Check download_dir for previously downloaded file with correct hash @@ -323,13 +300,14 @@ class RequirementPreparer(object): build_dir, # type: str download_dir, # type: Optional[str] src_dir, # type: str - wheel_download_dir, # type: Optional[str] build_isolation, # type: bool req_tracker, # type: RequirementTracker - downloader, # type: Downloader + session, # type: PipSession + progress_bar, # type: str finder, # type: PackageFinder require_hashes, # type: bool use_user_site, # type: bool + lazy_wheel, # type: bool ): # type: (...) -> None super(RequirementPreparer, self).__init__() @@ -337,23 +315,15 @@ class RequirementPreparer(object): self.src_dir = src_dir self.build_dir = build_dir self.req_tracker = req_tracker - self.downloader = downloader + self._session = session + self._download = Downloader(session, progress_bar) + self._batch_download = BatchDownloader(session, progress_bar) self.finder = finder # Where still-packed archives should be written to. If None, they are # not saved, and are deleted immediately after unpacking. self.download_dir = download_dir - # Where still-packed .whl files should be written to. If None, they are - # written to the download_dir parameter. Separate to download_dir to - # permit only keeping wheel archives for pip wheel. - self.wheel_download_dir = wheel_download_dir - - # NOTE - # download_dir and wheel_download_dir overlap semantically and may - # be combined if we're willing to have non-wheel archives present in - # the wheelhouse output by 'pip wheel'. - # Is build isolation allowed? self.build_isolation = build_isolation @@ -363,31 +333,35 @@ class RequirementPreparer(object): # Should install in user site-packages? self.use_user_site = use_user_site - @property - def _download_should_save(self): - # type: () -> bool - if not self.download_dir: - return False + # Should wheels be downloaded lazily? + self.use_lazy_wheel = lazy_wheel - if os.path.exists(self.download_dir): - return True + # Memoized downloaded files, as mapping of url: (path, mime type) + self._downloaded = {} # type: Dict[str, Tuple[str, str]] - logger.critical('Could not find download directory') - raise InstallationError( - "Could not find or access download directory '{}'" - .format(self.download_dir)) + # Previous "header" printed for a link-based InstallRequirement + self._previous_requirement_header = ("", "") def _log_preparing_link(self, req): # type: (InstallRequirement) -> None - """Log the way the link prepared.""" - if req.link.is_file: - path = req.link.file_path - logger.info('Processing %s', display_path(path)) + """Provide context for the requirement being prepared.""" + if req.link.is_file and not req.original_link_is_in_wheel_cache: + message = "Processing %s" + information = str(display_path(req.link.file_path)) else: - logger.info('Collecting %s', req.req or req) + message = "Collecting %s" + information = str(req.req or req) - def _ensure_link_req_src_dir(self, req, download_dir, parallel_builds): - # type: (InstallRequirement, Optional[str], bool) -> None + if (message, information) != self._previous_requirement_header: + self._previous_requirement_header = (message, information) + logger.info(message, information) + + if req.original_link_is_in_wheel_cache: + with indent_log(): + logger.info("Using cached %s", req.link.filename) + + def _ensure_link_req_src_dir(self, req, parallel_builds): + # type: (InstallRequirement, bool) -> None """Ensure source_dir of a linked InstallRequirement.""" # Since source_dir is only set for editable requirements. if req.link.is_wheel: @@ -449,63 +423,139 @@ class RequirementPreparer(object): # showing the user what the hash should be. return req.hashes(trust_internet=False) or MissingHashes() + def _fetch_metadata_using_lazy_wheel(self, link): + # type: (Link) -> Optional[Distribution] + """Fetch metadata using lazy wheel, if possible.""" + if not self.use_lazy_wheel: + return None + if self.require_hashes: + logger.debug('Lazy wheel is not used as hash checking is required') + return None + if link.is_file or not link.is_wheel: + logger.debug( + 'Lazy wheel is not used as ' + '%r does not points to a remote wheel', + link, + ) + return None + + wheel = Wheel(link.filename) + name = canonicalize_name(wheel.name) + logger.info( + 'Obtaining dependency information from %s %s', + name, wheel.version, + ) + url = link.url.split('#', 1)[0] + try: + return dist_from_wheel_url(name, url, self._session) + except HTTPRangeRequestUnsupported: + logger.debug('%s does not support range requests', url) + return None + def prepare_linked_requirement(self, req, parallel_builds=False): - # type: (InstallRequirement, bool) -> AbstractDistribution + # type: (InstallRequirement, bool) -> Distribution """Prepare a requirement to be obtained from req.link.""" assert req.link link = req.link self._log_preparing_link(req) - if link.is_wheel and self.wheel_download_dir: - # Download wheels to a dedicated dir when doing `pip wheel`. - download_dir = self.wheel_download_dir - else: - download_dir = self.download_dir - with indent_log(): - self._ensure_link_req_src_dir(req, download_dir, parallel_builds) + # Check if the relevant file is already available + # in the download directory + file_path = None + if self.download_dir is not None and link.is_wheel: + hashes = self._get_linked_req_hashes(req) + file_path = _check_download_dir(req.link, self.download_dir, hashes) + + if file_path is not None: + # The file is already available, so mark it as downloaded + self._downloaded[req.link.url] = file_path, None + else: + # The file is not available, attempt to fetch only metadata + wheel_dist = self._fetch_metadata_using_lazy_wheel(link) + if wheel_dist is not None: + req.needs_more_preparation = True + return wheel_dist + + # None of the optimizations worked, fully prepare the requirement + return self._prepare_linked_requirement(req, parallel_builds) + + def prepare_linked_requirements_more(self, reqs, parallel_builds=False): + # type: (Iterable[InstallRequirement], bool) -> None + """Prepare a linked requirement more, if needed.""" + reqs = [req for req in reqs if req.needs_more_preparation] + links = [req.link for req in reqs] + + # Let's download to a temporary directory. + tmpdir = TempDirectory(kind="unpack", globally_managed=True).path + self._downloaded.update(self._batch_download(links, tmpdir)) + for req in reqs: + self._prepare_linked_requirement(req, parallel_builds) + + def _prepare_linked_requirement(self, req, parallel_builds): + # type: (InstallRequirement, bool) -> Distribution + assert req.link + link = req.link + + self._ensure_link_req_src_dir(req, parallel_builds) + hashes = self._get_linked_req_hashes(req) + if link.url not in self._downloaded: try: local_file = unpack_url( - link, req.source_dir, self.downloader, download_dir, - hashes=self._get_linked_req_hashes(req) + link, req.source_dir, self._download, + self.download_dir, hashes, ) except NetworkConnectionError as exc: raise InstallationError( 'Could not install requirement {} because of HTTP ' 'error {} for URL {}'.format(req, exc, link) ) + else: + file_path, content_type = self._downloaded[link.url] + if hashes: + hashes.check_against_path(file_path) + local_file = File(file_path, content_type) - # For use in later processing, preserve the file path on the - # requirement. - if local_file: - req.local_file_path = local_file.path + # For use in later processing, + # preserve the file path on the requirement. + if local_file: + req.local_file_path = local_file.path - abstract_dist = _get_prepared_distribution( - req, self.req_tracker, self.finder, self.build_isolation, + dist = _get_prepared_distribution( + req, self.req_tracker, self.finder, self.build_isolation, + ) + return dist + + def save_linked_requirement(self, req): + # type: (InstallRequirement) -> None + assert self.download_dir is not None + assert req.link is not None + link = req.link + if link.is_vcs: + # Make a .zip of the source_dir we already created. + req.archive(self.download_dir) + return + + if link.is_existing_dir(): + logger.debug( + 'Not copying link to destination directory ' + 'since it is a directory: %s', link, ) + return + if req.local_file_path is None: + # No distribution was downloaded for this requirement. + return - if download_dir: - if link.is_existing_dir(): - logger.info('Link is a directory, ignoring download_dir') - elif local_file: - download_location = os.path.join( - download_dir, link.filename - ) - if not os.path.exists(download_location): - shutil.copy(local_file.path, download_location) - download_path = display_path(download_location) - logger.info('Saved %s', download_path) - - if self._download_should_save: - # Make a .zip of the source_dir we already created. - if link.is_vcs: - req.archive(self.download_dir) - return abstract_dist + download_location = os.path.join(self.download_dir, link.filename) + if not os.path.exists(download_location): + shutil.copy(req.local_file_path, download_location) + download_path = display_path(download_location) + logger.info('Saved %s', download_path) def prepare_editable_requirement( self, req, # type: InstallRequirement ): - # type: (...) -> AbstractDistribution + # type: (...) -> Distribution """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" @@ -520,24 +570,23 @@ class RequirementPreparer(object): 'hash.'.format(req) ) req.ensure_has_source_dir(self.src_dir) - req.update_editable(not self._download_should_save) + req.update_editable(self.download_dir is None) - abstract_dist = _get_prepared_distribution( + dist = _get_prepared_distribution( req, self.req_tracker, self.finder, self.build_isolation, ) - if self._download_should_save: - req.archive(self.download_dir) + req.archive(self.download_dir) req.check_if_exists(self.use_user_site) - return abstract_dist + return dist def prepare_installed_requirement( self, req, # type: InstallRequirement skip_reason # type: str ): - # type: (...) -> AbstractDistribution + # type: (...) -> Distribution """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" @@ -557,6 +606,4 @@ class RequirementPreparer(object): 'completely repeatable environment, install into an ' 'empty virtualenv.' ) - abstract_dist = InstalledDistribution(req) - - return abstract_dist + return InstalledDistribution(req).get_pkg_resources_distribution() diff --git a/src/pip/_internal/pyproject.py b/src/pip/_internal/pyproject.py index 6b4faf7a7..4144a9ed6 100644 --- a/src/pip/_internal/pyproject.py +++ b/src/pip/_internal/pyproject.py @@ -12,7 +12,7 @@ from pip._internal.exceptions import InstallationError from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Optional, List + from typing import Any, List, Optional def _is_list_of_str(obj): diff --git a/src/pip/_internal/req/constructors.py b/src/pip/_internal/req/constructors.py index 7a4641ef5..97420af6c 100644 --- a/src/pip/_internal/req/constructors.py +++ b/src/pip/_internal/req/constructors.py @@ -31,9 +31,8 @@ from pip._internal.utils.urls import path_to_url from pip._internal.vcs import is_url, vcs if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Optional, Set, Tuple, Union, - ) + from typing import Any, Dict, Optional, Set, Tuple, Union + from pip._internal.req.req_file import ParsedRequirement @@ -159,7 +158,7 @@ def deduce_helpful_msg(req): """ msg = "" if os.path.exists(req): - msg = " It does exist." + msg = " The path does exist. " # Try to parse and check if it is a requirements file. try: with open(req, 'r') as fp: diff --git a/src/pip/_internal/req/req_file.py b/src/pip/_internal/req/req_file.py index 105058228..c8c9165d3 100644 --- a/src/pip/_internal/req/req_file.py +++ b/src/pip/_internal/req/req_file.py @@ -13,20 +13,25 @@ import sys from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._internal.cli import cmdoptions -from pip._internal.exceptions import ( - InstallationError, - RequirementsFileParseError, -) +from pip._internal.exceptions import InstallationError, RequirementsFileParseError from pip._internal.models.search_scope import SearchScope from pip._internal.network.utils import raise_for_status from pip._internal.utils.encoding import auto_decode from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import get_url_scheme +from pip._internal.utils.urls import get_url_scheme, url_to_path if MYPY_CHECK_RUNNING: from optparse import Values from typing import ( - Any, Callable, Dict, Iterator, List, NoReturn, Optional, Text, Tuple, + Any, + Callable, + Dict, + Iterator, + List, + NoReturn, + Optional, + Text, + Tuple, ) from pip._internal.index.package_finder import PackageFinder @@ -256,6 +261,10 @@ def handle_option_line( value = relative_to_reqs_file find_links.append(value) + if session: + # We need to update the auth urls in session + session.update_index_urls(index_urls) + search_scope = SearchScope( find_links=find_links, index_urls=index_urls, @@ -568,16 +577,7 @@ def get_file_content(url, session, comes_from=None): 'Requirements file {} references URL {}, ' 'which is local'.format(comes_from, url) ) - - path = url.split(':', 1)[1] - path = path.replace('\\', '/') - match = _url_slash_drive_re.match(path) - if match: - path = match.group(1) + ':' + path.split('|', 1)[1] - path = urllib_parse.unquote(path) - if path.startswith('/'): - path = '/' + path.lstrip('/') - url = path + url = url_to_path(url) try: with open(url, 'rb') as f: @@ -587,6 +587,3 @@ def get_file_content(url, session, comes_from=None): 'Could not open requirements file: {}'.format(exc) ) return url, content - - -_url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index f25cec96a..8ce299503 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -22,10 +22,12 @@ from pip._internal.exceptions import InstallationError from pip._internal.locations import get_scheme from pip._internal.models.link import Link from pip._internal.operations.build.metadata import generate_metadata -from pip._internal.operations.build.metadata_legacy import \ - generate_metadata as generate_metadata_legacy -from pip._internal.operations.install.editable_legacy import \ - install_editable as install_editable_legacy +from pip._internal.operations.build.metadata_legacy import ( + generate_metadata as generate_metadata_legacy, +) +from pip._internal.operations.install.editable_legacy import ( + install_editable as install_editable_legacy, +) from pip._internal.operations.install.legacy import LegacyInstallFailure from pip._internal.operations.install.legacy import install as install_legacy from pip._internal.operations.install.wheel import install_wheel @@ -53,13 +55,13 @@ from pip._internal.utils.virtualenv import running_under_virtualenv from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, Optional, Sequence, Union, - ) - from pip._internal.build_env import BuildEnvironment - from pip._vendor.pkg_resources import Distribution - from pip._vendor.packaging.specifiers import SpecifierSet + from typing import Any, Dict, Iterable, List, Optional, Sequence, Union + from pip._vendor.packaging.markers import Marker + from pip._vendor.packaging.specifiers import SpecifierSet + from pip._vendor.pkg_resources import Distribution + + from pip._internal.build_env import BuildEnvironment logger = logging.getLogger(__name__) @@ -180,15 +182,6 @@ class InstallRequirement(object): # e.g. dependencies, extras or constraints. self.user_supplied = user_supplied - # Set by the legacy resolver when the requirement has been downloaded - # TODO: This introduces a strong coupling between the resolver and the - # requirement (the coupling was previously between the resolver - # and the requirement set). This should be refactored to allow - # the requirement to decide for itself when it has been - # successfully downloaded - but that is more tricky to get right, - # se we are making the change in stages. - self.successfully_downloaded = False - self.isolated = isolated self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment @@ -214,6 +207,9 @@ class InstallRequirement(object): # but after loading this flag should be treated as read only. self.use_pep517 = use_pep517 + # This requirement needs more preparation before it can be built + self.needs_more_preparation = False + def __str__(self): # type: () -> str if self.req: @@ -704,12 +700,14 @@ class InstallRequirement(object): return self.name + '/' + name def archive(self, build_dir): - # type: (str) -> None + # type: (Optional[str]) -> None """Saves archive to provided build_dir. Used for saving downloaded VCS requirements as part of `pip download`. """ assert self.source_dir + if build_dir is None: + return create_archive = True archive_name = '{}-{}.zip'.format(self.name, self.metadata["version"]) diff --git a/src/pip/_internal/req/req_set.py b/src/pip/_internal/req/req_set.py index ab4b6f849..c9ea3be5d 100644 --- a/src/pip/_internal/req/req_set.py +++ b/src/pip/_internal/req/req_set.py @@ -12,6 +12,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Dict, Iterable, List, Optional, Tuple + from pip._internal.req.req_install import InstallRequirement diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py index 13fb24563..7379c307b 100644 --- a/src/pip/_internal/req/req_tracker.py +++ b/src/pip/_internal/req/req_tracker.py @@ -14,8 +14,9 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from types import TracebackType from typing import Dict, Iterator, Optional, Set, Type, Union - from pip._internal.req.req_install import InstallRequirement + from pip._internal.models.link import Link + from pip._internal.req.req_install import InstallRequirement logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index 69719d338..2e7dfcc73 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -29,8 +29,17 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import ( - Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Set, + Tuple, ) + from pip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/resolution/base.py b/src/pip/_internal/resolution/base.py index 2fa118bd8..6d50555e5 100644 --- a/src/pip/_internal/resolution/base.py +++ b/src/pip/_internal/resolution/base.py @@ -2,6 +2,7 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import Callable, List + from pip._internal.req.req_install import InstallRequirement from pip._internal.req.req_set import RequirementSet diff --git a/src/pip/_internal/resolution/legacy/resolver.py b/src/pip/_internal/resolution/legacy/resolver.py index c9b4c6616..d0fc1a7b3 100644 --- a/src/pip/_internal/resolution/legacy/resolver.py +++ b/src/pip/_internal/resolution/legacy/resolver.py @@ -34,18 +34,15 @@ from pip._internal.resolution.base import BaseResolver from pip._internal.utils.compatibility_tags import get_supported from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_in_usersite, normalize_version_info -from pip._internal.utils.packaging import ( - check_requires_python, - get_requires_python, -) +from pip._internal.utils.packaging import check_requires_python, get_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from typing import DefaultDict, List, Optional, Set, Tuple - from pip._vendor import pkg_resources + + from pip._vendor.pkg_resources import Distribution from pip._internal.cache import WheelCache - from pip._internal.distributions import AbstractDistribution from pip._internal.index.package_finder import PackageFinder from pip._internal.models.link import Link from pip._internal.operations.prepare import RequirementPreparer @@ -58,7 +55,7 @@ logger = logging.getLogger(__name__) def _check_dist_requires_python( - dist, # type: pkg_resources.Distribution + dist, # type: Distribution version_info, # type: Tuple[int, int, int] ignore_requires_python=False, # type: bool ): @@ -317,8 +314,8 @@ class Resolver(BaseResolver): req.original_link_is_in_wheel_cache = True req.link = cache_entry.link - def _get_abstract_dist_for(self, req): - # type: (InstallRequirement) -> AbstractDistribution + def _get_dist_for(self, req): + # type: (InstallRequirement) -> Distribution """Takes a InstallRequirement and returns a single AbstractDist \ representing a prepared variant of the same. """ @@ -337,7 +334,7 @@ class Resolver(BaseResolver): # We eagerly populate the link, since that's our "legacy" behavior. self._populate_link(req) - abstract_dist = self.preparer.prepare_linked_requirement(req) + dist = self.preparer.prepare_linked_requirement(req) # NOTE # The following portion is for determining if a certain package is @@ -364,8 +361,7 @@ class Resolver(BaseResolver): 'Requirement already satisfied (use --upgrade to upgrade):' ' %s', req, ) - - return abstract_dist + return dist def _resolve_one( self, @@ -385,10 +381,8 @@ class Resolver(BaseResolver): req_to_install.prepared = True - abstract_dist = self._get_abstract_dist_for(req_to_install) - # Parse and return dependencies - dist = abstract_dist.get_pkg_resources_distribution() + dist = self._get_dist_for(req_to_install) # This will raise UnsupportedPythonVersion if the given Python # version isn't compatible with the distribution's Requires-Python. _check_dist_requires_python( @@ -448,12 +442,6 @@ class Resolver(BaseResolver): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) - if not req_to_install.editable and not req_to_install.satisfied_by: - # XXX: --no-install leads this to report 'Successfully - # downloaded' for only non-editable reqs, even though we took - # action on them. - req_to_install.successfully_downloaded = True - return more_reqs def get_installation_order(self, req_set): diff --git a/src/pip/_internal/resolution/resolvelib/candidates.py b/src/pip/_internal/resolution/resolvelib/candidates.py index 8b39d2dcb..ff2b336d9 100644 --- a/src/pip/_internal/resolution/resolvelib/candidates.py +++ b/src/pip/_internal/resolution/resolvelib/candidates.py @@ -1,22 +1,17 @@ import logging import sys -from pip._vendor.contextlib2 import suppress from pip._vendor.packaging.specifiers import InvalidSpecifier, SpecifierSet from pip._vendor.packaging.utils import canonicalize_name from pip._vendor.packaging.version import Version from pip._internal.exceptions import HashError, MetadataInconsistent -from pip._internal.network.lazy_wheel import ( - HTTPRangeRequestUnsupported, - dist_from_wheel_url, -) +from pip._internal.models.wheel import Wheel from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, ) from pip._internal.req.req_install import InstallRequirement -from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_is_editable, normalize_version_info from pip._internal.utils.packaging import get_requires_python from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -29,7 +24,6 @@ if MYPY_CHECK_RUNNING: from pip._vendor.packaging.version import _BaseVersion from pip._vendor.pkg_resources import Distribution - from pip._internal.distributions import AbstractDistribution from pip._internal.models.link import Link from .base import Requirement @@ -148,7 +142,6 @@ class _InstallRequirementBackedCandidate(Candidate): self._name = name self._version = version self._dist = None # type: Optional[Distribution] - self._prepared = False def __repr__(self): # type: () -> str @@ -200,16 +193,15 @@ class _InstallRequirementBackedCandidate(Candidate): self._link.file_path if self._link.is_file else self._link ) - def _prepare_abstract_distribution(self): - # type: () -> AbstractDistribution + def _prepare_distribution(self): + # type: () -> Distribution raise NotImplementedError("Override in subclass") - def _check_metadata_consistency(self): - # type: () -> None + def _check_metadata_consistency(self, dist): + # type: (Distribution) -> None """Check for consistency of project name and version of dist.""" # TODO: (Longer term) Rather than abort, reject this candidate # and backtrack. This would need resolvelib support. - dist = self._dist # type: Distribution name = canonicalize_name(dist.project_name) if self._name is not None and self._name != name: raise MetadataInconsistent(self._ireq, "name", dist.project_name) @@ -219,46 +211,23 @@ class _InstallRequirementBackedCandidate(Candidate): def _prepare(self): # type: () -> None - if self._prepared: + if self._dist is not None: return try: - abstract_dist = self._prepare_abstract_distribution() + dist = self._prepare_distribution() except HashError as e: e.req = self._ireq raise - self._dist = abstract_dist.get_pkg_resources_distribution() - assert self._dist is not None, "Distribution already installed" - self._check_metadata_consistency() - self._prepared = True - - def _fetch_metadata(self): - # type: () -> None - """Fetch metadata, using lazy wheel if possible.""" - preparer = self._factory.preparer - use_lazy_wheel = self._factory.use_lazy_wheel - remote_wheel = self._link.is_wheel and not self._link.is_file - if use_lazy_wheel and remote_wheel and not preparer.require_hashes: - assert self._name is not None - logger.info('Collecting %s', self._ireq.req or self._ireq) - # If HTTPRangeRequestUnsupported is raised, fallback silently. - with indent_log(), suppress(HTTPRangeRequestUnsupported): - logger.info( - 'Obtaining dependency information from %s %s', - self._name, self._version, - ) - url = self._link.url.split('#', 1)[0] - session = preparer.downloader._session - self._dist = dist_from_wheel_url(self._name, url, session) - self._check_metadata_consistency() - if self._dist is None: - self._prepare() + assert dist is not None, "Distribution already installed" + self._check_metadata_consistency(dist) + self._dist = dist @property def dist(self): # type: () -> Distribution if self._dist is None: - self._fetch_metadata() + self._prepare() return self._dist def _get_requires_python_dependency(self): @@ -305,6 +274,20 @@ class LinkCandidate(_InstallRequirementBackedCandidate): logger.debug("Using cached wheel link: %s", cache_entry.link) link = cache_entry.link ireq = make_install_req_from_link(link, template) + assert ireq.link == link + if ireq.link.is_wheel and not ireq.link.is_file: + wheel = Wheel(ireq.link.filename) + wheel_name = canonicalize_name(wheel.name) + assert name == wheel_name, ( + "{!r} != {!r} for wheel".format(name, wheel_name) + ) + # Version may not be present for PEP 508 direct URLs + if version is not None: + assert str(version) == wheel.version, ( + "{!r} != {!r} for wheel {}".format( + version, wheel.version, name + ) + ) if (cache_entry is not None and cache_entry.persistent and @@ -320,8 +303,8 @@ class LinkCandidate(_InstallRequirementBackedCandidate): version=version, ) - def _prepare_abstract_distribution(self): - # type: () -> AbstractDistribution + def _prepare_distribution(self): + # type: () -> Distribution return self._factory.preparer.prepare_linked_requirement( self._ireq, parallel_builds=True, ) @@ -348,8 +331,8 @@ class EditableCandidate(_InstallRequirementBackedCandidate): version=version, ) - def _prepare_abstract_distribution(self): - # type: () -> AbstractDistribution + def _prepare_distribution(self): + # type: () -> Distribution return self._factory.preparer.prepare_editable_requirement(self._ireq) diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py index 8813ab038..0ac8d1af9 100644 --- a/src/pip/_internal/resolution/resolvelib/factory.py +++ b/src/pip/_internal/resolution/resolvelib/factory.py @@ -37,8 +37,8 @@ from .requirements import ( if MYPY_CHECK_RUNNING: from typing import ( - FrozenSet, Dict, + FrozenSet, Iterable, Iterator, List, @@ -83,7 +83,6 @@ class Factory(object): ignore_installed, # type: bool ignore_requires_python, # type: bool py_version_info=None, # type: Optional[Tuple[int, ...]] - lazy_wheel=False, # type: bool ): # type: (...) -> None self._finder = finder @@ -94,7 +93,6 @@ class Factory(object): self._use_user_site = use_user_site self._force_reinstall = force_reinstall self._ignore_requires_python = ignore_requires_python - self.use_lazy_wheel = lazy_wheel self._link_candidate_cache = {} # type: Cache[LinkCandidate] self._editable_candidate_cache = {} # type: Cache[EditableCandidate] @@ -404,10 +402,6 @@ class Factory(object): return ", ".join(parts[:-1]) + " and " + parts[-1] - def readable_form(cand): - # type: (Candidate) -> str - return "{} {}".format(cand.name, cand.version) - def describe_trigger(parent): # type: (Candidate) -> str ireq = parent.get_install_requirement() diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py index 7d679e5fe..7f7d0e154 100644 --- a/src/pip/_internal/resolution/resolvelib/provider.py +++ b/src/pip/_internal/resolution/resolvelib/provider.py @@ -5,18 +5,9 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING from .base import Constraint if MYPY_CHECK_RUNNING: - from typing import ( - Any, - Dict, - Iterable, - Optional, - Sequence, - Set, - Tuple, - Union, - ) + from typing import Any, Dict, Iterable, Optional, Sequence, Set, Tuple, Union - from .base import Requirement, Candidate + from .base import Candidate, Requirement from .factory import Factory # Notes on the relationship between the provider, the factory, and the diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py index 449cfea28..cb7d1ae8a 100644 --- a/src/pip/_internal/resolution/resolvelib/resolver.py +++ b/src/pip/_internal/resolution/resolvelib/resolver.py @@ -49,17 +49,8 @@ class Resolver(BaseResolver): force_reinstall, # type: bool upgrade_strategy, # type: str py_version_info=None, # type: Optional[Tuple[int, ...]] - lazy_wheel=False, # type: bool ): super(Resolver, self).__init__() - if lazy_wheel: - logger.warning( - 'pip is using lazily downloaded wheels using HTTP ' - 'range requests to obtain dependency information. ' - 'This experimental feature is enabled through ' - '--use-feature=fast-deps and it is not ready for production.' - ) - assert upgrade_strategy in self._allowed_strategies self.factory = Factory( @@ -72,7 +63,6 @@ class Resolver(BaseResolver): ignore_installed=ignore_installed, ignore_requires_python=ignore_requires_python, py_version_info=py_version_info, - lazy_wheel=lazy_wheel, ) self.ignore_dependencies = ignore_dependencies self.upgrade_strategy = upgrade_strategy @@ -170,6 +160,8 @@ class Resolver(BaseResolver): req_set.add_named_requirement(ireq) + reqs = req_set.all_requirements + self.factory.preparer.prepare_linked_requirements_more(reqs) return req_set def get_installation_order(self, req_set): diff --git a/src/pip/_internal/self_outdated_check.py b/src/pip/_internal/self_outdated_check.py index fbd9dfd48..c2d166b18 100644 --- a/src/pip/_internal/self_outdated_check.py +++ b/src/pip/_internal/self_outdated_check.py @@ -13,16 +13,8 @@ from pip._vendor.six import ensure_binary from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder from pip._internal.models.selection_prefs import SelectionPreferences -from pip._internal.utils.filesystem import ( - adjacent_tmp_file, - check_path_owner, - replace, -) -from pip._internal.utils.misc import ( - ensure_dir, - get_distribution, - get_installed_version, -) +from pip._internal.utils.filesystem import adjacent_tmp_file, check_path_owner, replace +from pip._internal.utils.misc import ensure_dir, get_distribution, get_installed_version from pip._internal.utils.packaging import get_installer from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py index 9a2bb7800..2196e6e0a 100644 --- a/src/pip/_internal/utils/compat.py +++ b/src/pip/_internal/utils/compat.py @@ -255,8 +255,8 @@ else: def ioctl_GWINSZ(fd): try: import fcntl - import termios import struct + import termios cr = struct.unpack_from( 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') diff --git a/src/pip/_internal/utils/direct_url_helpers.py b/src/pip/_internal/utils/direct_url_helpers.py index f1fe209e9..a355a6c5e 100644 --- a/src/pip/_internal/utils/direct_url_helpers.py +++ b/src/pip/_internal/utils/direct_url_helpers.py @@ -20,10 +20,10 @@ except ImportError: if MYPY_CHECK_RUNNING: from typing import Optional - from pip._internal.models.link import Link - from pip._vendor.pkg_resources import Distribution + from pip._internal.models.link import Link + logger = logging.getLogger(__name__) diff --git a/src/pip/_internal/utils/encoding.py b/src/pip/_internal/utils/encoding.py index 5b83d61bb..42a57535a 100644 --- a/src/pip/_internal/utils/encoding.py +++ b/src/pip/_internal/utils/encoding.py @@ -6,7 +6,7 @@ import sys from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Tuple, Text + from typing import List, Text, Tuple BOMS = [ (codecs.BOM_UTF8, 'utf-8'), diff --git a/src/pip/_internal/utils/entrypoints.py b/src/pip/_internal/utils/entrypoints.py index befd01c89..64d1cb2bd 100644 --- a/src/pip/_internal/utils/entrypoints.py +++ b/src/pip/_internal/utils/entrypoints.py @@ -4,7 +4,7 @@ from pip._internal.cli.main import main from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, List + from typing import List, Optional def _wrapper(args=None): diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py index d9f74a640..b306dafe7 100644 --- a/src/pip/_internal/utils/hashes.py +++ b/src/pip/_internal/utils/hashes.py @@ -4,18 +4,13 @@ import hashlib from pip._vendor.six import iteritems, iterkeys, itervalues -from pip._internal.exceptions import ( - HashMismatch, - HashMissing, - InstallationError, -) +from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError from pip._internal.utils.misc import read_chunks from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Dict, List, BinaryIO, NoReturn, Iterator - ) + from typing import BinaryIO, Dict, Iterator, List, NoReturn + from pip._vendor.six import PY3 if PY3: from hashlib import _Hash diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 5629c60c1..c122beb32 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -20,6 +20,7 @@ from itertools import tee from pip._vendor import pkg_resources from pip._vendor.packaging.utils import canonicalize_name + # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is # why we ignore the type on this import. from pip._vendor.retrying import retry # type: ignore @@ -30,17 +31,8 @@ from pip._vendor.six.moves.urllib.parse import unquote as urllib_unquote from pip import __version__ from pip._internal.exceptions import CommandError -from pip._internal.locations import ( - get_major_minor_version, - site_packages, - user_site, -) -from pip._internal.utils.compat import ( - WINDOWS, - expanduser, - stdlib_pkgs, - str_to_display, -) +from pip._internal.locations import get_major_minor_version, site_packages, user_site +from pip._internal.utils.compat import WINDOWS, expanduser, stdlib_pkgs, str_to_display from pip._internal.utils.typing import MYPY_CHECK_RUNNING, cast from pip._internal.utils.virtualenv import ( running_under_virtualenv, @@ -54,9 +46,20 @@ else: if MYPY_CHECK_RUNNING: from typing import ( - Any, AnyStr, Callable, Container, Iterable, Iterator, List, Optional, - Text, Tuple, TypeVar, Union, + Any, + AnyStr, + Callable, + Container, + Iterable, + Iterator, + List, + Optional, + Text, + Tuple, + TypeVar, + Union, ) + from pip._vendor.pkg_resources import Distribution VersionInfo = Tuple[int, int, int] diff --git a/src/pip/_internal/utils/packaging.py b/src/pip/_internal/utils/packaging.py index 68aa86edb..27fd20423 100644 --- a/src/pip/_internal/utils/packaging.py +++ b/src/pip/_internal/utils/packaging.py @@ -11,8 +11,9 @@ from pip._internal.utils.misc import display_path from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple from email.message import Message + from typing import Optional, Tuple + from pip._vendor.pkg_resources import Distribution diff --git a/src/pip/_internal/utils/parallel.py b/src/pip/_internal/utils/parallel.py index 9fe1fe8b9..d4113bdc2 100644 --- a/src/pip/_internal/utils/parallel.py +++ b/src/pip/_internal/utils/parallel.py @@ -29,8 +29,8 @@ from pip._vendor.six.moves import map from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Callable, Iterable, Iterator, Union, TypeVar from multiprocessing import pool + from typing import Callable, Iterable, Iterator, TypeVar, Union Pool = Union[pool.Pool, pool.ThreadPool] S = TypeVar('S') diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index d398e68da..605e711e6 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -14,9 +14,7 @@ from pip._internal.utils.misc import HiddenText, path_to_display from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( - Any, Callable, Iterable, List, Mapping, Optional, Text, Union, - ) + from typing import Any, Callable, Iterable, List, Mapping, Optional, Text, Union CommandArgs = List[Union[str, HiddenText]] diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index 94408c52f..3180713f7 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -16,6 +16,7 @@ from pip._internal.vcs.versioncontrol import VersionControl, vcs if MYPY_CHECK_RUNNING: from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index a9c7fb66e..1831aede5 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -25,6 +25,7 @@ from pip._internal.vcs.versioncontrol import ( if MYPY_CHECK_RUNNING: from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions @@ -163,6 +164,29 @@ class Git(VersionControl): return (sha, False) + @classmethod + def _should_fetch(cls, dest, rev): + """ + Return true if rev is a ref or is a commit that we don't have locally. + + Branches and tags are not considered in this method because they are + assumed to be always available locally (which is a normal outcome of + ``git clone`` and ``git fetch --tags``). + """ + if rev.startswith("refs/"): + # Always fetch remote refs. + return True + + if not looks_like_hash(rev): + # Git fetch would fail with abbreviated commits. + return False + + if cls.has_commit(dest, rev): + # Don't fetch if we have the commit locally. + return False + + return True + @classmethod def resolve_revision(cls, dest, url, rev_options): # type: (str, HiddenText, RevOptions) -> RevOptions @@ -194,10 +218,10 @@ class Git(VersionControl): rev, ) - if not rev.startswith('refs/'): + if not cls._should_fetch(dest, rev): return rev_options - # If it looks like a ref, we have to fetch it explicitly. + # fetch the requested revision cls.run_command( make_command('fetch', '-q', url, rev_options.to_args()), cwd=dest, @@ -306,6 +330,20 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() + @classmethod + def has_commit(cls, location, rev): + """ + Check if rev is a commit that is available in the local repository. + """ + try: + cls.run_command( + ['rev-parse', '-q', '--verify', "sha^" + rev], cwd=location + ) + except SubProcessError: + return False + else: + return True + @classmethod def get_revision(cls, location, rev=None): if rev is None: @@ -349,7 +387,6 @@ class Git(VersionControl): urllib_request.url2pathname(path) .replace('\\', '/').lstrip('/') ) - url = urlunsplit((scheme, netloc, newpath, query, fragment)) after_plus = scheme.find('+') + 1 url = scheme[:after_plus] + urlunsplit( (scheme[after_plus:], netloc, newpath, query, fragment), diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index ab134970b..eae09c196 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -26,8 +26,9 @@ _svn_info_xml_url_re = re.compile(r'(.*)') if MYPY_CHECK_RUNNING: from typing import Optional, Tuple - from pip._internal.utils.subprocess import CommandArgs + from pip._internal.utils.misc import HiddenText + from pip._internal.utils.subprocess import CommandArgs from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 96f830f99..219f79673 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -12,11 +12,7 @@ import sys from pip._vendor import pkg_resources from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._internal.exceptions import ( - BadCommand, - InstallationError, - SubProcessError, -) +from pip._internal.exceptions import BadCommand, InstallationError, SubProcessError from pip._internal.utils.compat import console_to_str, samefile from pip._internal.utils.logging import subprocess_logger from pip._internal.utils.misc import ( @@ -38,9 +34,19 @@ from pip._internal.utils.urls import get_url_scheme if MYPY_CHECK_RUNNING: from typing import ( - Dict, Iterable, Iterator, List, Optional, Text, Tuple, - Type, Union, Mapping, Any + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Text, + Tuple, + Type, + Union, ) + from pip._internal.utils.misc import HiddenText from pip._internal.utils.subprocess import CommandArgs diff --git a/src/pip/_internal/wheel_builder.py b/src/pip/_internal/wheel_builder.py index fa08016bd..27fce66c2 100644 --- a/src/pip/_internal/wheel_builder.py +++ b/src/pip/_internal/wheel_builder.py @@ -19,9 +19,7 @@ from pip._internal.utils.urls import path_to_url from pip._internal.vcs import vcs if MYPY_CHECK_RUNNING: - from typing import ( - Any, Callable, Iterable, List, Optional, Tuple, - ) + from typing import Any, Callable, Iterable, List, Optional, Tuple from pip._internal.cache import WheelCache from pip._internal.req.req_install import InstallRequirement diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 581db54c8..c3db83ff6 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -32,11 +32,15 @@ def vendored(modulename): try: __import__(modulename, globals(), locals(), level=0) except ImportError: - # This error used to be silenced in earlier variants of this file, to instead - # raise the error when pip actually tries to use the missing module. - # Based on inputs in #5354, this was changed to explicitly raise the error. - # Re-raising the exception without modifying it is an intentional choice. - raise + # We can just silently allow import failures to pass here. If we + # got to this point it means that ``import pip._vendor.whatever`` + # failed and so did ``import whatever``. Since we're importing this + # upfront in an attempt to alias imports, not erroring here will + # just mean we get a regular import error whenever pip *actually* + # tries to import one of these modules to use it, which actually + # gives us a better error message than we would have otherwise + # gotten. + pass else: sys.modules[vendored_name] = sys.modules[modulename] base, head = vendored_name.rsplit(".", 1) diff --git a/tests/conftest.py b/tests/conftest.py index 32b6e6926..ffd9f4215 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,10 +6,12 @@ import re import shutil import subprocess import sys +import time from contextlib import contextmanager import pytest import six +from mock import patch from pip._vendor.contextlib2 import ExitStack, nullcontext from setuptools.wheel import Wheel @@ -25,7 +27,8 @@ from tests.lib.venv import VirtualEnvironment if MYPY_CHECK_RUNNING: from typing import Dict, Iterable - from tests.lib.server import MockServer as _MockServer, Responder + from tests.lib.server import MockServer as _MockServer + from tests.lib.server import Responder def pytest_addoption(parser): @@ -104,11 +107,13 @@ def use_new_resolver(request): """Set environment variable to make pip default to the new resolver. """ new_resolver = request.config.getoption("--new-resolver") + features = set(os.environ.get("PIP_USE_FEATURE", "").split()) if new_resolver: - os.environ["PIP_USE_FEATURE"] = "2020-resolver" + features.add("2020-resolver") else: - os.environ.pop("PIP_USE_FEATURE", None) - yield new_resolver + features.discard("2020-resolver") + with patch.dict(os.environ, {"PIP_USE_FEATURE": " ".join(features)}): + yield new_resolver @pytest.fixture(scope='session') @@ -151,7 +156,7 @@ def tmpdir(request, tmpdir): @pytest.fixture(autouse=True) -def isolate(tmpdir): +def isolate(tmpdir, monkeypatch): """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -174,45 +179,51 @@ def isolate(tmpdir): if sys.platform == 'win32': # Note: this will only take effect in subprocesses... home_drive, home_path = os.path.splitdrive(home_dir) - os.environ.update({ - 'USERPROFILE': home_dir, - 'HOMEDRIVE': home_drive, - 'HOMEPATH': home_path, - }) + monkeypatch.setenv('USERPROFILE', home_dir) + monkeypatch.setenv('HOMEDRIVE', home_drive) + monkeypatch.setenv('HOMEPATH', home_path) for env_var, sub_path in ( ('APPDATA', 'AppData/Roaming'), ('LOCALAPPDATA', 'AppData/Local'), ): path = os.path.join(home_dir, *sub_path.split('/')) - os.environ[env_var] = path + monkeypatch.setenv(env_var, path) os.makedirs(path) else: # Set our home directory to our temporary directory, this should force # all of our relative configuration files to be read from here instead # of the user's actual $HOME directory. - os.environ["HOME"] = home_dir + monkeypatch.setenv("HOME", home_dir) # Isolate ourselves from XDG directories - os.environ["XDG_DATA_HOME"] = os.path.join(home_dir, ".local", "share") - os.environ["XDG_CONFIG_HOME"] = os.path.join(home_dir, ".config") - os.environ["XDG_CACHE_HOME"] = os.path.join(home_dir, ".cache") - os.environ["XDG_RUNTIME_DIR"] = os.path.join(home_dir, ".runtime") - os.environ["XDG_DATA_DIRS"] = ":".join([ + monkeypatch.setenv("XDG_DATA_HOME", os.path.join( + home_dir, ".local", "share", + )) + monkeypatch.setenv("XDG_CONFIG_HOME", os.path.join( + home_dir, ".config", + )) + monkeypatch.setenv("XDG_CACHE_HOME", os.path.join(home_dir, ".cache")) + monkeypatch.setenv("XDG_RUNTIME_DIR", os.path.join( + home_dir, ".runtime", + )) + monkeypatch.setenv("XDG_DATA_DIRS", os.pathsep.join([ os.path.join(fake_root, "usr", "local", "share"), os.path.join(fake_root, "usr", "share"), - ]) - os.environ["XDG_CONFIG_DIRS"] = os.path.join(fake_root, "etc", "xdg") + ])) + monkeypatch.setenv("XDG_CONFIG_DIRS", os.path.join( + fake_root, "etc", "xdg", + )) # Configure git, because without an author name/email git will complain # and cause test failures. - os.environ["GIT_CONFIG_NOSYSTEM"] = "1" - os.environ["GIT_AUTHOR_NAME"] = "pip" - os.environ["GIT_AUTHOR_EMAIL"] = "distutils-sig@python.org" + monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") + monkeypatch.setenv("GIT_AUTHOR_NAME", "pip") + monkeypatch.setenv("GIT_AUTHOR_EMAIL", "distutils-sig@python.org") # We want to disable the version check from running in the tests - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + monkeypatch.setenv("PIP_DISABLE_PIP_VERSION_CHECK", "true") # Make sure tests don't share a requirements tracker. - os.environ.pop('PIP_REQ_TRACKER', None) + monkeypatch.delenv("PIP_REQ_TRACKER", False) # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) @@ -547,3 +558,13 @@ def mock_server(): test_server = MockServer(server) with test_server.context: yield test_server + + +@pytest.fixture +def utc(): + # time.tzset() is not implemented on some platforms, e.g. Windows. + tzset = getattr(time, 'tzset', lambda: None) + with patch.dict(os.environ, {'TZ': 'UTC'}): + tzset() + yield + tzset() diff --git a/tests/functional/test_cache.py b/tests/functional/test_cache.py index e30b2c079..872f55982 100644 --- a/tests/functional/test_cache.py +++ b/tests/functional/test_cache.py @@ -15,11 +15,30 @@ def cache_dir(script): return result.stdout.strip() +@pytest.fixture +def http_cache_dir(cache_dir): + return os.path.normcase(os.path.join(cache_dir, 'http')) + + @pytest.fixture def wheel_cache_dir(cache_dir): return os.path.normcase(os.path.join(cache_dir, 'wheels')) +@pytest.fixture +def http_cache_files(http_cache_dir): + destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname') + + if not os.path.exists(destination): + return [] + + filenames = glob(os.path.join(destination, '*')) + files = [] + for filename in filenames: + files.append(os.path.join(destination, filename)) + return files + + @pytest.fixture def wheel_cache_files(wheel_cache_dir): destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') @@ -34,6 +53,24 @@ def wheel_cache_files(wheel_cache_dir): return files +@pytest.fixture +def populate_http_cache(http_cache_dir): + destination = os.path.join(http_cache_dir, 'arbitrary', 'pathname') + os.makedirs(destination) + + files = [ + ('aaaaaaaaa', os.path.join(destination, 'aaaaaaaaa')), + ('bbbbbbbbb', os.path.join(destination, 'bbbbbbbbb')), + ('ccccccccc', os.path.join(destination, 'ccccccccc')), + ] + + for _name, filename in files: + with open(filename, 'w'): + pass + + return files + + @pytest.fixture def populate_wheel_cache(wheel_cache_dir): destination = os.path.join(wheel_cache_dir, 'arbitrary', 'pathname') @@ -70,6 +107,42 @@ def list_matches_wheel(wheel_name, result): return any(map(lambda l: l.startswith(expected), lines)) +def list_matches_wheel_abspath(wheel_name, result): + """Returns True if any line in `result`, which should be the output of + a `pip cache list --format=abspath` call, is a valid path and belongs to + `wheel_name`. + + E.g., If wheel_name is `foo-1.2.3` it searches for a line starting with + `foo-1.2.3-py3-none-any.whl`.""" + lines = result.stdout.splitlines() + expected = '{}-py3-none-any.whl'.format(wheel_name) + return any(map(lambda l: os.path.basename(l).startswith(expected) + and os.path.exists(l), lines)) + + +@pytest.fixture +def remove_matches_http(http_cache_dir): + """Returns True if any line in `result`, which should be the output of + a `pip cache purge` call, matches `http_filename`. + + E.g., If http_filename is `aaaaaaaaa`, it searches for a line equal to + `Removed /arbitrary/pathname/aaaaaaaaa`. + """ + + def _remove_matches_http(http_filename, result): + lines = result.stdout.splitlines() + + # The "/arbitrary/pathname/" bit is an implementation detail of how + # the `populate_http_cache` fixture is implemented. + path = os.path.join( + http_cache_dir, 'arbitrary', 'pathname', http_filename, + ) + expected = 'Removed {}'.format(path) + return expected in lines + + return _remove_matches_http + + @pytest.fixture def remove_matches_wheel(wheel_cache_dir): """Returns True if any line in `result`, which should be the output of @@ -111,11 +184,17 @@ def test_cache_dir_too_many_args(script, cache_dir): assert 'ERROR: Too many arguments' in result.stderr.splitlines() -@pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_info(script, wheel_cache_dir, wheel_cache_files): +@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") +def test_cache_info( + script, http_cache_dir, wheel_cache_dir, wheel_cache_files +): result = script.pip('cache', 'info') - assert 'Location: {}'.format(wheel_cache_dir) in result.stdout + assert ( + 'Package index page cache location: {}'.format(http_cache_dir) + in result.stdout + ) + assert 'Wheels location: {}'.format(wheel_cache_dir) in result.stdout num_wheels = len(wheel_cache_files) assert 'Number of wheels: {}'.format(num_wheels) in result.stdout @@ -132,6 +211,18 @@ def test_cache_list(script): assert list_matches_wheel('zzz-7.8.9', result) +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_abspath(script): + """Running `pip cache list --format=abspath` should return full + paths of exactly what the populate_wheel_cache fixture adds.""" + result = script.pip('cache', 'list', '--format=abspath') + + assert list_matches_wheel_abspath('yyy-1.2.3', result) + assert list_matches_wheel_abspath('zzz-4.5.6', result) + assert list_matches_wheel_abspath('zzz-4.5.7', result) + assert list_matches_wheel_abspath('zzz-7.8.9', result) + + @pytest.mark.usefixtures("empty_wheel_cache") def test_cache_list_with_empty_cache(script): """Running `pip cache list` with an empty cache should print @@ -140,6 +231,14 @@ def test_cache_list_with_empty_cache(script): assert result.stdout == "Nothing cached.\n" +@pytest.mark.usefixtures("empty_wheel_cache") +def test_cache_list_with_empty_cache_abspath(script): + """Running `pip cache list --format=abspath` with an empty cache should not + print anything and exit.""" + result = script.pip('cache', 'list', '--format=abspath') + assert result.stdout.strip() == "" + + def test_cache_list_too_many_args(script): """Passing `pip cache list` too many arguments should cause an error.""" script.pip('cache', 'list', 'aaa', 'bbb', @@ -158,6 +257,19 @@ def test_cache_list_name_match(script): assert list_matches_wheel('zzz-7.8.9', result) +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_name_match_abspath(script): + """Running `pip cache list zzz --format=abspath` should list paths of + zzz-4.5.6, zzz-4.5.7, zzz-7.8.9, but nothing else.""" + result = script.pip('cache', 'list', 'zzz', '--format=abspath', + '--verbose') + + assert not list_matches_wheel_abspath('yyy-1.2.3', result) + assert list_matches_wheel_abspath('zzz-4.5.6', result) + assert list_matches_wheel_abspath('zzz-4.5.7', result) + assert list_matches_wheel_abspath('zzz-7.8.9', result) + + @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_list_name_and_version_match(script): """Running `pip cache list zzz-4.5.6` should list zzz-4.5.6, but @@ -170,6 +282,19 @@ def test_cache_list_name_and_version_match(script): assert not list_matches_wheel('zzz-7.8.9', result) +@pytest.mark.usefixtures("populate_wheel_cache") +def test_cache_list_name_and_version_match_abspath(script): + """Running `pip cache list zzz-4.5.6 --format=abspath` should list path of + zzz-4.5.6, but nothing else.""" + result = script.pip('cache', 'list', 'zzz-4.5.6', '--format=abspath', + '--verbose') + + assert not list_matches_wheel_abspath('yyy-1.2.3', result) + assert list_matches_wheel_abspath('zzz-4.5.6', result) + assert not list_matches_wheel_abspath('zzz-4.5.7', result) + assert not list_matches_wheel_abspath('zzz-7.8.9', result) + + @pytest.mark.usefixtures("populate_wheel_cache") def test_cache_remove_no_arguments(script): """Running `pip cache remove` with no arguments should cause an error.""" @@ -206,21 +331,28 @@ def test_cache_remove_name_and_version_match(script, remove_matches_wheel): assert not remove_matches_wheel('zzz-7.8.9', result) -@pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_purge(script, remove_matches_wheel): - """Running `pip cache purge` should remove all cached wheels.""" +@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") +def test_cache_purge(script, remove_matches_http, remove_matches_wheel): + """Running `pip cache purge` should remove all cached http files and + wheels.""" result = script.pip('cache', 'purge', '--verbose') + assert remove_matches_http('aaaaaaaaa', result) + assert remove_matches_http('bbbbbbbbb', result) + assert remove_matches_http('ccccccccc', result) + assert remove_matches_wheel('yyy-1.2.3', result) assert remove_matches_wheel('zzz-4.5.6', result) assert remove_matches_wheel('zzz-4.5.7', result) assert remove_matches_wheel('zzz-7.8.9', result) -@pytest.mark.usefixtures("populate_wheel_cache") -def test_cache_purge_too_many_args(script, wheel_cache_files): +@pytest.mark.usefixtures("populate_http_cache", "populate_wheel_cache") +def test_cache_purge_too_many_args( + script, http_cache_files, wheel_cache_files +): """Running `pip cache purge aaa` should raise an error and remove no - cached wheels.""" + cached http files or wheels.""" result = script.pip('cache', 'purge', 'aaa', '--verbose', expect_error=True) assert result.stdout == '' @@ -230,7 +362,7 @@ def test_cache_purge_too_many_args(script, wheel_cache_files): assert 'ERROR: Too many arguments' in result.stderr.splitlines() # Make sure nothing was deleted. - for filename in wheel_cache_files: + for filename in http_cache_files + wheel_cache_files: assert os.path.exists(filename) diff --git a/tests/functional/test_configuration.py b/tests/functional/test_configuration.py index 63b243f78..f820bdc19 100644 --- a/tests/functional/test_configuration.py +++ b/tests/functional/test_configuration.py @@ -7,10 +7,7 @@ import textwrap import pytest from pip._internal.cli.status_codes import ERROR -from pip._internal.configuration import ( - CONFIG_BASENAME, - get_configuration_files, -) +from pip._internal.configuration import CONFIG_BASENAME, get_configuration_files from tests.lib.configuration_helpers import ConfigurationMixin, kinds diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py index 3e80fa5c5..3291d580d 100644 --- a/tests/functional/test_download.py +++ b/tests/functional/test_download.py @@ -4,6 +4,7 @@ import textwrap from hashlib import sha256 import pytest +from pip._vendor.six import PY2 from pip._internal.cli.status_codes import ERROR from pip._internal.utils.urls import path_to_url @@ -474,6 +475,7 @@ def make_wheel_with_python_requires(script, package_name, python_requires): package_dir.joinpath('setup.py').write_text(text) script.run( 'python', 'setup.py', 'bdist_wheel', '--universal', cwd=package_dir, + allow_stderr_warning=PY2, ) file_name = '{}-1.0-py2.py3-none-any.whl'.format(package_name) diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py index b41055c56..655440b88 100644 --- a/tests/functional/test_fast_deps.py +++ b/tests/functional/test_fast_deps.py @@ -48,3 +48,32 @@ def test_build_wheel_with_deps(data, script): assert fnmatch.filter(created, 'requiresPaste-3.1.4-*.whl') assert fnmatch.filter(created, 'Paste-3.4.2-*.whl') assert fnmatch.filter(created, 'six-*.whl') + + +@mark.network +def test_require_hash(script, tmp_path): + reqs = tmp_path / 'requirements.txt' + reqs.write_text( + u'idna==2.10' + ' --hash=sha256:' + 'b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0' + ' --hash=sha256:' + 'b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6' + ) + result = script.pip( + 'download', '--use-feature=fast-deps', '-r', str(reqs), + allow_stderr_warning=True, + ) + created = list(map(basename, result.files_created)) + assert fnmatch.filter(created, 'idna-2.10*') + + +@mark.network +def test_hash_mismatch(script, tmp_path): + reqs = tmp_path / 'requirements.txt' + reqs.write_text(u'idna==2.10 --hash=sha256:irna') + result = script.pip( + 'download', '--use-feature=fast-deps', '-r', str(reqs), + expect_error=True, + ) + assert 'DO NOT MATCH THE HASHES' in result.stderr diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index 2185251d2..17a72bca8 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -9,6 +9,7 @@ import textwrap from os.path import curdir, join, pardir import pytest +from pip._vendor.six import PY2 from pip._internal.cli.status_codes import ERROR, SUCCESS from pip._internal.models.index import PyPI, TestPyPI @@ -757,6 +758,7 @@ def test_install_using_install_option_and_editable(script, tmpdir): result.did_create(script_file) +@pytest.mark.xfail @pytest.mark.network @need_mercurial @windows_workaround_7667 @@ -1565,7 +1567,9 @@ def test_install_incompatible_python_requires_wheel(script, with_wheel): version='0.1') """)) script.run( - 'python', 'setup.py', 'bdist_wheel', '--universal', cwd=pkga_path) + 'python', 'setup.py', 'bdist_wheel', '--universal', + cwd=pkga_path, allow_stderr_warning=PY2, + ) result = script.pip('install', './pkga/dist/pkga-0.1-py2.py3-none-any.whl', expect_error=True) assert _get_expected_error_text() in result.stderr, str(result) diff --git a/tests/functional/test_install_check.py b/tests/functional/test_install_check.py index 88609422b..a173cb550 100644 --- a/tests/functional/test_install_check.py +++ b/tests/functional/test_install_check.py @@ -1,8 +1,9 @@ from tests.lib import create_test_package_with_setup -def contains_expected_lines(string, expected_lines): - return set(expected_lines) <= set(string.splitlines()) +def assert_contains_expected_lines(string, expected_lines): + for expected_line in expected_lines: + assert (expected_line + '\n') in string def test_check_install_canonicalization(script): @@ -38,7 +39,7 @@ def test_check_install_canonicalization(script): "pkga 1.0 requires SPECIAL.missing, which is not installed.", ] # Deprecated python versions produce an extra warning on stderr - assert contains_expected_lines(result.stderr, expected_lines) + assert_contains_expected_lines(result.stderr, expected_lines) assert result.returncode == 0 # Install the second missing package and expect that there is no warning @@ -55,7 +56,7 @@ def test_check_install_canonicalization(script): expected_lines = [ "No broken requirements found.", ] - assert contains_expected_lines(result.stdout, expected_lines) + assert_contains_expected_lines(result.stdout, expected_lines) assert result.returncode == 0 @@ -85,12 +86,10 @@ def test_check_install_does_not_warn_for_out_of_graph_issues(script): result = script.pip( 'install', '--no-index', pkg_conflict_path, allow_stderr_error=True, ) - assert contains_expected_lines(result.stderr, [ + assert_contains_expected_lines(result.stderr, [ "broken 1.0 requires missing, which is not installed.", - ( - "broken 1.0 requires conflict<1.0, but " - "you'll have conflict 1.0 which is incompatible." - ), + "broken 1.0 requires conflict<1.0, " + "but you'll have conflict 1.0 which is incompatible." ]) # Install unrelated package @@ -105,4 +104,4 @@ def test_check_install_does_not_warn_for_out_of_graph_issues(script): "broken 1.0 requires missing, which is not installed.", "broken 1.0 has requirement conflict<1.0, but you have conflict 1.0.", ] - assert contains_expected_lines(result.stdout, expected_lines) + assert_contains_expected_lines(result.stdout, expected_lines) diff --git a/tests/functional/test_install_reqs.py b/tests/functional/test_install_reqs.py index d12e19b21..c879b6903 100644 --- a/tests/functional/test_install_reqs.py +++ b/tests/functional/test_install_reqs.py @@ -159,6 +159,7 @@ def test_relative_requirements_file( result.did_create(egg_link_file) +@pytest.mark.xfail @pytest.mark.network @need_svn def test_multiple_requirements_files(script, tmpdir, with_wheel): diff --git a/tests/functional/test_install_user.py b/tests/functional/test_install_user.py index 24169470a..c5d7acced 100644 --- a/tests/functional/test_install_user.py +++ b/tests/functional/test_install_user.py @@ -45,6 +45,7 @@ class Tests_UserSite: project_name = result.stdout.strip() assert 'INITools' == project_name, project_name + @pytest.mark.xfail @pytest.mark.network @need_svn @pytest.mark.incompatible_with_test_venv diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py index 4b13ebc30..ad5e2f3d0 100644 --- a/tests/functional/test_new_resolver_hashes.py +++ b/tests/functional/test_new_resolver_hashes.py @@ -4,10 +4,7 @@ import hashlib import pytest from pip._internal.utils.urls import path_to_url -from tests.lib import ( - create_basic_sdist_for_package, - create_basic_wheel_for_package, -) +from tests.lib import create_basic_sdist_for_package, create_basic_wheel_for_package _FindLinks = collections.namedtuple( "_FindLinks", "index_html sdist_hash wheel_hash", diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py index 5918b4f64..1892e26b5 100644 --- a/tests/functional/test_search.py +++ b/tests/functional/test_search.py @@ -5,11 +5,7 @@ import pytest from pip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS from pip._internal.commands import create_command -from pip._internal.commands.search import ( - highest_version, - print_results, - transform_hits, -) +from pip._internal.commands.search import highest_version, print_results, transform_hits from tests.lib import pyversion if pyversion >= '3': diff --git a/tests/functional/test_uninstall.py b/tests/functional/test_uninstall.py index 1f2fe6912..6c687ada5 100644 --- a/tests/functional/test_uninstall.py +++ b/tests/functional/test_uninstall.py @@ -13,11 +13,7 @@ import pytest from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import rmtree -from tests.lib import ( - assert_all_changes, - create_test_package_with_setup, - need_svn, -) +from tests.lib import assert_all_changes, create_test_package_with_setup, need_svn from tests.lib.local_repos import local_checkout, local_repo @@ -307,6 +303,7 @@ def test_uninstall_easy_installed_console_scripts(script): ) +@pytest.mark.xfail @pytest.mark.network @need_svn def test_uninstall_editable_from_svn(script, tmpdir): @@ -372,6 +369,7 @@ def _test_uninstall_editable_with_source_outside_venv( ) +@pytest.mark.xfail @pytest.mark.network @need_svn def test_uninstall_from_reqs_file(script, tmpdir): @@ -420,9 +418,10 @@ def test_uninstallpathset_no_paths(caplog): Test UninstallPathSet logs notification when there are no paths to uninstall """ - from pip._internal.req.req_uninstall import UninstallPathSet from pkg_resources import get_distribution + from pip._internal.req.req_uninstall import UninstallPathSet + caplog.set_level(logging.INFO) test_dist = get_distribution('pip') diff --git a/tests/functional/test_vcs_git.py b/tests/functional/test_vcs_git.py index 37c35c4b5..8b07ae667 100644 --- a/tests/functional/test_vcs_git.py +++ b/tests/functional/test_vcs_git.py @@ -250,3 +250,35 @@ def test_get_repository_root(script): root2 = Git.get_repository_root(version_pkg_path.joinpath("tests")) assert os.path.normcase(root2) == os.path.normcase(version_pkg_path) + + +def test_resolve_commit_not_on_branch(script, tmp_path): + repo_path = tmp_path / "repo" + repo_file = repo_path / "file.txt" + clone_path = repo_path / "clone" + repo_path.mkdir() + script.run("git", "init", cwd=str(repo_path)) + + repo_file.write_text(u".") + script.run("git", "add", "file.txt", cwd=str(repo_path)) + script.run("git", "commit", "-m", "initial commit", cwd=str(repo_path)) + script.run("git", "checkout", "-b", "abranch", cwd=str(repo_path)) + + # create a commit + repo_file.write_text(u"..") + script.run("git", "commit", "-a", "-m", "commit 1", cwd=str(repo_path)) + commit = script.run( + "git", "rev-parse", "HEAD", cwd=str(repo_path) + ).stdout.strip() + + # make sure our commit is not on a branch + script.run("git", "checkout", "master", cwd=str(repo_path)) + script.run("git", "branch", "-D", "abranch", cwd=str(repo_path)) + + # create a ref that points to our commit + (repo_path / ".git" / "refs" / "myrefs").mkdir(parents=True) + (repo_path / ".git" / "refs" / "myrefs" / "myref").write_text(commit) + + # check we can fetch our commit + rev_options = Git.make_rev_options(commit) + Git().fetch_new(str(clone_path), repo_path.as_uri(), rev_options) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index a8ea86761..07569d814 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -31,6 +31,7 @@ from tests.lib.wheel import make_wheel if MYPY_CHECK_RUNNING: from typing import List, Optional + from pip._internal.models.target_python import TargetPython @@ -492,10 +493,7 @@ class PipTestEnvironment(TestFileEnvironment): kwargs.setdefault("cwd", self.scratch_path) # Setup our environment - environ = kwargs.get("environ") - if environ is None: - environ = os.environ.copy() - + environ = kwargs.setdefault("environ", os.environ.copy()) environ["PATH"] = Path.pathsep.join( [self.bin_path] + [environ.get("PATH", [])], ) @@ -504,7 +502,6 @@ class PipTestEnvironment(TestFileEnvironment): environ["PYTHONDONTWRITEBYTECODE"] = "1" # Make sure we get UTF-8 on output, even on Windows... environ["PYTHONIOENCODING"] = "UTF-8" - kwargs["environ"] = environ # Whether all pip invocations should expect stderr # (useful for Python version deprecation) diff --git a/tests/lib/configuration_helpers.py b/tests/lib/configuration_helpers.py index d33b8ec09..3e3692696 100644 --- a/tests/lib/configuration_helpers.py +++ b/tests/lib/configuration_helpers.py @@ -14,18 +14,6 @@ from pip._internal.utils.misc import ensure_dir kinds = pip._internal.configuration.kinds -def reset_os_environ(old_environ): - """ - Reset os.environ while preserving the same underlying mapping. - """ - # Preserving the same mapping is preferable to assigning a new mapping - # because the latter has interfered with test isolation by, for example, - # preventing time.tzset() from working in subsequent tests after - # changing os.environ['TZ'] in those tests. - os.environ.clear() - os.environ.update(old_environ) - - class ConfigurationMixin(object): def setup(self): @@ -34,14 +22,10 @@ class ConfigurationMixin(object): ) self._files_to_clear = [] - self._old_environ = os.environ.copy() - def teardown(self): for fname in self._files_to_clear: fname.stop() - reset_os_environ(self._old_environ) - def patch_configuration(self, variant, di): old = self.configuration._load_config_files diff --git a/tests/lib/options_helpers.py b/tests/lib/options_helpers.py index 120070abb..2354a818d 100644 --- a/tests/lib/options_helpers.py +++ b/tests/lib/options_helpers.py @@ -1,12 +1,9 @@ """Provides helper classes for testing option handling in pip """ -import os - from pip._internal.cli import cmdoptions from pip._internal.cli.base_command import Command from pip._internal.commands import CommandInfo, commands_dict -from tests.lib.configuration_helpers import reset_os_environ class FakeCommand(Command): @@ -23,11 +20,9 @@ class FakeCommand(Command): class AddFakeCommandMixin(object): def setup(self): - self.environ_before = os.environ.copy() commands_dict['fake'] = CommandInfo( 'tests.lib.options_helpers', 'FakeCommand', 'fake summary', ) def teardown(self): - reset_os_environ(self.environ_before) commands_dict.pop('fake') diff --git a/tests/lib/server.py b/tests/lib/server.py index 6a8862ac5..ebbf120d3 100644 --- a/tests/lib/server.py +++ b/tests/lib/server.py @@ -15,7 +15,16 @@ from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: from types import TracebackType from typing import ( - Any, Callable, Dict, Iterable, List, Optional, Text, Tuple, Type, Union + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Text, + Tuple, + Type, + Union, ) from werkzeug.serving import BaseWSGIServer diff --git a/tests/lib/wheel.py b/tests/lib/wheel.py index 8ea458658..d89a680a1 100644 --- a/tests/lib/wheel.py +++ b/tests/lib/wheel.py @@ -20,8 +20,16 @@ from tests.lib.path import Path if MYPY_CHECK_RUNNING: from typing import ( - AnyStr, Callable, Dict, List, Iterable, Optional, Tuple, Sequence, - TypeVar, Union, + AnyStr, + Callable, + Dict, + Iterable, + List, + Optional, + Sequence, + Tuple, + TypeVar, + Union, ) # path, digest, size diff --git a/tests/unit/resolution_resolvelib/conftest.py b/tests/unit/resolution_resolvelib/conftest.py index 87f5d129c..9c1c9e5c4 100644 --- a/tests/unit/resolution_resolvelib/conftest.py +++ b/tests/unit/resolution_resolvelib/conftest.py @@ -4,6 +4,7 @@ from pip._internal.cli.req_command import RequirementCommand from pip._internal.commands.install import InstallCommand from pip._internal.index.collector import LinkCollector from pip._internal.index.package_finder import PackageFinder + # from pip._internal.models.index import PyPI from pip._internal.models.search_scope import SearchScope from pip._internal.models.selection_prefs import SelectionPreferences diff --git a/tests/unit/test_base_command.py b/tests/unit/test_base_command.py index 27147e769..8ba3d9e25 100644 --- a/tests/unit/test_base_command.py +++ b/tests/unit/test_base_command.py @@ -1,6 +1,5 @@ import logging import os -import time import pytest from mock import Mock, patch @@ -12,6 +11,12 @@ from pip._internal.utils.logging import BrokenStdoutLoggingError from pip._internal.utils.temp_dir import TempDirectory +@pytest.fixture +def fixed_time(utc): + with patch('time.time', lambda: 1547704837.040001): + yield + + class FakeCommand(Command): _name = 'fake' @@ -91,67 +96,40 @@ def test_handle_pip_version_check_called(mock_handle_version_check): mock_handle_version_check.assert_called_once() -class Test_base_command_logging(object): +def test_log_command_success(fixed_time, tmpdir): + """Test the --log option logs when command succeeds.""" + cmd = FakeCommand() + log_path = tmpdir.joinpath('log') + cmd.main(['fake', '--log', log_path]) + with open(log_path) as f: + assert f.read().rstrip() == '2019-01-17T06:00:37,040 fake' + + +def test_log_command_error(fixed_time, tmpdir): + """Test the --log option logs when command fails.""" + cmd = FakeCommand(error=True) + log_path = tmpdir.joinpath('log') + cmd.main(['fake', '--log', log_path]) + with open(log_path) as f: + assert f.read().startswith('2019-01-17T06:00:37,040 fake') + + +def test_log_file_command_error(fixed_time, tmpdir): + """Test the --log-file option logs (when there's an error).""" + cmd = FakeCommand(error=True) + log_file_path = tmpdir.joinpath('log_file') + cmd.main(['fake', '--log-file', log_file_path]) + with open(log_file_path) as f: + assert f.read().startswith('2019-01-17T06:00:37,040 fake') + + +def test_log_unicode_messages(fixed_time, tmpdir): + """Tests that logging bytestrings and unicode objects + don't break logging. """ - Test `pip.base_command.Command` setting up logging consumers based on - options - """ - - def setup(self): - self.old_time = time.time - time.time = lambda: 1547704837.040001 - self.old_tz = os.environ.get('TZ') - os.environ['TZ'] = 'UTC' - # time.tzset() is not implemented on some platforms (notably, Windows). - if hasattr(time, 'tzset'): - time.tzset() - - def teardown(self): - if self.old_tz: - os.environ['TZ'] = self.old_tz - else: - del os.environ['TZ'] - if 'tzset' in dir(time): - time.tzset() - time.time = self.old_time - - def test_log_command_success(self, tmpdir): - """ - Test the --log option logs when command succeeds - """ - cmd = FakeCommand() - log_path = tmpdir.joinpath('log') - cmd.main(['fake', '--log', log_path]) - with open(log_path) as f: - assert f.read().rstrip() == '2019-01-17T06:00:37,040 fake' - - def test_log_command_error(self, tmpdir): - """ - Test the --log option logs when command fails - """ - cmd = FakeCommand(error=True) - log_path = tmpdir.joinpath('log') - cmd.main(['fake', '--log', log_path]) - with open(log_path) as f: - assert f.read().startswith('2019-01-17T06:00:37,040 fake') - - def test_log_file_command_error(self, tmpdir): - """ - Test the --log-file option logs (when there's an error). - """ - cmd = FakeCommand(error=True) - log_file_path = tmpdir.joinpath('log_file') - cmd.main(['fake', '--log-file', log_file_path]) - with open(log_file_path) as f: - assert f.read().startswith('2019-01-17T06:00:37,040 fake') - - def test_unicode_messages(self, tmpdir): - """ - Tests that logging bytestrings and unicode objects don't break logging - """ - cmd = FakeCommandWithUnicode() - log_path = tmpdir.joinpath('log') - cmd.main(['fake_unicode', '--log', log_path]) + cmd = FakeCommandWithUnicode() + log_path = tmpdir.joinpath('log') + cmd.main(['fake_unicode', '--log', log_path]) @pytest.mark.no_auto_tempdir_manager diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index f16252f44..0a45fc136 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -1,8 +1,6 @@ """Tests for all things related to the configuration """ -import os - import pytest from mock import MagicMock @@ -31,48 +29,48 @@ class TestConfigurationLoading(ConfigurationMixin): self.configuration.load() assert self.configuration.get_value("test.hello") == "3" - def test_environment_config_loading(self): + def test_environment_config_loading(self, monkeypatch): contents = """ [test] hello = 4 """ with self.tmpfile(contents) as config_file: - os.environ["PIP_CONFIG_FILE"] = config_file + monkeypatch.setenv("PIP_CONFIG_FILE", config_file) self.configuration.load() assert self.configuration.get_value("test.hello") == "4", \ self.configuration._config - def test_environment_var_loading(self): - os.environ["PIP_HELLO"] = "5" + def test_environment_var_loading(self, monkeypatch): + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value(":env:.hello") == "5" @pytest.mark.skipif("sys.platform == 'win32'") - def test_environment_var_does_not_load_lowercase(self): - os.environ["pip_hello"] = "5" + def test_environment_var_does_not_load_lowercase(self, monkeypatch): + monkeypatch.setenv("pip_hello", "5") self.configuration.load() with pytest.raises(ConfigurationError): self.configuration.get_value(":env:.hello") - def test_environment_var_does_not_load_version(self): - os.environ["PIP_VERSION"] = "True" + def test_environment_var_does_not_load_version(self, monkeypatch): + monkeypatch.setenv("PIP_VERSION", "True") self.configuration.load() with pytest.raises(ConfigurationError): self.configuration.get_value(":env:.version") - def test_environment_config_errors_if_malformed(self): + def test_environment_config_errors_if_malformed(self, monkeypatch): contents = """ test] hello = 4 """ with self.tmpfile(contents) as config_file: - os.environ["PIP_CONFIG_FILE"] = config_file + monkeypatch.setenv("PIP_CONFIG_FILE", config_file) with pytest.raises(ConfigurationError) as err: self.configuration.load() @@ -130,36 +128,36 @@ class TestConfigurationPrecedence(ConfigurationMixin): assert self.configuration.get_value("test.hello") == "2" - def test_env_not_overriden_by_environment_var(self): + def test_env_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.ENV, {"test.hello": "1"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value("test.hello") == "1" assert self.configuration.get_value(":env:.hello") == "5" - def test_site_not_overriden_by_environment_var(self): + def test_site_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.SITE, {"test.hello": "2"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value("test.hello") == "2" assert self.configuration.get_value(":env:.hello") == "5" - def test_user_not_overriden_by_environment_var(self): + def test_user_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.USER, {"test.hello": "3"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() assert self.configuration.get_value("test.hello") == "3" assert self.configuration.get_value(":env:.hello") == "5" - def test_global_not_overriden_by_environment_var(self): + def test_global_not_overriden_by_environment_var(self, monkeypatch): self.patch_configuration(kinds.GLOBAL, {"test.hello": "4"}) - os.environ["PIP_HELLO"] = "5" + monkeypatch.setenv("PIP_HELLO", "5") self.configuration.load() diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py index 853af723b..55fdab3b8 100644 --- a/tests/unit/test_finder.py +++ b/tests/unit/test_finder.py @@ -8,10 +8,7 @@ from pip._vendor.packaging.tags import Tag from pkg_resources import parse_version import pip._internal.utils.compatibility_tags -from pip._internal.exceptions import ( - BestVersionAlreadyInstalled, - DistributionNotFound, -) +from pip._internal.exceptions import BestVersionAlreadyInstalled, DistributionNotFound from pip._internal.index.package_finder import ( CandidateEvaluator, InstallationCandidate, diff --git a/tests/unit/test_locations.py b/tests/unit/test_locations.py index c9bbe7943..2ede236f9 100644 --- a/tests/unit/test_locations.py +++ b/tests/unit/test_locations.py @@ -100,6 +100,7 @@ class TestDistutilsScheme: f.parent.mkdir() f.write_text("[install]\ninstall-scripts=" + install_scripts) from distutils.dist import Distribution + # patch the function that returns what config files are present monkeypatch.setattr( Distribution, @@ -121,6 +122,7 @@ class TestDistutilsScheme: f.parent.mkdir() f.write_text("[install]\ninstall-lib=" + install_lib) from distutils.dist import Distribution + # patch the function that returns what config files are present monkeypatch.setattr( Distribution, diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index a62c18c77..10d47eb61 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -1,7 +1,5 @@ import errno import logging -import os -import time from threading import Thread import pytest @@ -33,24 +31,7 @@ def _make_broken_pipe_error(): class TestIndentingFormatter(object): - """ - Test `pip._internal.utils.logging.IndentingFormatter`. - """ - - def setup(self): - self.old_tz = os.environ.get('TZ') - os.environ['TZ'] = 'UTC' - # time.tzset() is not implemented on some platforms (notably, Windows). - if hasattr(time, 'tzset'): - time.tzset() - - def teardown(self): - if self.old_tz: - os.environ['TZ'] = self.old_tz - else: - del os.environ['TZ'] - if 'tzset' in dir(time): - time.tzset() + """Test ``pip._internal.utils.logging.IndentingFormatter``.""" def make_record(self, msg, level_name): level_number = getattr(logging, level_name) @@ -72,7 +53,7 @@ class TestIndentingFormatter(object): ('ERROR', 'ERROR: hello\nworld'), ('CRITICAL', 'ERROR: hello\nworld'), ]) - def test_format(self, level_name, expected): + def test_format(self, level_name, expected, utc): """ Args: level_name: a logging level name (e.g. "WARNING"). @@ -89,7 +70,7 @@ class TestIndentingFormatter(object): '2019-01-17T06:00:37,040 WARNING: hello\n' '2019-01-17T06:00:37,040 world'), ]) - def test_format_with_timestamp(self, level_name, expected): + def test_format_with_timestamp(self, level_name, expected, utc): record = self.make_record('hello\nworld', level_name=level_name) f = IndentingFormatter(fmt="%(message)s", add_timestamp=True) assert f.format(record) == expected @@ -99,7 +80,7 @@ class TestIndentingFormatter(object): ('ERROR', 'DEPRECATION: hello\nworld'), ('CRITICAL', 'DEPRECATION: hello\nworld'), ]) - def test_format_deprecated(self, level_name, expected): + def test_format_deprecated(self, level_name, expected, utc): """ Test that logged deprecation warnings coming from deprecated() don't get another prefix. @@ -110,7 +91,7 @@ class TestIndentingFormatter(object): f = IndentingFormatter(fmt="%(message)s") assert f.format(record) == expected - def test_thread_safety_base(self): + def test_thread_safety_base(self, utc): record = self.make_record( 'DEPRECATION: hello\nworld', level_name='WARNING', ) @@ -126,7 +107,7 @@ class TestIndentingFormatter(object): thread.join() assert results[0] == results[1] - def test_thread_safety_indent_log(self): + def test_thread_safety_indent_log(self, utc): record = self.make_record( 'DEPRECATION: hello\nworld', level_name='WARNING', ) diff --git a/tests/unit/test_operations_prepare.py b/tests/unit/test_operations_prepare.py index 41d8be260..af3ce72a1 100644 --- a/tests/unit/test_operations_prepare.py +++ b/tests/unit/test_operations_prepare.py @@ -10,18 +10,10 @@ from pip._internal.exceptions import HashMismatch from pip._internal.models.link import Link from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession -from pip._internal.operations.prepare import ( - _copy_source_tree, - _download_http_url, - unpack_url, -) +from pip._internal.operations.prepare import _copy_source_tree, unpack_url from pip._internal.utils.hashes import Hashes from pip._internal.utils.urls import path_to_url -from tests.lib.filesystem import ( - get_filelist, - make_socket_file, - make_unreadable_file, -) +from tests.lib.filesystem import get_filelist, make_socket_file, make_unreadable_file from tests.lib.path import Path from tests.lib.requests_mocks import MockResponse @@ -39,7 +31,7 @@ def test_unpack_url_with_urllib_response_without_content_type(data): session = Mock() session.get = _fake_session_get - downloader = Downloader(session, progress_bar="on") + download = Downloader(session, progress_bar="on") uri = path_to_url(data.packages.joinpath("simple-1.0.tar.gz")) link = Link(uri) @@ -48,7 +40,7 @@ def test_unpack_url_with_urllib_response_without_content_type(data): unpack_url( link, temp_dir, - downloader=downloader, + download=download, download_dir=None, ) assert set(os.listdir(temp_dir)) == { @@ -79,16 +71,11 @@ def test_download_http_url__no_directory_traversal(mock_raise_for_status, 'content-disposition': 'attachment;filename="../out_dir_file"' } session.get.return_value = resp - downloader = Downloader(session, progress_bar="on") + download = Downloader(session, progress_bar="on") download_dir = tmpdir.joinpath('download') os.mkdir(download_dir) - file_path, content_type = _download_http_url( - link, - downloader, - download_dir, - hashes=None, - ) + file_path, content_type = download(link, download_dir) # The file should be downloaded to download_dir. actual = os.listdir(download_dir) assert actual == ['out_dir_file'] @@ -187,11 +174,11 @@ class Test_unpack_url(object): self.dist_path2 = data.packages.joinpath(self.dist_file2) self.dist_url = Link(path_to_url(self.dist_path)) self.dist_url2 = Link(path_to_url(self.dist_path2)) - self.no_downloader = Mock(side_effect=AssertionError) + self.no_download = Mock(side_effect=AssertionError) def test_unpack_url_no_download(self, tmpdir, data): self.prep(tmpdir, data) - unpack_url(self.dist_url, self.build_dir, self.no_downloader) + unpack_url(self.dist_url, self.build_dir, self.no_download) assert os.path.isdir(os.path.join(self.build_dir, 'simple')) assert not os.path.isfile( os.path.join(self.download_dir, self.dist_file)) @@ -207,7 +194,7 @@ class Test_unpack_url(object): with pytest.raises(HashMismatch): unpack_url(dist_url, self.build_dir, - downloader=self.no_downloader, + download=self.no_download, hashes=Hashes({'md5': ['bogus']})) def test_unpack_url_thats_a_dir(self, tmpdir, data): @@ -215,7 +202,7 @@ class Test_unpack_url(object): dist_path = data.packages.joinpath("FSPkg") dist_url = Link(path_to_url(dist_path)) unpack_url(dist_url, self.build_dir, - downloader=self.no_downloader, + download=self.no_download, download_dir=self.download_dir) assert os.path.isdir(os.path.join(self.build_dir, 'fspkg')) diff --git a/tests/unit/test_options.py b/tests/unit/test_options.py index ce4fc9c25..533a4b8db 100644 --- a/tests/unit/test_options.py +++ b/tests/unit/test_options.py @@ -1,5 +1,6 @@ import os from contextlib import contextmanager +from tempfile import NamedTemporaryFile import pytest @@ -10,23 +11,6 @@ from pip._internal.exceptions import PipError from tests.lib.options_helpers import AddFakeCommandMixin -@contextmanager -def temp_environment_variable(name, value): - not_set = object() - original = os.environ[name] if name in os.environ else not_set - os.environ[name] = value - - try: - yield - finally: - # Return the environment variable to its original state. - if original is not_set: - if name in os.environ: - del os.environ[name] - else: - os.environ[name] = original - - @contextmanager def assert_option_error(capsys, expected): """ @@ -70,56 +54,48 @@ class TestOptionPrecedence(AddFakeCommandMixin): } return config[section] - def test_env_override_default_int(self): + def test_env_override_default_int(self, monkeypatch): """ Test that environment variable overrides an int option default. """ - os.environ['PIP_TIMEOUT'] = '-1' + monkeypatch.setenv('PIP_TIMEOUT', '-1') options, args = main(['fake']) assert options.timeout == -1 - def test_env_override_default_append(self): + @pytest.mark.parametrize('values', (['F1'], ['F1', 'F2'])) + def test_env_override_default_append(self, values, monkeypatch): """ Test that environment variable overrides an append option default. """ - os.environ['PIP_FIND_LINKS'] = 'F1' + monkeypatch.setenv('PIP_FIND_LINKS', ' '.join(values)) options, args = main(['fake']) - assert options.find_links == ['F1'] + assert options.find_links == values - os.environ['PIP_FIND_LINKS'] = 'F1 F2' - options, args = main(['fake']) - assert options.find_links == ['F1', 'F2'] - - def test_env_override_default_choice(self): + @pytest.mark.parametrize('choises', (['w'], ['s', 'w'])) + def test_env_override_default_choice(self, choises, monkeypatch): """ Test that environment variable overrides a choice option default. """ - os.environ['PIP_EXISTS_ACTION'] = 'w' + monkeypatch.setenv('PIP_EXISTS_ACTION', ' '.join(choises)) options, args = main(['fake']) - assert options.exists_action == ['w'] + assert options.exists_action == choises - os.environ['PIP_EXISTS_ACTION'] = 's w' - options, args = main(['fake']) - assert options.exists_action == ['s', 'w'] - - def test_env_alias_override_default(self): + @pytest.mark.parametrize('name', ('PIP_LOG_FILE', 'PIP_LOCAL_LOG')) + def test_env_alias_override_default(self, name, monkeypatch): """ When an option has multiple long forms, test that the technique of using the env variable, "PIP_" works for all cases. (e.g. PIP_LOG_FILE and PIP_LOCAL_LOG should all work) """ - os.environ['PIP_LOG_FILE'] = 'override.log' - options, args = main(['fake']) - assert options.log == 'override.log' - os.environ['PIP_LOCAL_LOG'] = 'override.log' + monkeypatch.setenv(name, 'override.log') options, args = main(['fake']) assert options.log == 'override.log' - def test_cli_override_environment(self): + def test_cli_override_environment(self, monkeypatch): """ Test the cli overrides and environment variable """ - os.environ['PIP_TIMEOUT'] = '-1' + monkeypatch.setenv('PIP_TIMEOUT', '-1') options, args = main(['fake', '--timeout', '-2']) assert options.timeout == -2 @@ -136,49 +112,49 @@ class TestOptionPrecedence(AddFakeCommandMixin): 'off', 'no', ]) - def test_cache_dir__PIP_NO_CACHE_DIR(self, pip_no_cache_dir): + def test_cache_dir__PIP_NO_CACHE_DIR(self, pip_no_cache_dir, monkeypatch): """ Test setting the PIP_NO_CACHE_DIR environment variable without passing any command-line flags. """ - os.environ['PIP_NO_CACHE_DIR'] = pip_no_cache_dir + monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) options, args = main(['fake']) assert options.cache_dir is False @pytest.mark.parametrize('pip_no_cache_dir', ['yes', 'no']) def test_cache_dir__PIP_NO_CACHE_DIR__with_cache_dir( - self, pip_no_cache_dir + self, pip_no_cache_dir, monkeypatch, ): """ Test setting PIP_NO_CACHE_DIR while also passing an explicit --cache-dir value. """ - os.environ['PIP_NO_CACHE_DIR'] = pip_no_cache_dir + monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) options, args = main(['--cache-dir', '/cache/dir', 'fake']) # The command-line flag takes precedence. assert options.cache_dir == '/cache/dir' @pytest.mark.parametrize('pip_no_cache_dir', ['yes', 'no']) def test_cache_dir__PIP_NO_CACHE_DIR__with_no_cache_dir( - self, pip_no_cache_dir + self, pip_no_cache_dir, monkeypatch, ): """ Test setting PIP_NO_CACHE_DIR while also passing --no-cache-dir. """ - os.environ['PIP_NO_CACHE_DIR'] = pip_no_cache_dir + monkeypatch.setenv('PIP_NO_CACHE_DIR', pip_no_cache_dir) options, args = main(['--no-cache-dir', 'fake']) # The command-line flag should take precedence (which has the same # value in this case). assert options.cache_dir is False def test_cache_dir__PIP_NO_CACHE_DIR_invalid__with_no_cache_dir( - self, capsys, + self, monkeypatch, capsys, ): """ Test setting PIP_NO_CACHE_DIR to an invalid value while also passing --no-cache-dir. """ - os.environ['PIP_NO_CACHE_DIR'] = 'maybe' + monkeypatch.setenv('PIP_NO_CACHE_DIR', 'maybe') expected_err = "--no-cache-dir error: invalid truth value 'maybe'" with assert_option_error(capsys, expected=expected_err): main(['--no-cache-dir', 'fake']) @@ -219,52 +195,49 @@ class TestUsePEP517Options(object): options = self.parse_args(['--no-use-pep517']) assert options.use_pep517 is False - def test_PIP_USE_PEP517_true(self): + def test_PIP_USE_PEP517_true(self, monkeypatch): """ Test setting PIP_USE_PEP517 to "true". """ - with temp_environment_variable('PIP_USE_PEP517', 'true'): - options = self.parse_args([]) + monkeypatch.setenv('PIP_USE_PEP517', 'true') + options = self.parse_args([]) # This is an int rather than a boolean because strtobool() in pip's # configuration code returns an int. assert options.use_pep517 == 1 - def test_PIP_USE_PEP517_false(self): + def test_PIP_USE_PEP517_false(self, monkeypatch): """ Test setting PIP_USE_PEP517 to "false". """ - with temp_environment_variable('PIP_USE_PEP517', 'false'): - options = self.parse_args([]) + monkeypatch.setenv('PIP_USE_PEP517', 'false') + options = self.parse_args([]) # This is an int rather than a boolean because strtobool() in pip's # configuration code returns an int. assert options.use_pep517 == 0 - def test_use_pep517_and_PIP_USE_PEP517_false(self): + def test_use_pep517_and_PIP_USE_PEP517_false(self, monkeypatch): """ Test passing --use-pep517 and setting PIP_USE_PEP517 to "false". """ - with temp_environment_variable('PIP_USE_PEP517', 'false'): - options = self.parse_args(['--use-pep517']) + monkeypatch.setenv('PIP_USE_PEP517', 'false') + options = self.parse_args(['--use-pep517']) assert options.use_pep517 is True - def test_no_use_pep517_and_PIP_USE_PEP517_true(self): + def test_no_use_pep517_and_PIP_USE_PEP517_true(self, monkeypatch): """ Test passing --no-use-pep517 and setting PIP_USE_PEP517 to "true". """ - with temp_environment_variable('PIP_USE_PEP517', 'true'): - options = self.parse_args(['--no-use-pep517']) + monkeypatch.setenv('PIP_USE_PEP517', 'true') + options = self.parse_args(['--no-use-pep517']) assert options.use_pep517 is False - def test_PIP_NO_USE_PEP517(self, capsys): + def test_PIP_NO_USE_PEP517(self, monkeypatch, capsys): """ Test setting PIP_NO_USE_PEP517, which isn't allowed. """ - expected_err = ( - '--no-use-pep517 error: A value was passed for --no-use-pep517,\n' - ) - with temp_environment_variable('PIP_NO_USE_PEP517', 'true'): - with assert_option_error(capsys, expected=expected_err): - self.parse_args([]) + monkeypatch.setenv('PIP_NO_USE_PEP517', 'true') + with assert_option_error(capsys, expected='--no-use-pep517 error'): + self.parse_args([]) class TestOptionsInterspersed(AddFakeCommandMixin): @@ -286,6 +259,107 @@ class TestOptionsInterspersed(AddFakeCommandMixin): main(['--find-links', 'F1', 'fake']) +@contextmanager +def tmpconfig(option, value, section='global'): + with NamedTemporaryFile(mode='w', delete=False) as f: + f.write('[{}]\n{}={}\n'.format(section, option, value)) + name = f.name + try: + yield name + finally: + os.unlink(name) + + +class TestCountOptions(AddFakeCommandMixin): + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(4)) + def test_cli_long(self, option, value): + flags = ['--{}'.format(option)] * value + opt1, args1 = main(flags+['fake']) + opt2, args2 = main(['fake']+flags) + assert getattr(opt1, option) == getattr(opt2, option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(1, 4)) + def test_cli_short(self, option, value): + flag = '-' + option[0]*value + opt1, args1 = main([flag, 'fake']) + opt2, args2 = main(['fake', flag]) + assert getattr(opt1, option) == getattr(opt2, option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(4)) + def test_env_var(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake'])[0], option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(3)) + def test_env_var_integrate_cli(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake', '--'+option])[0], option) == value + 1 + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', (-1, 'foobar')) + def test_env_var_invalid(self, option, value, monkeypatch, capsys): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + with assert_option_error(capsys, expected='a non-negative integer'): + main(['fake']) + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('no', 'false')) + def test_env_var_false(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake'])[0], option) == 0 + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('yes', 'true')) + def test_env_var_true(self, option, value, monkeypatch): + monkeypatch.setenv('PIP_'+option.upper(), str(value)) + assert getattr(main(['fake'])[0], option) == 1 + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(4)) + def test_config_file(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake'])[0], option) == value + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', range(3)) + def test_config_file_integrate_cli(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake', '--'+option])[0], option) == value + 1 + + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', (-1, 'foobar')) + def test_config_file_invalid(self, option, value, monkeypatch, capsys): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + with assert_option_error(capsys, expected='non-negative integer'): + main(['fake']) + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('no', 'false')) + def test_config_file_false(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake'])[0], option) == 0 + + # Undocumented, support for backward compatibility + @pytest.mark.parametrize('option', ('verbose', 'quiet')) + @pytest.mark.parametrize('value', ('yes', 'true')) + def test_config_file_true(self, option, value, monkeypatch): + with tmpconfig(option, value) as name: + monkeypatch.setenv('PIP_CONFIG_FILE', name) + assert getattr(main(['fake'])[0], option) == 1 + + class TestGeneralOptions(AddFakeCommandMixin): # the reason to specifically test general options is due to the @@ -310,24 +384,6 @@ class TestGeneralOptions(AddFakeCommandMixin): assert options1.require_venv assert options2.require_venv - def test_verbose(self): - options1, args1 = main(['--verbose', 'fake']) - options2, args2 = main(['fake', '--verbose']) - assert options1.verbose == options2.verbose == 1 - - def test_quiet(self): - options1, args1 = main(['--quiet', 'fake']) - options2, args2 = main(['fake', '--quiet']) - assert options1.quiet == options2.quiet == 1 - - options3, args3 = main(['--quiet', '--quiet', 'fake']) - options4, args4 = main(['fake', '--quiet', '--quiet']) - assert options3.quiet == options4.quiet == 2 - - options5, args5 = main(['--quiet', '--quiet', '--quiet', 'fake']) - options6, args6 = main(['fake', '--quiet', '--quiet', '--quiet']) - assert options5.quiet == options6.quiet == 3 - def test_log(self): options1, args1 = main(['--log', 'path', 'fake']) options2, args2 = main(['fake', '--log', 'path']) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index 1aee7fcdf..083d2c2c6 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -18,7 +18,6 @@ from pip._internal.exceptions import ( InvalidWheelFilename, PreviousBuildDirError, ) -from pip._internal.network.download import Downloader from pip._internal.network.session import PipSession from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import InstallRequirement, RequirementSet @@ -76,19 +75,21 @@ class TestRequirementSet(object): isolated=False, use_pep517=None, ) + session = PipSession() with get_requirement_tracker() as tracker: preparer = RequirementPreparer( build_dir=os.path.join(self.tempdir, 'build'), src_dir=os.path.join(self.tempdir, 'src'), download_dir=None, - wheel_download_dir=None, build_isolation=True, req_tracker=tracker, - downloader=Downloader(PipSession(), progress_bar="on"), + session=session, + progress_bar='on', finder=finder, require_hashes=require_hashes, use_user_site=False, + lazy_wheel=False, ) yield Resolver( preparer=preparer, @@ -570,7 +571,7 @@ class TestInstallRequirement(object): install_req_from_line(req_file_path) err_msg = e.value.args[0] assert "Invalid requirement" in err_msg - assert "It looks like a path. It does exist." in err_msg + assert "It looks like a path. The path does exist." in err_msg assert "appears to be a requirements file." in err_msg assert "If that is the case, use the '-r' flag to install" in err_msg diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 879f088a4..f995d05a6 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -10,10 +10,7 @@ from pip._vendor.six import PY2 from pretend import stub import pip._internal.req.req_file # this will be monkeypatched -from pip._internal.exceptions import ( - InstallationError, - RequirementsFileParseError, -) +from pip._internal.exceptions import InstallationError, RequirementsFileParseError from pip._internal.models.format_control import FormatControl from pip._internal.network.session import PipSession from pip._internal.req.constructors import ( @@ -341,17 +338,22 @@ class TestProcessLine(object): line_processor("--no-index", "file", 1, finder=finder) assert finder.index_urls == [] - def test_set_finder_index_url(self, line_processor, finder): - line_processor("--index-url=url", "file", 1, finder=finder) + def test_set_finder_index_url(self, line_processor, finder, session): + line_processor( + "--index-url=url", "file", 1, finder=finder, session=session) assert finder.index_urls == ['url'] + assert session.auth.index_urls == ['url'] def test_set_finder_find_links(self, line_processor, finder): line_processor("--find-links=url", "file", 1, finder=finder) assert finder.find_links == ['url'] - def test_set_finder_extra_index_urls(self, line_processor, finder): - line_processor("--extra-index-url=url", "file", 1, finder=finder) + def test_set_finder_extra_index_urls( + self, line_processor, finder, session): + line_processor( + "--extra-index-url=url", "file", 1, finder=finder, session=session) assert finder.index_urls == ['url'] + assert session.auth.index_urls == ['url'] def test_set_finder_trusted_host( self, line_processor, caplog, session, finder diff --git a/tests/unit/test_resolution_legacy_resolver.py b/tests/unit/test_resolution_legacy_resolver.py index 561313c00..0388b42be 100644 --- a/tests/unit/test_resolution_legacy_resolver.py +++ b/tests/unit/test_resolution_legacy_resolver.py @@ -4,10 +4,7 @@ import mock import pytest from pip._vendor import pkg_resources -from pip._internal.exceptions import ( - NoneMetadataError, - UnsupportedPythonVersion, -) +from pip._internal.exceptions import NoneMetadataError, UnsupportedPythonVersion from pip._internal.req.constructors import install_req_from_line from pip._internal.resolution.legacy.resolver import ( Resolver, diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 0a1c47cd7..14b4d7482 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -16,11 +16,7 @@ from io import BytesIO import pytest from mock import Mock, patch -from pip._internal.exceptions import ( - HashMismatch, - HashMissing, - InstallationError, -) +from pip._internal.exceptions import HashMismatch, HashMissing, InstallationError from pip._internal.utils.deprecation import PipDeprecationWarning, deprecated from pip._internal.utils.encoding import BOMS, auto_decode from pip._internal.utils.glibc import ( diff --git a/tests/unit/test_utils_unpacking.py b/tests/unit/test_utils_unpacking.py index d01ffb9cd..5c2be24d4 100644 --- a/tests/unit/test_utils_unpacking.py +++ b/tests/unit/test_utils_unpacking.py @@ -10,11 +10,7 @@ import zipfile import pytest from pip._internal.exceptions import InstallationError -from pip._internal.utils.unpacking import ( - is_within_directory, - untar_file, - unzip_file, -) +from pip._internal.utils.unpacking import is_within_directory, untar_file, unzip_file class TestUnpackArchives(object): diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 93598c367..b6ed86b62 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -173,8 +173,9 @@ def test_git_resolve_revision_not_found_warning(get_sha_mock, caplog): sha = 40 * 'a' rev_options = Git.make_rev_options(sha) - new_options = Git.resolve_revision('.', url, rev_options) - assert new_options.rev == sha + # resolve_revision with a full sha would fail here because + # it attempts a git fetch. This case is now covered by + # test_resolve_commit_not_on_branch. rev_options = Git.make_rev_options(sha[:6]) new_options = Git.resolve_revision('.', url, rev_options) diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 15dff94ca..35916058a 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -19,9 +19,7 @@ from pip._internal.models.direct_url import ( DirectUrl, ) from pip._internal.models.scheme import Scheme -from pip._internal.operations.build.wheel_legacy import ( - get_legacy_build_wheel_path, -) +from pip._internal.operations.build.wheel_legacy import get_legacy_build_wheel_path from pip._internal.operations.install import wheel from pip._internal.utils.compat import WINDOWS from pip._internal.utils.misc import hash_file @@ -600,16 +598,14 @@ class TestMessageAboutScriptsNotOnPATH(object): ) assert retval is None - def test_missing_PATH_env_treated_as_empty_PATH_env(self): + def test_missing_PATH_env_treated_as_empty_PATH_env(self, monkeypatch): scripts = ['a/b/foo'] - env = os.environ.copy() - del env['PATH'] - with patch.dict('os.environ', env, clear=True): - retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) + monkeypatch.delenv('PATH') + retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) - with patch.dict('os.environ', {'PATH': ''}): - retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) + monkeypatch.setenv('PATH', '') + retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) assert retval_missing == retval_empty diff --git a/tools/requirements/docs.txt b/tools/requirements/docs.txt index acbd33906..dc93a60ff 100644 --- a/tools/requirements/docs.txt +++ b/tools/requirements/docs.txt @@ -1,6 +1,7 @@ -sphinx == 2.4.3 +sphinx == 3.2.1 git+https://github.com/python/python-docs-theme.git#egg=python-docs-theme git+https://github.com/pypa/pypa-docs-theme.git#egg=pypa-docs-theme +sphinx-tabs == 1.1.13 # `docs.pipext` uses pip's internals to generate documentation. So, we install # the current directory to make it work. diff --git a/tools/requirements/tests.txt b/tools/requirements/tests.txt index 0c84f20aa..ef87225d6 100644 --- a/tools/requirements/tests.txt +++ b/tools/requirements/tests.txt @@ -1,19 +1,18 @@ +--use-feature=2020-resolver cryptography==2.8 csv23 enum34; python_version < '3.4' freezegun mock pretend -# pytest 5.x only supports python 3.5+ -pytest<5.0.0 +pytest pytest-cov -# Prevent installing 9.0 which has install_requires "pytest >= 5.0". -pytest-rerunfailures<9.0 +pytest-rerunfailures pytest-timeout pytest-xdist pyyaml -setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. scripttest +setuptools>=39.2.0 # Needed for `setuptools.wheel.Wheel` support. https://github.com/pypa/virtualenv/archive/legacy.zip#egg=virtualenv werkzeug==0.16.0 wheel diff --git a/tools/travis/run.sh b/tools/travis/run.sh index a531cbb56..df8f03e7a 100755 --- a/tools/travis/run.sh +++ b/tools/travis/run.sh @@ -49,15 +49,15 @@ if [[ "$GROUP" == "1" ]]; then # Unit tests tox -- --use-venv -m unit -n auto # Integration tests (not the ones for 'pip install') - tox -- -m integration -n auto --duration=5 -k "not test_install" \ + tox -- -m integration -n auto --durations=5 -k "not test_install" \ --use-venv $RESOLVER_SWITCH elif [[ "$GROUP" == "2" ]]; then # Separate Job for running integration tests for 'pip install' - tox -- -m integration -n auto --duration=5 -k "test_install" \ + tox -- -m integration -n auto --durations=5 -k "test_install" \ --use-venv $RESOLVER_SWITCH elif [[ "$GROUP" == "3" ]]; then # Separate Job for tests that fail with the new resolver - tox -- -m fails_on_new_resolver -n auto --duration=5 \ + tox -- -m fails_on_new_resolver -n auto --durations=5 \ --use-venv $RESOLVER_SWITCH --new-resolver-runtests else # Non-Testing Jobs should run once