diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a556ac046..ada7086d9 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,5 +6,5 @@ require either a news file fragment or a marker to indicate they don't require one. To read more about adding a news file fragment for your PR, please check out -our documentation at: https://pip.pypa.io/en/latest/development/#adding-a-news-entry +our documentation at: https://pip.pypa.io/en/latest/development/contributing/#adding-a-news-entry --> diff --git a/AUTHORS.txt b/AUTHORS.txt index fd8ed6d64..f02621e55 100644 --- a/AUTHORS.txt +++ b/AUTHORS.txt @@ -1,16 +1,20 @@ Adam Chainz Adam Wentz +Adrien Morison Alan Yee Aleks Bunin Alex Gaynor Alex Grönholm Alex Morega Alex Stachowiak +Alexander Shtyrov Alexandre Conrad Alli Anatoly Techtonik Andrei Geacar Andrey Bulgakov +Andrés Delfino <34587441+andresdelfino@users.noreply.github.com> +Andrés Delfino Andy Freeland Andy Kluger Anish Tambe @@ -33,6 +37,7 @@ Atsushi Odagiri Avner Cohen Baptiste Mispelon Bartek Ogryczak +Bastian Venthur Ben Darnell Ben Hoyt Ben Rosser @@ -45,6 +50,7 @@ Bogdan Opanchuk Brad Erickson Bradley Ayers Brandon L. Reiss +Brett Randall Brian Rosner BrownTruck Bruno Oliveira @@ -81,6 +87,7 @@ Craig Kerstiens Cristian Sorinel Curtis Doty Damian Quiroga +Dan Black Dan Savilonis Dan Sully daniel @@ -88,7 +95,9 @@ Daniel Collins Daniel Hahler Daniel Holth Daniel Jost +Daniel Shaulov Daniele Procida +Danny Hermes Dav Clark Dave Abrahams David Aguilar @@ -106,10 +115,12 @@ Domen Kožar Donald Stufft Dongweiming Douglas Thor +DrFeathers Dustin Ingram Dwayne Bailey Ed Morley <501702+edmorley@users.noreply.github.com> Ed Morley +Eli Schwartz Emil Styrke Endoh Takanao enoch @@ -118,6 +129,8 @@ Eric Hanchrow Eric Hopper Erik M. Bray Erik Rose +Ernest W Durbin III +Ernest W. Durbin III Erwin Janssen Eugene Vereshchagin fiber-space @@ -135,6 +148,7 @@ George Song Georgi Valkov Giftlin Rajaiah gizmoguy1 +gkdoc <40815324+gkdoc@users.noreply.github.com> GOTO Hayato <3532528+gh640@users.noreply.github.com> Guilherme Espada Guy Rozendorn @@ -171,6 +185,7 @@ Jay Graves Jeff Barber Jeff Dairiki Jeremy Stanley +Jeremy Zafran Jim Garrison Jivan Amara John-Scott Atlakson @@ -190,6 +205,7 @@ jwg4 Jyrki Pulliainen Kamal Bin Mustafa kaustav haldar +keanemind Kelsey Hightower Kenneth Belitzky Kenneth Reitz @@ -197,6 +213,7 @@ Kenneth Reitz Kevin Burke Kevin Carter Kevin Frommelt +Kexuan Sun Kit Randel kpinc Kumar McMillan @@ -246,16 +263,19 @@ Miguel Araujo Perez Mihir Singh Min RK MinRK +Miro Hrončok montefra Monty Taylor Nate Coraor Nathaniel J. Smith +Nehal J Wani Nick Coghlan Nick Stenning Nikhil Benesch Nowell Strite nvdv Ofekmeister +Oliver Jeeves Oliver Tonnhofer Olivier Girardot Olivier Grisel @@ -281,6 +301,7 @@ Phaneendra Chiruvella Phil Freo Phil Pennock Phil Whelan +Philip Molloy Philippe Ombredanne Pi Delport Pierre-Yves Rofes @@ -323,6 +344,7 @@ Sebastian Schaetz Segev Finer Sergey Vasilyev Seth Woodworth +Shlomi Fish Simeon Visser Simon Cross Simon Pichugin @@ -340,6 +362,7 @@ Stéphane Bidoul (ACSONE) Stéphane Bidoul Stéphane Klein Takayuki SHIMIZUKAWA +Thijs Triemstra Thomas Fenzl Thomas Grainger Thomas Guettler @@ -351,6 +374,7 @@ Tim Harder Tim Heap tim smith tinruufu +Tom Freudenheim Tom V Tomer Chachamu Tony Zhaocheng Tan @@ -380,4 +404,5 @@ Zearin Zearin Zhiping Deng Zvezdan Petkovic +Łukasz Langa Семён Марьясин diff --git a/NEWS.rst b/NEWS.rst index 2457576e0..6cd81393b 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -7,6 +7,83 @@ .. towncrier release notes start +18.0 (2018-07-22) +================= + +Process +------- + +- Switch to a Calendar based versioning scheme. +- Formally document our deprecation process as a minimum of 6 months of deprecation + warnings. +- Adopt and document NEWS fragment writing style. +- Switch to releasing a new, non bug fix version of pip every 3 months. + +Deprecations and Removals +------------------------- + +- Remove the legacy format from pip list. (#3651, #3654) +- Dropped support for Python 3.3. (#3796) +- Remove support for cleaning up #egg fragment postfixes. (#4174) +- Remove the shim for the old get-pip.py location. (#5520) + + For the past 2 years, it's only been redirecting users to use the newer + https://bootstrap.pypa.io/get-pip.py location. + +Features +-------- + +- Introduce a new --prefer-binary flag, to prefer older wheels over newer source packages. (#3785) +- Improve autocompletion function on file name completion after options + which have ````, ```` or ```` as metavar. (#4842, #5125) +- Add support for installing PEP 518 build dependencies from source. (#5229) +- Improve status message when upgrade is skipped due to only-if-needed strategy. (#5319) + +Bug Fixes +--------- + +- Update pip's self-check logic to not use a virtualenv specific file and honor cache-dir. (#3905) +- Remove compiled pyo files for wheel packages. (#4471) +- Speed up printing of newly installed package versions. (#5127) +- Restrict install time dependency warnings to directly-dependant packages. (#5196, #5457) + + Warning about the entire package set has resulted in users getting confused as + to why pip is printing these warnings. +- Improve handling of PEP 518 build requirements: support environment markers and extras. (#5230, #5265) +- Remove username/password from log message when using index with basic auth. (#5249) +- Remove trailing os.sep from PATH directories to avoid false negatives. (#5293) +- Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check. (#5311, #5312) +- Disable pip's version check (and upgrade message) when installed by a different package manager. (#5346) + + This works better with Linux distributions where pip's upgrade message may + result in users running pip in a manner that modifies files that should be + managed by the OS's package manager. +- Check for file existence and unlink first when clobbering existing files during a wheel install. (#5366) +- Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. (#5381) +- Always read ``pyproject.toml`` as UTF-8. This fixes Unicode handling on Windows and Python 2. (#5482) +- Fix a crash that occurs when PATH not set, while generating script location warning. (#5558) +- Disallow packages with ``pyproject.toml`` files that have an empty build-system table. (#5627) + +Vendored Libraries +------------------ + +- Update CacheControl to 0.12.5. +- Update certifi to 2018.4.16. +- Update distro to 1.3.0. +- Update idna to 2.7. +- Update ipaddress to 1.0.22. +- Update pkg_resources to 39.2.0 (via setuptools). +- Update progress to 1.4. +- Update pytoml to 0.1.16. +- Update requests to 2.19.1. +- Update urllib3 to 1.23. + +Improved Documentation +---------------------- + +- Document how to use pip with a proxy server. (#512, #5574) +- Document that the output of pip show is in RFC-compliant mail header format. (#5261) + 10.0.1 (2018-04-19) =================== diff --git a/appveyor.yml b/appveyor.yml index 078b4db0f..2a0a9d9a1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -28,10 +28,37 @@ cache: - '%LOCALAPPDATA%\pip\Cache' test_script: - # Shorten paths, workaround https://bugs.python.org/issue18199 - - "subst T: %TEMP%" - - "set TEMP=T:\\" - - "set TMP=T:\\" - - "tox -e py -- -m unit -n 3" - - "if \"%RUN_INTEGRATION_TESTS%\" == \"True\" ( - tox -e py -- -m integration -n 3 --duration=5 )" + - ps: | + function should_run_tests { + if ("$env:APPVEYOR_PULL_REQUEST_NUMBER" -eq "") { + Write-Host "Not a pull request - running tests" + return $true + } + Write-Host "Pull request $env:APPVEYOR_PULL_REQUEST_NUMBER based on branch $env:APPVEYOR_REPO_BRANCH" + git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH + $changes = (git diff --name-only HEAD (git merge-base HEAD FETCH_HEAD)) + Write-Host "Files changed:" + Write-Host $changes + $important = $changes | Where-Object { $_ -NotLike "*.rst" } | + Where-Object { $_ -NotLike "docs*" } | + Where-Object { $_ -NotLike "news*" } | + Where-Object { $_ -NotLike ".github*" } + if (!$important) { + Write-Host "Only documentation changes - skipping tests" + return $false + } + + Write-Host "Pull request $env:APPVEYOR_PULL_REQUEST_NUMBER alters code - running tests" + return $true + } + + if (should_run_tests) { + # Shorten paths, workaround https://bugs.python.org/issue18199 + subst T: $env:TEMP + $env:TEMP = "T:\" + $env:TMP = "T:\" + tox -e py -- -m unit -n 3 + if ($env:RUN_INTEGRATION_TESTS -eq "True") { + tox -e py -- -m integration -n 3 --duration=5 + } + } diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index bbf7afad0..000000000 --- a/docs/Makefile +++ /dev/null @@ -1,177 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) -$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/pip-installer.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/pip-installer.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/pip-installer" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/pip-installer" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/docs/conf.py b/docs/conf.py index e615e6be8..5e4842341 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -11,10 +11,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. +import glob import os import re import sys -import glob on_rtd = os.environ.get('READTHEDOCS', None) == 'True' @@ -127,6 +127,7 @@ pygments_style = 'sphinx' extlinks = { 'issue': ('https://github.com/pypa/pip/issues/%s', '#'), 'pull': ('https://github.com/pypa/pip/pull/%s', 'PR #'), + 'pypi': ('https://pypi.org/project/%s', ''), } # -- Options for HTML output -------------------------------------------------- diff --git a/docs/development.rst b/docs/development.rst deleted file mode 100644 index caac58048..000000000 --- a/docs/development.rst +++ /dev/null @@ -1,265 +0,0 @@ -=========== -Development -=========== - -Pull Requests -============= - -- Submit Pull Requests against the `master` branch. -- Provide a good description of what you're doing and why. -- Provide tests that cover your changes and try to run the tests locally first. - -**Example**. Assuming you set up GitHub account, forked pip repository from -https://github.com/pypa/pip to your own page via web interface, and your -fork is located at https://github.com/yourname/pip - -:: - - $ git clone git@github.com:pypa/pip.git - $ cd pip - # ... - $ git diff - $ git add ... - $ git status - $ git commit - -You may reference relevant issues in commit messages (like #1259) to -make GitHub link issues and commits together, and with phrase like -"fixes #1259" you can even close relevant issues automatically. Now -push the changes to your fork:: - - $ git push git@github.com:yourname/pip.git - -Open Pull Requests page at https://github.com/yourname/pip/pulls and -click "New pull request" and select your fork. That's it. - -Pull requests should be self-contained, and limited in scope. Before being -merged, a pull request must be reviewed, and keeping individual PRs limited -in scope makes this far easier. In particular, pull requests must not be -treated as "feature branches", with ongoing development work happening -within the PR. Instead, the feature should be broken up into smaller, -independent parts which can be reviewed and merged individually. - -When creating a pull request, avoid including "cosmetic" changes to -code that is unrelated to your change, as these make reviewing the PR -more difficult. Examples include re-flowing text in comments or -documentation, or addition or removal of blank lines or whitespace -within lines. Such changes can be made separately, as a "formatting -cleanup" PR, if needed. - - -Automated Testing -================= - -All pull requests and merges to 'master' branch are tested using `Travis CI`_ -and `Appveyor CI`_ based on our `.travis.yml`_ and `appveyor.yml`_ files. - -You can find the status and results to the CI runs for your PR on GitHub's Web -UI for the pull request. You can also find links to the CI services' pages for -the specific builds in the form of "Details" links, in case the CI run fails -and you wish to view the output. - -To trigger CI to run again for a pull request, you can close and open the pull -request or submit another change to the pull request. If needed, project -maintainers can manually trigger a restart of a job/build. - -Running tests -============= - -OS Requirements: subversion, bazaar, git, and mercurial. - -Python Requirements: tox or install all packages listed in -`tools/test-requirements.txt`_ - -Ways to run the tests locally:: - - $ tox -e py36 # The preferred way to run the tests, can use pyNN to - # run for a particular version or leave off the -e to - # run for all versions. - $ python setup.py test # Using the setuptools test plugin - $ py.test # Using py.test directly - $ tox # Using tox against pip's tox.ini - -If you are missing one of the VCS tools, you can tell ``py.test`` to skip it:: - - # When using tox - $ tox -e py36 -- -k 'not svn' - $ tox -e py36 -- -k 'not (svn or git)' - # Directly with py.test - $ py.test -k 'not svn' - $ py.test -k 'not (svn or git)' - - -Getting Involved -================ - -The pip project welcomes help in the following ways: - -- Making Pull Requests for code, tests, or docs. -- Commenting on open issues and pull requests. -- Helping to answer questions on the `mailing list`_. - -If you want to become an official maintainer, start by helping out. - -Later, when you think you're ready, get in touch with one of the maintainers, -and they will initiate a vote. - - -Adding a NEWS Entry -=================== - -The ``NEWS.rst`` file is managed using `towncrier`_ and all non trivial changes -must be accompanied by a news entry. - -To add an entry to the news file, first you need to have created an issue -describing the change you want to make. A Pull Request itself *may* function as -such, but it is preferred to have a dedicated issue (for example, in case the -PR ends up rejected due to code quality reasons). - -Once you have an issue or pull request, you take the number and you create a -file inside of the ``news/`` directory named after that issue number with an -extension of ``removal``, ``feature``, ``bugfix``, or ``doc``. Thus if your -issue or PR number is ``1234`` and this change is fixing a bug, then you would -create a file ``news/1234.bugfix``. PRs can span multiple categories by creating -multiple files (for instance, if you added a feature and deprecated/removed the -old feature at the same time, you would create ``news/NNNN.feature`` and -``news/NNNN.removal``). Likewise if a PR touches multiple issues/PRs you may -create a file for each of them with the exact same contents and Towncrier will -deduplicate them. - -The contents of this file are reStructuredText formatted text that will be used -as the content of the news file entry. You do not need to reference the issue -or PR numbers here as towncrier will automatically add a reference to all of -the affected issues when rendering the news file. - -In order to maintain a consistent style in the ``NEWS.rst`` file, it is -preferred to keep the news entry to the point, in sentence case, shorter than -80 characters and in an imperative tone -- an entry should complete the sentence -"This change will ...". In rare cases, where one line is not enough, use a -summary line in an imperative tone followed by a blank line separating it -from a description of the feature/change in one or more paragraphs, each wrapped -at 80 characters. Remember that a news entry is meant for end users and should -only contain details relevant to an end user. - -A trivial change is anything that does not warrant an entry in the news file. -Some examples are: Code refactors that don't change anything as far as the -public is concerned, typo fixes, white space modification, etc. To mark a PR -as trivial a contributor simply needs to add a randomly named, empty file to -the ``news/`` directory with the extension of ``.trivial``. If you are on a -POSIX like operating system, one can be added by running -``touch news/$(uuidgen).trivial``. On Windows, the same result can be achieved -in Powershell using ``New-Item "news/$([guid]::NewGuid()).trivial"``. Core -committers may also add a "trivial" label to the PR which will accomplish the -same thing. - -Upgrading, removing, or adding a new vendored library gets a special mention -using a ``news/.vendor`` file. This is in addition to any features, -bugfixes, or other kinds of news that pulling in this library may have. This -uses the library name as the key so that updating the same library twice doesn't -produce two news file entries. - -Changes to the processes, policies, or other non code related changed that are -otherwise notable can be done using a ``news/.process`` file. This is not -typically used, but can be used for things like changing version schemes, -updating deprecation policy, etc. - - -Release Cadence -=============== - -The pip project has a release cadence of releasing whatever is on ``master`` -every 3 months. This gives users a predictable pattern for when releases -are going to happen and prevents locking up improvements for fixes for long -periods of time, while still preventing massively fracturing the user base -with version numbers. - -Our release months are January, April, July, October. The release date within -that month will be up to the release manager for that release. If there are -no changes, then that release month is skipped and the next release will be -3 month later. - -The release manager may, at their discretion, choose whether or not there -will be a pre-release period for a release, and if there is may extend that -period into the next month if needed. - -Because releases are made direct from the ``master`` branch, it is essential -that ``master`` is always in a releasable state. It is acceptable to merge -PRs that partially implement a new feature, but only if the partially -implemented version is usable in that state (for example, with reduced -functionality or disabled by default). In the case where a merged PR is found -to need extra work before being released, the release manager always has the -option to back out the partial change prior to a release. The PR can then be -reworked and resubmitted for the next release. - - -Deprecation Policy -================== - -Any change to pip that removes or significantly alters user-visible behaviour -that is described in the pip documentation will be deprecated for a minimum of -one released version before the change occurs. Deprecation will take the form of -a warning being issued by pip when the feature is used. Longer deprecation -periods, or deprecation warnings for behaviour changes that would not normally -be covered by this policy, are also possible depending on circumstances, but -this is at the discretion of the pip developers. - -Note that the documentation is the sole reference for what counts as agreed -behaviour. If something isn't explicitly mentioned in the documentation, it can -be changed without warning, or any deprecation period, in a pip release. -However, we are aware that the documentation isn't always complete - PRs that -document existing behaviour with the intention of covering that behaviour with -the above deprecation process are always acceptable, and will be considered on -their merits. - - -Release Process -=============== - -#. On the current pip ``master`` branch, generate a new ``AUTHORS.txt`` by - running ``invoke generate.authors`` and commit the results. -#. On the current pip ``master`` branch, make a new commit which bumps the - version in ``pip/__init__.py`` to the release version and adjust the - ``CHANGES.txt`` file to reflect the current date. The release version should - follow a YY.N scheme, where YY is the two digit year, and N is the Nth - release within that year. -#. On the current pip ``master`` branch, generate a new ``NEWS.rst`` by running - ``invoke generate.news`` and commit the results. -#. Create a signed tag of the ``master`` branch of the form ``X.Y.Z`` using the - command ``git tag -s X.Y.Z``. -#. Checkout the tag using ``git checkout X.Y.Z`` and create the distribution - files using ``python setup.py sdist bdist_wheel``. -#. Upload the distribution files to PyPI using twine - (``twine upload -s dist/*``). The upload should include GPG signatures of - the distribution files. -#. Push all of the changes. -#. Regenerate the ``get-pip.py`` script by running - ``invoke generate.installer`` in the get-pip repository, and committing the - results. - - -Creating a Bugfix Release -========================= - -Sometimes we need to release a bugfix release of the form ``X.Y.Z+1``. In order -to create one of these the changes should already be merged into the -``master`` branch. - -#. Create a new ``release/YY.N.Z+1`` branch off of the ``YY.N`` tag using the - command ``git checkout -b release/YY.N.Z+1 YY.N``. -#. Cherry pick the fixed commits off of the ``master`` branch, fixing any - conflicts and moving any changelog entries from the development version's - changelog section to the ``YY.N.Z+1`` section. -#. Push the ``release/YY.N.Z+1`` branch to github and submit a PR for it against - the ``master`` branch and wait for the tests to run. -#. Once tests run, merge the ``release/YY.N.Z+1`` branch into master, and follow - the above release process starting with step 4. - - -.. _`mailing list`: https://mail.python.org/mailman/listinfo/distutils-sig -.. _`towncrier`: https://pypi.org/project/towncrier/ -.. _`Travis CI`: https://travis-ci.org/ -.. _`Appveyor CI`: https://www.appveyor.com/ -.. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml -.. _`appveyor.yml`: https://github.com/pypa/pip/blob/master/appveyor.yml -.. _`Travis CI Pull Requests`: https://travis-ci.org/pypa/pip/pull_requests -.. _`tools/test-requirements.txt`: https://github.com/pypa/pip/blob/master/tools/test-requirements.txt diff --git a/docs/development/contributing.rst b/docs/development/contributing.rst new file mode 100644 index 000000000..3dcf74726 --- /dev/null +++ b/docs/development/contributing.rst @@ -0,0 +1,249 @@ +============ +Contributing +============ + +.. todo + Create a "guide" to pip's internals and link to it from here saying + "you might want to take a look at the guide" + + +Submitting Pull Requests +======================== + +Submit pull requests against the ``master`` branch, providing a good +description of what you're doing and why. You must have legal permission to +distribute any code you contribute to pip and it must be available under the +MIT License. + +Provide tests that cover your changes and run the tests locally first. pip +:ref:`supports ` multiple Python versions and +operating systems. Any pull request must consider and work on all these +platforms. + +Pull Requests should be small to facilitate easier review. Keep them +self-contained, and limited in scope. `Studies have shown`_ that review quality +falls off as patch size grows. Sometimes this will result in many small PRs to +land a single large feature. In particular, pull requests must not be treated +as "feature branches", with ongoing development work happening within the PR. +Instead, the feature should be broken up into smaller, independent parts which +can be reviewed and merged individually. + +Additionally, avoid including "cosmetic" changes to code that +is unrelated to your change, as these make reviewing the PR more difficult. +Examples include re-flowing text in comments or documentation, or addition or +removal of blank lines or whitespace within lines. Such changes can be made +separately, as a "formatting cleanup" PR, if needed. + +Automated Testing +================= + +All pull requests and merges to 'master' branch are tested using `Travis CI`_ +and `Appveyor CI`_ based on our `.travis.yml`_ and `appveyor.yml`_ files. + +You can find the status and results to the CI runs for your PR on GitHub's Web +UI for the pull request. You can also find links to the CI services' pages for +the specific builds in the form of "Details" links, in case the CI run fails +and you wish to view the output. + +To trigger CI to run again for a pull request, you can close and open the pull +request or submit another change to the pull request. If needed, project +maintainers can manually trigger a restart of a job/build. + + +NEWS Entries +============ + +The ``NEWS.rst`` file is managed using `towncrier`_ and all non trivial changes +must be accompanied by a news entry. + +To add an entry to the news file, first you need to have created an issue +describing the change you want to make. A Pull Request itself *may* function as +such, but it is preferred to have a dedicated issue (for example, in case the +PR ends up rejected due to code quality reasons). + +Once you have an issue or pull request, you take the number and you create a +file inside of the ``news/`` directory named after that issue number with an +extension of ``removal``, ``feature``, ``bugfix``, or ``doc``. Thus if your +issue or PR number is ``1234`` and this change is fixing a bug, then you would +create a file ``news/1234.bugfix``. PRs can span multiple categories by creating +multiple files (for instance, if you added a feature and deprecated/removed the +old feature at the same time, you would create ``news/NNNN.feature`` and +``news/NNNN.removal``). Likewise if a PR touches multiple issues/PRs you may +create a file for each of them with the exact same contents and Towncrier will +deduplicate them. + +Contents of a NEWS entry +------------------------ + +The contents of this file are reStructuredText formatted text that will be used +as the content of the news file entry. You do not need to reference the issue +or PR numbers here as towncrier will automatically add a reference to all of +the affected issues when rendering the news file. + +In order to maintain a consistent style in the ``NEWS.rst`` file, it is +preferred to keep the news entry to the point, in sentence case, shorter than +80 characters and in an imperative tone -- an entry should complete the sentence +"This change will ...". In rare cases, where one line is not enough, use a +summary line in an imperative tone followed by a blank line separating it +from a description of the feature/change in one or more paragraphs, each wrapped +at 80 characters. Remember that a news entry is meant for end users and should +only contain details relevant to an end user. + +Choosing the type of NEWS entry +------------------------------- + +A trivial change is anything that does not warrant an entry in the news file. +Some examples are: Code refactors that don't change anything as far as the +public is concerned, typo fixes, white space modification, etc. To mark a PR +as trivial a contributor simply needs to add a randomly named, empty file to +the ``news/`` directory with the extension of ``.trivial``. If you are on a +POSIX like operating system, one can be added by running +``touch news/$(uuidgen).trivial``. On Windows, the same result can be achieved +in Powershell using ``New-Item "news/$([guid]::NewGuid()).trivial"``. Core +committers may also add a "trivial" label to the PR which will accomplish the +same thing. + +Upgrading, removing, or adding a new vendored library gets a special mention +using a ``news/.vendor`` file. This is in addition to any features, +bugfixes, or other kinds of news that pulling in this library may have. This +uses the library name as the key so that updating the same library twice doesn't +produce two news file entries. + +Changes to the processes, policies, or other non code related changed that are +otherwise notable can be done using a ``news/.process`` file. This is not +typically used, but can be used for things like changing version schemes, +updating deprecation policy, etc. + + +Updating your branch +==================== + +As you work, you might need to update your local master branch up-to-date with +the ``master`` branch in the main pip repository, which moves forward as the +maintainers merge pull requests. Most people working on the project use the +following workflow. + +This assumes that you have Git configured so that when you run the following +command: + +.. code-block:: console + + git remote -v + +Your output looks like this: + +.. code-block:: console + + origin https://github.com/USERNAME/pip.git (fetch) + origin https://github.com/USERNAME/pip.git (push) + upstream https://github.com/pypa/pip.git (fetch) + upstream https://github.com/pypa/pip.git (push) + +In the example above, ``USERNAME`` is your username on GitHub. + +First, fetch the latest changes from the main pip repository, ``upstream``: + +.. code-block:: console + + git fetch upstream + +Then, check out your local ``master`` branch, and rebase the changes on top of +it: + +.. code-block:: console + + git checkout master + git rebase upstream/master + +At this point, you might have to `resolve merge conflicts`_. Once this is done, +push the updates you have just made to your local ``master`` branch to your +``origin`` repository on GitHub: + +.. code-block:: console + + git checkout master + git push origin master + +Now your local ``master`` branch and the ``master`` branch in your ``origin`` +repo have been updated with the most recent changes from the main pip +repository. + +To keep your branches updated, the process is similar: + +.. code-block:: console + + git checkout awesome-feature + git fetch upstream + git rebase upstream/master + +Now your branch has been updated with the latest changes from the +``master`` branch on the upstream pip repository. + +It's good practice to back up your branches by pushing them to your +``origin`` on GitHub as you are working on them. To push a branch, +run this command: + +.. code-block:: console + + git push origin awesome-feature + +In this example, ```` is the name of your branch. This +will push the branch you are working on to GitHub, but will not +create a PR. + +Once you have pushed your branch to your ``origin``, if you need to +update it again, you will have to force push your changes by running the +following command: + +.. code-block:: console + + git push -f origin awesome-feature + +The ``-f`` (or ``--force``) flag after ``push`` forces updates from your local +branch to update your ``origin`` branch. If you have a PR open on your +branch, force pushing will update your PR. (This is a useful command +when someone requests changes on a PR.) + +If you get an error message like this: + +.. code-block:: console + + ! [rejected] awesome-feature -> awesome-feature (non-fast-forward) + error: failed to push some refs to 'https://github.com/USERNAME/pip.git' + hint: Updates were rejected because the tip of your current branch is behind + hint: its remote counterpart. Integrate the remote changes (e.g. + hint: 'git pull ...') before pushing again. + hint: See the 'Note about fast-forwards' in 'git push --help' for details. + +Try force-pushing your branch with ``push -f``. + +The ``master`` branch in the main pip repository gets updated frequently, so +you might have to update your branch at least once while you are working on it. + + +Becoming a maintainer +===================== + +If you want to become an official maintainer, start by helping out. + +Later, when you think you're ready, get in touch with one of the maintainers +and they will initiate a vote. + +.. note:: + + Upon becoming a maintainer, a person should be given access to various + pip-related tooling across multiple platforms. These are noted here for + future reference by the maintainers: + + - GitHub Push Access + - PyPI Publishing Access + - CI Administration capabilities + - ReadTheDocs Administration capabilities + +.. _`Studies have shown`: https://smartbear.com/smartbear/media/pdfs/wp-cc-11-best-practices-of-peer-code-review.pdf +.. _`resolve merge conflicts`: https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/ +.. _`Travis CI`: https://travis-ci.org/ +.. _`Appveyor CI`: https://www.appveyor.com/ +.. _`.travis.yml`: https://github.com/pypa/pip/blob/master/.travis.yml +.. _`appveyor.yml`: https://github.com/pypa/pip/blob/master/appveyor.yml +.. _`towncrier`: https://pypi.org/project/towncrier/ diff --git a/docs/development/getting-started.rst b/docs/development/getting-started.rst new file mode 100644 index 000000000..6a91b605d --- /dev/null +++ b/docs/development/getting-started.rst @@ -0,0 +1,110 @@ +=============== +Getting Started +=============== + +We’re pleased that you are interested in working on pip. + +This document is meant to get you setup to work on pip and to act as a guide and +reference to the the development setup. If you face any issues during this +process, please `open an issue`_ about it on the issue tracker. + +Development tools +================= + +pip uses :pypi:`tox` for testing against multiple different Python environments +and ensuring reproducible environments for linting and building documentation. + +For developing pip, you need to install ``tox`` on your system. Often, you can +just do ``python -m pip install tox`` to install and use it. + +Running Tests +------------- + +pip uses the :pypi:`pytest` test framework, :pypi:`mock` and :pypi:`pretend` +for testing. These are automatically installed by tox for running the tests. + +To run tests locally, run: + +.. code-block:: console + + $ tox -e py36 + +The example above runs tests against Python 3.6. You can also use other +versions like ``py27`` and ``pypy3``. + +``tox`` has been configured to any additional arguments it is given to +``pytest``. This enables the use of pytest's `rich CLI`_. As an example, you +can select tests using the various ways that pytest provides: + +.. code-block:: console + + $ # Using file name + $ tox -e py36 -- tests/functional/test_install.py + $ # Using markers + $ tox -e py36 -- -m unit + $ # Using keywords + $ tox -e py36 -- -k "install and not wheel" + +Running pip's test suite requires supported version control tools (subversion, +bazaar, git, and mercurial) to be installed. If you are missing one of the VCS +tools, you can tell pip to skip those tests: + +.. code-block:: console + + $ tox -e py36 -- -k "not svn" + $ tox -e py36 -- -k "not (svn or git)" + +Running Linters +--------------- + +pip uses :pypi:`flake8` and :pypi:`isort` for linting the codebase. These +ensure that the codebase is in compliance with :pep:`8` and the imports are +consistently ordered and styled. + +To use linters locally, run: + +.. code-block:: console + + $ tox -e lint-py2 + $ tox -e lint-py3 + +The above commands run the linters on Python 2 followed by Python 3. + +.. note:: + + Do not silence errors from flake8 with ``# noqa`` comments or otherwise. + The only exception to this is silencing unused-import errors for imports + related to static type checking as currently `flake8 does not understand + PEP 484 type-comments`_. + +Running mypy +------------ + +pip uses :pypi:`mypy` to run static type analysis, which helps catch certain +kinds of bugs. The codebase uses `PEP 484 type-comments`_ due to compatibility +requirements with Python 2.7. + +To run the ``mypy`` type checker, run: + +.. code-block:: console + + $ tox -e mypy + +Building Documentation +---------------------- + +pip's documentation is built using :pypi:`Sphinx`. The documentation is written +in reStructuredText. + +To build it locally, run: + +.. code-block:: console + + $ tox -e docs + +The built documentation can be found in the ``docs/build`` folder. + +.. _`open an issue`: https://github.com/pypa/pip/issues/new?title=Trouble+with+pip+development+environment +.. _`flake8 does not understand PEP 484 type-comments`: https://gitlab.com/pycqa/flake8/issues/118 +.. _`PEP 484 type-comments`: https://www.python.org/dev/peps/pep-0484/#suggested-syntax-for-python-2-7-and-straddling-code +.. _`rich CLI`: https://docs.pytest.org/en/latest/usage.html#specifying-tests-selecting-tests diff --git a/docs/development/index.rst b/docs/development/index.rst new file mode 100644 index 000000000..4124d831f --- /dev/null +++ b/docs/development/index.rst @@ -0,0 +1,25 @@ +Development +=========== + +pip is a volunteer maintained open source project and we welcome contributions +of all forms. The sections below will help you get started with development, +testing, and documentation. + +You can also join ``#pypa`` (general packaging discussion and user support) and +``#pypa-dev`` (discussion about development of packaging tools) `on Freenode`_, +or the `pypa-dev mailing list`_, to ask questions or get involved. + +.. toctree:: + :maxdepth: 2 + + getting-started + contributing + release-process + +.. note:: + + pip's development documentation has been rearranged and some older + references might be broken. + +.. _`on Freenode`: https://webchat.freenode.net/?channels=%23pypa-dev,pypa +.. _`pypa-dev mailing list`: https://groups.google.com/forum/#!forum/pypa-dev diff --git a/docs/development/release-process.rst b/docs/development/release-process.rst new file mode 100644 index 000000000..2a076fdca --- /dev/null +++ b/docs/development/release-process.rst @@ -0,0 +1,103 @@ +=============== +Release process +=============== + + +Release Cadence +=============== + +The pip project has a release cadence of releasing whatever is on ``master`` +every 3 months. This gives users a predictable pattern for when releases +are going to happen and prevents locking up improvements for fixes for long +periods of time, while still preventing massively fracturing the user base +with version numbers. + +Our release months are January, April, July, October. The release date within +that month will be up to the release manager for that release. If there are +no changes, then that release month is skipped and the next release will be +3 month later. + +The release manager may, at their discretion, choose whether or not there +will be a pre-release period for a release, and if there is may extend that +period into the next month if needed. + +Because releases are made direct from the ``master`` branch, it is essential +that ``master`` is always in a releasable state. It is acceptable to merge +PRs that partially implement a new feature, but only if the partially +implemented version is usable in that state (for example, with reduced +functionality or disabled by default). In the case where a merged PR is found +to need extra work before being released, the release manager always has the +option to back out the partial change prior to a release. The PR can then be +reworked and resubmitted for the next release. + + +Deprecation Policy +================== + +Any change to pip that removes or significantly alters user-visible behavior +that is described in the pip documentation will be deprecated for a minimum of +6 months before the change occurs. Deprecation will take the form of a warning +being issued by pip when the feature is used. Longer deprecation periods, or +deprecation warnings for behavior changes that would not normally be covered by +this policy, are also possible depending on circumstances, but this is at the +discretion of the pip developers. + +Note that the documentation is the sole reference for what counts as agreed +behavior. If something isn't explicitly mentioned in the documentation, it can +be changed without warning, or any deprecation period, in a pip release. +However, we are aware that the documentation isn't always complete - PRs that +document existing behavior with the intention of covering that behavior with +the above deprecation process are always acceptable, and will be considered on +their merits. + +.. note:: + + pip has a helper function for making deprecation easier for pip maintainers. + The supporting documentation can be found in the source code of + ``pip._internal.utils.deprecation.deprecated``. The function is not a part of + pip's public API. + + +Release Process +=============== + +Creating a new release +---------------------- + +#. On the current pip ``master`` branch, generate a new ``AUTHORS.txt`` by + running ``invoke generate.authors`` and commit the results. +#. On the current pip ``master`` branch, make a new commit which bumps the + version in ``pip/__init__.py`` to the release version and adjust the + ``CHANGES.txt`` file to reflect the current date. The release version should + follow a YY.N scheme, where YY is the two digit year, and N is the Nth + release within that year. +#. On the current pip ``master`` branch, generate a new ``NEWS.rst`` by running + ``invoke generate.news`` and commit the results. +#. Create a signed tag of the ``master`` branch of the form ``YY.N`` using the + command ``git tag -s YY.N``. +#. Checkout the tag using ``git checkout YY.N`` and create the distribution + files using ``python setup.py sdist bdist_wheel``. +#. Upload the distribution files to PyPI using twine + (``twine upload -s dist/*``). The upload should include GPG signatures of + the distribution files. +#. Push all of the changes. +#. Regenerate the ``get-pip.py`` script by running + ``invoke generate.installer`` in the get-pip repository, and committing the + results. + +Creating a bug-fix release +-------------------------- + +Sometimes we need to release a bugfix release of the form ``YY.N.Z+1``. In +order to create one of these the changes should already be merged into the +``master`` branch. + +#. Create a new ``release/YY.N.Z+1`` branch off of the ``YY.N`` tag using the + command ``git checkout -b release/YY.N.Z+1 YY.N``. +#. Cherry pick the fixed commits off of the ``master`` branch, fixing any + conflicts and moving any changelog entries from the development version's + changelog section to the ``YY.N.Z+1`` section. +#. Push the ``release/YY.N.Z+1`` branch to github and submit a PR for it against + the ``master`` branch and wait for the tests to run. +#. Once tests run, merge the ``release/YY.N.Z+1`` branch into master, and follow + the above release process starting with step 4. diff --git a/docs/docutils.conf b/docs/docutils.conf deleted file mode 100644 index dfd80b177..000000000 --- a/docs/docutils.conf +++ /dev/null @@ -1,2 +0,0 @@ -[restructuredtext parser] -smart_quotes = no diff --git a/docs/index.rst b/docs/index.rst index caf0725a6..64bbb79f3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,5 +18,5 @@ for installing Python packages. installing user_guide reference/index - development + development/index news diff --git a/docs/installing.rst b/docs/installing.rst index 6a103e84f..99eef7360 100644 --- a/docs/installing.rst +++ b/docs/installing.rst @@ -6,10 +6,10 @@ Installation Do I need to install pip? ------------------------- -pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4 -downloaded from `python.org `_ or if you are working -in a :ref:`Virtual Environment ` -created by :ref:`pypug:virtualenv` or :ref:`pyvenv `. +pip is already installed if you are using Python 2 >=2.7.9 or Python 3 >=3.4 +downloaded from `python.org `_ or if you are working +in a :ref:`Virtual Environment ` +created by :ref:`pypug:virtualenv` or :ref:`pyvenv `. Just make sure to :ref:`upgrade pip `. @@ -23,7 +23,9 @@ To install pip, securely download `get-pip.py curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -Inspect ``get-pip.py`` for any malevolence. Then run the following:: +As when running any script downloaded from the web, ensure that you have +reviewed the code and are happy that it works as you expect. +Then run the following:: python get-pip.py @@ -34,7 +36,7 @@ Inspect ``get-pip.py`` for any malevolence. Then run the following:: system or another package manager. ``get-pip.py`` does not coordinate with those tools, and may leave your system in an inconsistent state. -``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel` +``get-pip.py`` also installs :ref:`pypug:setuptools` [2]_ and :ref:`pypug:wheel` if they are not already. :ref:`pypug:setuptools` is required to install :term:`source distributions `. Both are required in order to build a :ref:`Wheel cache` (which improves installation @@ -104,6 +106,8 @@ On Windows [4]_:: python -m pip install -U pip +.. _compatibility-requirements: + Python and OS Compatibility --------------------------- diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 40c23cc7c..000000000 --- a/docs/make.bat +++ /dev/null @@ -1,253 +0,0 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% . -set I18NSPHINXOPTS=%SPHINXOPTS% . -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^` where ^ is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. xml to make Docutils-native XML files - echo. pseudoxml to make pseudoxml-XML files for display purposes - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - - -REM Check if sphinx-build is available and fallback to Python version if any -%SPHINXBUILD% 2> nul -if errorlevel 9009 goto sphinx_python -goto sphinx_ok - -:sphinx_python - -set SPHINXBUILD=python -m sphinx.__init__ -%SPHINXBUILD% 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -:sphinx_ok - - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\pip-installer.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\pip-installer.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdf" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "latexpdfja" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - cd %BUILDDIR%/latex - make all-pdf-ja - cd %BUILDDIR%/.. - echo. - echo.Build finished; the PDF files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -if "%1" == "xml" ( - %SPHINXBUILD% -b xml %ALLSPHINXOPTS% %BUILDDIR%/xml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The XML files are in %BUILDDIR%/xml. - goto end -) - -if "%1" == "pseudoxml" ( - %SPHINXBUILD% -b pseudoxml %ALLSPHINXOPTS% %BUILDDIR%/pseudoxml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The pseudo-XML files are in %BUILDDIR%/pseudoxml. - goto end -) - -:end diff --git a/docs/reference/pip.rst b/docs/reference/pip.rst index dd68f65ad..905be1ea6 100644 --- a/docs/reference/pip.rst +++ b/docs/reference/pip.rst @@ -154,8 +154,9 @@ appropriately. installation of build dependencies from source has been disabled until a safe resolution of this issue is found. -* ``pip<18.0`` does not support the use of environment markers and extras, only - version specifiers are respected. +* ``pip<18.0``: only support installing build requirements from wheels, and + does not support the use of environment markers and extras (only version + specifiers are respected). Future Developments diff --git a/docs/user_guide.rst b/docs/user_guide.rst index 8f5326643..d89ad5376 100644 --- a/docs/user_guide.rst +++ b/docs/user_guide.rst @@ -49,6 +49,21 @@ For more information and examples, see the :ref:`pip install` reference. .. _PyPI: https://pypi.org/ +Using a Proxy Server +******************** + +When installing packages from `PyPI`_, pip requires internet access, which +in many corporate environments requires an outbound HTTP proxy server. + +pip can be configured to connect through a proxy server in various ways: + +* using the ``--proxy`` command-line option to specify a proxy in the form + ``[user:passwd@]proxy.server:port`` +* using ``proxy`` in a :ref:`config-file` +* by setting the standard environment-variables ``http_proxy``, ``https_proxy`` + and ``no_proxy``. + + .. _`Requirements Files`: Requirements Files @@ -336,13 +351,13 @@ variable ``PIP_CONFIG_FILE``. If multiple configuration files are found by pip then they are combined in the following order: -1. Firstly the site-wide file is read, then -2. The per-user file is read, and finally -3. The virtualenv-specific file is read. +1. The site-wide file is read +2. The per-user file is read +3. The virtualenv-specific file is read Each file read overrides any values read from previous files, so if the global timeout is specified in both the site-wide file and the per-user file -then the latter value is the one that will be used. +then the latter value will be used. The names of the settings are derived from the long command line option, e.g. if you want to use a different package index (``--index-url``) and set the diff --git a/news/3785.feature b/news/3785.feature deleted file mode 100644 index ca5463ec8..000000000 --- a/news/3785.feature +++ /dev/null @@ -1 +0,0 @@ -Introduce a new --prefer-binary flag, to prefer older wheels over newer source packages. diff --git a/news/3796.removal b/news/3796.removal deleted file mode 100644 index c4c4bbb17..000000000 --- a/news/3796.removal +++ /dev/null @@ -1 +0,0 @@ -Dropped support for Python 3.3. diff --git a/news/3905.bugfix b/news/3905.bugfix deleted file mode 100644 index a719866fc..000000000 --- a/news/3905.bugfix +++ /dev/null @@ -1 +0,0 @@ -Adjust path to selfcheck.json - remove virtualenv specific path and honor cache-dir in pip.conf diff --git a/news/4187.feature b/news/4187.feature new file mode 100644 index 000000000..03a874bc3 --- /dev/null +++ b/news/4187.feature @@ -0,0 +1,5 @@ +Allow PEP 508 URL requirements to be used as dependencies. + +As a security measure, pip will raise an exception when installing packages from +PyPI if those packages depend on packages not also hosted on PyPI. +In the future, PyPI will block uploading packages with such external URL dependencies directly. diff --git a/news/4471.bugfix b/news/4471.bugfix deleted file mode 100644 index 997e59f59..000000000 --- a/news/4471.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove compiled pyo files for wheel packages. diff --git a/news/4842.feature b/news/4842.feature deleted file mode 100644 index f96a668ec..000000000 --- a/news/4842.feature +++ /dev/null @@ -1,2 +0,0 @@ -Improve autocompletion function on file name completion after options -which have ````, ```` or ```` as metavar. diff --git a/news/5125.feature b/news/5125.feature deleted file mode 100644 index f96a668ec..000000000 --- a/news/5125.feature +++ /dev/null @@ -1,2 +0,0 @@ -Improve autocompletion function on file name completion after options -which have ````, ```` or ```` as metavar. diff --git a/news/5127.bugfix b/news/5127.bugfix deleted file mode 100644 index 4c03f450a..000000000 --- a/news/5127.bugfix +++ /dev/null @@ -1 +0,0 @@ -Speed up printing of newly installed package versions diff --git a/news/5196.bugfix b/news/5196.bugfix deleted file mode 100644 index 02c31e25f..000000000 --- a/news/5196.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Restrict install time dependency warnings to directly-dependant packages - -Warning about the entire package set has resulted in users getting confused as -to why pip is printing these warnings. diff --git a/news/5230.bugfix b/news/5230.bugfix deleted file mode 100644 index 9f34eddf5..000000000 --- a/news/5230.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve handling of PEP 518 build requirements: support environment markers and extras. diff --git a/news/5249.bugfix b/news/5249.bugfix deleted file mode 100644 index 32f8a9721..000000000 --- a/news/5249.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove username/password from log message when using index with basic auth diff --git a/news/5261.doc b/news/5261.doc deleted file mode 100644 index 61b2918db..000000000 --- a/news/5261.doc +++ /dev/null @@ -1 +0,0 @@ -Clarify that the output of pip show is in RFC-compliant mail header format for people who want to parse the ouput. diff --git a/news/5265.bugfix b/news/5265.bugfix deleted file mode 100644 index 9f34eddf5..000000000 --- a/news/5265.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve handling of PEP 518 build requirements: support environment markers and extras. diff --git a/news/5293.bugfix b/news/5293.bugfix deleted file mode 100644 index dc1379ba1..000000000 --- a/news/5293.bugfix +++ /dev/null @@ -1 +0,0 @@ -Remove trailing os.sep from PATH directories to avoid false negatives \ No newline at end of file diff --git a/news/5311.bugfix b/news/5311.bugfix deleted file mode 100644 index b9f3d74b1..000000000 --- a/news/5311.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check diff --git a/news/5312.bugfix b/news/5312.bugfix deleted file mode 100644 index b9f3d74b1..000000000 --- a/news/5312.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix "pip wheel pip" being blocked by the "don't use pip to modify itself" check diff --git a/news/5319.feature b/news/5319.feature deleted file mode 100644 index 25041dcdb..000000000 --- a/news/5319.feature +++ /dev/null @@ -1 +0,0 @@ -Improve status message when upgrade is skipped due to only-if-needed strategy diff --git a/news/5325.bugfix b/news/5325.bugfix deleted file mode 100644 index 0d79eb054..000000000 --- a/news/5325.bugfix +++ /dev/null @@ -1 +0,0 @@ -Add test for PR 5293: Remove trailing os.sep to avoid false negatives diff --git a/news/5346.bugfix b/news/5346.bugfix deleted file mode 100644 index 3a652ac55..000000000 --- a/news/5346.bugfix +++ /dev/null @@ -1,5 +0,0 @@ -Disable pip's version check (and upgrade message) when installed by a different package manager. - -This works better with Linux distributions where pip's upgrade message may -result in users running pip in a manner that modifies files that should be -managed by the OS's package manager. diff --git a/news/5347.bugfix b/news/5347.bugfix deleted file mode 100644 index 195710772..000000000 --- a/news/5347.bugfix +++ /dev/null @@ -1 +0,0 @@ -Fix the revendoring script no to rewrite unrelated import that starts suspicious. \ No newline at end of file diff --git a/news/5366.bugfix b/news/5366.bugfix deleted file mode 100644 index af37e3369..000000000 --- a/news/5366.bugfix +++ /dev/null @@ -1 +0,0 @@ -Check for file existence and unlink first when clobbering existing files during a wheel install. diff --git a/news/5381.bugfix b/news/5381.bugfix deleted file mode 100644 index 899fd4908..000000000 --- a/news/5381.bugfix +++ /dev/null @@ -1 +0,0 @@ -Improve error message to be more specific when no files are found as listed in as listed in PKG-INFO. diff --git a/news/5416.bugfix b/news/5416.bugfix deleted file mode 100644 index 2f7707e0c..000000000 --- a/news/5416.bugfix +++ /dev/null @@ -1 +0,0 @@ -Start refusing to install packages with non PEP-518 compliant pyproject.toml diff --git a/news/5457.bugfix b/news/5457.bugfix deleted file mode 100644 index 02c31e25f..000000000 --- a/news/5457.bugfix +++ /dev/null @@ -1,4 +0,0 @@ -Restrict install time dependency warnings to directly-dependant packages - -Warning about the entire package set has resulted in users getting confused as -to why pip is printing these warnings. diff --git a/news/5482.bugfix b/news/5482.bugfix deleted file mode 100644 index dd8d65e0b..000000000 --- a/news/5482.bugfix +++ /dev/null @@ -1 +0,0 @@ -Always read ``pyproject.toml`` as UTF-8. This fixes Unicode handling on Windows and Python 2. diff --git a/news/5512.bugfix b/news/5512.bugfix deleted file mode 100644 index 2f7707e0c..000000000 --- a/news/5512.bugfix +++ /dev/null @@ -1 +0,0 @@ -Start refusing to install packages with non PEP-518 compliant pyproject.toml diff --git a/news/5520.removal b/news/5520.removal deleted file mode 100644 index 49fe4369f..000000000 --- a/news/5520.removal +++ /dev/null @@ -1,4 +0,0 @@ -Remove the shim for the old get-pip.py location. - -For the past 2 years, it's only been redirecting users to use the newer -https://bootstrap.pypa.io/get-pip.py location. diff --git a/news/03b68f4b-6e66-4e94-acbf-0228ca04b2b6.trivial b/news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial similarity index 100% rename from news/03b68f4b-6e66-4e94-acbf-0228ca04b2b6.trivial rename to news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial diff --git a/news/07E9718B-0155-413F-AC2A-1B75AA2934BC.trivial b/news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial similarity index 100% rename from news/07E9718B-0155-413F-AC2A-1B75AA2934BC.trivial rename to news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial diff --git a/news/bb227e37-7d24-4ecd-9ddd-78bbf0b2b456.trivial b/news/bb227e37-7d24-4ecd-9ddd-78bbf0b2b456.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/bbd0f55a-2fac-4e6f-b417-4d9d556027a4.trivial b/news/bbd0f55a-2fac-4e6f-b417-4d9d556027a4.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/calver.process b/news/calver.process deleted file mode 100644 index bfd7ba7a5..000000000 --- a/news/calver.process +++ /dev/null @@ -1 +0,0 @@ -Switch to a Calendar based versioning scheme. diff --git a/news/dd66ca44-097a-4ffb-8417-aa046e85eab7.trivial b/news/dd66ca44-097a-4ffb-8417-aa046e85eab7.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/deprecation.process b/news/deprecation.process deleted file mode 100644 index 0c3e042fe..000000000 --- a/news/deprecation.process +++ /dev/null @@ -1 +0,0 @@ -Formally document our deprecation process diff --git a/news/ed19045b-46b8-4ac0-a98a-60d2afa2c675.trivial b/news/ed19045b-46b8-4ac0-a98a-60d2afa2c675.trivial deleted file mode 100644 index e69de29bb..000000000 diff --git a/news/news-file-content.process b/news/news-file-content.process deleted file mode 100644 index cd89445fa..000000000 --- a/news/news-file-content.process +++ /dev/null @@ -1 +0,0 @@ -Adopt and document NEWS fragment writing style diff --git a/news/release-cadence.process b/news/release-cadence.process deleted file mode 100644 index f8f86f0e1..000000000 --- a/news/release-cadence.process +++ /dev/null @@ -1 +0,0 @@ -Switch to releasing a new, non bug fix version of pip every 3 months. diff --git a/news/user_guide_fix_requirements_file_ref.doc b/news/user_guide_fix_requirements_file_ref.doc new file mode 100644 index 000000000..07112d375 --- /dev/null +++ b/news/user_guide_fix_requirements_file_ref.doc @@ -0,0 +1 @@ +Fix "Requirements Files" reference in User Guide diff --git a/setup.cfg b/setup.cfg index 66c036df6..d811e7989 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,9 @@ [isort] skip = - _vendor - __main__.py + .tox, + .scratch, + _vendor, + data multi_line_output = 5 known_third_party = pip._vendor @@ -12,7 +14,11 @@ default_section = THIRDPARTY include_trailing_comma = true [flake8] -exclude = .tox,.idea,.scratch,*.egg,build,_vendor,data +exclude = + .tox, + .scratch, + _vendor, + data select = E,W,F [mypy] diff --git a/setup.py b/setup.py index de8cf1103..e2e14b496 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,7 @@ import os import re import sys -from setuptools import setup, find_packages - +from setuptools import find_packages, setup here = os.path.abspath(os.path.dirname(__file__)) diff --git a/src/pip/__init__.py b/src/pip/__init__.py index edcc99ba3..2e8c4e43d 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1 +1 @@ -__version__ = "18.0.dev0" +__version__ = "18.1.dev0" diff --git a/src/pip/__main__.py b/src/pip/__main__.py index 4609582c3..0c223f8c1 100644 --- a/src/pip/__main__.py +++ b/src/pip/__main__.py @@ -13,7 +13,7 @@ if __package__ == '': path = os.path.dirname(os.path.dirname(__file__)) sys.path.insert(0, path) -from pip._internal import main as _main # noqa +from pip._internal import main as _main # isort:skip # noqa if __name__ == '__main__': sys.exit(_main()) diff --git a/src/pip/_internal/__init__.py b/src/pip/_internal/__init__.py index 250ed1d90..4a0390d92 100755 --- a/src/pip/_internal/__init__.py +++ b/src/pip/_internal/__init__.py @@ -274,15 +274,6 @@ def parseopts(args): return cmd_name, cmd_args -def check_isolated(args): - isolated = False - - if "--isolated" in args: - isolated = True - - return isolated - - def main(args=None): if args is None: args = sys.argv[1:] @@ -306,5 +297,5 @@ def main(args=None): except locale.Error as e: # setlocale can apparently crash if locale are uninitialized logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=check_isolated(cmd_args)) + command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) return command.main(cmd_args) diff --git a/src/pip/_internal/basecommand.py b/src/pip/_internal/basecommand.py index 71261f55a..f0fe37ea5 100644 --- a/src/pip/_internal/basecommand.py +++ b/src/pip/_internal/basecommand.py @@ -24,7 +24,7 @@ from pip._internal.status_codes import ( ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND, ) -from pip._internal.utils.logging import IndentingFormatter +from pip._internal.utils.logging import setup_logging from pip._internal.utils.misc import get_prog, normalize_path from pip._internal.utils.outdated import pip_version_check from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -42,7 +42,6 @@ class Command(object): usage = None # type: Optional[str] hidden = False # type: bool ignore_require_venv = False # type: bool - log_streams = ("ext://sys.stdout", "ext://sys.stderr") def __init__(self, isolated=False): parser_kw = { @@ -114,89 +113,15 @@ class Command(object): # Set verbosity so that it can be used elsewhere. self.verbosity = options.verbose - options.quiet - if self.verbosity >= 1: - level = "DEBUG" - elif self.verbosity == -1: - level = "WARNING" - elif self.verbosity == -2: - level = "ERROR" - elif self.verbosity <= -3: - level = "CRITICAL" - else: - level = "INFO" + setup_logging( + verbosity=self.verbosity, + no_color=options.no_color, + user_log_file=options.log, + ) - # The root logger should match the "console" level *unless* we - # specified "--log" to send debug logs to a file. - root_level = level - if options.log: - root_level = "DEBUG" - - logger_class = "pip._internal.utils.logging.ColorizedStreamHandler" - handler_class = "pip._internal.utils.logging.BetterRotatingFileHandler" - - logging.config.dictConfig({ - "version": 1, - "disable_existing_loggers": False, - "filters": { - "exclude_warnings": { - "()": "pip._internal.utils.logging.MaxLevelFilter", - "level": logging.WARNING, - }, - }, - "formatters": { - "indent": { - "()": IndentingFormatter, - "format": "%(message)s", - }, - }, - "handlers": { - "console": { - "level": level, - "class": logger_class, - "no_color": options.no_color, - "stream": self.log_streams[0], - "filters": ["exclude_warnings"], - "formatter": "indent", - }, - "console_errors": { - "level": "WARNING", - "class": logger_class, - "no_color": options.no_color, - "stream": self.log_streams[1], - "formatter": "indent", - }, - "user_log": { - "level": "DEBUG", - "class": handler_class, - "filename": options.log or "/dev/null", - "delay": True, - "formatter": "indent", - }, - }, - "root": { - "level": root_level, - "handlers": list(filter(None, [ - "console", - "console_errors", - "user_log" if options.log else None, - ])), - }, - # Disable any logging besides WARNING unless we have DEBUG level - # logging enabled. These use both pip._vendor and the bare names - # for the case where someone unbundles our libraries. - "loggers": { - name: { - "level": ( - "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" - ) - } for name in [ - "pip._vendor", "distlib", "requests", "urllib3" - ] - }, - }) - - # TODO: try to get these passing down from the command? - # without resorting to os.environ to hold these. + # TODO: Try to get these passing down from the command? + # without resorting to os.environ to hold these. + # This also affects isolated builds and it should. if options.no_input: os.environ['PIP_NO_INPUT'] = '1' @@ -212,8 +137,6 @@ class Command(object): ) sys.exit(VIRTUALENV_NOT_FOUND) - original_root_handlers = set(logging.root.handlers) - try: status = self.run(options, args) # FIXME: all commands should return an exit status @@ -240,7 +163,7 @@ class Command(object): logger.debug('Exception information:', exc_info=True) return ERROR - except: + except BaseException: logger.critical('Exception:', exc_info=True) return UNKNOWN_ERROR @@ -258,10 +181,9 @@ class Command(object): ) with session: pip_version_check(session, options) - # Avoid leaking loggers - for handler in set(logging.root.handlers) - original_root_handlers: - # this method benefit from the Logger class internal lock - logging.root.removeHandler(handler) + + # Shutdown the logging module + logging.shutdown() return SUCCESS diff --git a/src/pip/_internal/build_env.py b/src/pip/_internal/build_env.py index 612b46352..f225f76d6 100644 --- a/src/pip/_internal/build_env.py +++ b/src/pip/_internal/build_env.py @@ -11,7 +11,6 @@ from pip._internal.utils.misc import call_subprocess from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.ui import open_spinner - logger = logging.getLogger(__name__) @@ -80,10 +79,13 @@ class BuildEnvironment(object): args = [ sys.executable, '-m', 'pip', 'install', '--ignore-installed', '--no-user', '--prefix', self.path, '--no-warn-script-location', - '--only-binary', ':all:', ] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') + for format_control in ('no_binary', 'only_binary'): + formats = getattr(finder.format_control, format_control) + args.extend(('--' + format_control.replace('_', '-'), + ','.join(sorted(formats or {':none:'})))) if finder.index_urls: args.extend(['-i', finder.index_urls[0]]) for extra_index in finder.index_urls[1:]: diff --git a/src/pip/_internal/cache.py b/src/pip/_internal/cache.py index 1aa17aa96..91d54b1b0 100644 --- a/src/pip/_internal/cache.py +++ b/src/pip/_internal/cache.py @@ -11,6 +11,7 @@ from pip._vendor.packaging.utils import canonicalize_name from pip._internal import index from pip._internal.compat import expanduser from pip._internal.download import path_to_url +from pip._internal.models.link import Link from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import InvalidWheelFilename, Wheel @@ -101,7 +102,7 @@ class Cache(object): root = self.get_path_for_link(link) path = os.path.join(root, candidate) - return index.Link(path_to_url(path)) + return Link(path_to_url(path)) def cleanup(self): pass diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index 66bcbd5c8..cf4827c58 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -9,6 +9,7 @@ from pip._internal.exceptions import CommandError from pip._internal.index import FormatControl from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.filesystem import check_path_owner from pip._internal.utils.misc import ensure_dir, normalize_path @@ -180,7 +181,7 @@ class DownloadCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="download" ) as directory: @@ -204,6 +205,7 @@ class DownloadCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/install.py b/src/pip/_internal/commands/install.py index da1146b9c..f42a1d137 100644 --- a/src/pip/_internal/commands/install.py +++ b/src/pip/_internal/commands/install.py @@ -19,6 +19,7 @@ from pip._internal.locations import distutils_scheme, virtualenv_no_global from pip._internal.operations.check import check_install_conflicts from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet, install_given_reqs +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.status_codes import ERROR from pip._internal.utils.filesystem import check_path_owner @@ -260,7 +261,7 @@ class InstallCommand(RequirementCommand): ) options.cache_dir = None - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="install" ) as directory: requirement_set = RequirementSet( @@ -279,6 +280,7 @@ class InstallCommand(RequirementCommand): wheel_download_dir=None, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/commands/list.py b/src/pip/_internal/commands/list.py index 09f633f39..53fb015f0 100644 --- a/src/pip/_internal/commands/list.py +++ b/src/pip/_internal/commands/list.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import json import logging -import warnings from pip._vendor import six from pip._vendor.six.moves import zip_longest @@ -11,7 +10,6 @@ from pip._internal.basecommand import Command from pip._internal.cmdoptions import index_group, make_option_group from pip._internal.exceptions import CommandError from pip._internal.index import PackageFinder -from pip._internal.utils.deprecation import RemovedInPip11Warning from pip._internal.utils.misc import ( dist_is_editable, get_installed_distributions, ) @@ -78,9 +76,9 @@ class ListCommand(Command): action='store', dest='list_format', default="columns", - choices=('legacy', 'columns', 'freeze', 'json'), + choices=('columns', 'freeze', 'json'), help="Select the output format among: columns (default), freeze, " - "json, or legacy.", + "or json", ) cmd_opts.add_option( @@ -123,13 +121,6 @@ class ListCommand(Command): ) def run(self, options, args): - if options.list_format == "legacy": - warnings.warn( - "The legacy format has been deprecated and will be removed " - "in the future.", - RemovedInPip11Warning, - ) - if options.outdated and options.uptodate: raise CommandError( "Options --outdated and --uptodate cannot be combined.") @@ -208,30 +199,6 @@ class ListCommand(Command): dist.latest_filetype = typ yield dist - def output_legacy(self, dist, options): - if options.verbose >= 1: - return '%s (%s, %s, %s)' % ( - dist.project_name, - dist.version, - dist.location, - get_installer(dist), - ) - elif dist_is_editable(dist): - return '%s (%s, %s)' % ( - dist.project_name, - dist.version, - dist.location, - ) - else: - return '%s (%s)' % (dist.project_name, dist.version) - - def output_legacy_latest(self, dist, options): - return '%s - Latest: %s [%s]' % ( - self.output_legacy(dist, options), - dist.latest_version, - dist.latest_filetype, - ) - def output_package_listing(self, packages, options): packages = sorted( packages, @@ -249,12 +216,6 @@ class ListCommand(Command): logger.info("%s==%s", dist.project_name, dist.version) elif options.list_format == 'json': logger.info(format_for_json(packages, options)) - elif options.list_format == "legacy": - for dist in packages: - if options.outdated: - logger.info(self.output_legacy_latest(dist, options)) - else: - logger.info(self.output_legacy(dist, options)) def output_package_listing_columns(self, data, header): # insert the header first: we need to know the size of column names diff --git a/src/pip/_internal/commands/wheel.py b/src/pip/_internal/commands/wheel.py index 0fb72c1a9..41893876d 100644 --- a/src/pip/_internal/commands/wheel.py +++ b/src/pip/_internal/commands/wheel.py @@ -10,6 +10,7 @@ from pip._internal.cache import WheelCache from pip._internal.exceptions import CommandError, PreviousBuildDirError from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import RequirementSet +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.temp_dir import TempDirectory from pip._internal.wheel import WheelBuilder @@ -120,9 +121,10 @@ class WheelCommand(RequirementCommand): build_delete = (not (options.no_clean or options.build_dir)) wheel_cache = WheelCache(options.cache_dir, options.format_control) - with TempDirectory( + with RequirementTracker() as req_tracker, TempDirectory( options.build_dir, delete=build_delete, kind="wheel" ) as directory: + requirement_set = RequirementSet( require_hashes=options.require_hashes, ) @@ -140,6 +142,7 @@ class WheelCommand(RequirementCommand): wheel_download_dir=options.wheel_dir, progress_bar=options.progress_bar, build_isolation=options.build_isolation, + req_tracker=req_tracker, ) resolver = Resolver( diff --git a/src/pip/_internal/compat.py b/src/pip/_internal/compat.py index 749115075..e6c008d3b 100644 --- a/src/pip/_internal/compat.py +++ b/src/pip/_internal/compat.py @@ -217,7 +217,7 @@ else: 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '12345678') ) - except: + except Exception: return None if cr == (0, 0): return None @@ -228,7 +228,7 @@ else: fd = os.open(os.ctermid(), os.O_RDONLY) cr = ioctl_GWINSZ(fd) os.close(fd) - except: + except Exception: pass if not cr: cr = (os.environ.get('LINES', 25), os.environ.get('COLUMNS', 80)) diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py index fe318a24f..5f4899e01 100644 --- a/src/pip/_internal/index.py +++ b/src/pip/_internal/index.py @@ -9,7 +9,6 @@ import os import posixpath import re import sys -import warnings from collections import namedtuple from pip._vendor import html5lib, requests, six @@ -27,13 +26,15 @@ from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, UnsupportedWheel, ) +from pip._internal.models.candidate import InstallationCandidate from pip._internal.models.index import PyPI +from pip._internal.models.link import Link from pip._internal.pep425tags import get_supported -from pip._internal.utils.deprecation import RemovedInPip11Warning +from pip._internal.utils.deprecation import deprecated from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path, - remove_auth_from_url, splitext, + remove_auth_from_url, ) from pip._internal.utils.packaging import check_requires_python from pip._internal.wheel import Wheel, wheel_ext @@ -57,47 +58,6 @@ SECURE_ORIGINS = [ logger = logging.getLogger(__name__) -class InstallationCandidate(object): - - def __init__(self, project, version, location): - self.project = project - self.version = parse_version(version) - self.location = location - self._key = (self.project, self.version, self.location) - - def __repr__(self): - return "".format( - self.project, self.version, self.location, - ) - - def __hash__(self): - return hash(self._key) - - def __lt__(self, other): - return self._compare(other, lambda s, o: s < o) - - def __le__(self, other): - return self._compare(other, lambda s, o: s <= o) - - def __eq__(self, other): - return self._compare(other, lambda s, o: s == o) - - def __ge__(self, other): - return self._compare(other, lambda s, o: s >= o) - - def __gt__(self, other): - return self._compare(other, lambda s, o: s > o) - - def __ne__(self, other): - return self._compare(other, lambda s, o: s != o) - - def _compare(self, other, method): - if not isinstance(other, InstallationCandidate): - return NotImplemented - - return method(self._key, other._key) - - class PackageFinder(object): """This finds packages. @@ -212,10 +172,12 @@ class PackageFinder(object): # # dependency_links value # # FIXME: also, we should track comes_from (i.e., use Link) if self.process_dependency_links: - warnings.warn( + deprecated( "Dependency Links processing has been deprecated and will be " "removed in a future release.", - RemovedInPip11Warning, + replacement=None, + gone_in="18.2", + issue=4187, ) self.dependency_links.extend(links) @@ -908,166 +870,6 @@ class HTMLPage(object): lambda match: '%%%2x' % ord(match.group(0)), url) -class Link(object): - - def __init__(self, url, comes_from=None, requires_python=None): - """ - Object representing a parsed link from https://pypi.org/simple/* - - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. - """ - - # url can be a UNC windows share - if url.startswith('\\\\'): - url = path_to_url(url) - - self.url = url - self.comes_from = comes_from - self.requires_python = requires_python if requires_python else None - - def __str__(self): - if self.requires_python: - rp = ' (requires-python:%s)' % self.requires_python - else: - rp = '' - if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) - else: - return str(self.url) - - def __repr__(self): - return '' % self - - def __eq__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url == other.url - - def __ne__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url != other.url - - def __lt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url < other.url - - def __le__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url <= other.url - - def __gt__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url > other.url - - def __ge__(self, other): - if not isinstance(other, Link): - return NotImplemented - return self.url >= other.url - - def __hash__(self): - return hash(self.url) - - @property - def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc - name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) - return name - - @property - def scheme(self): - return urllib_parse.urlsplit(self.url)[0] - - @property - def netloc(self): - return urllib_parse.urlsplit(self.url)[1] - - @property - def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) - - def splitext(self): - return splitext(posixpath.basename(self.path.rstrip('/'))) - - @property - def ext(self): - return self.splitext()[1] - - @property - def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) - return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) - - _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') - - @property - def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') - - @property - def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) - if not match: - return None - return match.group(1) - - _hash_re = re.compile( - r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' - ) - - @property - def hash(self): - match = self._hash_re.search(self.url) - if match: - return match.group(2) - return None - - @property - def hash_name(self): - match = self._hash_re.search(self.url) - if match: - return match.group(1) - return None - - @property - def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) - - @property - def is_wheel(self): - return self.ext == wheel_ext - - @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ - from pip._internal.vcs import vcs - - if self.scheme in vcs.all_schemes: - return False - - return True - - FormatControl = namedtuple('FormatControl', 'no_binary only_binary') """This object has two fields, no_binary and only_binary. diff --git a/src/pip/_internal/models/candidate.py b/src/pip/_internal/models/candidate.py new file mode 100644 index 000000000..c736de6c9 --- /dev/null +++ b/src/pip/_internal/models/candidate.py @@ -0,0 +1,23 @@ +from pip._vendor.packaging.version import parse as parse_version + +from pip._internal.utils.models import KeyBasedCompareMixin + + +class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + + def __init__(self, project, version, location): + self.project = project + self.version = parse_version(version) + self.location = location + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.location), + defining_class=InstallationCandidate + ) + + def __repr__(self): + return "".format( + self.project, self.version, self.location, + ) diff --git a/src/pip/_internal/models/index.py b/src/pip/_internal/models/index.py index a7f10c8db..870a315ed 100644 --- a/src/pip/_internal/models/index.py +++ b/src/pip/_internal/models/index.py @@ -1,15 +1,29 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse -class Index(object): - def __init__(self, url): +class PackageIndex(object): + """Represents a Package Index and provides easier access to endpoints + """ + + def __init__(self, url, file_storage_domain): + super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc - self.simple_url = self.url_to_path('simple') - self.pypi_url = self.url_to_path('pypi') + self.simple_url = self._url_for_path('simple') + self.pypi_url = self._url_for_path('pypi') - def url_to_path(self, path): + # This is part of a temporary hack used to block installs of PyPI + # packages which depend on external urls only necessary until PyPI can + # block such packages themselves + self.file_storage_domain = file_storage_domain + + def _url_for_path(self, path): return urllib_parse.urljoin(self.url, path) -PyPI = Index('https://pypi.org/') +PyPI = PackageIndex( + 'https://pypi.org/', file_storage_domain='files.pythonhosted.org' +) +TestPyPI = PackageIndex( + 'https://test.pypi.org/', file_storage_domain='test-files.pythonhosted.org' +) diff --git a/src/pip/_internal/models/link.py b/src/pip/_internal/models/link.py new file mode 100644 index 000000000..5decb7cc4 --- /dev/null +++ b/src/pip/_internal/models/link.py @@ -0,0 +1,141 @@ +import posixpath +import re + +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.download import path_to_url +from pip._internal.utils.misc import splitext +from pip._internal.utils.models import KeyBasedCompareMixin +from pip._internal.wheel import wheel_ext + + +class Link(KeyBasedCompareMixin): + """Represents a parsed link from a Package Index's simple URL + """ + + def __init__(self, url, comes_from=None, requires_python=None): + """ + url: + url of the resource pointed to (href of the link) + comes_from: + instance of HTMLPage where the link was found, or string. + requires_python: + String containing the `Requires-Python` metadata field, specified + in PEP 345. This may be specified by a data-requires-python + attribute in the HTML link tag, as described in PEP 503. + """ + + # url can be a UNC windows share + if url.startswith('\\\\'): + url = path_to_url(url) + + self.url = url + self.comes_from = comes_from + self.requires_python = requires_python if requires_python else None + + super(Link, self).__init__( + key=(self.url), + defining_class=Link + ) + + def __str__(self): + if self.requires_python: + rp = ' (requires-python:%s)' % self.requires_python + else: + rp = '' + if self.comes_from: + return '%s (from %s)%s' % (self.url, self.comes_from, rp) + else: + return str(self.url) + + def __repr__(self): + return '' % self + + @property + def filename(self): + _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) + name = posixpath.basename(path.rstrip('/')) or netloc + name = urllib_parse.unquote(name) + assert name, ('URL %r produced no filename' % self.url) + return name + + @property + def scheme(self): + return urllib_parse.urlsplit(self.url)[0] + + @property + def netloc(self): + return urllib_parse.urlsplit(self.url)[1] + + @property + def path(self): + return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + + def splitext(self): + return splitext(posixpath.basename(self.path.rstrip('/'))) + + @property + def ext(self): + return self.splitext()[1] + + @property + def url_without_fragment(self): + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) + + _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') + + @property + def egg_fragment(self): + match = self._egg_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _subdirectory_fragment_re = re.compile(r'[#&]subdirectory=([^&]*)') + + @property + def subdirectory_fragment(self): + match = self._subdirectory_fragment_re.search(self.url) + if not match: + return None + return match.group(1) + + _hash_re = re.compile( + r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)' + ) + + @property + def hash(self): + match = self._hash_re.search(self.url) + if match: + return match.group(2) + return None + + @property + def hash_name(self): + match = self._hash_re.search(self.url) + if match: + return match.group(1) + return None + + @property + def show_url(self): + return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_wheel(self): + return self.ext == wheel_ext + + @property + def is_artifact(self): + """ + Determines if this points to an actual artifact (e.g. a tarball) or if + it points to an "abstract" thing like a path or a VCS location. + """ + from pip._internal.vcs import vcs + + if self.scheme in vcs.all_schemes: + return False + + return True diff --git a/src/pip/_internal/operations/check.py b/src/pip/_internal/operations/check.py index cba18ad63..799257aea 100644 --- a/src/pip/_internal/operations/check.py +++ b/src/pip/_internal/operations/check.py @@ -6,7 +6,6 @@ from collections import namedtuple from pip._vendor.packaging.utils import canonicalize_name from pip._internal.operations.prepare import make_abstract_dist - from pip._internal.utils.misc import get_installed_distributions from pip._internal.utils.typing import MYPY_CHECK_RUNNING diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py index b6821c0e7..4bbc27b0a 100644 --- a/src/pip/_internal/operations/freeze.py +++ b/src/pip/_internal/operations/freeze.py @@ -4,7 +4,6 @@ import collections import logging import os import re -import warnings from pip._vendor import pkg_resources, six from pip._vendor.packaging.utils import canonicalize_name @@ -13,7 +12,7 @@ from pip._vendor.pkg_resources import RequirementParseError from pip._internal.exceptions import InstallationError from pip._internal.req import InstallRequirement from pip._internal.req.req_file import COMMENT_RE -from pip._internal.utils.deprecation import RemovedInPip11Warning +from pip._internal.utils.deprecation import deprecated from pip._internal.utils.misc import ( dist_is_editable, get_installed_distributions, ) @@ -216,10 +215,12 @@ class FrozenRequirement(object): 'for this package:' ) else: - warnings.warn( + deprecated( "SVN editable detection based on dependency links " "will be dropped in the future.", - RemovedInPip11Warning, + replacement=None, + gone_in="18.2", + issue=4187, ) comments.append( '# Installing as editable to satisfy requirement %s:' % diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 3ad721f01..7740c2843 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -141,11 +141,12 @@ class RequirementPreparer(object): """ def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation): + progress_bar, build_isolation, req_tracker): super(RequirementPreparer, self).__init__() self.src_dir = src_dir self.build_dir = build_dir + self.req_tracker = req_tracker # Where still packed archives should be written to. If None, they are # not saved, and are deleted immediately after unpacking. @@ -293,7 +294,8 @@ class RequirementPreparer(object): (req, exc, req.link) ) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: # Make a .zip of the source_dir we already created. if req.link.scheme in vcs.all_schemes: @@ -319,7 +321,8 @@ class RequirementPreparer(object): req.update_editable(not self._download_should_save) abstract_dist = make_abstract_dist(req) - abstract_dist.prep_for_dist(finder, self.build_isolation) + with self.req_tracker.track(req): + abstract_dist.prep_for_dist(finder, self.build_isolation) if self._download_should_save: req.archive(self.download_dir) diff --git a/src/pip/_internal/req/__init__.py b/src/pip/_internal/req/__init__.py index c9b4c3ce3..b270498e2 100644 --- a/src/pip/_internal/req/__init__.py +++ b/src/pip/_internal/req/__init__.py @@ -48,7 +48,7 @@ def install_given_reqs(to_install, install_options, global_options=(), *args, **kwargs ) - except: + except Exception: should_rollback = ( requirement.conflicts_with and not requirement.install_succeeded diff --git a/src/pip/_internal/req/req_install.py b/src/pip/_internal/req/req_install.py index 043353048..e90daf159 100644 --- a/src/pip/_internal/req/req_install.py +++ b/src/pip/_internal/req/req_install.py @@ -8,7 +8,6 @@ import shutil import sys import sysconfig import traceback -import warnings import zipfile from distutils.util import change_root from email.parser import FeedParser # type: ignore @@ -18,8 +17,8 @@ from pip._vendor.packaging import specifiers from pip._vendor.packaging.markers import Marker from pip._vendor.packaging.requirements import InvalidRequirement, Requirement from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.packaging.version import parse as parse_version from pip._vendor.packaging.version import Version +from pip._vendor.packaging.version import parse as parse_version from pip._vendor.pkg_resources import RequirementParseError, parse_requirements from pip._internal import wheel @@ -32,10 +31,9 @@ from pip._internal.exceptions import InstallationError from pip._internal.locations import ( PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, ) +from pip._internal.models.index import PyPI, TestPyPI +from pip._internal.models.link import Link from pip._internal.req.req_uninstall import UninstallPathSet -from pip._internal.utils.deprecation import ( - RemovedInPip11Warning, RemovedInPip12Warning, -) from pip._internal.utils.hashes import Hashes from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import ( @@ -137,8 +135,6 @@ class InstallRequirement(object): @classmethod def from_editable(cls, editable_req, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False): - from pip._internal.index import Link - name, url, extras_override = parse_editable(editable_req) if url.startswith('file:'): source_dir = url_to_path(url) @@ -169,11 +165,19 @@ class InstallRequirement(object): req = Requirement(req) except InvalidRequirement: raise InstallationError("Invalid requirement: '%s'" % req) - if req.url: + + domains_not_allowed = [ + PyPI.file_storage_domain, + TestPyPI.file_storage_domain, + ] + if req.url and comes_from.link.netloc in domains_not_allowed: + # Explicitly disallow pypi packages that depend on external urls raise InstallationError( - "Direct url requirement (like %s) are not allowed for " - "dependencies" % req + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI.\n" + "%s depends on %s " % (comes_from.name, req) ) + return cls(req, comes_from, isolated=isolated, wheel_cache=wheel_cache) @classmethod @@ -568,44 +572,40 @@ class InstallRequirement(object): specified as per PEP 518 within the package. If `pyproject.toml` is not present, returns None to signify not using the same. """ + # If pyproject.toml does not exist, don't do anything. if not os.path.isfile(self.pyproject_toml): return None + error_template = ( + "{package} has a pyproject.toml file that does not comply " + "with PEP 518: {reason}" + ) + with io.open(self.pyproject_toml, encoding="utf-8") as f: pp_toml = pytoml.load(f) - # Extract the build requirements - requires = pp_toml.get("build-system", {}).get("requires", None) + # If there is no build-system table, just use setuptools and wheel. + if "build-system" not in pp_toml: + return ["setuptools", "wheel"] - template = ( - "%s does not comply with PEP 518 since pyproject.toml " - "does not contain a valid '[build-system].requires' key: %s" - ) - - if requires is None: - logging.warn(template, self, "it is missing.") - warnings.warn( - "Future versions of pip will reject packages with " - "pyproject.toml files that do not comply with PEP 518.", - RemovedInPip12Warning, + # Specifying the build-system table but not the requires key is invalid + build_system = pp_toml["build-system"] + if "requires" not in build_system: + raise InstallationError( + error_template.format(package=self, reason=( + "it has a 'build-system' table but not " + "'build-system.requires' which is mandatory in the table" + )) ) - # NOTE: Currently allowing projects to skip this key so that they - # can transition to a PEP 518 compliant pyproject.toml or - # push to update the PEP. - # Come pip 19.0, bring this to compliance with PEP 518. - return None - else: - # Error out if it's not a list of strings - is_list_of_str = isinstance(requires, list) and all( - isinstance(req, six.string_types) for req in requires - ) - if not is_list_of_str: - raise InstallationError( - template % (self, "it is not a list of strings.") - ) + # Error out if it's not a list of strings + requires = build_system["requires"] + if not _is_list_of_str(requires): + raise InstallationError(error_template.format( + package=self, + reason="'build-system.requires' is not a list of strings.", + )) - # If control flow reaches here, we're good to go. return requires def run_egg_info(self): @@ -1049,22 +1049,6 @@ class InstallRequirement(object): return install_args -def _strip_postfix(req): - """ - Strip req postfix ( -dev, 0.2, etc ) - """ - # FIXME: use package_to_requirement? - match = re.search(r'^(.*?)(?:-dev|-\d.*)$', req) - if match: - # Strip off -dev, -0.2, etc. - warnings.warn( - "#egg cleanup for editable urls will be dropped in the future", - RemovedInPip11Warning, - ) - req = match.group(1) - return req - - def parse_editable(editable_req): """Parses an editable requirement into: - a requirement name @@ -1129,7 +1113,7 @@ def parse_editable(editable_req): "Could not detect requirement name for '%s', please specify one " "with #egg=your_package_name" % editable_req ) - return _strip_postfix(package_name), url, None + return package_name, url, None def deduce_helpful_msg(req): @@ -1157,3 +1141,10 @@ def deduce_helpful_msg(req): else: msg += " File '%s' does not exist." % (req) return msg + + +def _is_list_of_str(obj): + return ( + isinstance(obj, list) and + all(isinstance(item, six.string_types) for item in obj) + ) diff --git a/src/pip/_internal/req/req_tracker.py b/src/pip/_internal/req/req_tracker.py new file mode 100644 index 000000000..0a86f4cd3 --- /dev/null +++ b/src/pip/_internal/req/req_tracker.py @@ -0,0 +1,76 @@ +from __future__ import absolute_import + +import contextlib +import errno +import hashlib +import logging +import os + +from pip._internal.utils.temp_dir import TempDirectory + +logger = logging.getLogger(__name__) + + +class RequirementTracker(object): + + def __init__(self): + self._root = os.environ.get('PIP_REQ_TRACKER') + if self._root is None: + self._temp_dir = TempDirectory(delete=False, kind='req-tracker') + self._temp_dir.create() + self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path + logger.debug('Created requirements tracker %r', self._root) + else: + self._temp_dir = None + logger.debug('Re-using requirements tracker %r', self._root) + self._entries = set() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.cleanup() + + def _entry_path(self, link): + hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() + return os.path.join(self._root, hashed) + + def add(self, req): + link = req.link + info = str(req) + entry_path = self._entry_path(link) + try: + with open(entry_path) as fp: + # Error, these's already a build in progress. + raise LookupError('%s is already being built: %s' + % (link, fp.read())) + except IOError as e: + if e.errno != errno.ENOENT: + raise + assert req not in self._entries + with open(entry_path, 'w') as fp: + fp.write(info) + self._entries.add(req) + logger.debug('Added %s to build tracker %r', req, self._root) + + def remove(self, req): + link = req.link + self._entries.remove(req) + os.unlink(self._entry_path(link)) + logger.debug('Removed %s from build tracker %r', req, self._root) + + def cleanup(self): + for req in set(self._entries): + self.remove(req) + remove = self._temp_dir is not None + if remove: + self._temp_dir.cleanup() + logger.debug('%s build tracker %r', + 'Removed' if remove else 'Cleaned', + self._root) + + @contextlib.contextmanager + def track(self, req): + self.add(req) + yield + self.remove(req) diff --git a/src/pip/_internal/req/req_uninstall.py b/src/pip/_internal/req/req_uninstall.py index f7ec3a8f8..32ac6b2f7 100644 --- a/src/pip/_internal/req/req_uninstall.py +++ b/src/pip/_internal/req/req_uninstall.py @@ -120,6 +120,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) + _normcased_files = set(map(os.path.normcase, files)) + folders = compact(folders) # This walks the tree using os.walk to not miss extra folders @@ -130,8 +132,9 @@ def compress_for_output_listing(paths): if fname.endswith(".pyc"): continue - file_ = os.path.normcase(os.path.join(dirpath, fname)) - if os.path.isfile(file_) and file_ not in files: + file_ = os.path.join(dirpath, fname) + if (os.path.isfile(file_) and + os.path.normcase(file_) not in _normcased_files): # We are skipping this file. Add it to the set. will_skip.add(file_) diff --git a/src/pip/_internal/resolve.py b/src/pip/_internal/resolve.py index 6711f9a5b..8480e48c6 100644 --- a/src/pip/_internal/resolve.py +++ b/src/pip/_internal/resolve.py @@ -18,7 +18,6 @@ from pip._internal.exceptions import ( BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, UnsupportedPythonVersion, ) - from pip._internal.req.req_install import InstallRequirement from pip._internal.utils.logging import indent_log from pip._internal.utils.misc import dist_in_usersite, ensure_dir diff --git a/src/pip/_internal/utils/deprecation.py b/src/pip/_internal/utils/deprecation.py index 7ec40feeb..bd744cf2c 100644 --- a/src/pip/_internal/utils/deprecation.py +++ b/src/pip/_internal/utils/deprecation.py @@ -6,72 +6,84 @@ from __future__ import absolute_import import logging import warnings +from pip._vendor.packaging.version import parse + +from pip import __version__ as current_version from pip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Optional # noqa: F401 class PipDeprecationWarning(Warning): pass -class Pending(object): - pass - - -class RemovedInPip11Warning(PipDeprecationWarning): - pass - - -class RemovedInPip12Warning(PipDeprecationWarning, Pending): - pass +_original_showwarning = None # type: Any # Warnings <-> Logging Integration - - -_warnings_showwarning = None # type: Any - - def _showwarning(message, category, filename, lineno, file=None, line=None): if file is not None: - if _warnings_showwarning is not None: - _warnings_showwarning( + if _original_showwarning is not None: + _original_showwarning( message, category, filename, lineno, file, line, ) + elif issubclass(category, PipDeprecationWarning): + # We use a specially named logger which will handle all of the + # deprecation messages for pip. + logger = logging.getLogger("pip._internal.deprecations") + logger.warning(message) else: - if issubclass(category, PipDeprecationWarning): - # We use a specially named logger which will handle all of the - # deprecation messages for pip. - logger = logging.getLogger("pip._internal.deprecations") - - # This is purposely using the % formatter here instead of letting - # the logging module handle the interpolation. This is because we - # want it to appear as if someone typed this entire message out. - log_message = "DEPRECATION: %s" % message - - # PipDeprecationWarnings that are Pending still have at least 2 - # versions to go until they are removed so they can just be - # warnings. Otherwise, they will be removed in the very next - # version of pip. We want these to be more obvious so we use the - # ERROR logging level. - if issubclass(category, Pending): - logger.warning(log_message) - else: - logger.error(log_message) - else: - _warnings_showwarning( - message, category, filename, lineno, file, line, - ) + _original_showwarning( + message, category, filename, lineno, file, line, + ) def install_warning_logger(): # Enable our Deprecation Warnings warnings.simplefilter("default", PipDeprecationWarning, append=True) - global _warnings_showwarning + global _original_showwarning - if _warnings_showwarning is None: - _warnings_showwarning = warnings.showwarning + if _original_showwarning is None: + _original_showwarning = warnings.showwarning warnings.showwarning = _showwarning + + +def deprecated(reason, replacement, gone_in, issue=None): + # type: (str, Optional[str], Optional[str], Optional[int]) -> None + """Helper to deprecate existing functionality. + + reason: + Textual reason shown to the user about why this functionality has + been deprecated. + replacement: + Textual suggestion shown to the user about what alternative + functionality they can use. + gone_in: + The version of pip does this functionality should get removed in. + Raises errors if pip's current version is greater than or equal to + this. + issue: + Issue number on the tracker that would serve as a useful place for + users to find related discussion and provide feedback. + + Always pass replacement, gone_in and issue as keyword arguments for clarity + at the call site. + """ + + # Construct a nice message. + # This is purposely eagerly formatted as we want it to appear as if someone + # typed this entire message out. + message = "DEPRECATION: " + reason + if replacement is not None: + message += " A possible replacement is {}.".format(replacement) + if issue is not None: + url = "https://github.com/pypa/pip/issues/" + str(issue) + message += " You can find discussion regarding this at {}.".format(url) + + # Raise as an error if it has to be removed. + if gone_in is not None and parse(current_version) >= parse(gone_in): + raise PipDeprecationWarning(message) + warnings.warn(message, category=PipDeprecationWarning, stacklevel=2) diff --git a/src/pip/_internal/utils/logging.py b/src/pip/_internal/utils/logging.py index 0c1495dac..66c1d39d7 100644 --- a/src/pip/_internal/utils/logging.py +++ b/src/pip/_internal/utils/logging.py @@ -130,3 +130,96 @@ class MaxLevelFilter(logging.Filter): def filter(self, record): return record.levelno < self.level + + +def setup_logging(verbosity, no_color, user_log_file): + """Configures and sets up all of the logging + """ + + # Determine the level to be logging at. + if verbosity >= 1: + level = "DEBUG" + elif verbosity == -1: + level = "WARNING" + elif verbosity == -2: + level = "ERROR" + elif verbosity <= -3: + level = "CRITICAL" + else: + level = "INFO" + + # The "root" logger should match the "console" level *unless* we also need + # to log to a user log file. + include_user_log = user_log_file is not None + if include_user_log: + additional_log_file = user_log_file + root_level = "DEBUG" + else: + additional_log_file = "/dev/null" + root_level = level + + # Disable any logging besides WARNING unless we have DEBUG level logging + # enabled for vendored libraries. + vendored_log_level = "WARNING" if level in ["INFO", "ERROR"] else "DEBUG" + + # Shorthands for clarity + log_streams = { + "stdout": "ext://sys.stdout", + "stderr": "ext://sys.stderr", + } + handler_classes = { + "stream": "pip._internal.utils.logging.ColorizedStreamHandler", + "file": "pip._internal.utils.logging.BetterRotatingFileHandler", + } + + logging.config.dictConfig({ + "version": 1, + "disable_existing_loggers": False, + "filters": { + "exclude_warnings": { + "()": "pip._internal.utils.logging.MaxLevelFilter", + "level": logging.WARNING, + }, + }, + "formatters": { + "indent": { + "()": IndentingFormatter, + "format": "%(message)s", + }, + }, + "handlers": { + "console": { + "level": level, + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stdout"], + "filters": ["exclude_warnings"], + "formatter": "indent", + }, + "console_errors": { + "level": "WARNING", + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stderr"], + "formatter": "indent", + }, + "user_log": { + "level": "DEBUG", + "class": handler_classes["file"], + "filename": additional_log_file, + "delay": True, + "formatter": "indent", + }, + }, + "root": { + "level": root_level, + "handlers": ["console", "console_errors"] + ( + ["user_log"] if include_user_log else [] + ), + }, + "loggers": { + "pip._vendor": { + "level": vendored_log_level + } + }, + }) diff --git a/src/pip/_internal/utils/models.py b/src/pip/_internal/utils/models.py new file mode 100644 index 000000000..d5cb80a7c --- /dev/null +++ b/src/pip/_internal/utils/models.py @@ -0,0 +1,40 @@ +"""Utilities for defining models +""" + +import operator + + +class KeyBasedCompareMixin(object): + """Provides comparision capabilities that is based on a key + """ + + def __init__(self, key, defining_class): + self._compare_key = key + self._defining_class = defining_class + + def __hash__(self): + return hash(self._compare_key) + + def __lt__(self, other): + return self._compare(other, operator.__lt__) + + def __le__(self, other): + return self._compare(other, operator.__le__) + + def __gt__(self, other): + return self._compare(other, operator.__gt__) + + def __ge__(self, other): + return self._compare(other, operator.__ge__) + + def __eq__(self, other): + return self._compare(other, operator.__eq__) + + def __ne__(self, other): + return self._compare(other, operator.__ne__) + + def _compare(self, other, method): + if not isinstance(other, self._defining_class): + return NotImplemented + + return method(self._compare_key, other._compare_key) diff --git a/src/pip/_internal/utils/outdated.py b/src/pip/_internal/utils/outdated.py index 8c3b50b98..2b3fa95e3 100644 --- a/src/pip/_internal/utils/outdated.py +++ b/src/pip/_internal/utils/outdated.py @@ -58,14 +58,14 @@ class SelfCheckState(object): separators=(",", ":")) -def pip_installed_by_pip(): - """Checks whether pip was installed by pip +def was_installed_by_pip(pkg): + """Checks whether pkg was installed by pip This is used not to display the upgrade message when pip is in fact installed by system package manager, such as dnf on Fedora. """ try: - dist = pkg_resources.get_distribution('pip') + dist = pkg_resources.get_distribution(pkg) return (dist.has_metadata('INSTALLER') and 'pip' in dist.get_metadata_lines('INSTALLER')) except pkg_resources.DistributionNotFound: @@ -125,7 +125,7 @@ def pip_version_check(session, options): # Determine if our pypi_version is older if (pip_version < remote_version and pip_version.base_version != remote_version.base_version and - pip_installed_by_pip()): + was_installed_by_pip('pip')): # Advise "python -m pip" on Windows to avoid issues # with overwriting pip.exe. if WINDOWS: diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py index fd45fac93..768be3c97 100644 --- a/src/pip/_internal/vcs/__init__.py +++ b/src/pip/_internal/vcs/__init__.py @@ -200,12 +200,6 @@ class VersionControl(object): drive, tail = os.path.splitdrive(repo) return repo.startswith(os.path.sep) or drive - # See issue #1083 for why this method was introduced: - # https://github.com/pypa/pip/issues/1083 - def translate_egg_surname(self, surname): - # For example, Django has branches of the form "stable/1.7.x". - return surname.replace('/', '_') - def export(self, location): """ Export the repository at the url to the destination location @@ -213,32 +207,59 @@ class VersionControl(object): """ raise NotImplementedError - def get_url_rev(self): + def get_netloc_and_auth(self, netloc): """ - Returns the correct repository URL and revision by parsing the given - repository URL + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + def get_url_rev_and_auth(self, url): + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). """ error_message = ( "Sorry, '%s' is a malformed VCS url. " "The format is +://, " "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp" ) - assert '+' in self.url, error_message % self.url - url = self.url.split('+', 1)[1] + assert '+' in url, error_message % url + url = url.split('+', 1)[1] scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + netloc, user_pass = self.get_netloc_and_auth(netloc) rev = None if '@' in path: path, rev = path.rsplit('@', 1) url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev + return url, rev, user_pass - def get_info(self, location): + def make_rev_args(self, username, password): """ - Returns (url, revision), where both are strings + Return the RevOptions "extra arguments" to use in obtain(). """ - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - return self.get_url(location), self.get_revision(location) + return [] + + def get_url_rev_options(self, url): + """ + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). + """ + url, rev, user_pass = self.get_url_rev_and_auth(url) + username, password = user_pass + extra_args = self.make_rev_args(username, password) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return url, rev_options def normalize_url(self, url): """ @@ -253,10 +274,14 @@ class VersionControl(object): """ return (self.normalize_url(url1) == self.normalize_url(url2)) - def obtain(self, dest): + def fetch_new(self, dest, url, rev_options): """ - Called when installing or updating an editable package, takes the - source path of the checkout. + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. """ raise NotImplementedError @@ -288,21 +313,22 @@ class VersionControl(object): """ raise NotImplementedError - def check_destination(self, dest, url, rev_options): + def obtain(self, dest): """ - Prepare a location to receive a checkout/clone. - - Return True if the location is ready for (and requires) a - checkout/clone, False otherwise. + Install or update in editable mode the package represented by this + VersionControl object. Args: - rev_options: a RevOptions object. + dest: the repository directory in which to install or update. """ + url, rev_options = self.get_url_rev_options(self.url) + if not os.path.exists(dest): - return True + self.fetch_new(dest, url, rev_options) + return rev_display = rev_options.to_display() - if os.path.exists(os.path.join(dest, self.dirname)): + if self.is_repository_directory(dest): existing_url = self.get_url(dest) if self.compare_urls(existing_url, url): logger.debug( @@ -321,7 +347,7 @@ class VersionControl(object): self.update(dest, rev_options) else: logger.info('Skipping because already up-to-date.') - return False + return logger.warning( '%s %s in %s exists with URL %s', @@ -348,7 +374,25 @@ class VersionControl(object): ) response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) - checkout = False + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + self.fetch_new(dest, url, rev_options) + return + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + self.fetch_new(dest, url, rev_options) + return + + # Do nothing if the response is "i". if response == 's': logger.info( 'Switching %s %s to %s%s', @@ -358,24 +402,6 @@ class VersionControl(object): rev_display, ) self.switch(dest, url, rev_options) - elif response == 'i': - # do nothing - pass - elif response == 'w': - logger.warning('Deleting %s', display_path(dest)) - rmtree(dest) - checkout = True - elif response == 'b': - dest_dir = backup_dir(dest) - logger.warning( - 'Backing up %s to %s', display_path(dest), dest_dir, - ) - shutil.move(dest, dest_dir) - checkout = True - elif response == 'a': - sys.exit(-1) - - return checkout def unpack(self, location): """ @@ -398,7 +424,6 @@ class VersionControl(object): def get_url(self, location): """ Return the url used at location - Used in get_info or check_destination """ raise NotImplementedError @@ -435,17 +460,26 @@ class VersionControl(object): else: raise # re-raise exception if a different error occurred + @classmethod + def is_repository_directory(cls, path): + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + @classmethod def controls_location(cls, location): """ Check if a location is controlled by the vcs. It is meant to be overridden to implement smarter detection mechanisms for specific vcs. + + This can do more than is_repository_directory() alone. For example, + the Git override checks that Git is actually available. """ - logger.debug('Checking in %s for %s (%s)...', - location, cls.dirname, cls.name) - path = os.path.join(location, cls.dirname) - return os.path.exists(path) + return cls.is_repository_directory(location) def get_src_requirement(dist, location): diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py index b4e46e0e1..d83812000 100644 --- a/src/pip/_internal/vcs/bazaar.py +++ b/src/pip/_internal/vcs/bazaar.py @@ -48,6 +48,17 @@ class Bazaar(VersionControl): cwd=temp_dir.path, show_stdout=False, ) + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = ['branch', '-q'] + rev_options.to_args() + [url, dest] + self.run_command(cmd_args) + def switch(self, dest, url, rev_options): self.run_command(['switch', url], cwd=dest) @@ -55,26 +66,12 @@ class Bazaar(VersionControl): cmd_args = ['pull', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = self.make_rev_options(rev) - if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Checking out %s%s to %s', - url, - rev_display, - display_path(dest), - ) - cmd_args = ['branch', '-q'] + rev_options.to_args() + [url, dest] - self.run_command(cmd_args) - - def get_url_rev(self): + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev = super(Bazaar, self).get_url_rev() + url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url - return url, rev + return url, rev, user_pass def get_url(self, location): urls = self.run_command(['info'], show_stdout=False, cwd=location) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 7ef82d8ac..5376caa32 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -155,6 +155,33 @@ class Git(VersionControl): return self.get_revision(dest) == name + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Cloning %s%s to %s', url, rev_display, display_path(dest), + ) + self.run_command(['clone', '-q', url, dest]) + + if rev_options.rev: + # Then a specific revision was requested. + rev_options = self.check_rev_options(dest, rev_options) + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + rev = rev_options.rev + # Only fetch the revision if it's a ref + if rev.startswith('refs/'): + self.run_command( + ['fetch', '-q', url] + rev_options.to_args(), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + rev = 'FETCH_HEAD' + self.run_command(['checkout', '-q', rev], cwd=dest) + + #: repo may contain submodules + self.update_submodules(dest) + def switch(self, dest, url, rev_options): self.run_command(['config', 'remote.origin.url', url], cwd=dest) cmd_args = ['checkout', '-q'] + rev_options.to_args() @@ -176,35 +203,6 @@ class Git(VersionControl): #: update submodules self.update_submodules(dest) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = self.make_rev_options(rev) - if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Cloning %s%s to %s', url, rev_display, display_path(dest), - ) - self.run_command(['clone', '-q', url, dest]) - - if rev: - rev_options = self.check_rev_options(dest, rev_options) - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - rev = rev_options.rev - # Only fetch the revision if it's a ref - if rev.startswith('refs/'): - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - rev = 'FETCH_HEAD' - self.run_command(['checkout', '-q', rev], cwd=dest) - - #: repo may contain submodules - self.update_submodules(dest) - def get_url(self, location): """Return URL of the first remote encountered.""" remotes = self.run_command( @@ -267,22 +265,22 @@ class Git(VersionControl): req += '&subdirectory=' + subdirectory return req - def get_url_rev(self): + def get_url_rev_and_auth(self, url): """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't work with a ssh:// scheme (e.g. GitHub). But we need a scheme for parsing. Hence we remove it again afterwards and return it as a stub. """ - if '://' not in self.url: - assert 'file:' not in self.url - self.url = self.url.replace('git+', 'git+ssh://') - url, rev = super(Git, self).get_url_rev() + if '://' not in url: + assert 'file:' not in url + url = url.replace('git+', 'git+ssh://') + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev = super(Git, self).get_url_rev() + url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) - return url, rev + return url, rev, user_pass def update_submodules(self, location): if not os.path.exists(os.path.join(location, '.gitmodules')): diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 52a1cce54..2d0750cbb 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -31,6 +31,18 @@ class Mercurial(VersionControl): ['archive', location], show_stdout=False, cwd=temp_dir.path ) + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Cloning hg %s%s to %s', + url, + rev_display, + display_path(dest), + ) + self.run_command(['clone', '--noupdate', '-q', url, dest]) + cmd_args = ['update', '-q'] + rev_options.to_args() + self.run_command(cmd_args, cwd=dest) + def switch(self, dest, url, rev_options): repo_config = os.path.join(dest, self.dirname, 'hgrc') config = configparser.SafeConfigParser() @@ -52,21 +64,6 @@ class Mercurial(VersionControl): cmd_args = ['update', '-q'] + rev_options.to_args() self.run_command(cmd_args, cwd=dest) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = self.make_rev_options(rev) - if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Cloning hg %s%s to %s', - url, - rev_display, - display_path(dest), - ) - self.run_command(['clone', '--noupdate', '-q', url, dest]) - cmd_args = ['update', '-q'] + rev_options.to_args() - self.run_command(cmd_args, cwd=dest) - def get_url(self, location): url = self.run_command( ['showconfig', 'paths.default'], diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py index de448c7c6..a5aced280 100644 --- a/src/pip/_internal/vcs/subversion.py +++ b/src/pip/_internal/vcs/subversion.py @@ -4,17 +4,13 @@ import logging import os import re -from pip._vendor.six.moves.urllib import parse as urllib_parse - -from pip._internal.index import Link +from pip._internal.models.link import Link from pip._internal.utils.logging import indent_log -from pip._internal.utils.misc import display_path, remove_auth_from_url, rmtree +from pip._internal.utils.misc import display_path, rmtree from pip._internal.vcs import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') -_svn_url_re = re.compile(r'URL: (.+)') -_svn_revision_re = re.compile(r'Revision: (.+)') _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'(.*)') @@ -31,39 +27,10 @@ class Subversion(VersionControl): def get_base_rev_args(self, rev): return ['-r', rev] - def get_info(self, location): - """Returns (url, revision), where both are strings""" - assert not location.rstrip('/').endswith(self.dirname), \ - 'Bad directory: %s' % location - output = self.run_command( - ['info', location], - show_stdout=False, - extra_environ={'LANG': 'C'}, - ) - match = _svn_url_re.search(output) - if not match: - logger.warning( - 'Cannot determine URL of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return None, None - url = match.group(1).strip() - match = _svn_revision_re.search(output) - if not match: - logger.warning( - 'Cannot determine revision of svn checkout %s', - display_path(location), - ) - logger.debug('Output that cannot be parsed: \n%s', output) - return url, None - return url, match.group(1) - def export(self, location): """Export the svn repository at the url to the destination location""" - url, rev = self.get_url_rev() - rev_options = get_rev_options(self, url, rev) - url = remove_auth_from_url(url) + url, rev_options = self.get_url_rev_options(self.url) + logger.info('Exporting svn repository %s to %s', url, location) with indent_log(): if os.path.exists(location): @@ -73,6 +40,17 @@ class Subversion(VersionControl): cmd_args = ['export'] + rev_options.to_args() + [url, location] self.run_command(cmd_args, show_stdout=False) + def fetch_new(self, dest, url, rev_options): + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = ['checkout', '-q'] + rev_options.to_args() + [url, dest] + self.run_command(cmd_args) + def switch(self, dest, url, rev_options): cmd_args = ['switch'] + rev_options.to_args() + [url, dest] self.run_command(cmd_args) @@ -81,21 +59,6 @@ class Subversion(VersionControl): cmd_args = ['update'] + rev_options.to_args() + [dest] self.run_command(cmd_args) - def obtain(self, dest): - url, rev = self.get_url_rev() - rev_options = get_rev_options(self, url, rev) - url = remove_auth_from_url(url) - if self.check_destination(dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Checking out %s%s to %s', - url, - rev_display, - display_path(dest), - ) - cmd_args = ['checkout', '-q'] + rev_options.to_args() + [url, dest] - self.run_command(cmd_args) - def get_location(self, dist, dependency_links): for url in dependency_links: egg_fragment = Link(url).egg_fragment @@ -137,12 +100,45 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def get_url_rev(self): + def get_netloc_and_auth(self, netloc): + """ + Parse out and remove from the netloc the auth information. + + This allows the auth information to be provided via the --username + and --password options instead of via the URL. + """ + if '@' not in netloc: + return netloc, (None, None) + + # Split from the right because that's how urllib.parse.urlsplit() + # behaves if more than one @ is present (by checking the password + # attribute of urlsplit()'s return value). + auth, netloc = netloc.rsplit('@', 1) + if ':' in auth: + # Split from the left because that's how urllib.parse.urlsplit() + # behaves if more than one : is present (again by checking the + # password attribute of the return value) + user_pass = tuple(auth.split(':', 1)) + else: + user_pass = auth, None + + return netloc, user_pass + + def get_url_rev_and_auth(self, url): # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev = super(Subversion, self).get_url_rev() + url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url - return url, rev + return url, rev, user_pass + + def make_rev_args(self, username, password): + extra_args = [] + if username: + extra_args += ['--username', username] + if password: + extra_args += ['--password', password] + + return extra_args def get_url(self, location): # In cases where the source is in a subdirectory, not alongside @@ -222,32 +218,4 @@ class Subversion(VersionControl): return False -def get_rev_options(vcs, url, rev): - """ - Return a RevOptions object. - """ - r = urllib_parse.urlsplit(url) - if hasattr(r, 'username'): - # >= Python-2.5 - username, password = r.username, r.password - else: - netloc = r[1] - if '@' in netloc: - auth = netloc.split('@')[0] - if ':' in auth: - username, password = auth.split(':', 1) - else: - username, password = auth, None - else: - username, password = None, None - - extra_args = [] - if username: - extra_args += ['--username', username] - if password: - extra_args += ['--password', password] - - return vcs.make_rev_options(rev, extra_args=extra_args) - - vcs.register(Subversion) diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py index 667385976..60a0f295f 100644 --- a/src/pip/_internal/wheel.py +++ b/src/pip/_internal/wheel.py @@ -163,7 +163,7 @@ def message_about_scripts_not_on_PATH(scripts): # We don't want to warn for directories that are on PATH. not_warn_dirs = [ os.path.normcase(i).rstrip(os.sep) for i in - os.environ["PATH"].split(os.pathsep) + os.environ.get("PATH", "").split(os.pathsep) ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. @@ -506,8 +506,8 @@ if __name__ == '__main__': row[1], row[2] = rehash(row[0]) writer.writerow(row) for f in generated: - h, l = rehash(f) - writer.writerow((normpath(f, lib_dir), h, l)) + digest, length = rehash(f) + writer.writerow((normpath(f, lib_dir), digest, length)) for f in installed: writer.writerow((installed[f], '', '')) shutil.move(temp_record, record) @@ -528,7 +528,7 @@ def wheel_version(source_dir): version = wheel_data['Wheel-Version'].strip() version = tuple(map(int, version.split('.'))) return version - except: + except Exception: return False @@ -653,7 +653,7 @@ class WheelBuilder(object): ) logger.info('Stored in directory: %s', output_dir) return wheel_path - except: + except Exception: pass # Ignore return, we can't do anything else useful. self._clean_one(req) @@ -685,7 +685,7 @@ class WheelBuilder(object): call_subprocess(wheel_args, cwd=req.setup_py_dir, show_stdout=False, spinner=spinner) return True - except: + except Exception: spinner.finish("error") logger.error('Failed building wheel for %s', req.name) return False @@ -698,7 +698,7 @@ class WheelBuilder(object): try: call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False) return True - except: + except Exception: logger.error('Failed cleaning build dir for %s', req.name) return False @@ -710,6 +710,7 @@ class WheelBuilder(object): :return: True if all the wheels built correctly. """ from pip._internal import index + from pip._internal.models.link import Link building_is_possible = self._wheel_dir or ( autobuilding and self.wheel_cache.cache_dir @@ -802,7 +803,7 @@ class WheelBuilder(object): self.preparer.build_dir ) # Update the link for this. - req.link = index.Link(path_to_url(wheel_file)) + req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir unpack_url( diff --git a/src/pip/_vendor/README.rst b/src/pip/_vendor/README.rst index 0b0d27056..cf0ebddba 100644 --- a/src/pip/_vendor/README.rst +++ b/src/pip/_vendor/README.rst @@ -22,9 +22,9 @@ Policy Rationale --------- -Historically pip has not had any dependencies except for setuptools itself, +Historically pip has not had any dependencies except for ``setuptools`` itself, choosing instead to implement any functionality it needed to prevent needing -a dependency. However, starting with pip 1.5 we began to replace code that was +a dependency. However, starting with pip 1.5, we began to replace code that was implemented inside of pip with reusable libraries from PyPI. This brought the typical benefits of reusing libraries instead of reinventing the wheel like higher quality and more battle tested code, centralization of bug fixes @@ -43,7 +43,7 @@ way (via ``install_requires``) for pip. These issues are: * **Making other libraries uninstallable.** One of pip's current dependencies is the ``requests`` library, for which pip requires a fairly recent version to run. - If pip dependended on ``requests`` in the traditional manner, then we'd either + If pip depended on ``requests`` in the traditional manner, then we'd either have to maintain compatibility with every ``requests`` version that has ever existed (and ever will), OR allow pip to render certain versions of ``requests`` uninstallable. (The second issue, although technically true for any Python @@ -117,7 +117,7 @@ Debundling As mentioned in the rationale, we, the pip team, would prefer it if pip was not debundled (other than optionally ``pip/_vendor/requests/cacert.pem``) and that pip was left intact. However, if you insist on doing so, we have a -semi-supported method that we do test in our CI, but requires a bit of +semi-supported method (that we don't test in our CI) and requires a bit of extra work on your end in order to solve the problems described above. 1. Delete everything in ``pip/_vendor/`` **except** for @@ -131,6 +131,14 @@ extra work on your end in order to solve the problems described above. 3. Modify ``pip/_vendor/__init__.py`` so that the ``DEBUNDLED`` variable is ``True``. -4. *(Optional)* If you've placed the wheels in a location other than +4. Upon installation, the ``INSTALLER`` file in pip's own ``dist-info`` + directory should be set to something other than ``pip``, so that pip + can detect that it wasn't installed using itself. + +5. *(optional)* If you've placed the wheels in a location other than ``pip/_vendor/``, then modify ``pip/_vendor/__init__.py`` so that the ``WHEEL_DIR`` variable points to the location you've placed them. + +6. *(optional)* Update the ``pip_version_check`` logic to use the + appropriate logic for determining the latest available version of pip and + prompt the user with the correct upgrade message. diff --git a/src/pip/_vendor/__init__.py b/src/pip/_vendor/__init__.py index 1387dba3e..a0aae81ae 100644 --- a/src/pip/_vendor/__init__.py +++ b/src/pip/_vendor/__init__.py @@ -107,3 +107,4 @@ if DEBUNDLED: vendored("requests.packages.urllib3.util.ssl_") vendored("requests.packages.urllib3.util.timeout") vendored("requests.packages.urllib3.util.url") + vendored("urllib3") diff --git a/src/pip/_vendor/cachecontrol/__init__.py b/src/pip/_vendor/cachecontrol/__init__.py index f386d4929..8fdee66ff 100644 --- a/src/pip/_vendor/cachecontrol/__init__.py +++ b/src/pip/_vendor/cachecontrol/__init__.py @@ -2,9 +2,9 @@ Make it easy to import from cachecontrol without long namespaces. """ -__author__ = 'Eric Larson' -__email__ = 'eric@ionrock.org' -__version__ = '0.12.4' +__author__ = "Eric Larson" +__email__ = "eric@ionrock.org" +__version__ = "0.12.5" from .wrapper import CacheControl from .adapter import CacheControlAdapter diff --git a/src/pip/_vendor/cachecontrol/_cmd.py b/src/pip/_vendor/cachecontrol/_cmd.py index afdcc88c2..f1e0ad94a 100644 --- a/src/pip/_vendor/cachecontrol/_cmd.py +++ b/src/pip/_vendor/cachecontrol/_cmd.py @@ -17,14 +17,11 @@ def setup_logging(): def get_session(): adapter = CacheControlAdapter( - DictCache(), - cache_etags=True, - serializer=None, - heuristic=None, + DictCache(), cache_etags=True, serializer=None, heuristic=None ) sess = requests.Session() - sess.mount('http://', adapter) - sess.mount('https://', adapter) + sess.mount("http://", adapter) + sess.mount("https://", adapter) sess.cache_controller = adapter.controller return sess @@ -32,7 +29,7 @@ def get_session(): def get_args(): parser = ArgumentParser() - parser.add_argument('url', help='The URL to try and cache') + parser.add_argument("url", help="The URL to try and cache") return parser.parse_args() @@ -51,10 +48,10 @@ def main(args=None): # Now try to get it if sess.cache_controller.cached_request(resp.request): - print('Cached!') + print("Cached!") else: - print('Not cached :(') + print("Not cached :(") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/src/pip/_vendor/cachecontrol/adapter.py b/src/pip/_vendor/cachecontrol/adapter.py index ecb34a6db..780eb2883 100644 --- a/src/pip/_vendor/cachecontrol/adapter.py +++ b/src/pip/_vendor/cachecontrol/adapter.py @@ -10,25 +10,27 @@ from .filewrapper import CallbackFileWrapper class CacheControlAdapter(HTTPAdapter): - invalidating_methods = set(['PUT', 'DELETE']) + invalidating_methods = {"PUT", "DELETE"} - def __init__(self, cache=None, - cache_etags=True, - controller_class=None, - serializer=None, - heuristic=None, - cacheable_methods=None, - *args, **kw): + def __init__( + self, + cache=None, + cache_etags=True, + controller_class=None, + serializer=None, + heuristic=None, + cacheable_methods=None, + *args, + **kw + ): super(CacheControlAdapter, self).__init__(*args, **kw) self.cache = cache or DictCache() self.heuristic = heuristic - self.cacheable_methods = cacheable_methods or ('GET',) + self.cacheable_methods = cacheable_methods or ("GET",) controller_factory = controller_class or CacheController self.controller = controller_factory( - self.cache, - cache_etags=cache_etags, - serializer=serializer, + self.cache, cache_etags=cache_etags, serializer=serializer ) def send(self, request, cacheable_methods=None, **kw): @@ -43,20 +45,18 @@ class CacheControlAdapter(HTTPAdapter): except zlib.error: cached_response = None if cached_response: - return self.build_response(request, cached_response, - from_cache=True) + return self.build_response(request, cached_response, from_cache=True) # check for etags and add headers if appropriate - request.headers.update( - self.controller.conditional_headers(request) - ) + request.headers.update(self.controller.conditional_headers(request)) resp = super(CacheControlAdapter, self).send(request, **kw) return resp - def build_response(self, request, response, from_cache=False, - cacheable_methods=None): + def build_response( + self, request, response, from_cache=False, cacheable_methods=None + ): """ Build a response by making a request or using the cache. @@ -101,10 +101,8 @@ class CacheControlAdapter(HTTPAdapter): response._fp = CallbackFileWrapper( response._fp, functools.partial( - self.controller.cache_response, - request, - response, - ) + self.controller.cache_response, request, response + ), ) if response.chunked: super_update_chunk_length = response._update_chunk_length @@ -113,11 +111,12 @@ class CacheControlAdapter(HTTPAdapter): super_update_chunk_length() if self.chunk_left == 0: self._fp._close() - response._update_chunk_length = types.MethodType(_update_chunk_length, response) - resp = super(CacheControlAdapter, self).build_response( - request, response - ) + response._update_chunk_length = types.MethodType( + _update_chunk_length, response + ) + + resp = super(CacheControlAdapter, self).build_response(request, response) # See if we should invalidate the cache. if request.method in self.invalidating_methods and resp.ok: diff --git a/src/pip/_vendor/cachecontrol/cache.py b/src/pip/_vendor/cachecontrol/cache.py index 7389a73f8..94e07732d 100644 --- a/src/pip/_vendor/cachecontrol/cache.py +++ b/src/pip/_vendor/cachecontrol/cache.py @@ -8,13 +8,13 @@ from threading import Lock class BaseCache(object): def get(self, key): - raise NotImplemented() + raise NotImplementedError() def set(self, key, value): - raise NotImplemented() + raise NotImplementedError() def delete(self, key): - raise NotImplemented() + raise NotImplementedError() def close(self): pass diff --git a/src/pip/_vendor/cachecontrol/caches/file_cache.py b/src/pip/_vendor/cachecontrol/caches/file_cache.py index 885c8a653..1ba00806c 100644 --- a/src/pip/_vendor/cachecontrol/caches/file_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/file_cache.py @@ -9,7 +9,7 @@ try: FileNotFoundError except NameError: # py2.X - FileNotFoundError = OSError + FileNotFoundError = (IOError, OSError) def _secure_open_write(filename, fmode): @@ -46,6 +46,7 @@ def _secure_open_write(filename, fmode): fd = os.open(filename, flags, fmode) try: return os.fdopen(fd, "wb") + except: # An error occurred wrapping our FD in a file object os.close(fd) @@ -53,8 +54,16 @@ def _secure_open_write(filename, fmode): class FileCache(BaseCache): - def __init__(self, directory, forever=False, filemode=0o0600, - dirmode=0o0700, use_dir_lock=None, lock_class=None): + + def __init__( + self, + directory, + forever=False, + filemode=0o0600, + dirmode=0o0700, + use_dir_lock=None, + lock_class=None, + ): if use_dir_lock is not None and lock_class is not None: raise ValueError("Cannot use use_dir_lock and lock_class together") @@ -63,12 +72,15 @@ class FileCache(BaseCache): from pip._vendor.lockfile import LockFile from pip._vendor.lockfile.mkdirlockfile import MkdirLockFile except ImportError: - notice = dedent(""" + notice = dedent( + """ NOTE: In order to use the FileCache you must have lockfile installed. You can install it via pip: pip install lockfile - """) + """ + ) raise ImportError(notice) + else: if use_dir_lock: lock_class = MkdirLockFile @@ -95,11 +107,12 @@ class FileCache(BaseCache): def get(self, key): name = self._fn(key) - if not os.path.exists(name): - return None + try: + with open(name, "rb") as fh: + return fh.read() - with open(name, 'rb') as fh: - return fh.read() + except FileNotFoundError: + return None def set(self, key, value): name = self._fn(key) diff --git a/src/pip/_vendor/cachecontrol/caches/redis_cache.py b/src/pip/_vendor/cachecontrol/caches/redis_cache.py index b6285e9f1..ed705ce7d 100644 --- a/src/pip/_vendor/cachecontrol/caches/redis_cache.py +++ b/src/pip/_vendor/cachecontrol/caches/redis_cache.py @@ -4,16 +4,6 @@ from datetime import datetime from pip._vendor.cachecontrol.cache import BaseCache -def total_seconds(td): - """Python 2.6 compatability""" - if hasattr(td, 'total_seconds'): - return int(td.total_seconds()) - - ms = td.microseconds - secs = (td.seconds + td.days * 24 * 3600) - return int((ms + secs * 10**6) / 10**6) - - class RedisCache(BaseCache): def __init__(self, conn): @@ -27,7 +17,7 @@ class RedisCache(BaseCache): self.conn.set(key, value) else: expires = expires - datetime.utcnow() - self.conn.setex(key, total_seconds(expires), value) + self.conn.setex(key, int(expires.total_seconds()), value) def delete(self, key): self.conn.delete(key) diff --git a/src/pip/_vendor/cachecontrol/controller.py b/src/pip/_vendor/cachecontrol/controller.py index 0e2eb3c4f..1b2b943cb 100644 --- a/src/pip/_vendor/cachecontrol/controller.py +++ b/src/pip/_vendor/cachecontrol/controller.py @@ -30,8 +30,10 @@ def parse_uri(uri): class CacheController(object): """An interface to see if request should cached or not. """ - def __init__(self, cache=None, cache_etags=True, serializer=None, - status_codes=None): + + def __init__( + self, cache=None, cache_etags=True, serializer=None, status_codes=None + ): self.cache = cache or DictCache() self.cache_etags = cache_etags self.serializer = serializer or Serializer() @@ -64,34 +66,35 @@ class CacheController(object): def parse_cache_control(self, headers): known_directives = { # https://tools.ietf.org/html/rfc7234#section-5.2 - 'max-age': (int, True,), - 'max-stale': (int, False,), - 'min-fresh': (int, True,), - 'no-cache': (None, False,), - 'no-store': (None, False,), - 'no-transform': (None, False,), - 'only-if-cached' : (None, False,), - 'must-revalidate': (None, False,), - 'public': (None, False,), - 'private': (None, False,), - 'proxy-revalidate': (None, False,), - 's-maxage': (int, True,) + "max-age": (int, True), + "max-stale": (int, False), + "min-fresh": (int, True), + "no-cache": (None, False), + "no-store": (None, False), + "no-transform": (None, False), + "only-if-cached": (None, False), + "must-revalidate": (None, False), + "public": (None, False), + "private": (None, False), + "proxy-revalidate": (None, False), + "s-maxage": (int, True), } - cc_headers = headers.get('cache-control', - headers.get('Cache-Control', '')) + cc_headers = headers.get("cache-control", headers.get("Cache-Control", "")) retval = {} - for cc_directive in cc_headers.split(','): - parts = cc_directive.split('=', 1) + for cc_directive in cc_headers.split(","): + if not cc_directive.strip(): + continue + + parts = cc_directive.split("=", 1) directive = parts[0].strip() try: typ, required = known_directives[directive] except KeyError: - logger.debug('Ignoring unknown cache-control directive: %s', - directive) + logger.debug("Ignoring unknown cache-control directive: %s", directive) continue if not typ or not required: @@ -101,11 +104,16 @@ class CacheController(object): retval[directive] = typ(parts[1].strip()) except IndexError: if required: - logger.debug('Missing value for cache-control ' - 'directive: %s', directive) + logger.debug( + "Missing value for cache-control " "directive: %s", + directive, + ) except ValueError: - logger.debug('Invalid value for cache-control directive ' - '%s, must be %s', directive, typ.__name__) + logger.debug( + "Invalid value for cache-control directive " "%s, must be %s", + directive, + typ.__name__, + ) return retval @@ -119,24 +127,24 @@ class CacheController(object): cc = self.parse_cache_control(request.headers) # Bail out if the request insists on fresh data - if 'no-cache' in cc: + if "no-cache" in cc: logger.debug('Request header has "no-cache", cache bypassed') return False - if 'max-age' in cc and cc['max-age'] == 0: + if "max-age" in cc and cc["max-age"] == 0: logger.debug('Request header has "max_age" as 0, cache bypassed') return False # Request allows serving from the cache, let's see if we find something cache_data = self.cache.get(cache_url) if cache_data is None: - logger.debug('No cache entry available') + logger.debug("No cache entry available") return False # Check whether it can be deserialized resp = self.serializer.loads(request, cache_data) if not resp: - logger.warning('Cache entry deserialization failed, entry ignored') + logger.warning("Cache entry deserialization failed, entry ignored") return False # If we have a cached 301, return it immediately. We don't @@ -148,27 +156,27 @@ class CacheController(object): # Client can try to refresh the value by repeating the request # with cache busting headers as usual (ie no-cache). if resp.status == 301: - msg = ('Returning cached "301 Moved Permanently" response ' - '(ignoring date and etag information)') + msg = ( + 'Returning cached "301 Moved Permanently" response ' + "(ignoring date and etag information)" + ) logger.debug(msg) return resp headers = CaseInsensitiveDict(resp.headers) - if not headers or 'date' not in headers: - if 'etag' not in headers: + if not headers or "date" not in headers: + if "etag" not in headers: # Without date or etag, the cached response can never be used # and should be deleted. - logger.debug('Purging cached response: no date or etag') + logger.debug("Purging cached response: no date or etag") self.cache.delete(cache_url) - logger.debug('Ignoring cached response: no date') + logger.debug("Ignoring cached response: no date") return False now = time.time() - date = calendar.timegm( - parsedate_tz(headers['date']) - ) + date = calendar.timegm(parsedate_tz(headers["date"])) current_age = max(0, now - date) - logger.debug('Current age based on date: %i', current_age) + logger.debug("Current age based on date: %i", current_age) # TODO: There is an assumption that the result will be a # urllib3 response object. This may not be best since we @@ -180,45 +188,41 @@ class CacheController(object): freshness_lifetime = 0 # Check the max-age pragma in the cache control header - if 'max-age' in resp_cc: - freshness_lifetime = resp_cc['max-age'] - logger.debug('Freshness lifetime from max-age: %i', - freshness_lifetime) + if "max-age" in resp_cc: + freshness_lifetime = resp_cc["max-age"] + logger.debug("Freshness lifetime from max-age: %i", freshness_lifetime) # If there isn't a max-age, check for an expires header - elif 'expires' in headers: - expires = parsedate_tz(headers['expires']) + elif "expires" in headers: + expires = parsedate_tz(headers["expires"]) if expires is not None: expire_time = calendar.timegm(expires) - date freshness_lifetime = max(0, expire_time) - logger.debug("Freshness lifetime from expires: %i", - freshness_lifetime) + logger.debug("Freshness lifetime from expires: %i", freshness_lifetime) # Determine if we are setting freshness limit in the # request. Note, this overrides what was in the response. - if 'max-age' in cc: - freshness_lifetime = cc['max-age'] - logger.debug('Freshness lifetime from request max-age: %i', - freshness_lifetime) + if "max-age" in cc: + freshness_lifetime = cc["max-age"] + logger.debug( + "Freshness lifetime from request max-age: %i", freshness_lifetime + ) - if 'min-fresh' in cc: - min_fresh = cc['min-fresh'] + if "min-fresh" in cc: + min_fresh = cc["min-fresh"] # adjust our current age by our min fresh current_age += min_fresh - logger.debug('Adjusted current age from min-fresh: %i', - current_age) + logger.debug("Adjusted current age from min-fresh: %i", current_age) # Return entry if it is fresh enough if freshness_lifetime > current_age: logger.debug('The response is "fresh", returning cached response') - logger.debug('%i > %i', freshness_lifetime, current_age) + logger.debug("%i > %i", freshness_lifetime, current_age) return resp # we're not fresh. If we don't have an Etag, clear it out - if 'etag' not in headers: - logger.debug( - 'The cached response is "stale" with no etag, purging' - ) + if "etag" not in headers: + logger.debug('The cached response is "stale" with no etag, purging') self.cache.delete(cache_url) # return the original handler @@ -232,16 +236,15 @@ class CacheController(object): if resp: headers = CaseInsensitiveDict(resp.headers) - if 'etag' in headers: - new_headers['If-None-Match'] = headers['ETag'] + if "etag" in headers: + new_headers["If-None-Match"] = headers["ETag"] - if 'last-modified' in headers: - new_headers['If-Modified-Since'] = headers['Last-Modified'] + if "last-modified" in headers: + new_headers["If-Modified-Since"] = headers["Last-Modified"] return new_headers - def cache_response(self, request, response, body=None, - status_codes=None): + def cache_response(self, request, response, body=None, status_codes=None): """ Algorithm for caching requests. @@ -252,9 +255,7 @@ class CacheController(object): cacheable_status_codes = status_codes or self.cacheable_status_codes if response.status not in cacheable_status_codes: logger.debug( - 'Status code %s not in %s', - response.status, - cacheable_status_codes + "Status code %s not in %s", response.status, cacheable_status_codes ) return @@ -264,10 +265,12 @@ class CacheController(object): # Content-Length is valid then we can check to see if the body we've # been given matches the expected size, and if it doesn't we'll just # skip trying to cache it. - if (body is not None and - "content-length" in response_headers and - response_headers["content-length"].isdigit() and - int(response_headers["content-length"]) != len(body)): + if ( + body is not None + and "content-length" in response_headers + and response_headers["content-length"].isdigit() + and int(response_headers["content-length"]) != len(body) + ): return cc_req = self.parse_cache_control(request.headers) @@ -278,53 +281,49 @@ class CacheController(object): # Delete it from the cache if we happen to have it stored there no_store = False - if 'no-store' in cc: + if "no-store" in cc: no_store = True logger.debug('Response header has "no-store"') - if 'no-store' in cc_req: + if "no-store" in cc_req: no_store = True logger.debug('Request header has "no-store"') if no_store and self.cache.get(cache_url): logger.debug('Purging existing cache entry to honor "no-store"') self.cache.delete(cache_url) + if no_store: + return # If we've been given an etag, then keep the response - if self.cache_etags and 'etag' in response_headers: - logger.debug('Caching due to etag') + if self.cache_etags and "etag" in response_headers: + logger.debug("Caching due to etag") self.cache.set( - cache_url, - self.serializer.dumps(request, response, body=body), + cache_url, self.serializer.dumps(request, response, body=body) ) # Add to the cache any 301s. We do this before looking that # the Date headers. elif response.status == 301: - logger.debug('Caching permanant redirect') - self.cache.set( - cache_url, - self.serializer.dumps(request, response) - ) + logger.debug("Caching permanant redirect") + self.cache.set(cache_url, self.serializer.dumps(request, response)) # Add to the cache if the response headers demand it. If there # is no date header then we can't do anything about expiring # the cache. - elif 'date' in response_headers: + elif "date" in response_headers: # cache when there is a max-age > 0 - if 'max-age' in cc and cc['max-age'] > 0: - logger.debug('Caching b/c date exists and max-age > 0') + if "max-age" in cc and cc["max-age"] > 0: + logger.debug("Caching b/c date exists and max-age > 0") self.cache.set( - cache_url, - self.serializer.dumps(request, response, body=body), + cache_url, self.serializer.dumps(request, response, body=body) ) # If the request can expire, it means we should cache it # in the meantime. - elif 'expires' in response_headers: - if response_headers['expires']: - logger.debug('Caching b/c of expires header') + elif "expires" in response_headers: + if response_headers["expires"]: + logger.debug("Caching b/c of expires header") self.cache.set( - cache_url, - self.serializer.dumps(request, response, body=body), + cache_url, self.serializer.dumps(request, response, body=body) ) def update_cached_response(self, request, response): @@ -336,10 +335,7 @@ class CacheController(object): """ cache_url = self.cache_url(request.url) - cached_response = self.serializer.loads( - request, - self.cache.get(cache_url) - ) + cached_response = self.serializer.loads(request, self.cache.get(cache_url)) if not cached_response: # we didn't have a cached response @@ -352,22 +348,20 @@ class CacheController(object): # the cached body invalid. But... just in case, we'll be sure # to strip out ones we know that might be problmatic due to # typical assumptions. - excluded_headers = [ - "content-length", - ] + excluded_headers = ["content-length"] cached_response.headers.update( - dict((k, v) for k, v in response.headers.items() - if k.lower() not in excluded_headers) + dict( + (k, v) + for k, v in response.headers.items() + if k.lower() not in excluded_headers + ) ) # we want a 200 b/c we have content via the cache cached_response.status = 200 # update our cache - self.cache.set( - cache_url, - self.serializer.dumps(request, cached_response), - ) + self.cache.set(cache_url, self.serializer.dumps(request, cached_response)) return cached_response diff --git a/src/pip/_vendor/cachecontrol/filewrapper.py b/src/pip/_vendor/cachecontrol/filewrapper.py index f1e1ce055..30ed4c5a6 100644 --- a/src/pip/_vendor/cachecontrol/filewrapper.py +++ b/src/pip/_vendor/cachecontrol/filewrapper.py @@ -27,17 +27,19 @@ class CallbackFileWrapper(object): # self.__fp hasn't been set. # # [0] https://docs.python.org/2/reference/expressions.html#atom-identifiers - fp = self.__getattribute__('_CallbackFileWrapper__fp') + fp = self.__getattribute__("_CallbackFileWrapper__fp") return getattr(fp, name) def __is_fp_closed(self): try: return self.__fp.fp is None + except AttributeError: pass try: return self.__fp.closed + except AttributeError: pass @@ -66,7 +68,7 @@ class CallbackFileWrapper(object): def _safe_read(self, amt): data = self.__fp._safe_read(amt) - if amt == 2 and data == b'\r\n': + if amt == 2 and data == b"\r\n": # urllib executes this read to toss the CRLF at the end # of the chunk. return data diff --git a/src/pip/_vendor/cachecontrol/heuristics.py b/src/pip/_vendor/cachecontrol/heuristics.py index f182ff026..6c0e9790d 100644 --- a/src/pip/_vendor/cachecontrol/heuristics.py +++ b/src/pip/_vendor/cachecontrol/heuristics.py @@ -46,7 +46,7 @@ class BaseHeuristic(object): response.headers.update(updated_headers) warning_header_value = self.warning(response) if warning_header_value is not None: - response.headers.update({'Warning': warning_header_value}) + response.headers.update({"Warning": warning_header_value}) return response @@ -56,15 +56,15 @@ class OneDayCache(BaseHeuristic): Cache the response by providing an expires 1 day in the future. """ + def update_headers(self, response): headers = {} - if 'expires' not in response.headers: - date = parsedate(response.headers['date']) - expires = expire_after(timedelta(days=1), - date=datetime(*date[:6])) - headers['expires'] = datetime_to_header(expires) - headers['cache-control'] = 'public' + if "expires" not in response.headers: + date = parsedate(response.headers["date"]) + expires = expire_after(timedelta(days=1), date=datetime(*date[:6])) + headers["expires"] = datetime_to_header(expires) + headers["cache-control"] = "public" return headers @@ -78,13 +78,10 @@ class ExpiresAfter(BaseHeuristic): def update_headers(self, response): expires = expire_after(self.delta) - return { - 'expires': datetime_to_header(expires), - 'cache-control': 'public', - } + return {"expires": datetime_to_header(expires), "cache-control": "public"} def warning(self, response): - tmpl = '110 - Automatically cached for %s. Response might be stale' + tmpl = "110 - Automatically cached for %s. Response might be stale" return tmpl % self.delta @@ -100,27 +97,27 @@ class LastModified(BaseHeuristic): http://lxr.mozilla.org/mozilla-release/source/netwerk/protocol/http/nsHttpResponseHead.cpp#397 Unlike mozilla we limit this to 24-hr. """ - cacheable_by_default_statuses = set([ + cacheable_by_default_statuses = { 200, 203, 204, 206, 300, 301, 404, 405, 410, 414, 501 - ]) + } def update_headers(self, resp): headers = resp.headers - if 'expires' in headers: + if "expires" in headers: return {} - if 'cache-control' in headers and headers['cache-control'] != 'public': + if "cache-control" in headers and headers["cache-control"] != "public": return {} if resp.status not in self.cacheable_by_default_statuses: return {} - if 'date' not in headers or 'last-modified' not in headers: + if "date" not in headers or "last-modified" not in headers: return {} - date = calendar.timegm(parsedate_tz(headers['date'])) - last_modified = parsedate(headers['last-modified']) + date = calendar.timegm(parsedate_tz(headers["date"])) + last_modified = parsedate(headers["last-modified"]) if date is None or last_modified is None: return {} @@ -132,7 +129,7 @@ class LastModified(BaseHeuristic): return {} expires = date + freshness_lifetime - return {'expires': time.strftime(TIME_FMT, time.gmtime(expires))} + return {"expires": time.strftime(TIME_FMT, time.gmtime(expires))} def warning(self, resp): return None diff --git a/src/pip/_vendor/cachecontrol/serialize.py b/src/pip/_vendor/cachecontrol/serialize.py index 05b6e2474..ec43ff27a 100644 --- a/src/pip/_vendor/cachecontrol/serialize.py +++ b/src/pip/_vendor/cachecontrol/serialize.py @@ -48,23 +48,22 @@ class Serializer(object): u"response": { u"body": body, u"headers": dict( - (text_type(k), text_type(v)) - for k, v in response.headers.items() + (text_type(k), text_type(v)) for k, v in response.headers.items() ), u"status": response.status, u"version": response.version, u"reason": text_type(response.reason), u"strict": response.strict, u"decode_content": response.decode_content, - }, + } } # Construct our vary headers data[u"vary"] = {} if u"vary" in response_headers: - varied_headers = response_headers[u'vary'].split(',') + varied_headers = response_headers[u"vary"].split(",") for header in varied_headers: - header = header.strip() + header = text_type(header).strip() header_value = request.headers.get(header, None) if header_value is not None: header_value = text_type(header_value) @@ -95,7 +94,8 @@ class Serializer(object): # Dispatch to the actual load method for the given version try: - return getattr(self, "_loads_v{0}".format(ver))(request, data) + return getattr(self, "_loads_v{}".format(ver))(request, data) + except AttributeError: # This is a version we don't have a loads function for, so we'll # just treat it as a miss and return None @@ -118,11 +118,11 @@ class Serializer(object): body_raw = cached["response"].pop("body") - headers = CaseInsensitiveDict(data=cached['response']['headers']) - if headers.get('transfer-encoding', '') == 'chunked': - headers.pop('transfer-encoding') + headers = CaseInsensitiveDict(data=cached["response"]["headers"]) + if headers.get("transfer-encoding", "") == "chunked": + headers.pop("transfer-encoding") - cached['response']['headers'] = headers + cached["response"]["headers"] = headers try: body = io.BytesIO(body_raw) @@ -133,13 +133,9 @@ class Serializer(object): # fail with: # # TypeError: 'str' does not support the buffer interface - body = io.BytesIO(body_raw.encode('utf8')) + body = io.BytesIO(body_raw.encode("utf8")) - return HTTPResponse( - body=body, - preload_content=False, - **cached["response"] - ) + return HTTPResponse(body=body, preload_content=False, **cached["response"]) def _loads_v0(self, request, data): # The original legacy cache data. This doesn't contain enough @@ -162,16 +158,12 @@ class Serializer(object): return # We need to decode the items that we've base64 encoded - cached["response"]["body"] = _b64_decode_bytes( - cached["response"]["body"] - ) + cached["response"]["body"] = _b64_decode_bytes(cached["response"]["body"]) cached["response"]["headers"] = dict( (_b64_decode_str(k), _b64_decode_str(v)) for k, v in cached["response"]["headers"].items() ) - cached["response"]["reason"] = _b64_decode_str( - cached["response"]["reason"], - ) + cached["response"]["reason"] = _b64_decode_str(cached["response"]["reason"]) cached["vary"] = dict( (_b64_decode_str(k), _b64_decode_str(v) if v is not None else v) for k, v in cached["vary"].items() @@ -187,7 +179,7 @@ class Serializer(object): def _loads_v4(self, request, data): try: - cached = msgpack.loads(data, encoding='utf-8') + cached = msgpack.loads(data, encoding="utf-8") except ValueError: return diff --git a/src/pip/_vendor/cachecontrol/wrapper.py b/src/pip/_vendor/cachecontrol/wrapper.py index b50a6e27d..265bfc8bc 100644 --- a/src/pip/_vendor/cachecontrol/wrapper.py +++ b/src/pip/_vendor/cachecontrol/wrapper.py @@ -2,14 +2,16 @@ from .adapter import CacheControlAdapter from .cache import DictCache -def CacheControl(sess, - cache=None, - cache_etags=True, - serializer=None, - heuristic=None, - controller_class=None, - adapter_class=None, - cacheable_methods=None): +def CacheControl( + sess, + cache=None, + cache_etags=True, + serializer=None, + heuristic=None, + controller_class=None, + adapter_class=None, + cacheable_methods=None, +): cache = cache or DictCache() adapter_class = adapter_class or CacheControlAdapter @@ -19,9 +21,9 @@ def CacheControl(sess, serializer=serializer, heuristic=heuristic, controller_class=controller_class, - cacheable_methods=cacheable_methods + cacheable_methods=cacheable_methods, ) - sess.mount('http://', adapter) - sess.mount('https://', adapter) + sess.mount("http://", adapter) + sess.mount("https://", adapter) return sess diff --git a/src/pip/_vendor/certifi/__init__.py b/src/pip/_vendor/certifi/__init__.py index 556193cef..0c4963ef6 100644 --- a/src/pip/_vendor/certifi/__init__.py +++ b/src/pip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ from .core import where, old_where -__version__ = "2018.01.18" +__version__ = "2018.04.16" diff --git a/src/pip/_vendor/certifi/cacert.pem b/src/pip/_vendor/certifi/cacert.pem index 101ac98fa..2713f541c 100644 --- a/src/pip/_vendor/certifi/cacert.pem +++ b/src/pip/_vendor/certifi/cacert.pem @@ -3483,39 +3483,6 @@ AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su -----END CERTIFICATE----- -# Issuer: CN=T\xdcRKTRUST Elektronik Sertifika Hizmet Sa\u011flay\u0131c\u0131s\u0131 H5 O=T\xdcRKTRUST Bilgi \u0130leti\u015fim ve Bili\u015fim G\xfcvenli\u011fi Hizmetleri A.\u015e. -# Subject: CN=T\xdcRKTRUST Elektronik Sertifika Hizmet Sa\u011flay\u0131c\u0131s\u0131 H5 O=T\xdcRKTRUST Bilgi \u0130leti\u015fim ve Bili\u015fim G\xfcvenli\u011fi Hizmetleri A.\u015e. -# Label: "T\xdcRKTRUST Elektronik Sertifika Hizmet Sa\u011flay\u0131c\u0131s\u0131 H5" -# Serial: 156233699172481 -# MD5 Fingerprint: da:70:8e:f0:22:df:93:26:f6:5f:9f:d3:15:06:52:4e -# SHA1 Fingerprint: c4:18:f6:4d:46:d1:df:00:3d:27:30:13:72:43:a9:12:11:c6:75:fb -# SHA256 Fingerprint: 49:35:1b:90:34:44:c1:85:cc:dc:5c:69:3d:24:d8:55:5c:b2:08:d6:a8:14:13:07:69:9f:4a:f0:63:19:9d:78 ------BEGIN CERTIFICATE----- -MIIEJzCCAw+gAwIBAgIHAI4X/iQggTANBgkqhkiG9w0BAQsFADCBsTELMAkGA1UE -BhMCVFIxDzANBgNVBAcMBkFua2FyYTFNMEsGA1UECgxEVMOcUktUUlVTVCBCaWxn -aSDEsGxldGnFn2ltIHZlIEJpbGnFn2ltIEfDvHZlbmxpxJ9pIEhpem1ldGxlcmkg -QS7Fni4xQjBABgNVBAMMOVTDnFJLVFJVU1QgRWxla3Ryb25payBTZXJ0aWZpa2Eg -SGl6bWV0IFNhxJ9sYXnEsWPEsXPEsSBINTAeFw0xMzA0MzAwODA3MDFaFw0yMzA0 -MjgwODA3MDFaMIGxMQswCQYDVQQGEwJUUjEPMA0GA1UEBwwGQW5rYXJhMU0wSwYD -VQQKDERUw5xSS1RSVVNUIEJpbGdpIMSwbGV0acWfaW0gdmUgQmlsacWfaW0gR8O8 -dmVubGnEn2kgSGl6bWV0bGVyaSBBLsWeLjFCMEAGA1UEAww5VMOcUktUUlVTVCBF -bGVrdHJvbmlrIFNlcnRpZmlrYSBIaXptZXQgU2HEn2xhecSxY8Sxc8SxIEg1MIIB -IjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEApCUZ4WWe60ghUEoI5RHwWrom -/4NZzkQqL/7hzmAD/I0Dpe3/a6i6zDQGn1k19uwsu537jVJp45wnEFPzpALFp/kR -Gml1bsMdi9GYjZOHp3GXDSHHmflS0yxjXVW86B8BSLlg/kJK9siArs1mep5Fimh3 -4khon6La8eHBEJ/rPCmBp+EyCNSgBbGM+42WAA4+Jd9ThiI7/PS98wl+d+yG6w8z -5UNP9FR1bSmZLmZaQ9/LXMrI5Tjxfjs1nQ/0xVqhzPMggCTTV+wVunUlm+hkS7M0 -hO8EuPbJbKoCPrZV4jI3X/xml1/N1p7HIL9Nxqw/dV8c7TKcfGkAaZHjIxhT6QID -AQABo0IwQDAdBgNVHQ4EFgQUVpkHHtOsDGlktAxQR95DLL4gwPswDgYDVR0PAQH/ -BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAJ5FdnsX -SDLyOIspve6WSk6BGLFRRyDN0GSxDsnZAdkJzsiZ3GglE9Rc8qPoBP5yCccLqh0l -VX6Wmle3usURehnmp349hQ71+S4pL+f5bFgWV1Al9j4uPqrtd3GqqpmWRgqujuwq -URawXs3qZwQcWDD1YIq9pr1N5Za0/EKJAWv2cMhQOQwt1WbZyNKzMrcbGW3LM/nf -peYVhDfwwvJllpKQd/Ct9JDpEXjXk4nAPQu6KfTomZ1yju2dL+6SfaHx/126M2CF -Yv4HAqGEVka+lgqaE9chTLd8B59OTj+RdPsnnRHM3eaxynFNExc5JsUpISuTKWqW -+qtB4Uu2NQvAmxU= ------END CERTIFICATE----- - # Issuer: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 # Subject: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 # Label: "Certinomis - Root CA" diff --git a/src/pip/_vendor/distro.py b/src/pip/_vendor/distro.py index 39bfce797..aa4defc3b 100644 --- a/src/pip/_vendor/distro.py +++ b/src/pip/_vendor/distro.py @@ -1,4 +1,4 @@ -# Copyright 2015,2016 Nir Cohen +# Copyright 2015,2016,2017 Nir Cohen # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,7 +23,7 @@ functionality. An alternative implementation became necessary because Python 3.5 deprecated this function, and Python 3.7 is expected to remove it altogether. Its predecessor function :py:func:`platform.dist` was already deprecated since Python 2.6 and is also expected to be removed in Python 3.7. -Still, there are many cases in which access to Linux distribution information +Still, there are many cases in which access to OS distribution information is needed. See `Python issue 1322 `_ for more information. """ @@ -94,7 +94,7 @@ _DISTRO_RELEASE_IGNORE_BASENAMES = ( def linux_distribution(full_distribution_name=True): """ - Return information about the current Linux distribution as a tuple + Return information about the current OS distribution as a tuple ``(id_name, version, codename)`` with items as follows: * ``id_name``: If *full_distribution_name* is false, the result of @@ -110,22 +110,22 @@ def linux_distribution(full_distribution_name=True): The data it returns may not exactly be the same, because it uses more data sources than the original function, and that may lead to different data if - the Linux distribution is not consistent across multiple data sources it + the OS distribution is not consistent across multiple data sources it provides (there are indeed such distributions ...). Another reason for differences is the fact that the :func:`distro.id` method normalizes the distro ID string to a reliable machine-readable value - for a number of popular Linux distributions. + for a number of popular OS distributions. """ return _distro.linux_distribution(full_distribution_name) def id(): """ - Return the distro ID of the current Linux distribution, as a + Return the distro ID of the current distribution, as a machine-readable string. - For a number of Linux distributions, the returned distro ID value is + For a number of OS distributions, the returned distro ID value is *reliable*, in the sense that it is documented and that it does not change across releases of the distribution. @@ -158,6 +158,9 @@ def id(): "scientific" Scientific Linux "slackware" Slackware "xenserver" XenServer + "openbsd" OpenBSD + "netbsd" NetBSD + "freebsd" FreeBSD ============== ========================================= If you have a need to get distros for reliable IDs added into this set, @@ -187,7 +190,7 @@ def id(): * a normalization of the ID is performed, based upon `normalization tables`_. The purpose of this normalization is to ensure that the ID is as reliable as possible, even across incompatible changes - in the Linux distributions. A common reason for an incompatible change is + in the OS distributions. A common reason for an incompatible change is the addition of an os-release file, or the addition of the lsb_release command, with ID values that differ from what was previously determined from the distro release file name. @@ -197,7 +200,7 @@ def id(): def name(pretty=False): """ - Return the name of the current Linux distribution, as a human-readable + Return the name of the current OS distribution, as a human-readable string. If *pretty* is false, the name is returned without version or codename. @@ -236,7 +239,7 @@ def name(pretty=False): def version(pretty=False, best=False): """ - Return the version of the current Linux distribution, as a human-readable + Return the version of the current OS distribution, as a human-readable string. If *pretty* is false, the version is returned without codename (e.g. @@ -280,7 +283,7 @@ def version(pretty=False, best=False): def version_parts(best=False): """ - Return the version of the current Linux distribution as a tuple + Return the version of the current OS distribution as a tuple ``(major, minor, build_number)`` with items as follows: * ``major``: The result of :func:`distro.major_version`. @@ -297,7 +300,7 @@ def version_parts(best=False): def major_version(best=False): """ - Return the major version of the current Linux distribution, as a string, + Return the major version of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The major version is the first part of the dot-separated version string. @@ -310,7 +313,7 @@ def major_version(best=False): def minor_version(best=False): """ - Return the minor version of the current Linux distribution, as a string, + Return the minor version of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The minor version is the second part of the dot-separated version string. @@ -323,7 +326,7 @@ def minor_version(best=False): def build_number(best=False): """ - Return the build number of the current Linux distribution, as a string, + Return the build number of the current OS distribution, as a string, if provided. Otherwise, the empty string is returned. The build number is the third part of the dot-separated version string. @@ -337,7 +340,7 @@ def build_number(best=False): def like(): """ Return a space-separated list of distro IDs of distributions that are - closely related to the current Linux distribution in regards to packaging + closely related to the current OS distribution in regards to packaging and programming interfaces, for example distributions the current distribution is a derivative from. @@ -353,7 +356,7 @@ def like(): def codename(): """ - Return the codename for the release of the current Linux distribution, + Return the codename for the release of the current OS distribution, as a string. If the distribution does not have a codename, an empty string is returned. @@ -377,7 +380,7 @@ def codename(): def info(pretty=False, best=False): """ - Return certain machine-readable information items about the current Linux + Return certain machine-readable information items about the current OS distribution in a dictionary, as shown in the following example: .. sourcecode:: python @@ -422,7 +425,7 @@ def info(pretty=False, best=False): def os_release_info(): """ Return a dictionary containing key-value pairs for the information items - from the os-release file data source of the current Linux distribution. + from the os-release file data source of the current OS distribution. See `os-release file`_ for details about these information items. """ @@ -432,7 +435,7 @@ def os_release_info(): def lsb_release_info(): """ Return a dictionary containing key-value pairs for the information items - from the lsb_release command data source of the current Linux distribution. + from the lsb_release command data source of the current OS distribution. See `lsb_release command output`_ for details about these information items. @@ -443,17 +446,25 @@ def lsb_release_info(): def distro_release_info(): """ Return a dictionary containing key-value pairs for the information items - from the distro release file data source of the current Linux distribution. + from the distro release file data source of the current OS distribution. See `distro release file`_ for details about these information items. """ return _distro.distro_release_info() +def uname_info(): + """ + Return a dictionary containing key-value pairs for the information items + from the distro release file data source of the current OS distribution. + """ + return _distro.uname_info() + + def os_release_attr(attribute): """ Return a single named information item from the os-release file data source - of the current Linux distribution. + of the current OS distribution. Parameters: @@ -472,7 +483,7 @@ def os_release_attr(attribute): def lsb_release_attr(attribute): """ Return a single named information item from the lsb_release command output - data source of the current Linux distribution. + data source of the current OS distribution. Parameters: @@ -492,7 +503,7 @@ def lsb_release_attr(attribute): def distro_release_attr(attribute): """ Return a single named information item from the distro release file - data source of the current Linux distribution. + data source of the current OS distribution. Parameters: @@ -508,6 +519,23 @@ def distro_release_attr(attribute): return _distro.distro_release_attr(attribute) +def uname_attr(attribute): + """ + Return a single named information item from the distro release file + data source of the current OS distribution. + + Parameters: + + * ``attribute`` (string): Key of the information item. + + Returns: + + * (string): Value of the information item, if the item exists. + The empty string, if the item does not exist. + """ + return _distro.uname_attr(attribute) + + class cached_property(object): """A version of @property which caches the value. On access, it calls the underlying function and sets the value in `__dict__` so future accesses @@ -525,13 +553,13 @@ class cached_property(object): class LinuxDistribution(object): """ - Provides information about a Linux distribution. + Provides information about a OS distribution. This package creates a private module-global instance of this class with default initialization arguments, that is used by the `consolidated accessor functions`_ and `single source accessor functions`_. By using default initialization arguments, that module-global instance - returns data about the current Linux distribution (i.e. the distro this + returns data about the current OS distribution (i.e. the distro this package runs on). Normally, it is not necessary to create additional instances of this class. @@ -544,7 +572,8 @@ class LinuxDistribution(object): def __init__(self, include_lsb=True, os_release_file='', - distro_release_file=''): + distro_release_file='', + include_uname=True): """ The initialization method of this class gathers information from the available data sources, and stores that in private instance attributes. @@ -578,6 +607,11 @@ class LinuxDistribution(object): distro release file can be found, the data source for the distro release file will be empty. + * ``include_name`` (bool): Controls whether uname command output is + included as a data source. If the uname command is not available in + the program execution path the data source for the uname command will + be empty. + Public instance attributes: * ``os_release_file`` (string): The path name of the @@ -591,6 +625,10 @@ class LinuxDistribution(object): * ``include_lsb`` (bool): The result of the ``include_lsb`` parameter. This controls whether the lsb information will be loaded. + * ``include_uname`` (bool): The result of the ``include_uname`` + parameter. This controls whether the uname information will + be loaded. + Raises: * :py:exc:`IOError`: Some I/O issue with an os-release file or distro @@ -607,6 +645,7 @@ class LinuxDistribution(object): os.path.join(_UNIXCONFDIR, _OS_RELEASE_BASENAME) self.distro_release_file = distro_release_file or '' # updated later self.include_lsb = include_lsb + self.include_uname = include_uname def __repr__(self): """Return repr of all info @@ -616,14 +655,16 @@ class LinuxDistribution(object): "os_release_file={self.os_release_file!r}, " \ "distro_release_file={self.distro_release_file!r}, " \ "include_lsb={self.include_lsb!r}, " \ + "include_uname={self.include_uname!r}, " \ "_os_release_info={self._os_release_info!r}, " \ "_lsb_release_info={self._lsb_release_info!r}, " \ - "_distro_release_info={self._distro_release_info!r})".format( + "_distro_release_info={self._distro_release_info!r}, " \ + "_uname_info={self._uname_info!r})".format( self=self) def linux_distribution(self, full_distribution_name=True): """ - Return information about the Linux distribution that is compatible + Return information about the OS distribution that is compatible with Python's :func:`platform.linux_distribution`, supporting a subset of its parameters. @@ -636,7 +677,7 @@ class LinuxDistribution(object): ) def id(self): - """Return the distro ID of the Linux distribution, as a string. + """Return the distro ID of the OS distribution, as a string. For details, see :func:`distro.id`. """ @@ -656,22 +697,28 @@ class LinuxDistribution(object): if distro_id: return normalize(distro_id, NORMALIZED_DISTRO_ID) + distro_id = self.uname_attr('id') + if distro_id: + return normalize(distro_id, NORMALIZED_DISTRO_ID) + return '' def name(self, pretty=False): """ - Return the name of the Linux distribution, as a string. + Return the name of the OS distribution, as a string. For details, see :func:`distro.name`. """ name = self.os_release_attr('name') \ or self.lsb_release_attr('distributor_id') \ - or self.distro_release_attr('name') + or self.distro_release_attr('name') \ + or self.uname_attr('name') if pretty: name = self.os_release_attr('pretty_name') \ or self.lsb_release_attr('description') if not name: - name = self.distro_release_attr('name') + name = self.distro_release_attr('name') \ + or self.uname_attr('name') version = self.version(pretty=True) if version: name = name + ' ' + version @@ -679,7 +726,7 @@ class LinuxDistribution(object): def version(self, pretty=False, best=False): """ - Return the version of the Linux distribution, as a string. + Return the version of the OS distribution, as a string. For details, see :func:`distro.version`. """ @@ -690,7 +737,8 @@ class LinuxDistribution(object): self._parse_distro_release_content( self.os_release_attr('pretty_name')).get('version_id', ''), self._parse_distro_release_content( - self.lsb_release_attr('description')).get('version_id', '') + self.lsb_release_attr('description')).get('version_id', ''), + self.uname_attr('release') ] version = '' if best: @@ -712,7 +760,7 @@ class LinuxDistribution(object): def version_parts(self, best=False): """ - Return the version of the Linux distribution, as a tuple of version + Return the version of the OS distribution, as a tuple of version numbers. For details, see :func:`distro.version_parts`. @@ -736,7 +784,7 @@ class LinuxDistribution(object): def minor_version(self, best=False): """ - Return the minor version number of the Linux distribution. + Return the minor version number of the current distribution. For details, see :func:`distro.minor_version`. """ @@ -744,7 +792,7 @@ class LinuxDistribution(object): def build_number(self, best=False): """ - Return the build number of the Linux distribution. + Return the build number of the current distribution. For details, see :func:`distro.build_number`. """ @@ -752,7 +800,7 @@ class LinuxDistribution(object): def like(self): """ - Return the IDs of distributions that are like the Linux distribution. + Return the IDs of distributions that are like the OS distribution. For details, see :func:`distro.like`. """ @@ -760,7 +808,7 @@ class LinuxDistribution(object): def codename(self): """ - Return the codename of the Linux distribution. + Return the codename of the OS distribution. For details, see :func:`distro.codename`. """ @@ -771,7 +819,7 @@ class LinuxDistribution(object): def info(self, pretty=False, best=False): """ - Return certain machine-readable information about the Linux + Return certain machine-readable information about the OS distribution. For details, see :func:`distro.info`. @@ -791,7 +839,7 @@ class LinuxDistribution(object): def os_release_info(self): """ Return a dictionary containing key-value pairs for the information - items from the os-release file data source of the Linux distribution. + items from the os-release file data source of the OS distribution. For details, see :func:`distro.os_release_info`. """ @@ -800,7 +848,7 @@ class LinuxDistribution(object): def lsb_release_info(self): """ Return a dictionary containing key-value pairs for the information - items from the lsb_release command data source of the Linux + items from the lsb_release command data source of the OS distribution. For details, see :func:`distro.lsb_release_info`. @@ -810,17 +858,25 @@ class LinuxDistribution(object): def distro_release_info(self): """ Return a dictionary containing key-value pairs for the information - items from the distro release file data source of the Linux + items from the distro release file data source of the OS distribution. For details, see :func:`distro.distro_release_info`. """ return self._distro_release_info + def uname_info(self): + """ + Return a dictionary containing key-value pairs for the information + items from the uname command data source of the OS distribution. + + For details, see :func:`distro.uname_info`. + """ + def os_release_attr(self, attribute): """ Return a single named information item from the os-release file data - source of the Linux distribution. + source of the OS distribution. For details, see :func:`distro.os_release_attr`. """ @@ -829,7 +885,7 @@ class LinuxDistribution(object): def lsb_release_attr(self, attribute): """ Return a single named information item from the lsb_release command - output data source of the Linux distribution. + output data source of the OS distribution. For details, see :func:`distro.lsb_release_attr`. """ @@ -838,12 +894,21 @@ class LinuxDistribution(object): def distro_release_attr(self, attribute): """ Return a single named information item from the distro release file - data source of the Linux distribution. + data source of the OS distribution. For details, see :func:`distro.distro_release_attr`. """ return self._distro_release_info.get(attribute, '') + def uname_attr(self, attribute): + """ + Return a single named information item from the uname command + output data source of the OS distribution. + + For details, see :func:`distro.uname_release_attr`. + """ + return self._uname_info.get(attribute, '') + @cached_property def _os_release_info(self): """ @@ -960,6 +1025,34 @@ class LinuxDistribution(object): props.update({k.replace(' ', '_').lower(): v.strip()}) return props + @cached_property + def _uname_info(self): + with open(os.devnull, 'w') as devnull: + try: + cmd = ('uname', '-rs') + stdout = subprocess.check_output(cmd, stderr=devnull) + except OSError: + return {} + content = stdout.decode(sys.getfilesystemencoding()).splitlines() + return self._parse_uname_content(content) + + @staticmethod + def _parse_uname_content(lines): + props = {} + match = re.search(r'^([^\s]+)\s+([\d\.]+)', lines[0].strip()) + if match: + name, version = match.groups() + + # This is to prevent the Linux kernel version from + # appearing as the 'best' version on otherwise + # identifiable distributions. + if name == 'Linux': + return {} + props['id'] = name.lower() + props['name'] = name + props['release'] = version + return props + @cached_property def _distro_release_info(self): """ @@ -1082,7 +1175,7 @@ def main(): logger.setLevel(logging.DEBUG) logger.addHandler(logging.StreamHandler(sys.stdout)) - parser = argparse.ArgumentParser(description="Linux distro info tool") + parser = argparse.ArgumentParser(description="OS distro info tool") parser.add_argument( '--json', '-j', diff --git a/src/pip/_vendor/idna/__init__.py b/src/pip/_vendor/idna/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/codec.py b/src/pip/_vendor/idna/codec.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/compat.py b/src/pip/_vendor/idna/compat.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/core.py b/src/pip/_vendor/idna/core.py old mode 100644 new mode 100755 index b55b66456..090c2c18d --- a/src/pip/_vendor/idna/core.py +++ b/src/pip/_vendor/idna/core.py @@ -34,7 +34,11 @@ class InvalidCodepointContext(IDNAError): def _combining_class(cp): - return unicodedata.combining(unichr(cp)) + v = unicodedata.combining(unichr(cp)) + if v == 0: + if not unicodedata.name(unichr(cp)): + raise ValueError("Unknown character in unicodedata") + return v def _is_script(cp, script): return intranges_contain(ord(cp), idnadata.scripts[script]) @@ -71,7 +75,6 @@ def check_bidi(label, check_ltr=False): raise IDNABidiError('Unknown directionality in label {0} at position {1}'.format(repr(label), idx)) if direction in ['R', 'AL', 'AN']: bidi_label = True - break if not bidi_label and not check_ltr: return True @@ -244,8 +247,13 @@ def check_label(label): if intranges_contain(cp_value, idnadata.codepoint_classes['PVALID']): continue elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTJ']): - if not valid_contextj(label, pos): - raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label))) + try: + if not valid_contextj(label, pos): + raise InvalidCodepointContext('Joiner {0} not allowed at position {1} in {2}'.format( + _unot(cp_value), pos+1, repr(label))) + except ValueError: + raise IDNAError('Unknown codepoint adjacent to joiner {0} at position {1} in {2}'.format( + _unot(cp_value), pos+1, repr(label))) elif intranges_contain(cp_value, idnadata.codepoint_classes['CONTEXTO']): if not valid_contexto(label, pos): raise InvalidCodepointContext('Codepoint {0} not allowed at position {1} in {2}'.format(_unot(cp_value), pos+1, repr(label))) @@ -317,10 +325,10 @@ def uts46_remap(domain, std3_rules=True, transitional=False): replacement = uts46row[2] if len(uts46row) == 3 else None if (status == "V" or (status == "D" and not transitional) or - (status == "3" and std3_rules and replacement is None)): + (status == "3" and not std3_rules and replacement is None)): output += char elif replacement is not None and (status == "M" or - (status == "3" and std3_rules) or + (status == "3" and not std3_rules) or (status == "D" and transitional)): output += replacement elif status != "I": @@ -344,15 +352,17 @@ def encode(s, strict=False, uts46=False, std3_rules=False, transitional=False): labels = s.split('.') else: labels = _unicode_dots_re.split(s) - while labels and not labels[0]: - del labels[0] - if not labels: + if not labels or labels == ['']: raise IDNAError('Empty domain') if labels[-1] == '': del labels[-1] trailing_dot = True for label in labels: - result.append(alabel(label)) + s = alabel(label) + if s: + result.append(s) + else: + raise IDNAError('Empty label') if trailing_dot: result.append(b'') s = b'.'.join(result) @@ -373,15 +383,17 @@ def decode(s, strict=False, uts46=False, std3_rules=False): labels = _unicode_dots_re.split(s) else: labels = s.split(u'.') - while labels and not labels[0]: - del labels[0] - if not labels: + if not labels or labels == ['']: raise IDNAError('Empty domain') if not labels[-1]: del labels[-1] trailing_dot = True for label in labels: - result.append(ulabel(label)) + s = ulabel(label) + if s: + result.append(s) + else: + raise IDNAError('Empty label') if trailing_dot: result.append(u'') return u'.'.join(result) diff --git a/src/pip/_vendor/idna/idnadata.py b/src/pip/_vendor/idna/idnadata.py old mode 100644 new mode 100755 index c48f1b504..17974e233 --- a/src/pip/_vendor/idna/idnadata.py +++ b/src/pip/_vendor/idna/idnadata.py @@ -1,11 +1,12 @@ # This file is automatically generated by tools/idna-data -__version__ = "6.3.0" +__version__ = "10.0.0" scripts = { 'Greek': ( 0x37000000374, 0x37500000378, 0x37a0000037e, + 0x37f00000380, 0x38400000385, 0x38600000387, 0x3880000038b, @@ -34,7 +35,9 @@ scripts = { 0x1ff200001ff5, 0x1ff600001fff, 0x212600002127, - 0x101400001018b, + 0xab650000ab66, + 0x101400001018f, + 0x101a0000101a1, 0x1d2000001d246, ), 'Han': ( @@ -46,12 +49,14 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009fcd, + 0x4e0000009feb, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, 0x2a7000002b735, 0x2b7400002b81e, + 0x2b8200002cea2, + 0x2ceb00002ebe1, 0x2f8000002fa1e, ), 'Hebrew': ( @@ -68,7 +73,7 @@ scripts = { 'Hiragana': ( 0x304100003097, 0x309d000030a0, - 0x1b0010001b002, + 0x1b0010001b11f, 0x1f2000001f201, ), 'Katakana': ( @@ -88,6 +93,7 @@ joining_types = { 0x602: 85, 0x603: 85, 0x604: 85, + 0x605: 85, 0x608: 85, 0x60b: 85, 0x620: 68, @@ -365,7 +371,7 @@ joining_types = { 0x844: 68, 0x845: 68, 0x846: 82, - 0x847: 68, + 0x847: 82, 0x848: 68, 0x849: 82, 0x84a: 68, @@ -373,7 +379,7 @@ joining_types = { 0x84c: 68, 0x84d: 68, 0x84e: 68, - 0x84f: 82, + 0x84f: 68, 0x850: 68, 0x851: 68, 0x852: 68, @@ -383,7 +389,19 @@ joining_types = { 0x856: 85, 0x857: 85, 0x858: 85, + 0x860: 68, + 0x861: 85, + 0x862: 68, + 0x863: 68, + 0x864: 68, + 0x865: 68, + 0x866: 85, + 0x867: 82, + 0x868: 68, + 0x869: 82, + 0x86a: 82, 0x8a0: 68, + 0x8a1: 68, 0x8a2: 68, 0x8a3: 68, 0x8a4: 68, @@ -395,6 +413,23 @@ joining_types = { 0x8aa: 82, 0x8ab: 82, 0x8ac: 82, + 0x8ad: 85, + 0x8ae: 82, + 0x8af: 68, + 0x8b0: 68, + 0x8b1: 82, + 0x8b2: 82, + 0x8b3: 68, + 0x8b4: 68, + 0x8b6: 68, + 0x8b7: 68, + 0x8b8: 68, + 0x8b9: 82, + 0x8ba: 68, + 0x8bb: 68, + 0x8bc: 68, + 0x8bd: 68, + 0x8e2: 85, 0x1806: 85, 0x1807: 68, 0x180a: 67, @@ -492,8 +527,8 @@ joining_types = { 0x1882: 85, 0x1883: 85, 0x1884: 85, - 0x1885: 85, - 0x1886: 85, + 0x1885: 84, + 0x1886: 84, 0x1887: 68, 0x1888: 68, 0x1889: 68, @@ -531,6 +566,7 @@ joining_types = { 0x18aa: 68, 0x200c: 85, 0x200d: 67, + 0x202f: 85, 0x2066: 85, 0x2067: 85, 0x2068: 85, @@ -587,6 +623,141 @@ joining_types = { 0xa871: 68, 0xa872: 76, 0xa873: 85, + 0x10ac0: 68, + 0x10ac1: 68, + 0x10ac2: 68, + 0x10ac3: 68, + 0x10ac4: 68, + 0x10ac5: 82, + 0x10ac6: 85, + 0x10ac7: 82, + 0x10ac8: 85, + 0x10ac9: 82, + 0x10aca: 82, + 0x10acb: 85, + 0x10acc: 85, + 0x10acd: 76, + 0x10ace: 82, + 0x10acf: 82, + 0x10ad0: 82, + 0x10ad1: 82, + 0x10ad2: 82, + 0x10ad3: 68, + 0x10ad4: 68, + 0x10ad5: 68, + 0x10ad6: 68, + 0x10ad7: 76, + 0x10ad8: 68, + 0x10ad9: 68, + 0x10ada: 68, + 0x10adb: 68, + 0x10adc: 68, + 0x10add: 82, + 0x10ade: 68, + 0x10adf: 68, + 0x10ae0: 68, + 0x10ae1: 82, + 0x10ae2: 85, + 0x10ae3: 85, + 0x10ae4: 82, + 0x10aeb: 68, + 0x10aec: 68, + 0x10aed: 68, + 0x10aee: 68, + 0x10aef: 82, + 0x10b80: 68, + 0x10b81: 82, + 0x10b82: 68, + 0x10b83: 82, + 0x10b84: 82, + 0x10b85: 82, + 0x10b86: 68, + 0x10b87: 68, + 0x10b88: 68, + 0x10b89: 82, + 0x10b8a: 68, + 0x10b8b: 68, + 0x10b8c: 82, + 0x10b8d: 68, + 0x10b8e: 82, + 0x10b8f: 82, + 0x10b90: 68, + 0x10b91: 82, + 0x10ba9: 82, + 0x10baa: 82, + 0x10bab: 82, + 0x10bac: 82, + 0x10bad: 68, + 0x10bae: 68, + 0x10baf: 85, + 0x1e900: 68, + 0x1e901: 68, + 0x1e902: 68, + 0x1e903: 68, + 0x1e904: 68, + 0x1e905: 68, + 0x1e906: 68, + 0x1e907: 68, + 0x1e908: 68, + 0x1e909: 68, + 0x1e90a: 68, + 0x1e90b: 68, + 0x1e90c: 68, + 0x1e90d: 68, + 0x1e90e: 68, + 0x1e90f: 68, + 0x1e910: 68, + 0x1e911: 68, + 0x1e912: 68, + 0x1e913: 68, + 0x1e914: 68, + 0x1e915: 68, + 0x1e916: 68, + 0x1e917: 68, + 0x1e918: 68, + 0x1e919: 68, + 0x1e91a: 68, + 0x1e91b: 68, + 0x1e91c: 68, + 0x1e91d: 68, + 0x1e91e: 68, + 0x1e91f: 68, + 0x1e920: 68, + 0x1e921: 68, + 0x1e922: 68, + 0x1e923: 68, + 0x1e924: 68, + 0x1e925: 68, + 0x1e926: 68, + 0x1e927: 68, + 0x1e928: 68, + 0x1e929: 68, + 0x1e92a: 68, + 0x1e92b: 68, + 0x1e92c: 68, + 0x1e92d: 68, + 0x1e92e: 68, + 0x1e92f: 68, + 0x1e930: 68, + 0x1e931: 68, + 0x1e932: 68, + 0x1e933: 68, + 0x1e934: 68, + 0x1e935: 68, + 0x1e936: 68, + 0x1e937: 68, + 0x1e938: 68, + 0x1e939: 68, + 0x1e93a: 68, + 0x1e93b: 68, + 0x1e93c: 68, + 0x1e93d: 68, + 0x1e93e: 68, + 0x1e93f: 68, + 0x1e940: 68, + 0x1e941: 68, + 0x1e942: 68, + 0x1e943: 68, } codepoint_classes = { 'PVALID': ( @@ -858,6 +1029,10 @@ codepoint_classes = { 0x52300000524, 0x52500000526, 0x52700000528, + 0x5290000052a, + 0x52b0000052c, + 0x52d0000052e, + 0x52f00000530, 0x5590000055a, 0x56100000587, 0x591000005be, @@ -881,15 +1056,14 @@ codepoint_classes = { 0x7c0000007f6, 0x8000000082e, 0x8400000085c, - 0x8a0000008a1, - 0x8a2000008ad, - 0x8e4000008ff, - 0x90000000958, + 0x8600000086b, + 0x8a0000008b5, + 0x8b6000008be, + 0x8d4000008e2, + 0x8e300000958, 0x96000000964, 0x96600000970, - 0x97100000978, - 0x97900000980, - 0x98100000984, + 0x97100000984, 0x9850000098d, 0x98f00000991, 0x993000009a9, @@ -902,6 +1076,7 @@ codepoint_classes = { 0x9d7000009d8, 0x9e0000009e4, 0x9e6000009f2, + 0x9fc000009fd, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -930,6 +1105,7 @@ codepoint_classes = { 0xad000000ad1, 0xae000000ae4, 0xae600000af0, + 0xaf900000b00, 0xb0100000b04, 0xb0500000b0d, 0xb0f00000b11, @@ -960,20 +1136,19 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0100000c04, + 0xc0000000c04, 0xc0500000c0d, 0xc0e00000c11, 0xc1200000c29, - 0xc2a00000c34, - 0xc3500000c3a, + 0xc2a00000c3a, 0xc3d00000c45, 0xc4600000c49, 0xc4a00000c4e, 0xc5500000c57, - 0xc5800000c5a, + 0xc5800000c5b, 0xc6000000c64, 0xc6600000c70, - 0xc8200000c84, + 0xc8000000c84, 0xc8500000c8d, 0xc8e00000c91, 0xc9200000ca9, @@ -987,15 +1162,14 @@ codepoint_classes = { 0xce000000ce4, 0xce600000cf0, 0xcf100000cf3, - 0xd0200000d04, + 0xd0000000d04, 0xd0500000d0d, 0xd0e00000d11, - 0xd1200000d3b, - 0xd3d00000d45, + 0xd1200000d45, 0xd4600000d49, 0xd4a00000d4f, - 0xd5700000d58, - 0xd6000000d64, + 0xd5400000d58, + 0xd5f00000d64, 0xd6600000d70, 0xd7a00000d80, 0xd8200000d84, @@ -1008,6 +1182,7 @@ codepoint_classes = { 0xdcf00000dd5, 0xdd600000dd7, 0xdd800000de0, + 0xde600000df0, 0xdf200000df4, 0xe0100000e33, 0xe3400000e3b, @@ -1082,11 +1257,12 @@ codepoint_classes = { 0x13180000135b, 0x135d00001360, 0x138000001390, - 0x13a0000013f5, + 0x13a0000013f6, 0x14010000166d, 0x166f00001680, 0x16810000169b, 0x16a0000016eb, + 0x16f1000016f9, 0x17000000170d, 0x170e00001715, 0x172000001735, @@ -1103,7 +1279,7 @@ codepoint_classes = { 0x182000001878, 0x1880000018ab, 0x18b0000018f6, - 0x19000000191d, + 0x19000000191f, 0x19200000192c, 0x19300000193c, 0x19460000196e, @@ -1117,6 +1293,7 @@ codepoint_classes = { 0x1a7f00001a8a, 0x1a9000001a9a, 0x1aa700001aa8, + 0x1ab000001abe, 0x1b0000001b4c, 0x1b5000001b5a, 0x1b6b00001b74, @@ -1125,15 +1302,15 @@ codepoint_classes = { 0x1c4000001c4a, 0x1c4d00001c7e, 0x1cd000001cd3, - 0x1cd400001cf7, + 0x1cd400001cfa, 0x1d0000001d2c, 0x1d2f00001d30, 0x1d3b00001d3c, 0x1d4e00001d4f, 0x1d6b00001d78, 0x1d7900001d9b, - 0x1dc000001de7, - 0x1dfc00001e00, + 0x1dc000001dfa, + 0x1dfb00001e00, 0x1e0100001e02, 0x1e0300001e04, 0x1e0500001e06, @@ -1367,11 +1544,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312e, + 0x31050000312f, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009fcd, + 0x4e0000009feb, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1413,7 +1590,9 @@ codepoint_classes = { 0xa6930000a694, 0xa6950000a696, 0xa6970000a698, - 0xa69f0000a6e6, + 0xa6990000a69a, + 0xa69b0000a69c, + 0xa69e0000a6e6, 0xa6f00000a6f2, 0xa7170000a720, 0xa7230000a724, @@ -1463,30 +1642,39 @@ codepoint_classes = { 0xa7850000a786, 0xa7870000a789, 0xa78c0000a78d, - 0xa78e0000a78f, + 0xa78e0000a790, 0xa7910000a792, - 0xa7930000a794, + 0xa7930000a796, + 0xa7970000a798, + 0xa7990000a79a, + 0xa79b0000a79c, + 0xa79d0000a79e, + 0xa79f0000a7a0, 0xa7a10000a7a2, 0xa7a30000a7a4, 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7b50000a7b6, + 0xa7b70000a7b8, + 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, - 0xa8800000a8c5, + 0xa8800000a8c6, 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, + 0xa8fd0000a8fe, 0xa9000000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, + 0xa9e00000a9ff, 0xaa000000aa37, 0xaa400000aa4e, 0xaa500000aa5a, 0xaa600000aa77, - 0xaa7a0000aa7c, - 0xaa800000aac3, + 0xaa7a0000aac3, 0xaadb0000aade, 0xaae00000aaf0, 0xaaf20000aaf7, @@ -1495,6 +1683,8 @@ codepoint_classes = { 0xab110000ab17, 0xab200000ab27, 0xab280000ab2f, + 0xab300000ab5b, + 0xab600000ab66, 0xabc00000abeb, 0xabec0000abee, 0xabf00000abfa, @@ -1507,7 +1697,7 @@ codepoint_classes = { 0xfa230000fa25, 0xfa270000fa2a, 0xfb1e0000fb1f, - 0xfe200000fe27, + 0xfe200000fe30, 0xfe730000fe74, 0x100000001000c, 0x1000d00010027, @@ -1519,20 +1709,32 @@ codepoint_classes = { 0x101fd000101fe, 0x102800001029d, 0x102a0000102d1, - 0x103000001031f, - 0x1033000010341, + 0x102e0000102e1, + 0x1030000010320, + 0x1032d00010341, 0x103420001034a, + 0x103500001037b, 0x103800001039e, 0x103a0000103c4, 0x103c8000103d0, 0x104280001049e, 0x104a0000104aa, + 0x104d8000104fc, + 0x1050000010528, + 0x1053000010564, + 0x1060000010737, + 0x1074000010756, + 0x1076000010768, 0x1080000010806, 0x1080800010809, 0x1080a00010836, 0x1083700010839, 0x1083c0001083d, 0x1083f00010856, + 0x1086000010877, + 0x108800001089f, + 0x108e0000108f3, + 0x108f4000108f6, 0x1090000010916, 0x109200001093a, 0x10980000109b8, @@ -1545,31 +1747,137 @@ codepoint_classes = { 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, + 0x10a8000010a9d, + 0x10ac000010ac8, + 0x10ac900010ae7, 0x10b0000010b36, 0x10b4000010b56, 0x10b6000010b73, + 0x10b8000010b92, 0x10c0000010c49, + 0x10cc000010cf3, 0x1100000011047, 0x1106600011070, - 0x11080000110bb, + 0x1107f000110bb, 0x110d0000110e9, 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1115000011174, + 0x1117600011177, 0x11180000111c5, - 0x111d0000111da, + 0x111ca000111cd, + 0x111d0000111db, + 0x111dc000111dd, + 0x1120000011212, + 0x1121300011238, + 0x1123e0001123f, + 0x1128000011287, + 0x1128800011289, + 0x1128a0001128e, + 0x1128f0001129e, + 0x1129f000112a9, + 0x112b0000112eb, + 0x112f0000112fa, + 0x1130000011304, + 0x113050001130d, + 0x1130f00011311, + 0x1131300011329, + 0x1132a00011331, + 0x1133200011334, + 0x113350001133a, + 0x1133c00011345, + 0x1134700011349, + 0x1134b0001134e, + 0x1135000011351, + 0x1135700011358, + 0x1135d00011364, + 0x113660001136d, + 0x1137000011375, + 0x114000001144b, + 0x114500001145a, + 0x11480000114c6, + 0x114c7000114c8, + 0x114d0000114da, + 0x11580000115b6, + 0x115b8000115c1, + 0x115d8000115de, + 0x1160000011641, + 0x1164400011645, + 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x120000001236f, + 0x117000001171a, + 0x1171d0001172c, + 0x117300001173a, + 0x118c0000118ea, + 0x118ff00011900, + 0x11a0000011a3f, + 0x11a4700011a48, + 0x11a5000011a84, + 0x11a8600011a9a, + 0x11ac000011af9, + 0x11c0000011c09, + 0x11c0a00011c37, + 0x11c3800011c41, + 0x11c5000011c5a, + 0x11c7200011c90, + 0x11c9200011ca8, + 0x11ca900011cb7, + 0x11d0000011d07, + 0x11d0800011d0a, + 0x11d0b00011d37, + 0x11d3a00011d3b, + 0x11d3c00011d3e, + 0x11d3f00011d48, + 0x11d5000011d5a, + 0x120000001239a, + 0x1248000012544, 0x130000001342f, + 0x1440000014647, 0x1680000016a39, + 0x16a4000016a5f, + 0x16a6000016a6a, + 0x16ad000016aee, + 0x16af000016af5, + 0x16b0000016b37, + 0x16b4000016b44, + 0x16b5000016b5a, + 0x16b6300016b78, + 0x16b7d00016b90, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, - 0x1b0000001b002, + 0x16fe000016fe2, + 0x17000000187ed, + 0x1880000018af3, + 0x1b0000001b11f, + 0x1b1700001b2fc, + 0x1bc000001bc6b, + 0x1bc700001bc7d, + 0x1bc800001bc89, + 0x1bc900001bc9a, + 0x1bc9d0001bc9f, + 0x1da000001da37, + 0x1da3b0001da6d, + 0x1da750001da76, + 0x1da840001da85, + 0x1da9b0001daa0, + 0x1daa10001dab0, + 0x1e0000001e007, + 0x1e0080001e019, + 0x1e01b0001e022, + 0x1e0230001e025, + 0x1e0260001e02b, + 0x1e8000001e8c5, + 0x1e8d00001e8d7, + 0x1e9220001e94b, + 0x1e9500001e95a, 0x200000002a6d7, 0x2a7000002b735, 0x2b7400002b81e, + 0x2b8200002cea2, + 0x2ceb00002ebe1, ), 'CONTEXTJ': ( 0x200c0000200e, diff --git a/src/pip/_vendor/idna/intranges.py b/src/pip/_vendor/idna/intranges.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/idna/package_data.py b/src/pip/_vendor/idna/package_data.py old mode 100644 new mode 100755 index fc3313927..39c192bae --- a/src/pip/_vendor/idna/package_data.py +++ b/src/pip/_vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.6' +__version__ = '2.7' diff --git a/src/pip/_vendor/idna/uts46data.py b/src/pip/_vendor/idna/uts46data.py old mode 100644 new mode 100755 index f9b3236f4..79731cb9e --- a/src/pip/_vendor/idna/uts46data.py +++ b/src/pip/_vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "6.3.0" +__version__ = "10.0.0" def _seg_0(): return [ (0x0, '3'), @@ -635,7 +635,8 @@ def _seg_6(): (0x37A, '3', u' ι'), (0x37B, 'V'), (0x37E, '3', u';'), - (0x37F, 'X'), + (0x37F, 'M', u'ϳ'), + (0x380, 'X'), (0x384, '3', u' ́'), (0x385, '3', u' ̈́'), (0x386, 'M', u'ά'), @@ -730,11 +731,11 @@ def _seg_6(): (0x400, 'M', u'ѐ'), (0x401, 'M', u'ё'), (0x402, 'M', u'ђ'), - (0x403, 'M', u'ѓ'), ] def _seg_7(): return [ + (0x403, 'M', u'ѓ'), (0x404, 'M', u'є'), (0x405, 'M', u'ѕ'), (0x406, 'M', u'і'), @@ -834,11 +835,11 @@ def _seg_7(): (0x49B, 'V'), (0x49C, 'M', u'ҝ'), (0x49D, 'V'), - (0x49E, 'M', u'ҟ'), ] def _seg_8(): return [ + (0x49E, 'M', u'ҟ'), (0x49F, 'V'), (0x4A0, 'M', u'ҡ'), (0x4A1, 'V'), @@ -938,11 +939,11 @@ def _seg_8(): (0x500, 'M', u'ԁ'), (0x501, 'V'), (0x502, 'M', u'ԃ'), - (0x503, 'V'), ] def _seg_9(): return [ + (0x503, 'V'), (0x504, 'M', u'ԅ'), (0x505, 'V'), (0x506, 'M', u'ԇ'), @@ -979,7 +980,15 @@ def _seg_9(): (0x525, 'V'), (0x526, 'M', u'ԧ'), (0x527, 'V'), - (0x528, 'X'), + (0x528, 'M', u'ԩ'), + (0x529, 'V'), + (0x52A, 'M', u'ԫ'), + (0x52B, 'V'), + (0x52C, 'M', u'ԭ'), + (0x52D, 'V'), + (0x52E, 'M', u'ԯ'), + (0x52F, 'V'), + (0x530, 'X'), (0x531, 'M', u'ա'), (0x532, 'M', u'բ'), (0x533, 'M', u'գ'), @@ -1026,7 +1035,7 @@ def _seg_9(): (0x588, 'X'), (0x589, 'V'), (0x58B, 'X'), - (0x58F, 'V'), + (0x58D, 'V'), (0x590, 'X'), (0x591, 'V'), (0x5C8, 'X'), @@ -1034,6 +1043,10 @@ def _seg_9(): (0x5EB, 'X'), (0x5F0, 'V'), (0x5F5, 'X'), + ] + +def _seg_10(): + return [ (0x606, 'V'), (0x61C, 'X'), (0x61E, 'V'), @@ -1043,10 +1056,6 @@ def _seg_9(): (0x678, 'M', u'يٴ'), (0x679, 'V'), (0x6DD, 'X'), - ] - -def _seg_10(): - return [ (0x6DE, 'V'), (0x70E, 'X'), (0x710, 'V'), @@ -1063,13 +1072,15 @@ def _seg_10(): (0x85C, 'X'), (0x85E, 'V'), (0x85F, 'X'), + (0x860, 'V'), + (0x86B, 'X'), (0x8A0, 'V'), - (0x8A1, 'X'), - (0x8A2, 'V'), - (0x8AD, 'X'), - (0x8E4, 'V'), - (0x8FF, 'X'), - (0x900, 'V'), + (0x8B5, 'X'), + (0x8B6, 'V'), + (0x8BE, 'X'), + (0x8D4, 'V'), + (0x8E2, 'X'), + (0x8E3, 'V'), (0x958, 'M', u'क़'), (0x959, 'M', u'ख़'), (0x95A, 'M', u'ग़'), @@ -1079,10 +1090,6 @@ def _seg_10(): (0x95E, 'M', u'फ़'), (0x95F, 'M', u'य़'), (0x960, 'V'), - (0x978, 'X'), - (0x979, 'V'), - (0x980, 'X'), - (0x981, 'V'), (0x984, 'X'), (0x985, 'V'), (0x98D, 'X'), @@ -1111,7 +1118,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FC, 'X'), + (0x9FE, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1140,6 +1147,10 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + ] + +def _seg_11(): + return [ (0xA59, 'M', u'ਖ਼'), (0xA5A, 'M', u'ਗ਼'), (0xA5B, 'M', u'ਜ਼'), @@ -1147,10 +1158,6 @@ def _seg_10(): (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), - ] - -def _seg_11(): - return [ (0xA66, 'V'), (0xA76, 'X'), (0xA81, 'V'), @@ -1179,6 +1186,8 @@ def _seg_11(): (0xAE4, 'X'), (0xAE6, 'V'), (0xAF2, 'X'), + (0xAF9, 'V'), + (0xB00, 'X'), (0xB01, 'V'), (0xB04, 'X'), (0xB05, 'V'), @@ -1240,8 +1249,12 @@ def _seg_11(): (0xBD8, 'X'), (0xBE6, 'V'), (0xBFB, 'X'), - (0xC01, 'V'), + (0xC00, 'V'), (0xC04, 'X'), + ] + +def _seg_12(): + return [ (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), @@ -1249,12 +1262,6 @@ def _seg_11(): (0xC12, 'V'), (0xC29, 'X'), (0xC2A, 'V'), - (0xC34, 'X'), - (0xC35, 'V'), - ] - -def _seg_12(): - return [ (0xC3A, 'X'), (0xC3D, 'V'), (0xC45, 'X'), @@ -1265,14 +1272,12 @@ def _seg_12(): (0xC55, 'V'), (0xC57, 'X'), (0xC58, 'V'), - (0xC5A, 'X'), + (0xC5B, 'X'), (0xC60, 'V'), (0xC64, 'X'), (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC80, 'X'), - (0xC82, 'V'), (0xC84, 'X'), (0xC85, 'V'), (0xC8D, 'X'), @@ -1300,27 +1305,21 @@ def _seg_12(): (0xCF0, 'X'), (0xCF1, 'V'), (0xCF3, 'X'), - (0xD02, 'V'), + (0xD00, 'V'), (0xD04, 'X'), (0xD05, 'V'), (0xD0D, 'X'), (0xD0E, 'V'), (0xD11, 'X'), (0xD12, 'V'), - (0xD3B, 'X'), - (0xD3D, 'V'), (0xD45, 'X'), (0xD46, 'V'), (0xD49, 'X'), (0xD4A, 'V'), - (0xD4F, 'X'), - (0xD57, 'V'), - (0xD58, 'X'), - (0xD60, 'V'), + (0xD50, 'X'), + (0xD54, 'V'), (0xD64, 'X'), (0xD66, 'V'), - (0xD76, 'X'), - (0xD79, 'V'), (0xD80, 'X'), (0xD82, 'V'), (0xD84, 'X'), @@ -1342,6 +1341,8 @@ def _seg_12(): (0xDD7, 'X'), (0xDD8, 'V'), (0xDE0, 'X'), + (0xDE6, 'V'), + (0xDF0, 'X'), (0xDF2, 'V'), (0xDF5, 'X'), (0xE01, 'V'), @@ -1354,11 +1355,11 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - (0xE87, 'V'), ] def _seg_13(): return [ + (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), (0xE8B, 'X'), @@ -1458,11 +1459,11 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - (0x1258, 'V'), ] def _seg_14(): return [ + (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), (0x125E, 'X'), @@ -1493,13 +1494,20 @@ def _seg_14(): (0x1380, 'V'), (0x139A, 'X'), (0x13A0, 'V'), - (0x13F5, 'X'), + (0x13F6, 'X'), + (0x13F8, 'M', u'Ᏸ'), + (0x13F9, 'M', u'Ᏹ'), + (0x13FA, 'M', u'Ᏺ'), + (0x13FB, 'M', u'Ᏻ'), + (0x13FC, 'M', u'Ᏼ'), + (0x13FD, 'M', u'Ᏽ'), + (0x13FE, 'X'), (0x1400, 'V'), (0x1680, 'X'), (0x1681, 'V'), (0x169D, 'X'), (0x16A0, 'V'), - (0x16F1, 'X'), + (0x16F9, 'X'), (0x1700, 'V'), (0x170D, 'X'), (0x170E, 'V'), @@ -1536,7 +1544,7 @@ def _seg_14(): (0x18B0, 'V'), (0x18F6, 'X'), (0x1900, 'V'), - (0x191D, 'X'), + (0x191F, 'X'), (0x1920, 'V'), (0x192C, 'X'), (0x1930, 'V'), @@ -1555,6 +1563,10 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), + ] + +def _seg_15(): + return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1563,12 +1575,10 @@ def _seg_14(): (0x1A8A, 'X'), (0x1A90, 'V'), (0x1A9A, 'X'), - ] - -def _seg_15(): - return [ (0x1AA0, 'V'), (0x1AAE, 'X'), + (0x1AB0, 'V'), + (0x1ABF, 'X'), (0x1B00, 'V'), (0x1B4C, 'X'), (0x1B50, 'V'), @@ -1580,11 +1590,19 @@ def _seg_15(): (0x1C3B, 'V'), (0x1C4A, 'X'), (0x1C4D, 'V'), - (0x1C80, 'X'), + (0x1C80, 'M', u'в'), + (0x1C81, 'M', u'д'), + (0x1C82, 'M', u'о'), + (0x1C83, 'M', u'с'), + (0x1C84, 'M', u'т'), + (0x1C86, 'M', u'ъ'), + (0x1C87, 'M', u'ѣ'), + (0x1C88, 'M', u'ꙋ'), + (0x1C89, 'X'), (0x1CC0, 'V'), (0x1CC8, 'X'), (0x1CD0, 'V'), - (0x1CF7, 'X'), + (0x1CFA, 'X'), (0x1D00, 'V'), (0x1D2C, 'M', u'a'), (0x1D2D, 'M', u'æ'), @@ -1649,6 +1667,10 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), + ] + +def _seg_16(): + return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1667,10 +1689,6 @@ def _seg_15(): (0x1DA7, 'M', u'ᵻ'), (0x1DA8, 'M', u'ʝ'), (0x1DA9, 'M', u'ɭ'), - ] - -def _seg_16(): - return [ (0x1DAA, 'M', u'ᶅ'), (0x1DAB, 'M', u'ʟ'), (0x1DAC, 'M', u'ɱ'), @@ -1694,8 +1712,8 @@ def _seg_16(): (0x1DBE, 'M', u'ʒ'), (0x1DBF, 'M', u'θ'), (0x1DC0, 'V'), - (0x1DE7, 'X'), - (0x1DFC, 'V'), + (0x1DFA, 'X'), + (0x1DFB, 'V'), (0x1E00, 'M', u'ḁ'), (0x1E01, 'V'), (0x1E02, 'M', u'ḃ'), @@ -1753,6 +1771,10 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), + ] + +def _seg_17(): + return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1771,10 +1793,6 @@ def _seg_16(): (0x1E48, 'M', u'ṉ'), (0x1E49, 'V'), (0x1E4A, 'M', u'ṋ'), - ] - -def _seg_17(): - return [ (0x1E4B, 'V'), (0x1E4C, 'M', u'ṍ'), (0x1E4D, 'V'), @@ -1857,6 +1875,10 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), + ] + +def _seg_18(): + return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1875,10 +1897,6 @@ def _seg_17(): (0x1EB1, 'V'), (0x1EB2, 'M', u'ẳ'), (0x1EB3, 'V'), - ] - -def _seg_18(): - return [ (0x1EB4, 'M', u'ẵ'), (0x1EB5, 'V'), (0x1EB6, 'M', u'ặ'), @@ -1961,6 +1979,10 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), + ] + +def _seg_19(): + return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1979,10 +2001,6 @@ def _seg_18(): (0x1F2B, 'M', u'ἣ'), (0x1F2C, 'M', u'ἤ'), (0x1F2D, 'M', u'ἥ'), - ] - -def _seg_19(): - return [ (0x1F2E, 'M', u'ἦ'), (0x1F2F, 'M', u'ἧ'), (0x1F30, 'V'), @@ -2065,6 +2083,10 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), + ] + +def _seg_20(): + return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2083,10 +2105,6 @@ def _seg_19(): (0x1FAC, 'M', u'ὤι'), (0x1FAD, 'M', u'ὥι'), (0x1FAE, 'M', u'ὦι'), - ] - -def _seg_20(): - return [ (0x1FAF, 'M', u'ὧι'), (0x1FB0, 'V'), (0x1FB2, 'M', u'ὰι'), @@ -2169,6 +2187,10 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), + ] + +def _seg_21(): + return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2187,10 +2209,6 @@ def _seg_20(): (0x204A, 'V'), (0x2057, 'M', u'′′′′'), (0x2058, 'V'), - ] - -def _seg_21(): - return [ (0x205F, '3', u' '), (0x2060, 'I'), (0x2061, 'X'), @@ -2244,7 +2262,7 @@ def _seg_21(): (0x20A0, 'V'), (0x20A8, 'M', u'rs'), (0x20A9, 'V'), - (0x20BB, 'X'), + (0x20C0, 'X'), (0x20D0, 'V'), (0x20F1, 'X'), (0x2100, '3', u'a/c'), @@ -2273,6 +2291,10 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), + ] + +def _seg_22(): + return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2291,10 +2313,6 @@ def _seg_21(): (0x2133, 'M', u'm'), (0x2134, 'M', u'o'), (0x2135, 'M', u'א'), - ] - -def _seg_22(): - return [ (0x2136, 'M', u'ב'), (0x2137, 'M', u'ג'), (0x2138, 'M', u'ד'), @@ -2363,7 +2381,8 @@ def _seg_22(): (0x2183, 'X'), (0x2184, 'V'), (0x2189, 'M', u'0⁄3'), - (0x218A, 'X'), + (0x218A, 'V'), + (0x218C, 'X'), (0x2190, 'V'), (0x222C, 'M', u'∫∫'), (0x222D, 'M', u'∫∫∫'), @@ -2376,10 +2395,12 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), + ] + +def _seg_23(): + return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), - (0x23F4, 'X'), - (0x2400, 'V'), (0x2427, 'X'), (0x2440, 'V'), (0x244B, 'X'), @@ -2395,10 +2416,6 @@ def _seg_22(): (0x2469, 'M', u'10'), (0x246A, 'M', u'11'), (0x246B, 'M', u'12'), - ] - -def _seg_23(): - return [ (0x246C, 'M', u'13'), (0x246D, 'M', u'14'), (0x246E, 'M', u'15'), @@ -2482,6 +2499,10 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), + ] + +def _seg_24(): + return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2499,10 +2520,6 @@ def _seg_23(): (0x24E0, 'M', u'q'), (0x24E1, 'M', u'r'), (0x24E2, 'M', u's'), - ] - -def _seg_24(): - return [ (0x24E3, 'M', u't'), (0x24E4, 'M', u'u'), (0x24E5, 'M', u'v'), @@ -2512,8 +2529,6 @@ def _seg_24(): (0x24E9, 'M', u'z'), (0x24EA, 'M', u'0'), (0x24EB, 'V'), - (0x2700, 'X'), - (0x2701, 'V'), (0x2A0C, 'M', u'∫∫∫∫'), (0x2A0D, 'V'), (0x2A74, '3', u'::='), @@ -2522,9 +2537,17 @@ def _seg_24(): (0x2A77, 'V'), (0x2ADC, 'M', u'⫝̸'), (0x2ADD, 'V'), - (0x2B4D, 'X'), - (0x2B50, 'V'), - (0x2B5A, 'X'), + (0x2B74, 'X'), + (0x2B76, 'V'), + (0x2B96, 'X'), + (0x2B98, 'V'), + (0x2BBA, 'X'), + (0x2BBD, 'V'), + (0x2BC9, 'X'), + (0x2BCA, 'V'), + (0x2BD3, 'X'), + (0x2BEC, 'V'), + (0x2BF0, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2580,6 +2603,10 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), + ] + +def _seg_25(): + return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2603,10 +2630,6 @@ def _seg_24(): (0x2C80, 'M', u'ⲁ'), (0x2C81, 'V'), (0x2C82, 'M', u'ⲃ'), - ] - -def _seg_25(): - return [ (0x2C83, 'V'), (0x2C84, 'M', u'ⲅ'), (0x2C85, 'V'), @@ -2684,6 +2707,10 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), + ] + +def _seg_26(): + return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2707,10 +2734,6 @@ def _seg_25(): (0x2CEB, 'M', u'ⳬ'), (0x2CEC, 'V'), (0x2CED, 'M', u'ⳮ'), - ] - -def _seg_26(): - return [ (0x2CEE, 'V'), (0x2CF2, 'M', u'ⳳ'), (0x2CF3, 'V'), @@ -2745,7 +2768,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E3C, 'X'), + (0x2E4A, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2788,6 +2811,10 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), + ] + +def _seg_27(): + return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2811,10 +2838,6 @@ def _seg_26(): (0x2F37, 'M', u'弋'), (0x2F38, 'M', u'弓'), (0x2F39, 'M', u'彐'), - ] - -def _seg_27(): - return [ (0x2F3A, 'M', u'彡'), (0x2F3B, 'M', u'彳'), (0x2F3C, 'M', u'心'), @@ -2892,6 +2915,10 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), + ] + +def _seg_28(): + return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2915,10 +2942,6 @@ def _seg_27(): (0x2F9B, 'M', u'走'), (0x2F9C, 'M', u'足'), (0x2F9D, 'M', u'身'), - ] - -def _seg_28(): - return [ (0x2F9E, 'M', u'車'), (0x2F9F, 'M', u'辛'), (0x2FA0, 'M', u'辰'), @@ -2996,9 +3019,13 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), + ] + +def _seg_29(): + return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312E, 'X'), + (0x312F, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3019,10 +3046,6 @@ def _seg_28(): (0x3142, 'M', u'ᄇ'), (0x3143, 'M', u'ᄈ'), (0x3144, 'M', u'ᄡ'), - ] - -def _seg_29(): - return [ (0x3145, 'M', u'ᄉ'), (0x3146, 'M', u'ᄊ'), (0x3147, 'M', u'ᄋ'), @@ -3100,6 +3123,10 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), + ] + +def _seg_30(): + return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3123,10 +3150,6 @@ def _seg_29(): (0x3202, '3', u'(ᄃ)'), (0x3203, '3', u'(ᄅ)'), (0x3204, '3', u'(ᄆ)'), - ] - -def _seg_30(): - return [ (0x3205, '3', u'(ᄇ)'), (0x3206, '3', u'(ᄉ)'), (0x3207, '3', u'(ᄋ)'), @@ -3204,6 +3227,10 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), + ] + +def _seg_31(): + return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3227,10 +3254,6 @@ def _seg_30(): (0x326D, 'M', u'ᄒ'), (0x326E, 'M', u'가'), (0x326F, 'M', u'나'), - ] - -def _seg_31(): - return [ (0x3270, 'M', u'다'), (0x3271, 'M', u'라'), (0x3272, 'M', u'마'), @@ -3308,6 +3331,10 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), + ] + +def _seg_32(): + return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3331,10 +3358,6 @@ def _seg_31(): (0x32D1, 'M', u'イ'), (0x32D2, 'M', u'ウ'), (0x32D3, 'M', u'エ'), - ] - -def _seg_32(): - return [ (0x32D4, 'M', u'オ'), (0x32D5, 'M', u'カ'), (0x32D6, 'M', u'キ'), @@ -3412,6 +3435,10 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), + ] + +def _seg_33(): + return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3435,10 +3462,6 @@ def _seg_32(): (0x3335, 'M', u'フラン'), (0x3336, 'M', u'ヘクタール'), (0x3337, 'M', u'ペソ'), - ] - -def _seg_33(): - return [ (0x3338, 'M', u'ペニヒ'), (0x3339, 'M', u'ヘルツ'), (0x333A, 'M', u'ペンス'), @@ -3516,6 +3539,10 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), + ] + +def _seg_34(): + return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3539,10 +3566,6 @@ def _seg_33(): (0x3399, 'M', u'fm'), (0x339A, 'M', u'nm'), (0x339B, 'M', u'μm'), - ] - -def _seg_34(): - return [ (0x339C, 'M', u'mm'), (0x339D, 'M', u'cm'), (0x339E, 'M', u'km'), @@ -3620,6 +3643,10 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), + ] + +def _seg_35(): + return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3643,14 +3670,10 @@ def _seg_34(): (0x33FD, 'M', u'30日'), (0x33FE, 'M', u'31日'), (0x33FF, 'M', u'gal'), - ] - -def _seg_35(): - return [ (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FCD, 'X'), + (0x9FEB, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3724,11 +3747,20 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), + ] + +def _seg_36(): + return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), - (0xA698, 'X'), - (0xA69F, 'V'), + (0xA698, 'M', u'ꚙ'), + (0xA699, 'V'), + (0xA69A, 'M', u'ꚛ'), + (0xA69B, 'V'), + (0xA69C, 'M', u'ъ'), + (0xA69D, 'M', u'ь'), + (0xA69E, 'V'), (0xA6F8, 'X'), (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), @@ -3747,10 +3779,6 @@ def _seg_35(): (0xA72F, 'V'), (0xA732, 'M', u'ꜳ'), (0xA733, 'V'), - ] - -def _seg_36(): - return [ (0xA734, 'M', u'ꜵ'), (0xA735, 'V'), (0xA736, 'M', u'ꜷ'), @@ -3823,6 +3851,10 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), + ] + +def _seg_37(): + return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3832,12 +3864,20 @@ def _seg_36(): (0xA78C, 'V'), (0xA78D, 'M', u'ɥ'), (0xA78E, 'V'), - (0xA78F, 'X'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), - (0xA794, 'X'), + (0xA796, 'M', u'ꞗ'), + (0xA797, 'V'), + (0xA798, 'M', u'ꞙ'), + (0xA799, 'V'), + (0xA79A, 'M', u'ꞛ'), + (0xA79B, 'V'), + (0xA79C, 'M', u'ꞝ'), + (0xA79D, 'V'), + (0xA79E, 'M', u'ꞟ'), + (0xA79F, 'V'), (0xA7A0, 'M', u'ꞡ'), (0xA7A1, 'V'), (0xA7A2, 'M', u'ꞣ'), @@ -3849,12 +3889,22 @@ def _seg_36(): (0xA7A8, 'M', u'ꞩ'), (0xA7A9, 'V'), (0xA7AA, 'M', u'ɦ'), - (0xA7AB, 'X'), + (0xA7AB, 'M', u'ɜ'), + (0xA7AC, 'M', u'ɡ'), + (0xA7AD, 'M', u'ɬ'), + (0xA7AE, 'M', u'ɪ'), + (0xA7AF, 'X'), + (0xA7B0, 'M', u'ʞ'), + (0xA7B1, 'M', u'ʇ'), + (0xA7B2, 'M', u'ʝ'), + (0xA7B3, 'M', u'ꭓ'), + (0xA7B4, 'M', u'ꞵ'), + (0xA7B5, 'V'), + (0xA7B6, 'M', u'ꞷ'), + (0xA7B7, 'V'), + (0xA7B8, 'X'), + (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), - ] - -def _seg_37(): - return [ (0xA7F9, 'M', u'œ'), (0xA7FA, 'V'), (0xA82C, 'X'), @@ -3863,11 +3913,11 @@ def _seg_37(): (0xA840, 'V'), (0xA878, 'X'), (0xA880, 'V'), - (0xA8C5, 'X'), + (0xA8C6, 'X'), (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FC, 'X'), + (0xA8FE, 'X'), (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), @@ -3877,7 +3927,7 @@ def _seg_37(): (0xA9CF, 'V'), (0xA9DA, 'X'), (0xA9DE, 'V'), - (0xA9E0, 'X'), + (0xA9FF, 'X'), (0xAA00, 'V'), (0xAA37, 'X'), (0xAA40, 'V'), @@ -3885,8 +3935,6 @@ def _seg_37(): (0xAA50, 'V'), (0xAA5A, 'X'), (0xAA5C, 'V'), - (0xAA7C, 'X'), - (0xAA80, 'V'), (0xAAC3, 'X'), (0xAADB, 'V'), (0xAAF7, 'X'), @@ -3900,6 +3948,97 @@ def _seg_37(): (0xAB27, 'X'), (0xAB28, 'V'), (0xAB2F, 'X'), + (0xAB30, 'V'), + (0xAB5C, 'M', u'ꜧ'), + (0xAB5D, 'M', u'ꬷ'), + (0xAB5E, 'M', u'ɫ'), + (0xAB5F, 'M', u'ꭒ'), + (0xAB60, 'V'), + (0xAB66, 'X'), + ] + +def _seg_38(): + return [ + (0xAB70, 'M', u'Ꭰ'), + (0xAB71, 'M', u'Ꭱ'), + (0xAB72, 'M', u'Ꭲ'), + (0xAB73, 'M', u'Ꭳ'), + (0xAB74, 'M', u'Ꭴ'), + (0xAB75, 'M', u'Ꭵ'), + (0xAB76, 'M', u'Ꭶ'), + (0xAB77, 'M', u'Ꭷ'), + (0xAB78, 'M', u'Ꭸ'), + (0xAB79, 'M', u'Ꭹ'), + (0xAB7A, 'M', u'Ꭺ'), + (0xAB7B, 'M', u'Ꭻ'), + (0xAB7C, 'M', u'Ꭼ'), + (0xAB7D, 'M', u'Ꭽ'), + (0xAB7E, 'M', u'Ꭾ'), + (0xAB7F, 'M', u'Ꭿ'), + (0xAB80, 'M', u'Ꮀ'), + (0xAB81, 'M', u'Ꮁ'), + (0xAB82, 'M', u'Ꮂ'), + (0xAB83, 'M', u'Ꮃ'), + (0xAB84, 'M', u'Ꮄ'), + (0xAB85, 'M', u'Ꮅ'), + (0xAB86, 'M', u'Ꮆ'), + (0xAB87, 'M', u'Ꮇ'), + (0xAB88, 'M', u'Ꮈ'), + (0xAB89, 'M', u'Ꮉ'), + (0xAB8A, 'M', u'Ꮊ'), + (0xAB8B, 'M', u'Ꮋ'), + (0xAB8C, 'M', u'Ꮌ'), + (0xAB8D, 'M', u'Ꮍ'), + (0xAB8E, 'M', u'Ꮎ'), + (0xAB8F, 'M', u'Ꮏ'), + (0xAB90, 'M', u'Ꮐ'), + (0xAB91, 'M', u'Ꮑ'), + (0xAB92, 'M', u'Ꮒ'), + (0xAB93, 'M', u'Ꮓ'), + (0xAB94, 'M', u'Ꮔ'), + (0xAB95, 'M', u'Ꮕ'), + (0xAB96, 'M', u'Ꮖ'), + (0xAB97, 'M', u'Ꮗ'), + (0xAB98, 'M', u'Ꮘ'), + (0xAB99, 'M', u'Ꮙ'), + (0xAB9A, 'M', u'Ꮚ'), + (0xAB9B, 'M', u'Ꮛ'), + (0xAB9C, 'M', u'Ꮜ'), + (0xAB9D, 'M', u'Ꮝ'), + (0xAB9E, 'M', u'Ꮞ'), + (0xAB9F, 'M', u'Ꮟ'), + (0xABA0, 'M', u'Ꮠ'), + (0xABA1, 'M', u'Ꮡ'), + (0xABA2, 'M', u'Ꮢ'), + (0xABA3, 'M', u'Ꮣ'), + (0xABA4, 'M', u'Ꮤ'), + (0xABA5, 'M', u'Ꮥ'), + (0xABA6, 'M', u'Ꮦ'), + (0xABA7, 'M', u'Ꮧ'), + (0xABA8, 'M', u'Ꮨ'), + (0xABA9, 'M', u'Ꮩ'), + (0xABAA, 'M', u'Ꮪ'), + (0xABAB, 'M', u'Ꮫ'), + (0xABAC, 'M', u'Ꮬ'), + (0xABAD, 'M', u'Ꮭ'), + (0xABAE, 'M', u'Ꮮ'), + (0xABAF, 'M', u'Ꮯ'), + (0xABB0, 'M', u'Ꮰ'), + (0xABB1, 'M', u'Ꮱ'), + (0xABB2, 'M', u'Ꮲ'), + (0xABB3, 'M', u'Ꮳ'), + (0xABB4, 'M', u'Ꮴ'), + (0xABB5, 'M', u'Ꮵ'), + (0xABB6, 'M', u'Ꮶ'), + (0xABB7, 'M', u'Ꮷ'), + (0xABB8, 'M', u'Ꮸ'), + (0xABB9, 'M', u'Ꮹ'), + (0xABBA, 'M', u'Ꮺ'), + (0xABBB, 'M', u'Ꮻ'), + (0xABBC, 'M', u'Ꮼ'), + (0xABBD, 'M', u'Ꮽ'), + (0xABBE, 'M', u'Ꮾ'), + (0xABBF, 'M', u'Ꮿ'), (0xABC0, 'V'), (0xABEE, 'X'), (0xABF0, 'V'), @@ -3920,6 +4059,10 @@ def _seg_37(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), + ] + +def _seg_39(): + return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -3955,10 +4098,6 @@ def _seg_37(): (0xF92B, 'M', u'狼'), (0xF92C, 'M', u'郎'), (0xF92D, 'M', u'來'), - ] - -def _seg_38(): - return [ (0xF92E, 'M', u'冷'), (0xF92F, 'M', u'勞'), (0xF930, 'M', u'擄'), @@ -4024,6 +4163,10 @@ def _seg_38(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), + ] + +def _seg_40(): + return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4059,10 +4202,6 @@ def _seg_38(): (0xF98F, 'M', u'憐'), (0xF990, 'M', u'戀'), (0xF991, 'M', u'撚'), - ] - -def _seg_39(): - return [ (0xF992, 'M', u'漣'), (0xF993, 'M', u'煉'), (0xF994, 'M', u'璉'), @@ -4128,6 +4267,10 @@ def _seg_39(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), + ] + +def _seg_41(): + return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4163,10 +4306,6 @@ def _seg_39(): (0xF9F3, 'M', u'麟'), (0xF9F4, 'M', u'林'), (0xF9F5, 'M', u'淋'), - ] - -def _seg_40(): - return [ (0xF9F6, 'M', u'臨'), (0xF9F7, 'M', u'立'), (0xF9F8, 'M', u'笠'), @@ -4232,6 +4371,10 @@ def _seg_40(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), + ] + +def _seg_42(): + return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4267,10 +4410,6 @@ def _seg_40(): (0xFA5C, 'M', u'臭'), (0xFA5D, 'M', u'艹'), (0xFA5F, 'M', u'著'), - ] - -def _seg_41(): - return [ (0xFA60, 'M', u'褐'), (0xFA61, 'M', u'視'), (0xFA62, 'M', u'謁'), @@ -4336,6 +4475,10 @@ def _seg_41(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), + ] + +def _seg_43(): + return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4371,10 +4514,6 @@ def _seg_41(): (0xFAC2, 'M', u'輸'), (0xFAC3, 'M', u'遲'), (0xFAC4, 'M', u'醙'), - ] - -def _seg_42(): - return [ (0xFAC5, 'M', u'鉶'), (0xFAC6, 'M', u'陼'), (0xFAC7, 'M', u'難'), @@ -4440,6 +4579,10 @@ def _seg_42(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), + ] + +def _seg_44(): + return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4475,10 +4618,6 @@ def _seg_42(): (0xFB7A, 'M', u'چ'), (0xFB7E, 'M', u'ڇ'), (0xFB82, 'M', u'ڍ'), - ] - -def _seg_43(): - return [ (0xFB84, 'M', u'ڌ'), (0xFB86, 'M', u'ڎ'), (0xFB88, 'M', u'ڈ'), @@ -4544,6 +4683,10 @@ def _seg_43(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), + ] + +def _seg_45(): + return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4579,10 +4722,6 @@ def _seg_43(): (0xFC3C, 'M', u'كم'), (0xFC3D, 'M', u'كى'), (0xFC3E, 'M', u'كي'), - ] - -def _seg_44(): - return [ (0xFC3F, 'M', u'لج'), (0xFC40, 'M', u'لح'), (0xFC41, 'M', u'لخ'), @@ -4648,6 +4787,10 @@ def _seg_44(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), + ] + +def _seg_46(): + return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4683,10 +4826,6 @@ def _seg_44(): (0xFCA0, 'M', u'به'), (0xFCA1, 'M', u'تج'), (0xFCA2, 'M', u'تح'), - ] - -def _seg_45(): - return [ (0xFCA3, 'M', u'تخ'), (0xFCA4, 'M', u'تم'), (0xFCA5, 'M', u'ته'), @@ -4752,6 +4891,10 @@ def _seg_45(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), + ] + +def _seg_47(): + return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4787,10 +4930,6 @@ def _seg_45(): (0xFD04, 'M', u'خي'), (0xFD05, 'M', u'صى'), (0xFD06, 'M', u'صي'), - ] - -def _seg_46(): - return [ (0xFD07, 'M', u'ضى'), (0xFD08, 'M', u'ضي'), (0xFD09, 'M', u'شج'), @@ -4856,6 +4995,10 @@ def _seg_46(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), + ] + +def _seg_48(): + return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -4891,10 +5034,6 @@ def _seg_46(): (0xFD87, 'M', u'لمح'), (0xFD89, 'M', u'محج'), (0xFD8A, 'M', u'محم'), - ] - -def _seg_47(): - return [ (0xFD8B, 'M', u'محي'), (0xFD8C, 'M', u'مجح'), (0xFD8D, 'M', u'مجم'), @@ -4960,6 +5099,10 @@ def _seg_47(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), + ] + +def _seg_49(): + return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -4981,7 +5124,7 @@ def _seg_47(): (0xFE18, 'M', u'〗'), (0xFE19, 'X'), (0xFE20, 'V'), - (0xFE27, 'X'), + (0xFE30, 'X'), (0xFE31, 'M', u'—'), (0xFE32, 'M', u'–'), (0xFE33, '3', u'_'), @@ -4995,10 +5138,6 @@ def _seg_47(): (0xFE3C, 'M', u'】'), (0xFE3D, 'M', u'《'), (0xFE3E, 'M', u'》'), - ] - -def _seg_48(): - return [ (0xFE3F, 'M', u'〈'), (0xFE40, 'M', u'〉'), (0xFE41, 'M', u'「'), @@ -5064,6 +5203,10 @@ def _seg_48(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), + ] + +def _seg_50(): + return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5099,10 +5242,6 @@ def _seg_48(): (0xFF00, 'X'), (0xFF01, '3', u'!'), (0xFF02, '3', u'"'), - ] - -def _seg_49(): - return [ (0xFF03, '3', u'#'), (0xFF04, '3', u'$'), (0xFF05, '3', u'%'), @@ -5168,6 +5307,10 @@ def _seg_49(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), + ] + +def _seg_51(): + return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5203,10 +5346,6 @@ def _seg_49(): (0xFF64, 'M', u'、'), (0xFF65, 'M', u'・'), (0xFF66, 'M', u'ヲ'), - ] - -def _seg_50(): - return [ (0xFF67, 'M', u'ァ'), (0xFF68, 'M', u'ィ'), (0xFF69, 'M', u'ゥ'), @@ -5272,6 +5411,10 @@ def _seg_50(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), + ] + +def _seg_52(): + return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5307,10 +5450,6 @@ def _seg_50(): (0xFFCB, 'M', u'ᅨ'), (0xFFCC, 'M', u'ᅩ'), (0xFFCD, 'M', u'ᅪ'), - ] - -def _seg_51(): - return [ (0xFFCE, 'M', u'ᅫ'), (0xFFCF, 'M', u'ᅬ'), (0xFFD0, 'X'), @@ -5360,21 +5499,29 @@ def _seg_51(): (0x10107, 'V'), (0x10134, 'X'), (0x10137, 'V'), - (0x1018B, 'X'), + (0x1018F, 'X'), (0x10190, 'V'), (0x1019C, 'X'), + (0x101A0, 'V'), + (0x101A1, 'X'), (0x101D0, 'V'), (0x101FE, 'X'), (0x10280, 'V'), (0x1029D, 'X'), (0x102A0, 'V'), (0x102D1, 'X'), + (0x102E0, 'V'), + (0x102FC, 'X'), (0x10300, 'V'), - (0x1031F, 'X'), - (0x10320, 'V'), (0x10324, 'X'), - (0x10330, 'V'), + (0x1032D, 'V'), + ] + +def _seg_53(): + return [ (0x1034B, 'X'), + (0x10350, 'V'), + (0x1037B, 'X'), (0x10380, 'V'), (0x1039E, 'X'), (0x1039F, 'V'), @@ -5411,10 +5558,6 @@ def _seg_51(): (0x1041B, 'M', u'𐑃'), (0x1041C, 'M', u'𐑄'), (0x1041D, 'M', u'𐑅'), - ] - -def _seg_52(): - return [ (0x1041E, 'M', u'𐑆'), (0x1041F, 'M', u'𐑇'), (0x10420, 'M', u'𐑈'), @@ -5429,6 +5572,61 @@ def _seg_52(): (0x1049E, 'X'), (0x104A0, 'V'), (0x104AA, 'X'), + (0x104B0, 'M', u'𐓘'), + (0x104B1, 'M', u'𐓙'), + (0x104B2, 'M', u'𐓚'), + (0x104B3, 'M', u'𐓛'), + (0x104B4, 'M', u'𐓜'), + (0x104B5, 'M', u'𐓝'), + (0x104B6, 'M', u'𐓞'), + (0x104B7, 'M', u'𐓟'), + (0x104B8, 'M', u'𐓠'), + (0x104B9, 'M', u'𐓡'), + (0x104BA, 'M', u'𐓢'), + (0x104BB, 'M', u'𐓣'), + (0x104BC, 'M', u'𐓤'), + (0x104BD, 'M', u'𐓥'), + (0x104BE, 'M', u'𐓦'), + (0x104BF, 'M', u'𐓧'), + (0x104C0, 'M', u'𐓨'), + (0x104C1, 'M', u'𐓩'), + (0x104C2, 'M', u'𐓪'), + (0x104C3, 'M', u'𐓫'), + (0x104C4, 'M', u'𐓬'), + (0x104C5, 'M', u'𐓭'), + (0x104C6, 'M', u'𐓮'), + (0x104C7, 'M', u'𐓯'), + (0x104C8, 'M', u'𐓰'), + (0x104C9, 'M', u'𐓱'), + (0x104CA, 'M', u'𐓲'), + (0x104CB, 'M', u'𐓳'), + (0x104CC, 'M', u'𐓴'), + (0x104CD, 'M', u'𐓵'), + (0x104CE, 'M', u'𐓶'), + (0x104CF, 'M', u'𐓷'), + (0x104D0, 'M', u'𐓸'), + (0x104D1, 'M', u'𐓹'), + (0x104D2, 'M', u'𐓺'), + (0x104D3, 'M', u'𐓻'), + (0x104D4, 'X'), + (0x104D8, 'V'), + (0x104FC, 'X'), + (0x10500, 'V'), + (0x10528, 'X'), + (0x10530, 'V'), + (0x10564, 'X'), + (0x1056F, 'V'), + (0x10570, 'X'), + (0x10600, 'V'), + (0x10737, 'X'), + ] + +def _seg_54(): + return [ + (0x10740, 'V'), + (0x10756, 'X'), + (0x10760, 'V'), + (0x10768, 'X'), (0x10800, 'V'), (0x10806, 'X'), (0x10808, 'V'), @@ -5442,8 +5640,14 @@ def _seg_52(): (0x1083F, 'V'), (0x10856, 'X'), (0x10857, 'V'), - (0x10860, 'X'), - (0x10900, 'V'), + (0x1089F, 'X'), + (0x108A7, 'V'), + (0x108B0, 'X'), + (0x108E0, 'V'), + (0x108F3, 'X'), + (0x108F4, 'V'), + (0x108F6, 'X'), + (0x108FB, 'V'), (0x1091C, 'X'), (0x1091F, 'V'), (0x1093A, 'X'), @@ -5451,9 +5655,9 @@ def _seg_52(): (0x10940, 'X'), (0x10980, 'V'), (0x109B8, 'X'), - (0x109BE, 'V'), - (0x109C0, 'X'), - (0x10A00, 'V'), + (0x109BC, 'V'), + (0x109D0, 'X'), + (0x109D2, 'V'), (0x10A04, 'X'), (0x10A05, 'V'), (0x10A07, 'X'), @@ -5470,7 +5674,11 @@ def _seg_52(): (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), - (0x10A80, 'X'), + (0x10AA0, 'X'), + (0x10AC0, 'V'), + (0x10AE7, 'X'), + (0x10AEB, 'V'), + (0x10AF7, 'X'), (0x10B00, 'V'), (0x10B36, 'X'), (0x10B39, 'V'), @@ -5478,16 +5686,80 @@ def _seg_52(): (0x10B58, 'V'), (0x10B73, 'X'), (0x10B78, 'V'), - (0x10B80, 'X'), + (0x10B92, 'X'), + (0x10B99, 'V'), + (0x10B9D, 'X'), + (0x10BA9, 'V'), + (0x10BB0, 'X'), (0x10C00, 'V'), (0x10C49, 'X'), + (0x10C80, 'M', u'𐳀'), + (0x10C81, 'M', u'𐳁'), + (0x10C82, 'M', u'𐳂'), + (0x10C83, 'M', u'𐳃'), + (0x10C84, 'M', u'𐳄'), + (0x10C85, 'M', u'𐳅'), + (0x10C86, 'M', u'𐳆'), + (0x10C87, 'M', u'𐳇'), + (0x10C88, 'M', u'𐳈'), + (0x10C89, 'M', u'𐳉'), + (0x10C8A, 'M', u'𐳊'), + (0x10C8B, 'M', u'𐳋'), + (0x10C8C, 'M', u'𐳌'), + (0x10C8D, 'M', u'𐳍'), + (0x10C8E, 'M', u'𐳎'), + (0x10C8F, 'M', u'𐳏'), + (0x10C90, 'M', u'𐳐'), + (0x10C91, 'M', u'𐳑'), + (0x10C92, 'M', u'𐳒'), + (0x10C93, 'M', u'𐳓'), + (0x10C94, 'M', u'𐳔'), + (0x10C95, 'M', u'𐳕'), + (0x10C96, 'M', u'𐳖'), + (0x10C97, 'M', u'𐳗'), + (0x10C98, 'M', u'𐳘'), + (0x10C99, 'M', u'𐳙'), + (0x10C9A, 'M', u'𐳚'), + (0x10C9B, 'M', u'𐳛'), + (0x10C9C, 'M', u'𐳜'), + (0x10C9D, 'M', u'𐳝'), + ] + +def _seg_55(): + return [ + (0x10C9E, 'M', u'𐳞'), + (0x10C9F, 'M', u'𐳟'), + (0x10CA0, 'M', u'𐳠'), + (0x10CA1, 'M', u'𐳡'), + (0x10CA2, 'M', u'𐳢'), + (0x10CA3, 'M', u'𐳣'), + (0x10CA4, 'M', u'𐳤'), + (0x10CA5, 'M', u'𐳥'), + (0x10CA6, 'M', u'𐳦'), + (0x10CA7, 'M', u'𐳧'), + (0x10CA8, 'M', u'𐳨'), + (0x10CA9, 'M', u'𐳩'), + (0x10CAA, 'M', u'𐳪'), + (0x10CAB, 'M', u'𐳫'), + (0x10CAC, 'M', u'𐳬'), + (0x10CAD, 'M', u'𐳭'), + (0x10CAE, 'M', u'𐳮'), + (0x10CAF, 'M', u'𐳯'), + (0x10CB0, 'M', u'𐳰'), + (0x10CB1, 'M', u'𐳱'), + (0x10CB2, 'M', u'𐳲'), + (0x10CB3, 'X'), + (0x10CC0, 'V'), + (0x10CF3, 'X'), + (0x10CFA, 'V'), + (0x10D00, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), (0x11070, 'X'), - (0x11080, 'V'), + (0x1107F, 'V'), (0x110BD, 'X'), (0x110BE, 'V'), (0x110C2, 'X'), @@ -5499,36 +5771,235 @@ def _seg_52(): (0x11135, 'X'), (0x11136, 'V'), (0x11144, 'X'), + (0x11150, 'V'), + (0x11177, 'X'), (0x11180, 'V'), - (0x111C9, 'X'), + (0x111CE, 'X'), (0x111D0, 'V'), - (0x111DA, 'X'), + (0x111E0, 'X'), + (0x111E1, 'V'), + (0x111F5, 'X'), + (0x11200, 'V'), + (0x11212, 'X'), + (0x11213, 'V'), + (0x1123F, 'X'), + (0x11280, 'V'), + (0x11287, 'X'), + (0x11288, 'V'), + (0x11289, 'X'), + (0x1128A, 'V'), + (0x1128E, 'X'), + (0x1128F, 'V'), + (0x1129E, 'X'), + (0x1129F, 'V'), + (0x112AA, 'X'), + (0x112B0, 'V'), + (0x112EB, 'X'), + (0x112F0, 'V'), + (0x112FA, 'X'), + (0x11300, 'V'), + (0x11304, 'X'), + (0x11305, 'V'), + (0x1130D, 'X'), + (0x1130F, 'V'), + (0x11311, 'X'), + (0x11313, 'V'), + (0x11329, 'X'), + (0x1132A, 'V'), + (0x11331, 'X'), + (0x11332, 'V'), + (0x11334, 'X'), + (0x11335, 'V'), + (0x1133A, 'X'), + (0x1133C, 'V'), + (0x11345, 'X'), + (0x11347, 'V'), + (0x11349, 'X'), + (0x1134B, 'V'), + (0x1134E, 'X'), + (0x11350, 'V'), + (0x11351, 'X'), + (0x11357, 'V'), + (0x11358, 'X'), + (0x1135D, 'V'), + (0x11364, 'X'), + (0x11366, 'V'), + (0x1136D, 'X'), + (0x11370, 'V'), + (0x11375, 'X'), + ] + +def _seg_56(): + return [ + (0x11400, 'V'), + (0x1145A, 'X'), + (0x1145B, 'V'), + (0x1145C, 'X'), + (0x1145D, 'V'), + (0x1145E, 'X'), + (0x11480, 'V'), + (0x114C8, 'X'), + (0x114D0, 'V'), + (0x114DA, 'X'), + (0x11580, 'V'), + (0x115B6, 'X'), + (0x115B8, 'V'), + (0x115DE, 'X'), + (0x11600, 'V'), + (0x11645, 'X'), + (0x11650, 'V'), + (0x1165A, 'X'), + (0x11660, 'V'), + (0x1166D, 'X'), (0x11680, 'V'), (0x116B8, 'X'), (0x116C0, 'V'), (0x116CA, 'X'), - (0x12000, 'V'), - (0x1236F, 'X'), - (0x12400, 'V'), - (0x12463, 'X'), - (0x12470, 'V'), - (0x12474, 'X'), - (0x13000, 'V'), - (0x1342F, 'X'), + (0x11700, 'V'), + (0x1171A, 'X'), + (0x1171D, 'V'), + (0x1172C, 'X'), + (0x11730, 'V'), + (0x11740, 'X'), + (0x118A0, 'M', u'𑣀'), + (0x118A1, 'M', u'𑣁'), + (0x118A2, 'M', u'𑣂'), + (0x118A3, 'M', u'𑣃'), + (0x118A4, 'M', u'𑣄'), + (0x118A5, 'M', u'𑣅'), + (0x118A6, 'M', u'𑣆'), + (0x118A7, 'M', u'𑣇'), + (0x118A8, 'M', u'𑣈'), + (0x118A9, 'M', u'𑣉'), + (0x118AA, 'M', u'𑣊'), + (0x118AB, 'M', u'𑣋'), + (0x118AC, 'M', u'𑣌'), + (0x118AD, 'M', u'𑣍'), + (0x118AE, 'M', u'𑣎'), + (0x118AF, 'M', u'𑣏'), + (0x118B0, 'M', u'𑣐'), + (0x118B1, 'M', u'𑣑'), + (0x118B2, 'M', u'𑣒'), + (0x118B3, 'M', u'𑣓'), + (0x118B4, 'M', u'𑣔'), + (0x118B5, 'M', u'𑣕'), + (0x118B6, 'M', u'𑣖'), + (0x118B7, 'M', u'𑣗'), + (0x118B8, 'M', u'𑣘'), + (0x118B9, 'M', u'𑣙'), + (0x118BA, 'M', u'𑣚'), + (0x118BB, 'M', u'𑣛'), + (0x118BC, 'M', u'𑣜'), + (0x118BD, 'M', u'𑣝'), + (0x118BE, 'M', u'𑣞'), + (0x118BF, 'M', u'𑣟'), + (0x118C0, 'V'), + (0x118F3, 'X'), + (0x118FF, 'V'), + (0x11900, 'X'), + (0x11A00, 'V'), + (0x11A48, 'X'), + (0x11A50, 'V'), + (0x11A84, 'X'), + (0x11A86, 'V'), + (0x11A9D, 'X'), + (0x11A9E, 'V'), + (0x11AA3, 'X'), + (0x11AC0, 'V'), + (0x11AF9, 'X'), + (0x11C00, 'V'), + (0x11C09, 'X'), + (0x11C0A, 'V'), + (0x11C37, 'X'), + (0x11C38, 'V'), + (0x11C46, 'X'), + (0x11C50, 'V'), + (0x11C6D, 'X'), + (0x11C70, 'V'), + (0x11C90, 'X'), + (0x11C92, 'V'), + (0x11CA8, 'X'), + (0x11CA9, 'V'), + (0x11CB7, 'X'), + (0x11D00, 'V'), + (0x11D07, 'X'), + (0x11D08, 'V'), + (0x11D0A, 'X'), + (0x11D0B, 'V'), + (0x11D37, 'X'), + (0x11D3A, 'V'), + (0x11D3B, 'X'), + (0x11D3C, 'V'), + (0x11D3E, 'X'), ] -def _seg_53(): +def _seg_57(): return [ + (0x11D3F, 'V'), + (0x11D48, 'X'), + (0x11D50, 'V'), + (0x11D5A, 'X'), + (0x12000, 'V'), + (0x1239A, 'X'), + (0x12400, 'V'), + (0x1246F, 'X'), + (0x12470, 'V'), + (0x12475, 'X'), + (0x12480, 'V'), + (0x12544, 'X'), + (0x13000, 'V'), + (0x1342F, 'X'), + (0x14400, 'V'), + (0x14647, 'X'), (0x16800, 'V'), (0x16A39, 'X'), + (0x16A40, 'V'), + (0x16A5F, 'X'), + (0x16A60, 'V'), + (0x16A6A, 'X'), + (0x16A6E, 'V'), + (0x16A70, 'X'), + (0x16AD0, 'V'), + (0x16AEE, 'X'), + (0x16AF0, 'V'), + (0x16AF6, 'X'), + (0x16B00, 'V'), + (0x16B46, 'X'), + (0x16B50, 'V'), + (0x16B5A, 'X'), + (0x16B5B, 'V'), + (0x16B62, 'X'), + (0x16B63, 'V'), + (0x16B78, 'X'), + (0x16B7D, 'V'), + (0x16B90, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), (0x16F7F, 'X'), (0x16F8F, 'V'), (0x16FA0, 'X'), + (0x16FE0, 'V'), + (0x16FE2, 'X'), + (0x17000, 'V'), + (0x187ED, 'X'), + (0x18800, 'V'), + (0x18AF3, 'X'), (0x1B000, 'V'), - (0x1B002, 'X'), + (0x1B11F, 'X'), + (0x1B170, 'V'), + (0x1B2FC, 'X'), + (0x1BC00, 'V'), + (0x1BC6B, 'X'), + (0x1BC70, 'V'), + (0x1BC7D, 'X'), + (0x1BC80, 'V'), + (0x1BC89, 'X'), + (0x1BC90, 'V'), + (0x1BC9A, 'X'), + (0x1BC9C, 'V'), + (0x1BCA0, 'I'), + (0x1BCA4, 'X'), (0x1D000, 'V'), (0x1D0F6, 'X'), (0x1D100, 'V'), @@ -5551,7 +6022,7 @@ def _seg_53(): (0x1D1BF, 'M', u'𝆹𝅥𝅯'), (0x1D1C0, 'M', u'𝆺𝅥𝅯'), (0x1D1C1, 'V'), - (0x1D1DE, 'X'), + (0x1D1E9, 'X'), (0x1D200, 'V'), (0x1D246, 'X'), (0x1D300, 'V'), @@ -5564,6 +6035,10 @@ def _seg_53(): (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), + ] + +def _seg_58(): + return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -5619,10 +6094,6 @@ def _seg_53(): (0x1D43A, 'M', u'g'), (0x1D43B, 'M', u'h'), (0x1D43C, 'M', u'i'), - ] - -def _seg_54(): - return [ (0x1D43D, 'M', u'j'), (0x1D43E, 'M', u'k'), (0x1D43F, 'M', u'l'), @@ -5668,6 +6139,10 @@ def _seg_54(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), + ] + +def _seg_59(): + return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -5723,10 +6198,6 @@ def _seg_54(): (0x1D49E, 'M', u'c'), (0x1D49F, 'M', u'd'), (0x1D4A0, 'X'), - ] - -def _seg_55(): - return [ (0x1D4A2, 'M', u'g'), (0x1D4A3, 'X'), (0x1D4A5, 'M', u'j'), @@ -5772,6 +6243,10 @@ def _seg_55(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), + ] + +def _seg_60(): + return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -5827,10 +6302,6 @@ def _seg_55(): (0x1D505, 'M', u'b'), (0x1D506, 'X'), (0x1D507, 'M', u'd'), - ] - -def _seg_56(): - return [ (0x1D508, 'M', u'e'), (0x1D509, 'M', u'f'), (0x1D50A, 'M', u'g'), @@ -5876,6 +6347,10 @@ def _seg_56(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), + ] + +def _seg_61(): + return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -5931,10 +6406,6 @@ def _seg_56(): (0x1D56C, 'M', u'a'), (0x1D56D, 'M', u'b'), (0x1D56E, 'M', u'c'), - ] - -def _seg_57(): - return [ (0x1D56F, 'M', u'd'), (0x1D570, 'M', u'e'), (0x1D571, 'M', u'f'), @@ -5980,6 +6451,10 @@ def _seg_57(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), + ] + +def _seg_62(): + return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6035,10 +6510,6 @@ def _seg_57(): (0x1D5D0, 'M', u'w'), (0x1D5D1, 'M', u'x'), (0x1D5D2, 'M', u'y'), - ] - -def _seg_58(): - return [ (0x1D5D3, 'M', u'z'), (0x1D5D4, 'M', u'a'), (0x1D5D5, 'M', u'b'), @@ -6084,6 +6555,10 @@ def _seg_58(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), + ] + +def _seg_63(): + return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6139,10 +6614,6 @@ def _seg_58(): (0x1D634, 'M', u's'), (0x1D635, 'M', u't'), (0x1D636, 'M', u'u'), - ] - -def _seg_59(): - return [ (0x1D637, 'M', u'v'), (0x1D638, 'M', u'w'), (0x1D639, 'M', u'x'), @@ -6188,6 +6659,10 @@ def _seg_59(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), + ] + +def _seg_64(): + return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6243,10 +6718,6 @@ def _seg_59(): (0x1D698, 'M', u'o'), (0x1D699, 'M', u'p'), (0x1D69A, 'M', u'q'), - ] - -def _seg_60(): - return [ (0x1D69B, 'M', u'r'), (0x1D69C, 'M', u's'), (0x1D69D, 'M', u't'), @@ -6292,6 +6763,10 @@ def _seg_60(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), + ] + +def _seg_65(): + return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6347,10 +6822,6 @@ def _seg_60(): (0x1D6FE, 'M', u'γ'), (0x1D6FF, 'M', u'δ'), (0x1D700, 'M', u'ε'), - ] - -def _seg_61(): - return [ (0x1D701, 'M', u'ζ'), (0x1D702, 'M', u'η'), (0x1D703, 'M', u'θ'), @@ -6396,6 +6867,10 @@ def _seg_61(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), + ] + +def _seg_66(): + return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6451,10 +6926,6 @@ def _seg_61(): (0x1D764, 'M', u'ο'), (0x1D765, 'M', u'π'), (0x1D766, 'M', u'ρ'), - ] - -def _seg_62(): - return [ (0x1D767, 'M', u'θ'), (0x1D768, 'M', u'σ'), (0x1D769, 'M', u'τ'), @@ -6500,6 +6971,10 @@ def _seg_62(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), + ] + +def _seg_67(): + return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -6555,10 +7030,6 @@ def _seg_62(): (0x1D7CA, 'M', u'ϝ'), (0x1D7CC, 'X'), (0x1D7CE, 'M', u'0'), - ] - -def _seg_63(): - return [ (0x1D7CF, 'M', u'1'), (0x1D7D0, 'M', u'2'), (0x1D7D1, 'M', u'3'), @@ -6604,11 +7075,74 @@ def _seg_63(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), + ] + +def _seg_68(): + return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), (0x1D7FF, 'M', u'9'), - (0x1D800, 'X'), + (0x1D800, 'V'), + (0x1DA8C, 'X'), + (0x1DA9B, 'V'), + (0x1DAA0, 'X'), + (0x1DAA1, 'V'), + (0x1DAB0, 'X'), + (0x1E000, 'V'), + (0x1E007, 'X'), + (0x1E008, 'V'), + (0x1E019, 'X'), + (0x1E01B, 'V'), + (0x1E022, 'X'), + (0x1E023, 'V'), + (0x1E025, 'X'), + (0x1E026, 'V'), + (0x1E02B, 'X'), + (0x1E800, 'V'), + (0x1E8C5, 'X'), + (0x1E8C7, 'V'), + (0x1E8D7, 'X'), + (0x1E900, 'M', u'𞤢'), + (0x1E901, 'M', u'𞤣'), + (0x1E902, 'M', u'𞤤'), + (0x1E903, 'M', u'𞤥'), + (0x1E904, 'M', u'𞤦'), + (0x1E905, 'M', u'𞤧'), + (0x1E906, 'M', u'𞤨'), + (0x1E907, 'M', u'𞤩'), + (0x1E908, 'M', u'𞤪'), + (0x1E909, 'M', u'𞤫'), + (0x1E90A, 'M', u'𞤬'), + (0x1E90B, 'M', u'𞤭'), + (0x1E90C, 'M', u'𞤮'), + (0x1E90D, 'M', u'𞤯'), + (0x1E90E, 'M', u'𞤰'), + (0x1E90F, 'M', u'𞤱'), + (0x1E910, 'M', u'𞤲'), + (0x1E911, 'M', u'𞤳'), + (0x1E912, 'M', u'𞤴'), + (0x1E913, 'M', u'𞤵'), + (0x1E914, 'M', u'𞤶'), + (0x1E915, 'M', u'𞤷'), + (0x1E916, 'M', u'𞤸'), + (0x1E917, 'M', u'𞤹'), + (0x1E918, 'M', u'𞤺'), + (0x1E919, 'M', u'𞤻'), + (0x1E91A, 'M', u'𞤼'), + (0x1E91B, 'M', u'𞤽'), + (0x1E91C, 'M', u'𞤾'), + (0x1E91D, 'M', u'𞤿'), + (0x1E91E, 'M', u'𞥀'), + (0x1E91F, 'M', u'𞥁'), + (0x1E920, 'M', u'𞥂'), + (0x1E921, 'M', u'𞥃'), + (0x1E922, 'V'), + (0x1E94B, 'X'), + (0x1E950, 'V'), + (0x1E95A, 'X'), + (0x1E95E, 'V'), + (0x1E960, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -6645,6 +7179,10 @@ def _seg_63(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), + ] + +def _seg_69(): + return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -6659,10 +7197,6 @@ def _seg_63(): (0x1EE30, 'M', u'ف'), (0x1EE31, 'M', u'ص'), (0x1EE32, 'M', u'ق'), - ] - -def _seg_64(): - return [ (0x1EE33, 'X'), (0x1EE34, 'M', u'ش'), (0x1EE35, 'M', u'ت'), @@ -6749,6 +7283,10 @@ def _seg_64(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), + ] + +def _seg_70(): + return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -6763,10 +7301,6 @@ def _seg_64(): (0x1EEA2, 'M', u'ج'), (0x1EEA3, 'M', u'د'), (0x1EEA4, 'X'), - ] - -def _seg_65(): - return [ (0x1EEA5, 'M', u'و'), (0x1EEA6, 'M', u'ز'), (0x1EEA7, 'M', u'ح'), @@ -6800,11 +7334,11 @@ def _seg_65(): (0x1F0A0, 'V'), (0x1F0AF, 'X'), (0x1F0B1, 'V'), - (0x1F0BF, 'X'), + (0x1F0C0, 'X'), (0x1F0C1, 'V'), (0x1F0D0, 'X'), (0x1F0D1, 'V'), - (0x1F0E0, 'X'), + (0x1F0F6, 'X'), (0x1F101, '3', u'0,'), (0x1F102, '3', u'1,'), (0x1F103, '3', u'2,'), @@ -6815,7 +7349,8 @@ def _seg_65(): (0x1F108, '3', u'7,'), (0x1F109, '3', u'8,'), (0x1F10A, '3', u'9,'), - (0x1F10B, 'X'), + (0x1F10B, 'V'), + (0x1F10D, 'X'), (0x1F110, '3', u'(a)'), (0x1F111, '3', u'(b)'), (0x1F112, '3', u'(c)'), @@ -6852,6 +7387,10 @@ def _seg_65(): (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), + ] + +def _seg_71(): + return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -6867,10 +7406,6 @@ def _seg_65(): (0x1F140, 'M', u'q'), (0x1F141, 'M', u'r'), (0x1F142, 'M', u's'), - ] - -def _seg_66(): - return [ (0x1F143, 'M', u't'), (0x1F144, 'M', u'u'), (0x1F145, 'M', u'v'), @@ -6891,7 +7426,7 @@ def _seg_66(): (0x1F170, 'V'), (0x1F190, 'M', u'dj'), (0x1F191, 'V'), - (0x1F19B, 'X'), + (0x1F1AD, 'X'), (0x1F1E6, 'V'), (0x1F200, 'M', u'ほか'), (0x1F201, 'M', u'ココ'), @@ -6940,7 +7475,8 @@ def _seg_66(): (0x1F238, 'M', u'申'), (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), - (0x1F23B, 'X'), + (0x1F23B, 'M', u'配'), + (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), (0x1F242, 'M', u'〔二〕'), @@ -6954,52 +7490,56 @@ def _seg_66(): (0x1F250, 'M', u'得'), (0x1F251, 'M', u'可'), (0x1F252, 'X'), - (0x1F300, 'V'), - (0x1F321, 'X'), - (0x1F330, 'V'), - (0x1F336, 'X'), - (0x1F337, 'V'), - (0x1F37D, 'X'), - (0x1F380, 'V'), - (0x1F394, 'X'), - (0x1F3A0, 'V'), - (0x1F3C5, 'X'), - (0x1F3C6, 'V'), - (0x1F3CB, 'X'), - (0x1F3E0, 'V'), - (0x1F3F1, 'X'), - (0x1F400, 'V'), - (0x1F43F, 'X'), - (0x1F440, 'V'), + (0x1F260, 'V'), ] -def _seg_67(): +def _seg_72(): return [ - (0x1F441, 'X'), - (0x1F442, 'V'), - (0x1F4F8, 'X'), - (0x1F4F9, 'V'), - (0x1F4FD, 'X'), - (0x1F500, 'V'), - (0x1F53E, 'X'), - (0x1F540, 'V'), - (0x1F544, 'X'), - (0x1F550, 'V'), - (0x1F568, 'X'), - (0x1F5FB, 'V'), - (0x1F641, 'X'), - (0x1F645, 'V'), - (0x1F650, 'X'), - (0x1F680, 'V'), - (0x1F6C6, 'X'), + (0x1F266, 'X'), + (0x1F300, 'V'), + (0x1F6D5, 'X'), + (0x1F6E0, 'V'), + (0x1F6ED, 'X'), + (0x1F6F0, 'V'), + (0x1F6F9, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), + (0x1F780, 'V'), + (0x1F7D5, 'X'), + (0x1F800, 'V'), + (0x1F80C, 'X'), + (0x1F810, 'V'), + (0x1F848, 'X'), + (0x1F850, 'V'), + (0x1F85A, 'X'), + (0x1F860, 'V'), + (0x1F888, 'X'), + (0x1F890, 'V'), + (0x1F8AE, 'X'), + (0x1F900, 'V'), + (0x1F90C, 'X'), + (0x1F910, 'V'), + (0x1F93F, 'X'), + (0x1F940, 'V'), + (0x1F94D, 'X'), + (0x1F950, 'V'), + (0x1F96C, 'X'), + (0x1F980, 'V'), + (0x1F998, 'X'), + (0x1F9C0, 'V'), + (0x1F9C1, 'X'), + (0x1F9D0, 'V'), + (0x1F9E7, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), (0x2B735, 'X'), (0x2B740, 'V'), (0x2B81E, 'X'), + (0x2B820, 'V'), + (0x2CEA2, 'X'), + (0x2CEB0, 'V'), + (0x2EBE1, 'X'), (0x2F800, 'M', u'丽'), (0x2F801, 'M', u'丸'), (0x2F802, 'M', u'乁'), @@ -7055,6 +7595,10 @@ def _seg_67(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), + ] + +def _seg_73(): + return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7075,10 +7619,6 @@ def _seg_67(): (0x2F84B, 'M', u'圖'), (0x2F84C, 'M', u'嘆'), (0x2F84D, 'M', u'圗'), - ] - -def _seg_68(): - return [ (0x2F84E, 'M', u'噑'), (0x2F84F, 'M', u'噴'), (0x2F850, 'M', u'切'), @@ -7159,6 +7699,10 @@ def _seg_68(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), + ] + +def _seg_74(): + return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7179,10 +7723,6 @@ def _seg_68(): (0x2F8B2, 'M', u'成'), (0x2F8B3, 'M', u'戛'), (0x2F8B4, 'M', u'扝'), - ] - -def _seg_69(): - return [ (0x2F8B5, 'M', u'抱'), (0x2F8B6, 'M', u'拔'), (0x2F8B7, 'M', u'捐'), @@ -7263,6 +7803,10 @@ def _seg_69(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), + ] + +def _seg_75(): + return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7283,10 +7827,6 @@ def _seg_69(): (0x2F916, 'M', u'㶖'), (0x2F917, 'M', u'灊'), (0x2F918, 'M', u'災'), - ] - -def _seg_70(): - return [ (0x2F919, 'M', u'灷'), (0x2F91A, 'M', u'炭'), (0x2F91B, 'M', u'𠔥'), @@ -7367,6 +7907,10 @@ def _seg_70(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), + ] + +def _seg_76(): + return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7387,10 +7931,6 @@ def _seg_70(): (0x2F97D, 'M', u'聠'), (0x2F97E, 'M', u'𦖨'), (0x2F97F, 'M', u'聰'), - ] - -def _seg_71(): - return [ (0x2F980, 'M', u'𣍟'), (0x2F981, 'M', u'䏕'), (0x2F982, 'M', u'育'), @@ -7471,6 +8011,10 @@ def _seg_71(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), + ] + +def _seg_77(): + return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -7491,10 +8035,6 @@ def _seg_71(): (0x2F9E1, 'M', u'𨗭'), (0x2F9E2, 'M', u'邔'), (0x2F9E3, 'M', u'郱'), - ] - -def _seg_72(): - return [ (0x2F9E4, 'M', u'鄑'), (0x2F9E5, 'M', u'𨜮'), (0x2F9E6, 'M', u'鄛'), @@ -7631,4 +8171,9 @@ uts46data = tuple( + _seg_70() + _seg_71() + _seg_72() + + _seg_73() + + _seg_74() + + _seg_75() + + _seg_76() + + _seg_77() ) diff --git a/src/pip/_vendor/ipaddress.py b/src/pip/_vendor/ipaddress.py index 8cfdd58ac..f2d076684 100644 --- a/src/pip/_vendor/ipaddress.py +++ b/src/pip/_vendor/ipaddress.py @@ -14,7 +14,7 @@ from __future__ import unicode_literals import itertools import struct -__version__ = '1.0.19' +__version__ = '1.0.22' # Compatibility functions _compat_int_types = (int,) diff --git a/src/pip/_vendor/pkg_resources/__init__.py b/src/pip/_vendor/pkg_resources/__init__.py index 19d3d0562..f2815cc6b 100644 --- a/src/pip/_vendor/pkg_resources/__init__.py +++ b/src/pip/_vendor/pkg_resources/__init__.py @@ -377,11 +377,7 @@ def get_build_platform(): XXX Currently this is the same as ``distutils.util.get_platform()``, but it needs some hacks for Linux and Mac OS X. """ - try: - # Python 2.7 or >=3.2 - from sysconfig import get_platform - except ImportError: - from distutils.util import get_platform + from sysconfig import get_platform plat = get_platform() if sys.platform == "darwin" and not plat.startswith('macosx-'): @@ -1518,12 +1514,10 @@ class DefaultProvider(EggProvider): @classmethod def _register(cls): - loader_cls = getattr( - importlib_machinery, - 'SourceFileLoader', - type(None), - ) - register_loader_type(loader_cls, cls) + loader_names = 'SourceFileLoader', 'SourcelessFileLoader', + for name in loader_names: + loader_cls = getattr(importlib_machinery, name, type(None)) + register_loader_type(loader_cls, cls) DefaultProvider._register() @@ -2669,6 +2663,19 @@ class Distribution(object): raise AttributeError(attr) return getattr(self._provider, attr) + def __dir__(self): + return list( + set(super(Distribution, self).__dir__()) + | set( + attr for attr in self._provider.__dir__() + if not attr.startswith('_') + ) + ) + + if not hasattr(object, '__dir__'): + # python 2.7 not supported + del __dir__ + @classmethod def from_filename(cls, filename, metadata=None, **kw): return cls.from_location( diff --git a/src/pip/_vendor/progress/__init__.py b/src/pip/_vendor/progress/__init__.py index 09dfc1ebe..a41f65dc5 100644 --- a/src/pip/_vendor/progress/__init__.py +++ b/src/pip/_vendor/progress/__init__.py @@ -21,7 +21,7 @@ from sys import stderr from time import time -__version__ = '1.3' +__version__ = '1.4' class Infinite(object): diff --git a/src/pip/_vendor/progress/bar.py b/src/pip/_vendor/progress/bar.py index 5ee968f0c..025e61c45 100644 --- a/src/pip/_vendor/progress/bar.py +++ b/src/pip/_vendor/progress/bar.py @@ -15,6 +15,9 @@ # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. from __future__ import unicode_literals + +import sys + from . import Progress from .helpers import WritelnMixin @@ -61,7 +64,10 @@ class FillingCirclesBar(ChargingBar): class IncrementalBar(Bar): - phases = (' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█') + if sys.platform.startswith('win'): + phases = (u' ', u'▌', u'█') + else: + phases = (' ', '▏', '▎', '▍', '▌', '▋', '▊', '▉', '█') def update(self): nphases = len(self.phases) diff --git a/src/pip/_vendor/progress/helpers.py b/src/pip/_vendor/progress/helpers.py index 9ed90b2bc..0cde44ec2 100644 --- a/src/pip/_vendor/progress/helpers.py +++ b/src/pip/_vendor/progress/helpers.py @@ -28,14 +28,14 @@ class WriteMixin(object): if message: self.message = message - if self.file.isatty(): + if self.file and self.file.isatty(): if self.hide_cursor: print(HIDE_CURSOR, end='', file=self.file) print(self.message, end='', file=self.file) self.file.flush() def write(self, s): - if self.file.isatty(): + if self.file and self.file.isatty(): b = '\b' * self._width c = s.ljust(self._width) print(b + c, end='', file=self.file) @@ -43,7 +43,7 @@ class WriteMixin(object): self.file.flush() def finish(self): - if self.file.isatty() and self.hide_cursor: + if self.file and self.file.isatty() and self.hide_cursor: print(SHOW_CURSOR, end='', file=self.file) @@ -55,21 +55,21 @@ class WritelnMixin(object): if message: self.message = message - if self.file.isatty() and self.hide_cursor: + if self.file and self.file.isatty() and self.hide_cursor: print(HIDE_CURSOR, end='', file=self.file) def clearln(self): - if self.file.isatty(): + if self.file and self.file.isatty(): print('\r\x1b[K', end='', file=self.file) def writeln(self, line): - if self.file.isatty(): + if self.file and self.file.isatty(): self.clearln() print(line, end='', file=self.file) self.file.flush() def finish(self): - if self.file.isatty(): + if self.file and self.file.isatty(): print(file=self.file) if self.hide_cursor: print(SHOW_CURSOR, end='', file=self.file) diff --git a/src/pip/_vendor/pytoml/parser.py b/src/pip/_vendor/pytoml/parser.py index 7fc3d34d5..e03a03fbd 100644 --- a/src/pip/_vendor/pytoml/parser.py +++ b/src/pip/_vendor/pytoml/parser.py @@ -6,35 +6,35 @@ if sys.version_info[0] == 2: else: _chr = chr -def load(fin, translate=lambda t, x, v: v): - return loads(fin.read(), translate=translate, filename=getattr(fin, 'name', repr(fin))) +def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): + return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) -def loads(s, filename='', translate=lambda t, x, v: v): +def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): if isinstance(s, bytes): s = s.decode('utf-8') s = s.replace('\r\n', '\n') - root = {} - tables = {} + root = object_pairs_hook() + tables = object_pairs_hook() scope = root src = _Source(s, filename=filename) - ast = _p_toml(src) + ast = _p_toml(src, object_pairs_hook=object_pairs_hook) def error(msg): raise TomlError(msg, pos[0], pos[1], filename) - def process_value(v): + def process_value(v, object_pairs_hook): kind, text, value, pos = v if kind == 'str' and value.startswith('\n'): value = value[1:] if kind == 'array': if value and any(k != value[0][0] for k, t, v, p in value[1:]): error('array-type-mismatch') - value = [process_value(item) for item in value] + value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] elif kind == 'table': - value = dict([(k, process_value(value[k])) for k in value]) + value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) return translate(kind, text, value) for kind, value, pos in ast: @@ -42,7 +42,7 @@ def loads(s, filename='', translate=lambda t, x, v: v): k, v = value if k in scope: error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) - scope[k] = process_value(v) + scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) else: is_table_array = (kind == 'table_array') cur = tables @@ -50,19 +50,19 @@ def loads(s, filename='', translate=lambda t, x, v: v): if isinstance(cur.get(name), list): d, cur = cur[name][-1] else: - d, cur = cur.setdefault(name, (None, {})) + d, cur = cur.setdefault(name, (None, object_pairs_hook())) - scope = {} + scope = object_pairs_hook() name = value[-1] if name not in cur: if is_table_array: - cur[name] = [(scope, {})] + cur[name] = [(scope, object_pairs_hook())] else: - cur[name] = (scope, {}) + cur[name] = (scope, object_pairs_hook()) elif isinstance(cur[name], list): if not is_table_array: error('table_type_mismatch') - cur[name].append((scope, {})) + cur[name].append((scope, object_pairs_hook())) else: if is_table_array: error('table_type_mismatch') @@ -73,7 +73,7 @@ def loads(s, filename='', translate=lambda t, x, v: v): def merge_tables(scope, tables): if scope is None: - scope = {} + scope = object_pairs_hook() for k in tables: if k in scope: error('key_table_conflict') @@ -225,7 +225,7 @@ _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') _litstr_re = re.compile(r"[^'\000-\037]*") _litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*") -def _p_value(s): +def _p_value(s, object_pairs_hook): pos = s.pos() if s.consume('true'): @@ -283,7 +283,7 @@ def _p_value(s): with s: while True: _p_ews(s) - items.append(_p_value(s)) + items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) s.commit() _p_ews(s) s.expect(',') @@ -294,13 +294,13 @@ def _p_value(s): if s.consume('{'): _p_ws(s) - items = {} + items = object_pairs_hook() if not s.consume('}'): k = _p_key(s) _p_ws(s) s.expect('=') _p_ws(s) - items[k] = _p_value(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) _p_ws(s) while s.consume(','): _p_ws(s) @@ -308,14 +308,14 @@ def _p_value(s): _p_ws(s) s.expect('=') _p_ws(s) - items[k] = _p_value(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) _p_ws(s) s.expect('}') return 'table', None, items, pos s.fail() -def _p_stmt(s): +def _p_stmt(s, object_pairs_hook): pos = s.pos() if s.consume( '['): is_array = s.consume('[') @@ -335,19 +335,19 @@ def _p_stmt(s): _p_ws(s) s.expect('=') _p_ws(s) - value = _p_value(s) + value = _p_value(s, object_pairs_hook=object_pairs_hook) return 'kv', (key, value), pos _stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') -def _p_toml(s): +def _p_toml(s, object_pairs_hook): stmts = [] _p_ews(s) with s: - stmts.append(_p_stmt(s)) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) while True: s.commit() s.expect_re(_stmtsep_re) - stmts.append(_p_stmt(s)) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) _p_ews(s) s.expect_eof() return stmts diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__init__.py index ccd361adf..9fb6633ed 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.22 + # urllib3 >= 1.21.1, <= 1.23 assert major == 1 assert minor >= 21 - assert minor <= 22 + assert minor <= 23 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -71,6 +71,17 @@ def check_compatibility(urllib3_version, chardet_version): assert patch >= 2 +def _check_cryptography(cryptography_version): + # cryptography < 1.3.4 + try: + cryptography_version = list(map(int, cryptography_version.split('.'))) + except ValueError: + return + + if cryptography_version < [1, 3, 4]: + warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warnings.warn(warning, RequestsDependencyWarning) + # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) @@ -85,6 +96,10 @@ if not WINDOWS: try: from pip._vendor.urllib3.contrib import pyopenssl pyopenssl.inject_into_urllib3() + + # Check cryptography version + from cryptography import __version__ as cryptography_version + _check_cryptography(cryptography_version) except ImportError: pass diff --git a/src/pip/_vendor/requests/__version__.py b/src/pip/_vendor/requests/__version__.py index dc33eef65..ef61ec0f5 100644 --- a/src/pip/_vendor/requests/__version__.py +++ b/src/pip/_vendor/requests/__version__.py @@ -5,10 +5,10 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.18.4' -__build__ = 0x021804 +__version__ = '2.19.1' +__build__ = 0x021901 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2017 Kenneth Reitz' +__copyright__ = 'Copyright 2018 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/src/pip/_vendor/requests/adapters.py b/src/pip/_vendor/requests/adapters.py index 57876383c..f6f3f9965 100644 --- a/src/pip/_vendor/requests/adapters.py +++ b/src/pip/_vendor/requests/adapters.py @@ -13,6 +13,7 @@ import socket from pip._vendor.urllib3.poolmanager import PoolManager, proxy_from_url from pip._vendor.urllib3.response import HTTPResponse +from pip._vendor.urllib3.util import parse_url from pip._vendor.urllib3.util import Timeout as TimeoutSauce from pip._vendor.urllib3.util.retry import Retry from pip._vendor.urllib3.exceptions import ClosedPoolError @@ -28,13 +29,13 @@ from pip._vendor.urllib3.exceptions import ResponseError from .models import Response from .compat import urlparse, basestring -from .utils import (DEFAULT_CA_BUNDLE_PATH, get_encoding_from_headers, - prepend_scheme_if_needed, get_auth_from_url, urldefragauth, - select_proxy) +from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, + get_encoding_from_headers, prepend_scheme_if_needed, + get_auth_from_url, urldefragauth, select_proxy) from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL) from .auth import _basic_auth_str try: @@ -219,7 +220,7 @@ class HTTPAdapter(BaseAdapter): cert_loc = verify if not cert_loc: - cert_loc = DEFAULT_CA_BUNDLE_PATH + cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " @@ -300,6 +301,10 @@ class HTTPAdapter(BaseAdapter): if proxy: proxy = prepend_scheme_if_needed(proxy, 'http') + proxy_url = parse_url(proxy) + if not proxy_url.host: + raise InvalidProxyURL("Please check proxy URL. It is malformed" + " and could be missing the host.") proxy_manager = self.proxy_manager_for(proxy) conn = proxy_manager.connection_from_url(url) else: @@ -406,7 +411,7 @@ class HTTPAdapter(BaseAdapter): self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) - self.add_headers(request) + self.add_headers(request, stream=stream, timeout=timeout, verify=verify, cert=cert, proxies=proxies) chunked = not (request.body is None or 'Content-Length' in request.headers) diff --git a/src/pip/_vendor/requests/api.py b/src/pip/_vendor/requests/api.py index bc2115c15..a2cc84d76 100644 --- a/src/pip/_vendor/requests/api.py +++ b/src/pip/_vendor/requests/api.py @@ -20,7 +20,7 @@ def request(method, url, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. - :param json: (optional) json data to send in the body of the :class:`Request`. + :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': file-tuple}``) for multipart encoding upload. diff --git a/src/pip/_vendor/requests/auth.py b/src/pip/_vendor/requests/auth.py index 1a182dffd..4ae459474 100644 --- a/src/pip/_vendor/requests/auth.py +++ b/src/pip/_vendor/requests/auth.py @@ -153,6 +153,18 @@ class HTTPDigestAuth(AuthBase): x = x.encode('utf-8') return hashlib.sha1(x).hexdigest() hash_utf8 = sha_utf8 + elif _algorithm == 'SHA-256': + def sha256_utf8(x): + if isinstance(x, str): + x = x.encode('utf-8') + return hashlib.sha256(x).hexdigest() + hash_utf8 = sha256_utf8 + elif _algorithm == 'SHA-512': + def sha512_utf8(x): + if isinstance(x, str): + x = x.encode('utf-8') + return hashlib.sha512(x).hexdigest() + hash_utf8 = sha512_utf8 KD = lambda s, d: hash_utf8("%s:%s" % (s, d)) diff --git a/src/pip/_vendor/requests/compat.py b/src/pip/_vendor/requests/compat.py index 20da3e0e0..ec5d30585 100644 --- a/src/pip/_vendor/requests/compat.py +++ b/src/pip/_vendor/requests/compat.py @@ -47,6 +47,7 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO + from collections import Callable, Mapping, MutableMapping from pip._vendor.urllib3.packages.ordered_dict import OrderedDict @@ -64,6 +65,7 @@ elif is_py3: from http.cookies import Morsel from io import StringIO from collections import OrderedDict + from collections.abc import Callable, Mapping, MutableMapping builtin_str = str str = str diff --git a/src/pip/_vendor/requests/cookies.py b/src/pip/_vendor/requests/cookies.py index ab3c88b9b..50883a84f 100644 --- a/src/pip/_vendor/requests/cookies.py +++ b/src/pip/_vendor/requests/cookies.py @@ -12,10 +12,9 @@ requests.utils imports from here, so be careful with imports. import copy import time import calendar -import collections from ._internal_utils import to_native_string -from .compat import cookielib, urlparse, urlunparse, Morsel +from .compat import cookielib, urlparse, urlunparse, Morsel, MutableMapping try: import threading @@ -169,7 +168,7 @@ class CookieConflictError(RuntimeError): """ -class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): +class RequestsCookieJar(cookielib.CookieJar, MutableMapping): """Compatibility class; is a cookielib.CookieJar, but exposes a dict interface. @@ -415,9 +414,14 @@ class RequestsCookieJar(cookielib.CookieJar, collections.MutableMapping): def copy(self): """Return a copy of this RequestsCookieJar.""" new_cj = RequestsCookieJar() + new_cj.set_policy(self.get_policy()) new_cj.update(self) return new_cj + def get_policy(self): + """Return the CookiePolicy instance used.""" + return self._policy + def _copy_cookie_jar(jar): if jar is None: diff --git a/src/pip/_vendor/requests/exceptions.py b/src/pip/_vendor/requests/exceptions.py index 3e5d0b2db..a91e1fd11 100644 --- a/src/pip/_vendor/requests/exceptions.py +++ b/src/pip/_vendor/requests/exceptions.py @@ -85,6 +85,10 @@ class InvalidHeader(RequestException, ValueError): """The header value provided was somehow invalid.""" +class InvalidProxyURL(InvalidURL): + """The proxy URL provided is invalid.""" + + class ChunkedEncodingError(RequestException): """The server declared chunked encoding but sent an invalid chunk.""" diff --git a/src/pip/_vendor/requests/help.py b/src/pip/_vendor/requests/help.py index 7c4b193c9..df1b4ebcf 100644 --- a/src/pip/_vendor/requests/help.py +++ b/src/pip/_vendor/requests/help.py @@ -13,7 +13,7 @@ from pip._vendor import chardet from . import __version__ as requests_version try: - from .packages.urllib3.contrib import pyopenssl + from pip._vendor.urllib3.contrib import pyopenssl except ImportError: pyopenssl = None OpenSSL = None diff --git a/src/pip/_vendor/requests/models.py b/src/pip/_vendor/requests/models.py index 4ab4feca9..4230535d2 100644 --- a/src/pip/_vendor/requests/models.py +++ b/src/pip/_vendor/requests/models.py @@ -7,7 +7,6 @@ requests.models This module contains the primary objects that power Requests. """ -import collections import datetime import sys @@ -37,6 +36,7 @@ from .utils import ( stream_decode_response_unicode, to_key_val_list, parse_header_links, iter_slices, guess_json_utf, super_len, check_header_validity) from .compat import ( + Callable, Mapping, cookielib, urlunparse, urlsplit, urlencode, str, bytes, is_py2, chardet, builtin_str, basestring) from .compat import json as complexjson @@ -155,8 +155,12 @@ class RequestEncodingMixin(object): if isinstance(fp, (str, bytes, bytearray)): fdata = fp - else: + elif hasattr(fp, 'read'): fdata = fp.read() + elif fp is None: + continue + else: + fdata = fp rf = RequestField(name=k, data=fdata, filename=fn, headers=fh) rf.make_multipart(content_type=ft) @@ -174,10 +178,10 @@ class RequestHooksMixin(object): if event not in self.hooks: raise ValueError('Unsupported event specified, with event name "%s"' % (event)) - if isinstance(hook, collections.Callable): + if isinstance(hook, Callable): self.hooks[event].append(hook) elif hasattr(hook, '__iter__'): - self.hooks[event].extend(h for h in hook if isinstance(h, collections.Callable)) + self.hooks[event].extend(h for h in hook if isinstance(h, Callable)) def deregister_hook(self, event, hook): """Deregister a previously registered hook. @@ -461,7 +465,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): is_stream = all([ hasattr(data, '__iter__'), - not isinstance(data, (basestring, list, tuple, collections.Mapping)) + not isinstance(data, (basestring, list, tuple, Mapping)) ]) try: @@ -686,11 +690,11 @@ class Response(object): @property def ok(self): - """Returns True if :attr:`status_code` is less than 400. + """Returns True if :attr:`status_code` is less than 400, False if not. This attribute checks if the status code of the response is between 400 and 600 to see if there was a client error or a server error. If - the status code, is between 200 and 400, this will return True. This + the status code is between 200 and 400, this will return True. This is **not** a check to see if the response code is ``200 OK``. """ try: @@ -820,7 +824,7 @@ class Response(object): if self.status_code == 0 or self.raw is None: self._content = None else: - self._content = bytes().join(self.iter_content(CONTENT_CHUNK_SIZE)) or bytes() + self._content = b''.join(self.iter_content(CONTENT_CHUNK_SIZE)) or b'' self._content_consumed = True # don't need to release the connection; that's been handled by urllib3 diff --git a/src/pip/_vendor/requests/sessions.py b/src/pip/_vendor/requests/sessions.py index 6570e7334..ba135268a 100644 --- a/src/pip/_vendor/requests/sessions.py +++ b/src/pip/_vendor/requests/sessions.py @@ -8,13 +8,12 @@ This module provides a Session object to manage and persist settings across requests (cookies, auth, proxies). """ import os -import platform +import sys import time -from collections import Mapping from datetime import timedelta from .auth import _basic_auth_str -from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse +from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse, Mapping from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT @@ -38,8 +37,8 @@ from .status_codes import codes from .models import REDIRECT_STATI # Preferred clock, based on which one is more accurate on a given system. -if platform.system() == 'Windows': - try: # Python 3.3+ +if sys.platform == 'win32': + try: # Python 3.4+ preferred_clock = time.perf_counter except AttributeError: # Earlier than Python 3. preferred_clock = time.clock @@ -123,6 +122,7 @@ class SessionRedirectMixin(object): hist = [] # keep track of history url = self.get_redirect_target(resp) + previous_fragment = urlparse(req.url).fragment while url: prepared_request = req.copy() @@ -147,8 +147,12 @@ class SessionRedirectMixin(object): parsed_rurl = urlparse(resp.url) url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) - # The scheme should be lower case... + # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) parsed = urlparse(url) + if parsed.fragment == '' and previous_fragment: + parsed = parsed._replace(fragment=previous_fragment) + elif parsed.fragment: + previous_fragment = parsed.fragment url = parsed.geturl() # Facilitate relative 'location' headers, as allowed by RFC 7231. @@ -696,7 +700,7 @@ class Session(SessionRedirectMixin): """ for (prefix, adapter) in self.adapters.items(): - if url.lower().startswith(prefix): + if url.lower().startswith(prefix.lower()): return adapter # Nothing matches :-/ diff --git a/src/pip/_vendor/requests/status_codes.py b/src/pip/_vendor/requests/status_codes.py index dee89190c..ff462c6c6 100644 --- a/src/pip/_vendor/requests/status_codes.py +++ b/src/pip/_vendor/requests/status_codes.py @@ -1,5 +1,22 @@ # -*- coding: utf-8 -*- +""" +The ``codes`` object defines a mapping from common names for HTTP statuses +to their numerical codes, accessible either as attributes or as dictionary +items. + +>>> requests.codes['temporary_redirect'] +307 +>>> requests.codes.teapot +418 +>>> requests.codes['\o/'] +200 + +Some codes have multiple names, and both upper- and lower-case versions of +the names are allowed. For example, ``codes.ok``, ``codes.OK``, and +``codes.okay`` all correspond to the HTTP status code 200. +""" + from .structures import LookupDict _codes = { @@ -84,8 +101,20 @@ _codes = { codes = LookupDict(name='status_codes') -for code, titles in _codes.items(): - for title in titles: - setattr(codes, title, code) - if not title.startswith(('\\', '/')): - setattr(codes, title.upper(), code) +def _init(): + for code, titles in _codes.items(): + for title in titles: + setattr(codes, title, code) + if not title.startswith(('\\', '/')): + setattr(codes, title.upper(), code) + + def doc(code): + names = ', '.join('``%s``' % n for n in _codes[code]) + return '* %d: %s' % (code, names) + + global __doc__ + __doc__ = (__doc__ + '\n' + + '\n'.join(doc(code) for code in sorted(_codes)) + if __doc__ is not None else None) + +_init() diff --git a/src/pip/_vendor/requests/structures.py b/src/pip/_vendor/requests/structures.py index 05d2b3f57..da930e285 100644 --- a/src/pip/_vendor/requests/structures.py +++ b/src/pip/_vendor/requests/structures.py @@ -7,16 +7,14 @@ requests.structures Data structures that power Requests. """ -import collections - -from .compat import OrderedDict +from .compat import OrderedDict, Mapping, MutableMapping -class CaseInsensitiveDict(collections.MutableMapping): +class CaseInsensitiveDict(MutableMapping): """A case-insensitive ``dict``-like object. Implements all methods and operations of - ``collections.MutableMapping`` as well as dict's ``copy``. Also + ``MutableMapping`` as well as dict's ``copy``. Also provides ``lower_items``. All keys are expected to be strings. The structure remembers the @@ -71,7 +69,7 @@ class CaseInsensitiveDict(collections.MutableMapping): ) def __eq__(self, other): - if isinstance(other, collections.Mapping): + if isinstance(other, Mapping): other = CaseInsensitiveDict(other) else: return NotImplemented diff --git a/src/pip/_vendor/requests/utils.py b/src/pip/_vendor/requests/utils.py index 5c47de989..431f6be07 100644 --- a/src/pip/_vendor/requests/utils.py +++ b/src/pip/_vendor/requests/utils.py @@ -8,17 +8,17 @@ This module provides utility functions that are used within Requests that are also useful for external consumption. """ -import cgi import codecs -import collections import contextlib import io import os -import platform import re import socket import struct +import sys +import tempfile import warnings +import zipfile from .__version__ import __version__ from . import certs @@ -28,7 +28,7 @@ from .compat import parse_http_list as _parse_list_header from .compat import ( quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, proxy_bypass, urlunparse, basestring, integer_types, is_py3, - proxy_bypass_environment, getproxies_environment) + proxy_bypass_environment, getproxies_environment, Mapping) from .cookies import cookiejar_from_dict from .structures import CaseInsensitiveDict from .exceptions import ( @@ -39,19 +39,25 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() -if platform.system() == 'Windows': +if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups def proxy_bypass_registry(host): - if is_py3: - import winreg - else: - import _winreg as winreg + try: + if is_py3: + import winreg + else: + import _winreg as winreg + except ImportError: + return False + try: internetSettings = winreg.OpenKey(winreg.HKEY_CURRENT_USER, r'Software\Microsoft\Windows\CurrentVersion\Internet Settings') - proxyEnable = winreg.QueryValueEx(internetSettings, - 'ProxyEnable')[0] + # ProxyEnable could be REG_SZ or REG_DWORD, normalizing it + proxyEnable = int(winreg.QueryValueEx(internetSettings, + 'ProxyEnable')[0]) + # ProxyOverride is almost always a string proxyOverride = winreg.QueryValueEx(internetSettings, 'ProxyOverride')[0] except OSError: @@ -216,6 +222,38 @@ def guess_filename(obj): return os.path.basename(name) +def extract_zipped_paths(path): + """Replace nonexistent paths that look like they refer to a member of a zip + archive with the location of an extracted copy of the target, or else + just return the provided path unchanged. + """ + if os.path.exists(path): + # this is already a valid path, no need to do anything further + return path + + # find the first valid part of the provided path and treat that as a zip archive + # assume the rest of the path is the name of a member in the archive + archive, member = os.path.split(path) + while archive and not os.path.exists(archive): + archive, prefix = os.path.split(archive) + member = '/'.join([prefix, member]) + + if not zipfile.is_zipfile(archive): + return path + + zip_file = zipfile.ZipFile(archive) + if member not in zip_file.namelist(): + return path + + # we have a valid zip archive and a valid member of that archive + tmp = tempfile.gettempdir() + extracted_path = os.path.join(tmp, *member.split('/')) + if not os.path.exists(extracted_path): + extracted_path = zip_file.extract(member, path=tmp) + + return extracted_path + + def from_key_val_list(value): """Take an object and test to see if it can be represented as a dictionary. Unless it can not be represented as such, return an @@ -262,7 +300,7 @@ def to_key_val_list(value): if isinstance(value, (str, bytes, bool, int)): raise ValueError('cannot encode objects that are not 2-tuples') - if isinstance(value, collections.Mapping): + if isinstance(value, Mapping): value = value.items() return list(value) @@ -407,6 +445,31 @@ def get_encodings_from_content(content): xml_re.findall(content)) +def _parse_content_type_header(header): + """Returns content type and parameters from given header + + :param header: string + :return: tuple containing content type and dictionary of + parameters + """ + + tokens = header.split(';') + content_type, params = tokens[0].strip(), tokens[1:] + params_dict = {} + items_to_strip = "\"' " + + for param in params: + param = param.strip() + if param: + key, value = param, True + index_of_equals = param.find("=") + if index_of_equals != -1: + key = param[:index_of_equals].strip(items_to_strip) + value = param[index_of_equals + 1:].strip(items_to_strip) + params_dict[key] = value + return content_type, params_dict + + def get_encoding_from_headers(headers): """Returns encodings from given HTTP Header Dict. @@ -419,7 +482,7 @@ def get_encoding_from_headers(headers): if not content_type: return None - content_type, params = cgi.parse_header(content_type) + content_type, params = _parse_content_type_header(content_type) if 'charset' in params: return params['charset'].strip("'\"") @@ -632,6 +695,8 @@ def should_bypass_proxies(url, no_proxy): :rtype: bool """ + # Prioritize lowercase environment variables over uppercase + # to keep a consistent behaviour with other http projects (curl, wget). get_proxy = lambda k: os.environ.get(k) or os.environ.get(k.upper()) # First check whether no_proxy is defined. If it is, check that the URL @@ -639,28 +704,31 @@ def should_bypass_proxies(url, no_proxy): no_proxy_arg = no_proxy if no_proxy is None: no_proxy = get_proxy('no_proxy') - netloc = urlparse(url).netloc + parsed = urlparse(url) if no_proxy: # We need to check whether we match here. We need to see if we match - # the end of the netloc, both with and without the port. + # the end of the hostname, both with and without the port. no_proxy = ( host for host in no_proxy.replace(' ', '').split(',') if host ) - ip = netloc.split(':')[0] - if is_ipv4_address(ip): + if is_ipv4_address(parsed.hostname): for proxy_ip in no_proxy: if is_valid_cidr(proxy_ip): - if address_in_network(ip, proxy_ip): + if address_in_network(parsed.hostname, proxy_ip): return True - elif ip == proxy_ip: + elif parsed.hostname == proxy_ip: # If no_proxy ip was defined in plain IP notation instead of cidr notation & # matches the IP of the index return True else: + host_with_port = parsed.hostname + if parsed.port: + host_with_port += ':{0}'.format(parsed.port) + for host in no_proxy: - if netloc.endswith(host) or netloc.split(':')[0].endswith(host): + if parsed.hostname.endswith(host) or host_with_port.endswith(host): # The URL does match something in no_proxy, so we don't want # to apply the proxies on this URL. return True @@ -673,7 +741,7 @@ def should_bypass_proxies(url, no_proxy): # legitimate problems. with set_environ('no_proxy', no_proxy_arg): try: - bypass = proxy_bypass(netloc) + bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): bypass = False @@ -743,7 +811,7 @@ def default_headers(): def parse_header_links(value): - """Return a dict of parsed link headers proxies. + """Return a list of parsed link headers proxies. i.e. Link: ; rel=front; type="image/jpeg",; rel=back;type="image/jpeg" @@ -754,6 +822,10 @@ def parse_header_links(value): replace_chars = ' \'"' + value = value.strip(replace_chars) + if not value: + return links + for val in re.split(', *<', value): try: url, params = val.split(';', 1) diff --git a/src/pip/_vendor/urllib3/__init__.py b/src/pip/_vendor/urllib3/__init__.py old mode 100644 new mode 100755 index aaa6b1c6d..4bd533b5b --- a/src/pip/_vendor/urllib3/__init__.py +++ b/src/pip/_vendor/urllib3/__init__.py @@ -32,7 +32,7 @@ except ImportError: __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.22' +__version__ = '1.23' __all__ = ( 'HTTPConnectionPool', diff --git a/src/pip/_vendor/urllib3/_collections.py b/src/pip/_vendor/urllib3/_collections.py old mode 100644 new mode 100755 index 5df2372c4..6e36b84e5 --- a/src/pip/_vendor/urllib3/_collections.py +++ b/src/pip/_vendor/urllib3/_collections.py @@ -1,5 +1,8 @@ from __future__ import absolute_import -from collections import Mapping, MutableMapping +try: + from collections.abc import Mapping, MutableMapping +except ImportError: + from collections import Mapping, MutableMapping try: from threading import RLock except ImportError: # Platform-specific: No threads available @@ -15,6 +18,7 @@ try: # Python 2.7+ from collections import OrderedDict except ImportError: from .packages.ordered_dict import OrderedDict +from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 @@ -305,13 +309,22 @@ class HTTPHeaderDict(MutableMapping): # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. + obs_fold_continued_leaders = (' ', '\t') headers = [] for line in message.headers: - if line.startswith((' ', '\t')): - key, value = headers[-1] - headers[-1] = (key, value + '\r\n' + line.rstrip()) - continue + if line.startswith(obs_fold_continued_leaders): + if not headers: + # We received a header line that starts with OWS as described + # in RFC-7230 S3.2.4. This indicates a multiline header, but + # there exists no previous header to which we can attach it. + raise InvalidHeader( + 'Header continuation with no previous header: %s' % line + ) + else: + key, value = headers[-1] + headers[-1] = (key, value + ' ' + line.strip()) + continue key, value = line.split(':', 1) headers.append((key, value.strip())) diff --git a/src/pip/_vendor/urllib3/connection.py b/src/pip/_vendor/urllib3/connection.py old mode 100644 new mode 100755 index c0d832998..a03b573f0 --- a/src/pip/_vendor/urllib3/connection.py +++ b/src/pip/_vendor/urllib3/connection.py @@ -56,10 +56,11 @@ port_by_scheme = { 'https': 443, } -# When updating RECENT_DATE, move it to -# within two years of the current date, and no -# earlier than 6 months ago. -RECENT_DATE = datetime.date(2016, 1, 1) +# When updating RECENT_DATE, move it to within two years of the current date, +# and not less than 6 months ago. +# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or +# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months) +RECENT_DATE = datetime.date(2017, 6, 30) class DummyConnection(object): @@ -124,6 +125,35 @@ class HTTPConnection(_HTTPConnection, object): # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) + @property + def host(self): + """ + Getter method to remove any trailing dots that indicate the hostname is an FQDN. + + In general, SSL certificates don't include the trailing dot indicating a + fully-qualified domain name, and thus, they don't validate properly when + checked against a domain name that includes the dot. In addition, some + servers may not expect to receive the trailing dot when provided. + + However, the hostname with trailing dot is critical to DNS resolution; doing a + lookup with the trailing dot will properly only resolve the appropriate FQDN, + whereas a lookup without a trailing dot will search the system's search domain + list. Thus, it's important to keep the original host around for use only in + those cases where it's appropriate (i.e., when doing DNS lookup to establish the + actual TCP connection across which we're going to send HTTP requests). + """ + return self._dns_host.rstrip('.') + + @host.setter + def host(self, value): + """ + Setter for the `host` property. + + We assume that only urllib3 uses the _dns_host attribute; httplib itself + only uses `host`, and it seems reasonable that other libraries follow suit. + """ + self._dns_host = value + def _new_conn(self): """ Establish a socket connection and set nodelay settings on it. @@ -138,7 +168,7 @@ class HTTPConnection(_HTTPConnection, object): try: conn = connection.create_connection( - (self.host, self.port), self.timeout, **extra_kw) + (self._dns_host, self.port), self.timeout, **extra_kw) except SocketTimeout as e: raise ConnectTimeoutError( diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py old mode 100644 new mode 100755 index ec9600f8f..8fcb0bce7 --- a/src/pip/_vendor/urllib3/connectionpool.py +++ b/src/pip/_vendor/urllib3/connectionpool.py @@ -40,13 +40,10 @@ from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout -from .util.url import get_host, Url +from .util.url import get_host, Url, NORMALIZABLE_SCHEMES +from .util.queue import LifoQueue -if six.PY2: - # Queue is imported for side effects on MS Windows - import Queue as _unused_module_Queue # noqa: F401 - xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -62,13 +59,13 @@ class ConnectionPool(object): """ scheme = None - QueueCls = queue.LifoQueue + QueueCls = LifoQueue def __init__(self, host, port=None): if not host: raise LocationValueError("No host specified.") - self.host = _ipv6_host(host).lower() + self.host = _ipv6_host(host, self.scheme) self._proxy_host = host.lower() self.port = port @@ -204,8 +201,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTP connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTP connection (%d): %s:%s", + self.num_connections, self.host, self.port or "80") conn = self.ConnectionCls(host=self.host, port=self.port, timeout=self.timeout.connect_timeout, @@ -411,6 +408,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """ Close all pooled connections and disable the pool. """ + if self.pool is None: + return # Disable access to the pool old_pool, self.pool = self.pool, None @@ -434,7 +433,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - host = _ipv6_host(host).lower() + host = _ipv6_host(host, self.scheme) # Use explicit default port for comparison when none is given if self.port and not port: @@ -820,8 +819,8 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTPS connection (%d): %s", - self.num_connections, self.host) + log.debug("Starting new HTTPS connection (%d): %s:%s", + self.num_connections, self.host, self.port or "443") if not self.ConnectionCls or self.ConnectionCls is DummyConnection: raise SSLError("Can't connect to HTTPS URL because the SSL " @@ -886,7 +885,7 @@ def connection_from_url(url, **kw): return HTTPConnectionPool(host, port=port, **kw) -def _ipv6_host(host): +def _ipv6_host(host, scheme): """ Process IPv6 address literals """ @@ -902,4 +901,6 @@ def _ipv6_host(host): # percent sign might be URIencoded, convert it back into ASCII if host.startswith('[') and host.endswith(']'): host = host.replace('%25', '%').strip('[]') + if scheme in NORMALIZABLE_SCHEMES: + host = host.lower() return host diff --git a/src/pip/_vendor/urllib3/contrib/__init__.py b/src/pip/_vendor/urllib3/contrib/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py b/src/pip/_vendor/urllib3/contrib/_securetransport/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py b/src/pip/_vendor/urllib3/contrib/_securetransport/bindings.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py b/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py old mode 100644 new mode 100755 index 5e3494bce..b13cd9e72 --- a/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py +++ b/src/pip/_vendor/urllib3/contrib/_securetransport/low_level.py @@ -111,6 +111,9 @@ def _cert_array_from_pem(pem_bundle): Given a bundle of certs in PEM format, turns them into a CFArray of certs that can be used to validate a cert chain. """ + # Normalize the PEM bundle's line endings. + pem_bundle = pem_bundle.replace(b"\r\n", b"\n") + der_certs = [ base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) @@ -183,8 +186,8 @@ def _temporary_keychain(): # some random bytes to password-protect the keychain we're creating, so we # ask for 40 random bytes. random_bytes = os.urandom(40) - filename = base64.b64encode(random_bytes[:8]).decode('utf-8') - password = base64.b64encode(random_bytes[8:]) # Must be valid UTF-8 + filename = base64.b16encode(random_bytes[:8]).decode('utf-8') + password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 tempdirectory = tempfile.mkdtemp() keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') diff --git a/src/pip/_vendor/urllib3/contrib/appengine.py b/src/pip/_vendor/urllib3/contrib/appengine.py old mode 100644 new mode 100755 index ce17e83f5..59f2a617c --- a/src/pip/_vendor/urllib3/contrib/appengine.py +++ b/src/pip/_vendor/urllib3/contrib/appengine.py @@ -236,15 +236,24 @@ class AppEngineManager(RequestMethods): encodings.remove('chunked') urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) - return HTTPResponse( + original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. body=BytesIO(urlfetch_resp.content), + msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, **response_kw ) + return HTTPResponse( + body=BytesIO(urlfetch_resp.content), + headers=urlfetch_resp.headers, + status=urlfetch_resp.status_code, + original_response=original_response, + **response_kw + ) + def _get_absolute_timeout(self, timeout): if timeout is Timeout.DEFAULT_TIMEOUT: return None # Defer to URLFetch's default. diff --git a/src/pip/_vendor/urllib3/contrib/ntlmpool.py b/src/pip/_vendor/urllib3/contrib/ntlmpool.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/contrib/pyopenssl.py b/src/pip/_vendor/urllib3/contrib/pyopenssl.py old mode 100644 new mode 100755 index 1bb3787d3..6dd3a01e2 --- a/src/pip/_vendor/urllib3/contrib/pyopenssl.py +++ b/src/pip/_vendor/urllib3/contrib/pyopenssl.py @@ -47,6 +47,12 @@ import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend from cryptography.hazmat.backends.openssl.x509 import _Certificate +try: + from cryptography.x509 import UnsupportedExtension +except ImportError: + # UnsupportedExtension is gone in cryptography >= 2.1.0 + class UnsupportedExtension(Exception): + pass from socket import timeout, error as SocketError from io import BytesIO @@ -199,7 +205,7 @@ def get_subj_alt_name(peer_cert): except x509.ExtensionNotFound: # No such extension, return the empty list. return [] - except (x509.DuplicateExtension, x509.UnsupportedExtension, + except (x509.DuplicateExtension, UnsupportedExtension, x509.UnsupportedGeneralNameType, UnicodeError) as e: # A problem has been found with the quality of the certificate. Assume # no SAN field is present. @@ -267,8 +273,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd = util.wait_for_read(self.socket, self.socket.gettimeout()) - if not rd: + if not util.wait_for_read(self.socket, self.socket.gettimeout()): raise timeout('The read operation timed out') else: return self.recv(*args, **kwargs) @@ -289,8 +294,7 @@ class WrappedSocket(object): else: raise except OpenSSL.SSL.WantReadError: - rd = util.wait_for_read(self.socket, self.socket.gettimeout()) - if not rd: + if not util.wait_for_read(self.socket, self.socket.gettimeout()): raise timeout('The read operation timed out') else: return self.recv_into(*args, **kwargs) @@ -303,8 +307,7 @@ class WrappedSocket(object): try: return self.connection.send(data) except OpenSSL.SSL.WantWriteError: - wr = util.wait_for_write(self.socket, self.socket.gettimeout()) - if not wr: + if not util.wait_for_write(self.socket, self.socket.gettimeout()): raise timeout() continue except OpenSSL.SSL.SysCallError as e: @@ -418,7 +421,7 @@ class PyOpenSSLContext(object): self._ctx.load_verify_locations(BytesIO(cadata)) def load_cert_chain(self, certfile, keyfile=None, password=None): - self._ctx.use_certificate_file(certfile) + self._ctx.use_certificate_chain_file(certfile) if password is not None: self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) self._ctx.use_privatekey_file(keyfile or certfile) @@ -440,8 +443,7 @@ class PyOpenSSLContext(object): try: cnx.do_handshake() except OpenSSL.SSL.WantReadError: - rd = util.wait_for_read(sock, sock.gettimeout()) - if not rd: + if not util.wait_for_read(sock, sock.gettimeout()): raise timeout('select timed out') continue except OpenSSL.SSL.Error as e: diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py old mode 100644 new mode 100755 index 2cac70f7f..77cb59ed7 --- a/src/pip/_vendor/urllib3/contrib/securetransport.py +++ b/src/pip/_vendor/urllib3/contrib/securetransport.py @@ -51,11 +51,6 @@ except ImportError: # Platform-specific: Python 3 _fileobject = None from ..packages.backports.makefile import backport_makefile -try: - memoryview(b'') -except NameError: - raise ImportError("SecureTransport only works on Pythons with memoryview") - __all__ = ['inject_into_urllib3', 'extract_from_urllib3'] # SNI always works @@ -88,7 +83,7 @@ _connection_ref_lock = threading.Lock() SSL_WRITE_BLOCKSIZE = 16384 # This is our equivalent of util.ssl_.DEFAULT_CIPHERS, but expanded out to -# individual cipher suites. We need to do this becuase this is how +# individual cipher suites. We need to do this because this is how # SecureTransport wants them. CIPHER_SUITES = [ SecurityConst.TLS_AES_256_GCM_SHA384, @@ -195,21 +190,18 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): timeout = wrapped_socket.gettimeout() error = None read_count = 0 - buffer = (ctypes.c_char * requested_length).from_address(data_buffer) - buffer_view = memoryview(buffer) try: while read_count < requested_length: if timeout is None or timeout >= 0: - readables = util.wait_for_read([base_socket], timeout) - if not readables: + if not util.wait_for_read(base_socket, timeout): raise socket.error(errno.EAGAIN, 'timed out') - # We need to tell ctypes that we have a buffer that can be - # written to. Upsettingly, we do that like this: - chunk_size = base_socket.recv_into( - buffer_view[read_count:requested_length] + remaining = requested_length - read_count + buffer = (ctypes.c_char * remaining).from_address( + data_buffer + read_count ) + chunk_size = base_socket.recv_into(buffer, remaining) read_count += chunk_size if not chunk_size: if not read_count: @@ -219,7 +211,8 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): error = e.errno if error is not None and error != errno.EAGAIN: - if error == errno.ECONNRESET: + data_length_pointer[0] = read_count + if error == errno.ECONNRESET or error == errno.EPIPE: return SecurityConst.errSSLClosedAbort raise @@ -257,8 +250,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): try: while sent < bytes_to_write: if timeout is None or timeout >= 0: - writables = util.wait_for_write([base_socket], timeout) - if not writables: + if not util.wait_for_write(base_socket, timeout): raise socket.error(errno.EAGAIN, 'timed out') chunk_sent = base_socket.send(data) sent += chunk_sent @@ -270,11 +262,13 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): error = e.errno if error is not None and error != errno.EAGAIN: - if error == errno.ECONNRESET: + data_length_pointer[0] = sent + if error == errno.ECONNRESET or error == errno.EPIPE: return SecurityConst.errSSLClosedAbort raise data_length_pointer[0] = sent + if sent != bytes_to_write: return SecurityConst.errSSLWouldBlock @@ -399,7 +393,7 @@ class WrappedSocket(object): if trust: CoreFoundation.CFRelease(trust) - if cert_array is None: + if cert_array is not None: CoreFoundation.CFRelease(cert_array) # Ok, now we can look at what the result was. diff --git a/src/pip/_vendor/urllib3/contrib/socks.py b/src/pip/_vendor/urllib3/contrib/socks.py old mode 100644 new mode 100755 index 39e92fde1..811e312ec --- a/src/pip/_vendor/urllib3/contrib/socks.py +++ b/src/pip/_vendor/urllib3/contrib/socks.py @@ -152,6 +152,10 @@ class SOCKSProxyManager(PoolManager): num_pools=10, headers=None, **connection_pool_kw): parsed = parse_url(proxy_url) + if username is None and password is None and parsed.auth is not None: + split = parsed.auth.split(':') + if len(split) == 2: + username, password = split if parsed.scheme == 'socks5': socks_version = socks.PROXY_TYPE_SOCKS5 rdns = False diff --git a/src/pip/_vendor/urllib3/exceptions.py b/src/pip/_vendor/urllib3/exceptions.py old mode 100644 new mode 100755 index 6c4be5810..7bbaa9871 --- a/src/pip/_vendor/urllib3/exceptions.py +++ b/src/pip/_vendor/urllib3/exceptions.py @@ -154,7 +154,7 @@ class ResponseError(HTTPError): class SecurityWarning(HTTPWarning): - "Warned when perfoming security reducing actions" + "Warned when performing security reducing actions" pass diff --git a/src/pip/_vendor/urllib3/fields.py b/src/pip/_vendor/urllib3/fields.py old mode 100644 new mode 100755 index 19b0ae0c8..37fe64a3e --- a/src/pip/_vendor/urllib3/fields.py +++ b/src/pip/_vendor/urllib3/fields.py @@ -121,7 +121,7 @@ class RequestField(object): 'Content-Disposition' fields. :param header_parts: - A sequence of (k, v) typles or a :class:`dict` of (k, v) to format + A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format as `k1="v1"; k2="v2"; ...`. """ parts = [] diff --git a/src/pip/_vendor/urllib3/filepost.py b/src/pip/_vendor/urllib3/filepost.py old mode 100644 new mode 100755 index cd11cee46..78f1e19b0 --- a/src/pip/_vendor/urllib3/filepost.py +++ b/src/pip/_vendor/urllib3/filepost.py @@ -1,7 +1,8 @@ from __future__ import absolute_import +import binascii import codecs +import os -from uuid import uuid4 from io import BytesIO from .packages import six @@ -15,7 +16,10 @@ def choose_boundary(): """ Our embarrassingly-simple replacement for mimetools.choose_boundary. """ - return uuid4().hex + boundary = binascii.hexlify(os.urandom(16)) + if six.PY3: + boundary = boundary.decode('ascii') + return boundary def iter_field_objects(fields): @@ -65,7 +69,7 @@ def encode_multipart_formdata(fields, boundary=None): :param boundary: If not specified, then a random boundary will be generated using - :func:`mimetools.choose_boundary`. + :func:`urllib3.filepost.choose_boundary`. """ body = BytesIO() if boundary is None: diff --git a/src/pip/_vendor/urllib3/packages/__init__.py b/src/pip/_vendor/urllib3/packages/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/backports/__init__.py b/src/pip/_vendor/urllib3/packages/backports/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/backports/makefile.py b/src/pip/_vendor/urllib3/packages/backports/makefile.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/ordered_dict.py b/src/pip/_vendor/urllib3/packages/ordered_dict.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/six.py b/src/pip/_vendor/urllib3/packages/six.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/src/pip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/poolmanager.py b/src/pip/_vendor/urllib3/poolmanager.py old mode 100644 new mode 100755 index 4ae91744d..506a3c9b8 --- a/src/pip/_vendor/urllib3/poolmanager.py +++ b/src/pip/_vendor/urllib3/poolmanager.py @@ -312,8 +312,9 @@ class PoolManager(RequestMethods): kw['assert_same_host'] = False kw['redirect'] = False + if 'headers' not in kw: - kw['headers'] = self.headers + kw['headers'] = self.headers.copy() if self.proxy is not None and u.scheme == "http": response = conn.urlopen(method, url, **kw) @@ -335,6 +336,14 @@ class PoolManager(RequestMethods): if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) + # Strip headers marked as unsafe to forward to the redirected location. + # Check remove_headers_on_redirect to avoid a potential network call within + # conn.is_same_host() which may use socket.gethostbyname() in the future. + if (retries.remove_headers_on_redirect + and not conn.is_same_host(redirect_location)): + for header in retries.remove_headers_on_redirect: + kw['headers'].pop(header, None) + try: retries = retries.increment(method, url, response=response, _pool=conn) except MaxRetryError: @@ -358,7 +367,7 @@ class ProxyManager(PoolManager): The URL of the proxy to be used. :param proxy_headers: - A dictionary contaning headers that will be sent to the proxy. In case + A dictionary containing headers that will be sent to the proxy. In case of HTTP they are being sent with each request, while in the HTTPS/CONNECT case they are sent only once. Could be used for proxy authentication. diff --git a/src/pip/_vendor/urllib3/request.py b/src/pip/_vendor/urllib3/request.py old mode 100644 new mode 100755 index c0fddff04..1be333411 --- a/src/pip/_vendor/urllib3/request.py +++ b/src/pip/_vendor/urllib3/request.py @@ -44,8 +44,8 @@ class RequestMethods(object): def urlopen(self, method, url, body=None, headers=None, encode_multipart=True, multipart_boundary=None, **kw): # Abstract - raise NotImplemented("Classes extending RequestMethods must implement " - "their own ``urlopen`` method.") + raise NotImplementedError("Classes extending RequestMethods must implement " + "their own ``urlopen`` method.") def request(self, method, url, fields=None, headers=None, **urlopen_kw): """ @@ -60,6 +60,8 @@ class RequestMethods(object): """ method = method.upper() + urlopen_kw['request_url'] = url + if method in self._encode_url_methods: return self.request_encode_url(method, url, fields=fields, headers=headers, @@ -117,7 +119,7 @@ class RequestMethods(object): } When uploading a file, providing a filename (the first parameter of the - tuple) is optional but recommended to best mimick behavior of browsers. + tuple) is optional but recommended to best mimic behavior of browsers. Note that if ``headers`` are supplied, the 'Content-Type' header will be overwritten because it depends on the dynamic random boundary string diff --git a/src/pip/_vendor/urllib3/response.py b/src/pip/_vendor/urllib3/response.py old mode 100644 new mode 100755 index d3e5a1e60..9873cb942 --- a/src/pip/_vendor/urllib3/response.py +++ b/src/pip/_vendor/urllib3/response.py @@ -52,18 +52,42 @@ class DeflateDecoder(object): self._data = None +class GzipDecoderState(object): + + FIRST_MEMBER = 0 + OTHER_MEMBERS = 1 + SWALLOW_DATA = 2 + + class GzipDecoder(object): def __init__(self): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + self._state = GzipDecoderState.FIRST_MEMBER def __getattr__(self, name): return getattr(self._obj, name) def decompress(self, data): - if not data: - return data - return self._obj.decompress(data) + ret = binary_type() + if self._state == GzipDecoderState.SWALLOW_DATA or not data: + return ret + while True: + try: + ret += self._obj.decompress(data) + except zlib.error: + previous_state = self._state + # Ignore data after the first error + self._state = GzipDecoderState.SWALLOW_DATA + if previous_state == GzipDecoderState.OTHER_MEMBERS: + # Allow trailing garbage acceptable in other gzip clients + return ret + raise + data = self._obj.unused_data + if not data: + return ret + self._state = GzipDecoderState.OTHER_MEMBERS + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) def _get_decoder(mode): @@ -89,9 +113,8 @@ class HTTPResponse(io.IOBase): If True, the response's body will be preloaded during construction. :param decode_content: - If True, attempts to decode specific content-encoding's based on headers - (like 'gzip' and 'deflate') will be skipped and raw data will be used - instead. + If True, will attempt to decode the body based on the + 'content-encoding' header. :param original_response: When this HTTPResponse wrapper is generated from an httplib.HTTPResponse @@ -112,8 +135,9 @@ class HTTPResponse(io.IOBase): def __init__(self, body='', headers=None, status=0, version=0, reason=None, strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None, - retries=None, enforce_content_length=False, request_method=None): + original_response=None, pool=None, connection=None, msg=None, + retries=None, enforce_content_length=False, + request_method=None, request_url=None): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -132,6 +156,8 @@ class HTTPResponse(io.IOBase): self._fp = None self._original_response = original_response self._fp_bytes_read = 0 + self.msg = msg + self._request_url = request_url if body and isinstance(body, (basestring, binary_type)): self._body = body @@ -191,6 +217,9 @@ class HTTPResponse(io.IOBase): def connection(self): return self._connection + def isclosed(self): + return is_fp_closed(self._fp) + def tell(self): """ Obtain the number of bytes pulled over the wire so far. May differ from @@ -205,18 +234,18 @@ class HTTPResponse(io.IOBase): """ length = self.headers.get('content-length') - if length is not None and self.chunked: - # This Response will fail with an IncompleteRead if it can't be - # received as chunked. This method falls back to attempt reading - # the response before raising an exception. - log.warning("Received response with both Content-Length and " - "Transfer-Encoding set. This is expressly forbidden " - "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " - "attempting to process response as Transfer-Encoding: " - "chunked.") - return None + if length is not None: + if self.chunked: + # This Response will fail with an IncompleteRead if it can't be + # received as chunked. This method falls back to attempt reading + # the response before raising an exception. + log.warning("Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked.") + return None - elif length is not None: try: # RFC 7230 section 3.3.2 specifies multiple content lengths can # be sent in a single Content-Length header @@ -573,6 +602,11 @@ class HTTPResponse(io.IOBase): Similar to :meth:`HTTPResponse.read`, but with an additional parameter: ``decode_content``. + :param amt: + How much of the content to read. If specified, caching is skipped + because it doesn't make sense to cache partial content as the full + response. + :param decode_content: If True, will attempt to decode the body based on the 'content-encoding' header. @@ -588,12 +622,17 @@ class HTTPResponse(io.IOBase): "Body should be httplib.HTTPResponse like. " "It should have have an fp attribute which returns raw chunks.") - # Don't bother reading the body of a HEAD request. - if self._original_response and is_response_to_head(self._original_response): - self._original_response.close() - return - with self._error_catcher(): + # Don't bother reading the body of a HEAD request. + if self._original_response and is_response_to_head(self._original_response): + self._original_response.close() + return + + # If a response is already read and closed + # then return immediately. + if self._fp.fp is None: + return + while True: self._update_chunk_length() if self.chunk_left == 0: @@ -624,3 +663,14 @@ class HTTPResponse(io.IOBase): # We read everything; close the "file". if self._original_response: self._original_response.close() + + def geturl(self): + """ + Returns the URL that was the source of this response. + If the request that generated this response redirected, this method + will return the final redirect location. + """ + if self.retries is not None and len(self.retries.history): + return self.retries.history[-1].redirect_location + else: + return self._request_url diff --git a/src/pip/_vendor/urllib3/util/__init__.py b/src/pip/_vendor/urllib3/util/__init__.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/connection.py b/src/pip/_vendor/urllib3/util/connection.py old mode 100644 new mode 100755 index bf699cfd0..5cf488f4b --- a/src/pip/_vendor/urllib3/util/connection.py +++ b/src/pip/_vendor/urllib3/util/connection.py @@ -1,7 +1,6 @@ from __future__ import absolute_import import socket -from .wait import wait_for_read -from .selectors import HAS_SELECT, SelectorError +from .wait import NoWayToWaitForSocketError, wait_for_read def is_connection_dropped(conn): # Platform-specific @@ -19,14 +18,11 @@ def is_connection_dropped(conn): # Platform-specific return False if sock is None: # Connection already closed (such as by httplib). return True - - if not HAS_SELECT: - return False - try: - return bool(wait_for_read(sock, timeout=0.0)) - except SelectorError: - return True + # Returns True if readable, which here means it's been dropped + return wait_for_read(sock, timeout=0.0) + except NoWayToWaitForSocketError: # Platform-specific: AppEngine + return False # This function is copied from socket.py in the Python 2.7 standard diff --git a/src/pip/_vendor/urllib3/util/queue.py b/src/pip/_vendor/urllib3/util/queue.py new file mode 100755 index 000000000..d3d379a19 --- /dev/null +++ b/src/pip/_vendor/urllib3/util/queue.py @@ -0,0 +1,21 @@ +import collections +from ..packages import six +from ..packages.six.moves import queue + +if six.PY2: + # Queue is imported for side effects on MS Windows. See issue #229. + import Queue as _unused_module_Queue # noqa: F401 + + +class LifoQueue(queue.Queue): + def _init(self, _): + self.queue = collections.deque() + + def _qsize(self, len=len): + return len(self.queue) + + def _put(self, item): + self.queue.append(item) + + def _get(self): + return self.queue.pop() diff --git a/src/pip/_vendor/urllib3/util/request.py b/src/pip/_vendor/urllib3/util/request.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/response.py b/src/pip/_vendor/urllib3/util/response.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py old mode 100644 new mode 100755 index c603cb490..7ad3dc660 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -19,6 +19,7 @@ from ..packages import six log = logging.getLogger(__name__) + # Data structure for representing the metadata of requests that result in a retry. RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", "status", "redirect_location"]) @@ -139,6 +140,10 @@ class Retry(object): Whether to respect Retry-After header on status codes defined as :attr:`Retry.RETRY_AFTER_STATUS_CODES` or not. + :param iterable remove_headers_on_redirect: + Sequence of headers to remove from the request when a response + indicating a redirect is returned before firing off the redirected + request. """ DEFAULT_METHOD_WHITELIST = frozenset([ @@ -146,13 +151,16 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) + DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) + #: Maximum backoff time. BACKOFF_MAX = 120 def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - history=None, respect_retry_after_header=True): + history=None, respect_retry_after_header=True, + remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): self.total = total self.connect = connect @@ -171,6 +179,7 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header + self.remove_headers_on_redirect = remove_headers_on_redirect def new(self, **kw): params = dict( @@ -182,6 +191,7 @@ class Retry(object): raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, + remove_headers_on_redirect=self.remove_headers_on_redirect ) params.update(kw) return type(self)(**params) diff --git a/src/pip/_vendor/urllib3/util/selectors.py b/src/pip/_vendor/urllib3/util/selectors.py deleted file mode 100644 index d75cb266b..000000000 --- a/src/pip/_vendor/urllib3/util/selectors.py +++ /dev/null @@ -1,581 +0,0 @@ -# Backport of selectors.py from Python 3.5+ to support Python < 3.4 -# Also has the behavior specified in PEP 475 which is to retry syscalls -# in the case of an EINTR error. This module is required because selectors34 -# does not follow this behavior and instead returns that no dile descriptor -# events have occurred rather than retry the syscall. The decision to drop -# support for select.devpoll is made to maintain 100% test coverage. - -import errno -import math -import select -import socket -import sys -import time -from collections import namedtuple, Mapping - -try: - monotonic = time.monotonic -except (AttributeError, ImportError): # Python 3.3< - monotonic = time.time - -EVENT_READ = (1 << 0) -EVENT_WRITE = (1 << 1) - -HAS_SELECT = True # Variable that shows whether the platform has a selector. -_SYSCALL_SENTINEL = object() # Sentinel in case a system call returns None. -_DEFAULT_SELECTOR = None - - -class SelectorError(Exception): - def __init__(self, errcode): - super(SelectorError, self).__init__() - self.errno = errcode - - def __repr__(self): - return "".format(self.errno) - - def __str__(self): - return self.__repr__() - - -def _fileobj_to_fd(fileobj): - """ Return a file descriptor from a file object. If - given an integer will simply return that integer back. """ - if isinstance(fileobj, int): - fd = fileobj - else: - try: - fd = int(fileobj.fileno()) - except (AttributeError, TypeError, ValueError): - raise ValueError("Invalid file object: {0!r}".format(fileobj)) - if fd < 0: - raise ValueError("Invalid file descriptor: {0}".format(fd)) - return fd - - -# Determine which function to use to wrap system calls because Python 3.5+ -# already handles the case when system calls are interrupted. -if sys.version_info >= (3, 5): - def _syscall_wrapper(func, _, *args, **kwargs): - """ This is the short-circuit version of the below logic - because in Python 3.5+ all system calls automatically restart - and recalculate their timeouts. """ - try: - return func(*args, **kwargs) - except (OSError, IOError, select.error) as e: - errcode = None - if hasattr(e, "errno"): - errcode = e.errno - raise SelectorError(errcode) -else: - def _syscall_wrapper(func, recalc_timeout, *args, **kwargs): - """ Wrapper function for syscalls that could fail due to EINTR. - All functions should be retried if there is time left in the timeout - in accordance with PEP 475. """ - timeout = kwargs.get("timeout", None) - if timeout is None: - expires = None - recalc_timeout = False - else: - timeout = float(timeout) - if timeout < 0.0: # Timeout less than 0 treated as no timeout. - expires = None - else: - expires = monotonic() + timeout - - args = list(args) - if recalc_timeout and "timeout" not in kwargs: - raise ValueError( - "Timeout must be in args or kwargs to be recalculated") - - result = _SYSCALL_SENTINEL - while result is _SYSCALL_SENTINEL: - try: - result = func(*args, **kwargs) - # OSError is thrown by select.select - # IOError is thrown by select.epoll.poll - # select.error is thrown by select.poll.poll - # Aren't we thankful for Python 3.x rework for exceptions? - except (OSError, IOError, select.error) as e: - # select.error wasn't a subclass of OSError in the past. - errcode = None - if hasattr(e, "errno"): - errcode = e.errno - elif hasattr(e, "args"): - errcode = e.args[0] - - # Also test for the Windows equivalent of EINTR. - is_interrupt = (errcode == errno.EINTR or (hasattr(errno, "WSAEINTR") and - errcode == errno.WSAEINTR)) - - if is_interrupt: - if expires is not None: - current_time = monotonic() - if current_time > expires: - raise OSError(errno=errno.ETIMEDOUT) - if recalc_timeout: - if "timeout" in kwargs: - kwargs["timeout"] = expires - current_time - continue - if errcode: - raise SelectorError(errcode) - else: - raise - return result - - -SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) - - -class _SelectorMapping(Mapping): - """ Mapping of file objects to selector keys """ - - def __init__(self, selector): - self._selector = selector - - def __len__(self): - return len(self._selector._fd_to_key) - - def __getitem__(self, fileobj): - try: - fd = self._selector._fileobj_lookup(fileobj) - return self._selector._fd_to_key[fd] - except KeyError: - raise KeyError("{0!r} is not registered.".format(fileobj)) - - def __iter__(self): - return iter(self._selector._fd_to_key) - - -class BaseSelector(object): - """ Abstract Selector class - - A selector supports registering file objects to be monitored - for specific I/O events. - - A file object is a file descriptor or any object with a - `fileno()` method. An arbitrary object can be attached to the - file object which can be used for example to store context info, - a callback, etc. - - A selector can use various implementations (select(), poll(), epoll(), - and kqueue()) depending on the platform. The 'DefaultSelector' class uses - the most efficient implementation for the current platform. - """ - def __init__(self): - # Maps file descriptors to keys. - self._fd_to_key = {} - - # Read-only mapping returned by get_map() - self._map = _SelectorMapping(self) - - def _fileobj_lookup(self, fileobj): - """ Return a file descriptor from a file object. - This wraps _fileobj_to_fd() to do an exhaustive - search in case the object is invalid but we still - have it in our map. Used by unregister() so we can - unregister an object that was previously registered - even if it is closed. It is also used by _SelectorMapping - """ - try: - return _fileobj_to_fd(fileobj) - except ValueError: - - # Search through all our mapped keys. - for key in self._fd_to_key.values(): - if key.fileobj is fileobj: - return key.fd - - # Raise ValueError after all. - raise - - def register(self, fileobj, events, data=None): - """ Register a file object for a set of events to monitor. """ - if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): - raise ValueError("Invalid events: {0!r}".format(events)) - - key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) - - if key.fd in self._fd_to_key: - raise KeyError("{0!r} (FD {1}) is already registered" - .format(fileobj, key.fd)) - - self._fd_to_key[key.fd] = key - return key - - def unregister(self, fileobj): - """ Unregister a file object from being monitored. """ - try: - key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) - except KeyError: - raise KeyError("{0!r} is not registered".format(fileobj)) - - # Getting the fileno of a closed socket on Windows errors with EBADF. - except socket.error as e: # Platform-specific: Windows. - if e.errno != errno.EBADF: - raise - else: - for key in self._fd_to_key.values(): - if key.fileobj is fileobj: - self._fd_to_key.pop(key.fd) - break - else: - raise KeyError("{0!r} is not registered".format(fileobj)) - return key - - def modify(self, fileobj, events, data=None): - """ Change a registered file object monitored events and data. """ - # NOTE: Some subclasses optimize this operation even further. - try: - key = self._fd_to_key[self._fileobj_lookup(fileobj)] - except KeyError: - raise KeyError("{0!r} is not registered".format(fileobj)) - - if events != key.events: - self.unregister(fileobj) - key = self.register(fileobj, events, data) - - elif data != key.data: - # Use a shortcut to update the data. - key = key._replace(data=data) - self._fd_to_key[key.fd] = key - - return key - - def select(self, timeout=None): - """ Perform the actual selection until some monitored file objects - are ready or the timeout expires. """ - raise NotImplementedError() - - def close(self): - """ Close the selector. This must be called to ensure that all - underlying resources are freed. """ - self._fd_to_key.clear() - self._map = None - - def get_key(self, fileobj): - """ Return the key associated with a registered file object. """ - mapping = self.get_map() - if mapping is None: - raise RuntimeError("Selector is closed") - try: - return mapping[fileobj] - except KeyError: - raise KeyError("{0!r} is not registered".format(fileobj)) - - def get_map(self): - """ Return a mapping of file objects to selector keys """ - return self._map - - def _key_from_fd(self, fd): - """ Return the key associated to a given file descriptor - Return None if it is not found. """ - try: - return self._fd_to_key[fd] - except KeyError: - return None - - def __enter__(self): - return self - - def __exit__(self, *args): - self.close() - - -# Almost all platforms have select.select() -if hasattr(select, "select"): - class SelectSelector(BaseSelector): - """ Select-based selector. """ - def __init__(self): - super(SelectSelector, self).__init__() - self._readers = set() - self._writers = set() - - def register(self, fileobj, events, data=None): - key = super(SelectSelector, self).register(fileobj, events, data) - if events & EVENT_READ: - self._readers.add(key.fd) - if events & EVENT_WRITE: - self._writers.add(key.fd) - return key - - def unregister(self, fileobj): - key = super(SelectSelector, self).unregister(fileobj) - self._readers.discard(key.fd) - self._writers.discard(key.fd) - return key - - def _select(self, r, w, timeout=None): - """ Wrapper for select.select because timeout is a positional arg """ - return select.select(r, w, [], timeout) - - def select(self, timeout=None): - # Selecting on empty lists on Windows errors out. - if not len(self._readers) and not len(self._writers): - return [] - - timeout = None if timeout is None else max(timeout, 0.0) - ready = [] - r, w, _ = _syscall_wrapper(self._select, True, self._readers, - self._writers, timeout) - r = set(r) - w = set(w) - for fd in r | w: - events = 0 - if fd in r: - events |= EVENT_READ - if fd in w: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - -if hasattr(select, "poll"): - class PollSelector(BaseSelector): - """ Poll-based selector """ - def __init__(self): - super(PollSelector, self).__init__() - self._poll = select.poll() - - def register(self, fileobj, events, data=None): - key = super(PollSelector, self).register(fileobj, events, data) - event_mask = 0 - if events & EVENT_READ: - event_mask |= select.POLLIN - if events & EVENT_WRITE: - event_mask |= select.POLLOUT - self._poll.register(key.fd, event_mask) - return key - - def unregister(self, fileobj): - key = super(PollSelector, self).unregister(fileobj) - self._poll.unregister(key.fd) - return key - - def _wrap_poll(self, timeout=None): - """ Wrapper function for select.poll.poll() so that - _syscall_wrapper can work with only seconds. """ - if timeout is not None: - if timeout <= 0: - timeout = 0 - else: - # select.poll.poll() has a resolution of 1 millisecond, - # round away from zero to wait *at least* timeout seconds. - timeout = math.ceil(timeout * 1e3) - - result = self._poll.poll(timeout) - return result - - def select(self, timeout=None): - ready = [] - fd_events = _syscall_wrapper(self._wrap_poll, True, timeout=timeout) - for fd, event_mask in fd_events: - events = 0 - if event_mask & ~select.POLLIN: - events |= EVENT_WRITE - if event_mask & ~select.POLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - - return ready - - -if hasattr(select, "epoll"): - class EpollSelector(BaseSelector): - """ Epoll-based selector """ - def __init__(self): - super(EpollSelector, self).__init__() - self._epoll = select.epoll() - - def fileno(self): - return self._epoll.fileno() - - def register(self, fileobj, events, data=None): - key = super(EpollSelector, self).register(fileobj, events, data) - events_mask = 0 - if events & EVENT_READ: - events_mask |= select.EPOLLIN - if events & EVENT_WRITE: - events_mask |= select.EPOLLOUT - _syscall_wrapper(self._epoll.register, False, key.fd, events_mask) - return key - - def unregister(self, fileobj): - key = super(EpollSelector, self).unregister(fileobj) - try: - _syscall_wrapper(self._epoll.unregister, False, key.fd) - except SelectorError: - # This can occur when the fd was closed since registry. - pass - return key - - def select(self, timeout=None): - if timeout is not None: - if timeout <= 0: - timeout = 0.0 - else: - # select.epoll.poll() has a resolution of 1 millisecond - # but luckily takes seconds so we don't need a wrapper - # like PollSelector. Just for better rounding. - timeout = math.ceil(timeout * 1e3) * 1e-3 - timeout = float(timeout) - else: - timeout = -1.0 # epoll.poll() must have a float. - - # We always want at least 1 to ensure that select can be called - # with no file descriptors registered. Otherwise will fail. - max_events = max(len(self._fd_to_key), 1) - - ready = [] - fd_events = _syscall_wrapper(self._epoll.poll, True, - timeout=timeout, - maxevents=max_events) - for fd, event_mask in fd_events: - events = 0 - if event_mask & ~select.EPOLLIN: - events |= EVENT_WRITE - if event_mask & ~select.EPOLLOUT: - events |= EVENT_READ - - key = self._key_from_fd(fd) - if key: - ready.append((key, events & key.events)) - return ready - - def close(self): - self._epoll.close() - super(EpollSelector, self).close() - - -if hasattr(select, "kqueue"): - class KqueueSelector(BaseSelector): - """ Kqueue / Kevent-based selector """ - def __init__(self): - super(KqueueSelector, self).__init__() - self._kqueue = select.kqueue() - - def fileno(self): - return self._kqueue.fileno() - - def register(self, fileobj, events, data=None): - key = super(KqueueSelector, self).register(fileobj, events, data) - if events & EVENT_READ: - kevent = select.kevent(key.fd, - select.KQ_FILTER_READ, - select.KQ_EV_ADD) - - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - - if events & EVENT_WRITE: - kevent = select.kevent(key.fd, - select.KQ_FILTER_WRITE, - select.KQ_EV_ADD) - - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - - return key - - def unregister(self, fileobj): - key = super(KqueueSelector, self).unregister(fileobj) - if key.events & EVENT_READ: - kevent = select.kevent(key.fd, - select.KQ_FILTER_READ, - select.KQ_EV_DELETE) - try: - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - except SelectorError: - pass - if key.events & EVENT_WRITE: - kevent = select.kevent(key.fd, - select.KQ_FILTER_WRITE, - select.KQ_EV_DELETE) - try: - _syscall_wrapper(self._kqueue.control, False, [kevent], 0, 0) - except SelectorError: - pass - - return key - - def select(self, timeout=None): - if timeout is not None: - timeout = max(timeout, 0) - - max_events = len(self._fd_to_key) * 2 - ready_fds = {} - - kevent_list = _syscall_wrapper(self._kqueue.control, True, - None, max_events, timeout) - - for kevent in kevent_list: - fd = kevent.ident - event_mask = kevent.filter - events = 0 - if event_mask == select.KQ_FILTER_READ: - events |= EVENT_READ - if event_mask == select.KQ_FILTER_WRITE: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) - if key: - if key.fd not in ready_fds: - ready_fds[key.fd] = (key, events & key.events) - else: - old_events = ready_fds[key.fd][1] - ready_fds[key.fd] = (key, (events | old_events) & key.events) - - return list(ready_fds.values()) - - def close(self): - self._kqueue.close() - super(KqueueSelector, self).close() - - -if not hasattr(select, 'select'): # Platform-specific: AppEngine - HAS_SELECT = False - - -def _can_allocate(struct): - """ Checks that select structs can be allocated by the underlying - operating system, not just advertised by the select module. We don't - check select() because we'll be hopeful that most platforms that - don't have it available will not advertise it. (ie: GAE) """ - try: - # select.poll() objects won't fail until used. - if struct == 'poll': - p = select.poll() - p.poll(0) - - # All others will fail on allocation. - else: - getattr(select, struct)().close() - return True - except (OSError, AttributeError) as e: - return False - - -# Choose the best implementation, roughly: -# kqueue == epoll > poll > select. Devpoll not supported. (See above) -# select() also can't accept a FD > FD_SETSIZE (usually around 1024) -def DefaultSelector(): - """ This function serves as a first call for DefaultSelector to - detect if the select module is being monkey-patched incorrectly - by eventlet, greenlet, and preserve proper behavior. """ - global _DEFAULT_SELECTOR - if _DEFAULT_SELECTOR is None: - if _can_allocate('kqueue'): - _DEFAULT_SELECTOR = KqueueSelector - elif _can_allocate('epoll'): - _DEFAULT_SELECTOR = EpollSelector - elif _can_allocate('poll'): - _DEFAULT_SELECTOR = PollSelector - elif hasattr(select, 'select'): - _DEFAULT_SELECTOR = SelectSelector - else: # Platform-specific: AppEngine - raise ValueError('Platform does not have a selector') - return _DEFAULT_SELECTOR() diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py old mode 100644 new mode 100755 index dafc75b5b..325428052 --- a/src/pip/_vendor/urllib3/util/ssl_.py +++ b/src/pip/_vendor/urllib3/util/ssl_.py @@ -2,11 +2,13 @@ from __future__ import absolute_import import errno import warnings import hmac +import socket from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning +from ..packages import six SSLContext = None @@ -53,6 +55,27 @@ except ImportError: OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 OP_NO_COMPRESSION = 0x20000 + +# Python 2.7 and earlier didn't have inet_pton on non-Linux +# so we fallback on inet_aton in those cases. This means that +# we can only detect IPv4 addresses in this case. +if hasattr(socket, 'inet_pton'): + inet_pton = socket.inet_pton +else: + # Maybe we can use ipaddress if the user has urllib3[secure]? + try: + from pip._vendor import ipaddress + + def inet_pton(_, host): + if isinstance(host, six.binary_type): + host = host.decode('ascii') + return ipaddress.ip_address(host) + + except ImportError: # Platform-specific: Non-Linux + def inet_pton(_, host): + return socket.inet_aton(host) + + # A secure default. # Sources for more information on TLS ciphers: # @@ -183,7 +206,7 @@ def resolve_cert_reqs(candidate): the wrap_socket function/method from the ssl module. Defaults to :data:`ssl.CERT_NONE`. If given a string it is assumed to be the name of the constant in the - :mod:`ssl` module or its abbrevation. + :mod:`ssl` module or its abbreviation. (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. If it's neither `None` nor a string we assume it is already the numeric constant which can directly be passed to wrap_socket. @@ -325,17 +348,49 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if certfile: context.load_cert_chain(certfile, keyfile) - if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI - return context.wrap_socket(sock, server_hostname=server_hostname) - warnings.warn( - 'An HTTPS request has been made, but the SNI (Subject Name ' - 'Indication) extension to TLS is not available on this platform. ' - 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. You can upgrade to ' - 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - SNIMissingWarning - ) + # If we detect server_hostname is an IP address then the SNI + # extension should not be used according to RFC3546 Section 3.1 + # We shouldn't warn the user if SNI isn't available but we would + # not be using SNI anyways due to IP address for server_hostname. + if ((server_hostname is not None and not is_ipaddress(server_hostname)) + or IS_SECURETRANSPORT): + if HAS_SNI and server_hostname is not None: + return context.wrap_socket(sock, server_hostname=server_hostname) + + warnings.warn( + 'An HTTPS request has been made, but the SNI (Server Name ' + 'Indication) extension to TLS is not available on this platform. ' + 'This may cause the server to present an incorrect TLS ' + 'certificate, which can cause validation failures. You can upgrade to ' + 'a newer version of Python to solve this. For more information, see ' + 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' + '#ssl-warnings', + SNIMissingWarning + ) + return context.wrap_socket(sock) + + +def is_ipaddress(hostname): + """Detects whether the hostname given is an IP address. + + :param str hostname: Hostname to examine. + :return: True if the hostname is an IP address, False otherwise. + """ + if six.PY3 and isinstance(hostname, six.binary_type): + # IDN A-label bytes are ASCII compatible. + hostname = hostname.decode('ascii') + + families = [socket.AF_INET] + if hasattr(socket, 'AF_INET6'): + families.append(socket.AF_INET6) + + for af in families: + try: + inet_pton(af, hostname) + except (socket.error, ValueError, OSError): + pass + else: + return True + return False diff --git a/src/pip/_vendor/urllib3/util/timeout.py b/src/pip/_vendor/urllib3/util/timeout.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/url.py b/src/pip/_vendor/urllib3/util/url.py old mode 100644 new mode 100755 diff --git a/src/pip/_vendor/urllib3/util/wait.py b/src/pip/_vendor/urllib3/util/wait.py old mode 100644 new mode 100755 index cb396e508..fa686eff4 --- a/src/pip/_vendor/urllib3/util/wait.py +++ b/src/pip/_vendor/urllib3/util/wait.py @@ -1,40 +1,153 @@ -from .selectors import ( - HAS_SELECT, - DefaultSelector, - EVENT_READ, - EVENT_WRITE -) +import errno +from functools import partial +import select +import sys +try: + from time import monotonic +except ImportError: + from time import time as monotonic + +__all__ = ["NoWayToWaitForSocketError", "wait_for_read", "wait_for_write"] -def _wait_for_io_events(socks, events, timeout=None): - """ Waits for IO events to be available from a list of sockets - or optionally a single socket if passed in. Returns a list of - sockets that can be interacted with immediately. """ - if not HAS_SELECT: - raise ValueError('Platform does not have a selector') - if not isinstance(socks, list): - # Probably just a single socket. - if hasattr(socks, "fileno"): - socks = [socks] - # Otherwise it might be a non-list iterable. +class NoWayToWaitForSocketError(Exception): + pass + + +# How should we wait on sockets? +# +# There are two types of APIs you can use for waiting on sockets: the fancy +# modern stateful APIs like epoll/kqueue, and the older stateless APIs like +# select/poll. The stateful APIs are more efficient when you have a lots of +# sockets to keep track of, because you can set them up once and then use them +# lots of times. But we only ever want to wait on a single socket at a time +# and don't want to keep track of state, so the stateless APIs are actually +# more efficient. So we want to use select() or poll(). +# +# Now, how do we choose between select() and poll()? On traditional Unixes, +# select() has a strange calling convention that makes it slow, or fail +# altogether, for high-numbered file descriptors. The point of poll() is to fix +# that, so on Unixes, we prefer poll(). +# +# On Windows, there is no poll() (or at least Python doesn't provide a wrapper +# for it), but that's OK, because on Windows, select() doesn't have this +# strange calling convention; plain select() works fine. +# +# So: on Windows we use select(), and everywhere else we use poll(). We also +# fall back to select() in case poll() is somehow broken or missing. + +if sys.version_info >= (3, 5): + # Modern Python, that retries syscalls by default + def _retry_on_intr(fn, timeout): + return fn(timeout) +else: + # Old and broken Pythons. + def _retry_on_intr(fn, timeout): + if timeout is not None and timeout <= 0: + return fn(timeout) + + if timeout is None: + deadline = float("inf") else: - socks = list(socks) - with DefaultSelector() as selector: - for sock in socks: - selector.register(sock, events) - return [key[0].fileobj for key in - selector.select(timeout) if key[1] & events] + deadline = monotonic() + timeout + + while True: + try: + return fn(timeout) + # OSError for 3 <= pyver < 3.5, select.error for pyver <= 2.7 + except (OSError, select.error) as e: + # 'e.args[0]' incantation works for both OSError and select.error + if e.args[0] != errno.EINTR: + raise + else: + timeout = deadline - monotonic() + if timeout < 0: + timeout = 0 + if timeout == float("inf"): + timeout = None + continue -def wait_for_read(socks, timeout=None): - """ Waits for reading to be available from a list of sockets - or optionally a single socket if passed in. Returns a list of - sockets that can be read from immediately. """ - return _wait_for_io_events(socks, EVENT_READ, timeout) +def select_wait_for_socket(sock, read=False, write=False, timeout=None): + if not read and not write: + raise RuntimeError("must specify at least one of read=True, write=True") + rcheck = [] + wcheck = [] + if read: + rcheck.append(sock) + if write: + wcheck.append(sock) + # When doing a non-blocking connect, most systems signal success by + # marking the socket writable. Windows, though, signals success by marked + # it as "exceptional". We paper over the difference by checking the write + # sockets for both conditions. (The stdlib selectors module does the same + # thing.) + fn = partial(select.select, rcheck, wcheck, wcheck) + rready, wready, xready = _retry_on_intr(fn, timeout) + return bool(rready or wready or xready) -def wait_for_write(socks, timeout=None): - """ Waits for writing to be available from a list of sockets - or optionally a single socket if passed in. Returns a list of - sockets that can be written to immediately. """ - return _wait_for_io_events(socks, EVENT_WRITE, timeout) +def poll_wait_for_socket(sock, read=False, write=False, timeout=None): + if not read and not write: + raise RuntimeError("must specify at least one of read=True, write=True") + mask = 0 + if read: + mask |= select.POLLIN + if write: + mask |= select.POLLOUT + poll_obj = select.poll() + poll_obj.register(sock, mask) + + # For some reason, poll() takes timeout in milliseconds + def do_poll(t): + if t is not None: + t *= 1000 + return poll_obj.poll(t) + + return bool(_retry_on_intr(do_poll, timeout)) + + +def null_wait_for_socket(*args, **kwargs): + raise NoWayToWaitForSocketError("no select-equivalent available") + + +def _have_working_poll(): + # Apparently some systems have a select.poll that fails as soon as you try + # to use it, either due to strange configuration or broken monkeypatching + # from libraries like eventlet/greenlet. + try: + poll_obj = select.poll() + poll_obj.poll(0) + except (AttributeError, OSError): + return False + else: + return True + + +def wait_for_socket(*args, **kwargs): + # We delay choosing which implementation to use until the first time we're + # called. We could do it at import time, but then we might make the wrong + # decision if someone goes wild with monkeypatching select.poll after + # we're imported. + global wait_for_socket + if _have_working_poll(): + wait_for_socket = poll_wait_for_socket + elif hasattr(select, "select"): + wait_for_socket = select_wait_for_socket + else: # Platform-specific: Appengine. + wait_for_socket = null_wait_for_socket + return wait_for_socket(*args, **kwargs) + + +def wait_for_read(sock, timeout=None): + """ Waits for reading to be available on a given socket. + Returns True if the socket is readable, or False if the timeout expired. + """ + return wait_for_socket(sock, read=True, timeout=timeout) + + +def wait_for_write(sock, timeout=None): + """ Waits for writing to be available on a given socket. + Returns True if the socket is readable, or False if the timeout expired. + """ + return wait_for_socket(sock, write=True, timeout=timeout) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 4a937e8d7..b9854e9ad 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -1,22 +1,22 @@ appdirs==1.4.3 distlib==0.2.7 -distro==1.2.0 +distro==1.3.0 html5lib==1.0.1 six==1.11.0 colorama==0.3.9 -CacheControl==0.12.4 +CacheControl==0.12.5 msgpack-python==0.5.6 lockfile==0.12.2 -progress==1.3 -ipaddress==1.0.19 # Only needed on 2.6 and 2.7 +progress==1.4 +ipaddress==1.0.22 # Only needed on 2.6 and 2.7 packaging==17.1 pyparsing==2.2.0 -pytoml==0.1.14 +pytoml==0.1.16 retrying==1.3.3 -requests==2.18.4 +requests==2.19.1 chardet==3.0.4 - idna==2.6 - urllib3==1.22 - certifi==2018.1.18 -setuptools==39.0.1 + idna==2.7 + urllib3==1.23 + certifi==2018.4.16 +setuptools==39.2.0 webencodings==0.5.1 diff --git a/tasks/vendoring/patches/requests.patch b/tasks/vendoring/patches/requests.patch index d7cc8c8d6..ceee9e833 100644 --- a/tasks/vendoring/patches/requests.patch +++ b/tasks/vendoring/patches/requests.patch @@ -24,13 +24,17 @@ diff --git a/src/pip/_vendor/requests/__init__.py b/src/pip/_vendor/requests/__i index 9c3b769..36a4ef40 100644 --- a/src/pip/_vendor/requests/__init__.py +++ b/src/pip/_vendor/requests/__init__.py -@@ -80,10 +80,12 @@ except (AssertionError, ValueError): +@@ -80,13 +80,15 @@ except (AssertionError, ValueError): RequestsDependencyWarning) # Attempt to enable urllib3's SNI support, if possible -try: - from pip._vendor.urllib3.contrib import pyopenssl - pyopenssl.inject_into_urllib3() +- +- # Check cryptography version +- from cryptography import __version__ as cryptography_version +- _check_cryptography(cryptography_version) -except ImportError: - pass +from pip._internal.compat import WINDOWS @@ -38,6 +42,10 @@ index 9c3b769..36a4ef40 100644 + try: + from pip._vendor.urllib3.contrib import pyopenssl + pyopenssl.inject_into_urllib3() ++ ++ # Check cryptography version ++ from cryptography import __version__ as cryptography_version ++ _check_cryptography(cryptography_version) + except ImportError: + pass diff --git a/tests/conftest.py b/tests/conftest.py index 1e927fc13..f8e3a41b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -129,6 +129,9 @@ def isolate(tmpdir): # We want to disable the version check from running in the tests os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "true" + # Make sure tests don't share a requirements tracker. + os.environ.pop('PIP_REQ_TRACKER', None) + # FIXME: Windows... os.makedirs(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: diff --git a/tests/data/packages/pep518_forkbomb-235.tar.gz b/tests/data/packages/pep518_forkbomb-235.tar.gz new file mode 100644 index 000000000..1edce52d8 Binary files /dev/null and b/tests/data/packages/pep518_forkbomb-235.tar.gz differ diff --git a/tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz b/tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz new file mode 100644 index 000000000..2ca9452b1 Binary files /dev/null and b/tests/data/packages/pep518_twin_forkbombs_first-234.tar.gz differ diff --git a/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz b/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz new file mode 100644 index 000000000..75998c3d4 Binary files /dev/null and b/tests/data/packages/pep518_twin_forkbombs_second-238.tar.gz differ diff --git a/tests/data/packages4/simple-1.0-py2.py3-none-any.whl b/tests/data/packages4/simple-1.0-py2.py3-none-any.whl deleted file mode 100644 index 3b91f5e01..000000000 Binary files a/tests/data/packages4/simple-1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/data/src/pep518_forkbomb-235/MANIFEST.in b/tests/data/src/pep518_forkbomb-235/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_forkbomb-235/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/news/2b9004e2-70dd-49df-91c3-29deb506cfbe.trivial b/tests/data/src/pep518_forkbomb-235/pep518_forkbomb.py similarity index 100% rename from news/2b9004e2-70dd-49df-91c3-29deb506cfbe.trivial rename to tests/data/src/pep518_forkbomb-235/pep518_forkbomb.py diff --git a/tests/data/src/pep518_forkbomb-235/pyproject.toml b/tests/data/src/pep518_forkbomb-235/pyproject.toml new file mode 100644 index 000000000..1138bfb0d --- /dev/null +++ b/tests/data/src/pep518_forkbomb-235/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "pep518_forkbomb"] diff --git a/news/438e7c9b-3755-4983-9c91-9fd47b1b178c.trivial b/tests/data/src/pep518_forkbomb-235/setup.cfg similarity index 100% rename from news/438e7c9b-3755-4983-9c91-9fd47b1b178c.trivial rename to tests/data/src/pep518_forkbomb-235/setup.cfg diff --git a/tests/data/src/pep518_forkbomb-235/setup.py b/tests/data/src/pep518_forkbomb-235/setup.py new file mode 100644 index 000000000..c8bc29287 --- /dev/null +++ b/tests/data/src/pep518_forkbomb-235/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup(name='pep518_forkbomb', + version='235', + py_modules=['pep518_forkbomb']) diff --git a/tests/data/src/pep518_invalid_build_system/MANIFEST.in b/tests/data/src/pep518_invalid_build_system/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/tests/data/src/pep518_invalid_build_system/pep518.py b/tests/data/src/pep518_invalid_build_system/pep518.py new file mode 100644 index 000000000..7986d1137 --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/pep518.py @@ -0,0 +1 @@ +#dummy diff --git a/tests/data/src/pep518_invalid_build_system/pyproject.toml b/tests/data/src/pep518_invalid_build_system/pyproject.toml new file mode 100644 index 000000000..86fcebfa8 --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +# This table is intentionally empty. diff --git a/tests/data/src/pep518_invalid_build_system/setup.py b/tests/data/src/pep518_invalid_build_system/setup.py new file mode 100644 index 000000000..ba23cf24a --- /dev/null +++ b/tests/data/src/pep518_invalid_build_system/setup.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +from setuptools import setup + +setup( + name='pep518_invalid_build_system', + version='1.0.0', + py_modules=['pep518'], +) diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in b/tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_first-234/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/news/4BCF6244-41AE-4085-9E20-7C2A0336FB1D.trivial b/tests/data/src/pep518_twin_forkbombs_first-234/pep518_twin_forkbombs_first.py similarity index 100% rename from news/4BCF6244-41AE-4085-9E20-7C2A0336FB1D.trivial rename to tests/data/src/pep518_twin_forkbombs_first-234/pep518_twin_forkbombs_first.py diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml b/tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml new file mode 100644 index 000000000..ca106ad69 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_first-234/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "pep518_twin_forkbombs_second"] diff --git a/news/64b003f8-f701-4df9-a8b5-a869a652327e.trivial b/tests/data/src/pep518_twin_forkbombs_first-234/setup.cfg similarity index 100% rename from news/64b003f8-f701-4df9-a8b5-a869a652327e.trivial rename to tests/data/src/pep518_twin_forkbombs_first-234/setup.cfg diff --git a/tests/data/src/pep518_twin_forkbombs_first-234/setup.py b/tests/data/src/pep518_twin_forkbombs_first-234/setup.py new file mode 100644 index 000000000..55e9bbfb1 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_first-234/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup(name='pep518_twin_forkbombs_first', + version='234', + py_modules=['pep518_twin_forkbombs_first']) diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in b/tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in new file mode 100644 index 000000000..bec201fc8 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_second-238/MANIFEST.in @@ -0,0 +1 @@ +include pyproject.toml diff --git a/news/776692fe-c2b3-41af-867b-ee11411e0756.trivial b/tests/data/src/pep518_twin_forkbombs_second-238/pep518_twin_forkbombs_second.py similarity index 100% rename from news/776692fe-c2b3-41af-867b-ee11411e0756.trivial rename to tests/data/src/pep518_twin_forkbombs_second-238/pep518_twin_forkbombs_second.py diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml b/tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml new file mode 100644 index 000000000..194a86848 --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_second-238/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "pep518_twin_forkbombs_first"] diff --git a/news/951c2359-ff29-4ed8-ba91-2e82dff5e5ad.trivial b/tests/data/src/pep518_twin_forkbombs_second-238/setup.cfg similarity index 100% rename from news/951c2359-ff29-4ed8-ba91-2e82dff5e5ad.trivial rename to tests/data/src/pep518_twin_forkbombs_second-238/setup.cfg diff --git a/tests/data/src/pep518_twin_forkbombs_second-238/setup.py b/tests/data/src/pep518_twin_forkbombs_second-238/setup.py new file mode 100644 index 000000000..985af51df --- /dev/null +++ b/tests/data/src/pep518_twin_forkbombs_second-238/setup.py @@ -0,0 +1,5 @@ +from setuptools import setup + +setup(name='pep518_twin_forkbombs_second', + version='238', + py_modules=['pep518_twin_forkbombs_second']) diff --git a/tests/data/src/withpyproject/setup.py b/tests/data/src/withpyproject/setup.py index 326f7fd1e..1ea9e3e41 100644 --- a/tests/data/src/withpyproject/setup.py +++ b/tests/data/src/withpyproject/setup.py @@ -1,2 +1,3 @@ from setuptools import setup + setup(name='withpyproject', version='0.0.1') diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py index b10251907..86a14cea9 100644 --- a/tests/functional/test_install.py +++ b/tests/functional/test_install.py @@ -8,6 +8,7 @@ from os.path import curdir, join, pardir import pytest from pip._internal import pep425tags +from pip._internal.models.index import PyPI, TestPyPI from pip._internal.status_codes import ERROR from pip._internal.utils.misc import rmtree from tests.lib import ( @@ -47,14 +48,28 @@ def test_pep518_refuses_invalid_requires(script, data, common_wheels): assert "does not comply with PEP 518" in result.stderr -def test_pep518_allows_but_warns_missing_requires(script, data, common_wheels): +def test_pep518_refuses_invalid_build_system(script, data, common_wheels): + result = script.pip( + 'install', '-f', common_wheels, + data.src.join("pep518_invalid_build_system"), + expect_error=True + ) + assert result.returncode == 1 + assert "does not comply with PEP 518" in result.stderr + + +def test_pep518_allows_missing_requires(script, data, common_wheels): result = script.pip( 'install', '-f', common_wheels, data.src.join("pep518_missing_requires"), expect_stderr=True ) - assert "does not comply with PEP 518" in result.stderr - assert "DEPRECATION" in result.stderr + # Make sure we don't warn when this occurs. + assert "does not comply with PEP 518" not in result.stderr + + # We want it to go through isolation for now. + assert "Installing build dependencies" in result.stdout, result.stdout + assert result.returncode == 0 assert result.files_created @@ -80,14 +95,30 @@ def test_pep518_with_extra_and_markers(script, data, common_wheels): 'wheel', '--no-index', '-f', common_wheels, '-f', data.find_links, - # Add tests/data/packages4, which contains a wheel for - # simple==1.0 (needed by requires_simple_extra[extra]). - '-f', data.find_links4, data.src.join("pep518_with_extra_and_markers-1.0"), use_module=True, ) +@pytest.mark.timeout(60) +@pytest.mark.parametrize('command', ('install', 'wheel')) +@pytest.mark.parametrize('package', ('pep518_forkbomb', + 'pep518_twin_forkbombs_first', + 'pep518_twin_forkbombs_second')) +def test_pep518_forkbombs(script, data, common_wheels, command, package): + package_source = next(data.packages.glob(package + '-[0-9]*.tar.gz')) + result = script.pip( + 'wheel', '--no-index', '-v', + '-f', common_wheels, + '-f', data.find_links, + package, + expect_error=True, + ) + assert '{1} is already being built: {0} from {1}'.format( + package, path_to_url(package_source), + ) in result.stdout, str(result) + + @pytest.mark.network def test_pip_second_command_line_interface_works(script, data): """ @@ -1267,9 +1298,31 @@ def test_install_pep508_with_url_in_install_requires(script): 'ce1a869fe039fbf7e217df36c4653d1dbe657778b2d41709593a0003584405f4' ], ) - res = script.pip('install', pkga_path, expect_error=True) - assert "Direct url requirement " in res.stderr, str(res) - assert "are not allowed for dependencies" in res.stderr, str(res) + res = script.pip('install', pkga_path) + assert "Successfully installed packaging-15.3" in str(res), str(res) + + +@pytest.mark.network +@pytest.mark.parametrize('index', (PyPI.simple_url, TestPyPI.simple_url)) +def test_install_from_test_pypi_with_ext_url_dep_is_blocked(script, index): + res = script.pip( + 'install', + '--index-url', + index, + 'pep-508-url-deps', + expect_error=True, + ) + error_message = ( + "Packages installed from PyPI cannot depend on packages " + "which are not also hosted on PyPI." + ) + error_cause = ( + "pep-508-url-deps depends on sampleproject@ " + "https://github.com/pypa/sampleproject/archive/master.zip" + ) + assert res.returncode == 1 + assert error_message in res.stderr, str(res) + assert error_cause in res.stderr, str(res) def test_installing_scripts_outside_path_prints_warning(script): diff --git a/tests/functional/test_list.py b/tests/functional/test_list.py index 234119b95..1f12d91e0 100644 --- a/tests/functional/test_list.py +++ b/tests/functional/test_list.py @@ -51,19 +51,6 @@ def test_columns_flag(script, data): assert 'simple2 3.0' in result.stdout, str(result) -def test_legacy_format(script, data): - """ - Test that legacy format - """ - script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', - ) - result = script.pip('list', '--format=legacy', expect_stderr=True) - assert 'simple (1.0)' in result.stdout, str(result) - assert 'simple2 (3.0)' in result.stdout, str(result) - - def test_format_priority(script, data): """ Test that latest format has priority over previous ones. @@ -72,18 +59,18 @@ def test_format_priority(script, data): 'install', '-f', data.find_links, '--no-index', 'simple==1.0', 'simple2==3.0', ) - result = script.pip('list', '--format=columns', '--format=legacy', + result = script.pip('list', '--format=columns', '--format=freeze', expect_stderr=True) - assert 'simple (1.0)' in result.stdout, str(result) - assert 'simple2 (3.0)' in result.stdout, str(result) + assert 'simple==1.0' in result.stdout, str(result) + assert 'simple2==3.0' in result.stdout, str(result) assert 'simple 1.0' not in result.stdout, str(result) assert 'simple2 3.0' not in result.stdout, str(result) - result = script.pip('list', '--format=legacy', '--format=columns') + result = script.pip('list', '--format=freeze', '--format=columns') assert 'Package' in result.stdout, str(result) assert 'Version' in result.stdout, str(result) - assert 'simple (1.0)' not in result.stdout, str(result) - assert 'simple2 (3.0)' not in result.stdout, str(result) + assert 'simple==1.0' not in result.stdout, str(result) + assert 'simple2==3.0' not in result.stdout, str(result) assert 'simple 1.0' in result.stdout, str(result) assert 'simple2 3.0' in result.stdout, str(result) @@ -111,17 +98,6 @@ def test_local_columns_flag(script, data): assert 'simple 1.0' in result.stdout, str(result) -def test_local_legacy_flag(script, data): - """ - Test the behavior of --local --format=legacy flags in the list - command. - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - result = script.pip('list', '--local', '--format=legacy', - expect_stderr=True) - assert 'simple (1.0)' in result.stdout - - @pytest.mark.network def test_user_flag(script, data, virtualenv): """ @@ -157,23 +133,6 @@ def test_user_columns_flag(script, data, virtualenv): assert 'simple2 2.0' in result.stdout, str(result) -@pytest.mark.network -def test_user_legacy(script, data, virtualenv): - """ - Test the behavior of --user flag in the list command - - """ - virtualenv.system_site_packages = True - script.pip('download', 'setuptools', 'wheel', '-d', data.packages) - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip('install', '-f', data.find_links, '--no-index', - '--user', 'simple2==2.0') - result = script.pip('list', '--user', '--format=legacy', - expect_stderr=True) - assert 'simple (1.0)' not in result.stdout - assert 'simple2 (2.0)' in result.stdout, str(result) - - @pytest.mark.network def test_uptodate_flag(script, data): """ @@ -225,30 +184,6 @@ def test_uptodate_columns_flag(script, data): assert 'simple2 3.0' in result.stdout, str(result) -@pytest.mark.network -def test_uptodate_legacy_flag(script, data): - """ - Test the behavior of --uptodate --format=legacy flag in the list command - - """ - script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', - ) - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--uptodate', - '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout # 3.0 is latest - assert 'pip-test-package (0.1.1,' in result.stdout # editables included - assert 'simple2 (3.0)' in result.stdout, str(result) - - @pytest.mark.network def test_outdated_flag(script, data): """ @@ -314,33 +249,6 @@ def test_outdated_columns_flag(script, data): assert 'simple2' not in result.stdout, str(result) # 3.0 is latest -@pytest.mark.network -def test_outdated_legacy(script, data): - """ - Test the behavior of --outdated --format=legacy flag in the list command - - """ - script.pip( - 'install', '-f', data.find_links, '--no-index', 'simple==1.0', - 'simple2==3.0', 'simplewheel==1.0', - ) - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--outdated', - '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0) - Latest: 3.0 [sdist]' in result.stdout - assert 'simplewheel (1.0) - Latest: 2.0 [wheel]' in result.stdout - assert 'pip-test-package (0.1, ' in result.stdout - assert ' Latest: 0.1.1 [sdist]' in result.stdout - assert 'simple2' not in result.stdout, str(result) # 3.0 is latest - - @pytest.mark.network def test_editables_flag(script, data): """ @@ -393,25 +301,6 @@ def test_editables_columns_flag(script, data): ) -@pytest.mark.network -def test_editables_legacy(script, data): - """ - Test the behavior of --editables flag in the list command - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' - ) - result = script.pip( - 'list', '--editable', '--format=legacy', expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout, str(result) - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) - - @pytest.mark.network def test_uptodate_editables_flag(script, data): """ @@ -455,28 +344,6 @@ def test_uptodate_editables_columns_flag(script, data): ) -@pytest.mark.network -def test_uptodate_editables_legacy(script, data): - """ - test the behavior of --editable --uptodate --format=columns --format=legacy - flag in the list command - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', '--editable', - '--uptodate', '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout, str(result) - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) - - @pytest.mark.network def test_outdated_editables_flag(script, data): """ @@ -519,28 +386,6 @@ def test_outdated_editables_columns_flag(script, data): ) -@pytest.mark.network -def test_outdated_editables_legacy(script, data): - """ - test the behavior of --editable --outdated flag in the list command - """ - script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') - script.pip( - 'install', '-e', - 'git+https://github.com/pypa/pip-test-package.git' - '@0.1#egg=pip-test-package' - ) - result = script.pip( - 'list', '-f', data.find_links, '--no-index', - '--editable', '--outdated', '--format=legacy', - expect_stderr=True, - ) - assert 'simple (1.0)' not in result.stdout, str(result) - assert os.path.join('src', 'pip-test-package') in result.stdout, ( - str(result) - ) - - def test_outdated_pre(script, data): script.pip('install', '-f', data.find_links, '--no-index', 'simple==1.0') @@ -583,11 +428,6 @@ def test_outdated_formats(script, data): ) assert 'simple==1.0' in result.stdout - # Check legacy - result = script.pip('list', '--no-index', '--find-links', wheelhouse_path, - '--outdated', '--format=legacy', expect_stderr=True) - assert 'simple (1.0) - Latest: 1.1 [wheel]' in result.stdout - # Check columns result = script.pip( 'list', '--no-index', '--find-links', wheelhouse_path, diff --git a/tests/functional/test_warning.py b/tests/functional/test_warning.py index 20f246795..6c097c75b 100644 --- a/tests/functional/test_warning.py +++ b/tests/functional/test_warning.py @@ -1,21 +1,22 @@ +import textwrap + def test_environ(script, tmpdir): """$PYTHONWARNINGS was added in python2.7""" demo = tmpdir.join('warnings_demo.py') - demo.write(''' -from pip._internal.utils import deprecation -deprecation.install_warning_logger() + demo.write(textwrap.dedent(''' + from logging import basicConfig + from pip._internal.utils import deprecation -from logging import basicConfig -basicConfig() + deprecation.install_warning_logger() + basicConfig() -from warnings import warn -warn("deprecated!", deprecation.PipDeprecationWarning) -''') + deprecation.deprecated("deprecated!", replacement=None, gone_in=None) + ''')) result = script.run('python', demo, expect_stderr=True) - assert result.stderr == \ - 'ERROR:pip._internal.deprecations:DEPRECATION: deprecated!\n' + expected = 'WARNING:pip._internal.deprecations:DEPRECATION: deprecated!\n' + assert result.stderr == expected script.environ['PYTHONWARNINGS'] = 'ignore' result = script.run('python', demo) diff --git a/tests/lib/__init__.py b/tests/lib/__init__.py index b445e89e8..e0e08fd76 100644 --- a/tests/lib/__init__.py +++ b/tests/lib/__init__.py @@ -100,10 +100,6 @@ class TestData(object): def packages3(self): return self.root.join("packages3") - @property - def packages4(self): - return self.root.join("packages4") - @property def src(self): return self.root.join("src") @@ -132,10 +128,6 @@ class TestData(object): def find_links3(self): return path_to_url(self.packages3) - @property - def find_links4(self): - return path_to_url(self.packages4) - def index_url(self, index="simple"): return path_to_url(self.root.join("indexes", index)) diff --git a/tests/unit/test_check.py b/tests/unit/test_check.py index 4dc893ada..1d1921484 100644 --- a/tests/unit/test_check.py +++ b/tests/unit/test_check.py @@ -2,6 +2,7 @@ """ import mock + from pip._internal.operations import check diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py new file mode 100644 index 000000000..ab9a2c9dd --- /dev/null +++ b/tests/unit/test_models.py @@ -0,0 +1,60 @@ +"""Tests for various classes in pip._internal.models +""" + +from pip._vendor.packaging.version import parse as parse_version + +from pip._internal.models import candidate, index + + +class TestPackageIndex(object): + """Tests for pip._internal.models.index.PackageIndex + """ + + def test_gives_right_urls(self): + url = "https://mypypi.internal/path/" + file_storage_domain = "files.mypypi.internal" + pack_index = index.PackageIndex(url, file_storage_domain) + + assert pack_index.url == url + assert pack_index.file_storage_domain == file_storage_domain + + assert pack_index.netloc == "mypypi.internal" + assert pack_index.simple_url == url + "simple" + assert pack_index.pypi_url == url + "pypi" + + def test_PyPI_urls_are_correct(self): + pack_index = index.PyPI + + assert pack_index.netloc == "pypi.org" + assert pack_index.url == "https://pypi.org/" + assert pack_index.simple_url == "https://pypi.org/simple" + assert pack_index.pypi_url == "https://pypi.org/pypi" + assert pack_index.file_storage_domain == "files.pythonhosted.org" + + def test_TestPyPI_urls_are_correct(self): + pack_index = index.TestPyPI + + assert pack_index.netloc == "test.pypi.org" + assert pack_index.url == "https://test.pypi.org/" + assert pack_index.simple_url == "https://test.pypi.org/simple" + assert pack_index.pypi_url == "https://test.pypi.org/pypi" + assert pack_index.file_storage_domain == "test-files.pythonhosted.org" + + +class TestInstallationCandidate(object): + + def test_sets_correct_variables(self): + obj = candidate.InstallationCandidate( + "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" + ) + assert obj.project == "A" + assert obj.version == parse_version("1.0.0") + assert obj.location == "https://somewhere.com/path/A-1.0.0.tar.gz" + + # NOTE: This isn't checking the ordering logic; only the data provided to + # it is correct. + def test_sets_the_right_key(self): + obj = candidate.InstallationCandidate( + "A", "1.0.0", "https://somewhere.com/path/A-1.0.0.tar.gz" + ) + assert obj._compare_key == (obj.project, obj.version, obj.location) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index f9e73afc3..218eb21be 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -19,6 +19,7 @@ from pip._internal.operations.prepare import RequirementPreparer from pip._internal.req import InstallRequirement, RequirementSet from pip._internal.req.req_file import process_line from pip._internal.req.req_install import parse_editable +from pip._internal.req.req_tracker import RequirementTracker from pip._internal.resolve import Resolver from pip._internal.utils.misc import read_text_file from tests.lib import DATA_DIR, assert_raises_regexp, requirements_file @@ -47,6 +48,7 @@ class TestRequirementSet(object): wheel_download_dir=None, progress_bar="on", build_isolation=True, + req_tracker=RequirementTracker(), ) return Resolver( preparer=preparer, wheel_cache=None, diff --git a/tests/unit/test_req_file.py b/tests/unit/test_req_file.py index 93fec8e73..3f7c293b5 100644 --- a/tests/unit/test_req_file.py +++ b/tests/unit/test_req_file.py @@ -639,7 +639,7 @@ class TestParseRequirements(object): popen.return_value.stdout.readline.return_value = b"" try: req.install([]) - except: + except Exception: pass last_call = popen.call_args_list[-1] diff --git a/tests/unit/test_req_uninstall.py b/tests/unit/test_req_uninstall.py index 5e165d8c1..8f9994579 100644 --- a/tests/unit/test_req_uninstall.py +++ b/tests/unit/test_req_uninstall.py @@ -50,9 +50,9 @@ def test_compressed_listing(tmpdir): def in_tmpdir(paths): li = [] for path in paths: - li.append(str(os.path.normcase( - os.path.join(tmpdir, path.replace("/", os.path.sep)) - ))) + li.append( + str(os.path.join(tmpdir, path.replace("/", os.path.sep))) + ) return li sample = in_tmpdir([ @@ -121,12 +121,12 @@ class TestUninstallPathSet(object): f = os.path.join(tmpdir, 'foo') with open(f, 'w'): pass - l = os.path.join(tmpdir, 'foo_link') - os.symlink(f, l) + foo_link = os.path.join(tmpdir, 'foo_link') + os.symlink(f, foo_link) ups = UninstallPathSet(dist=Mock()) - ups.add(l) - assert ups.paths == {l} + ups.add(foo_link) + assert ups.paths == {foo_link} def test_compact_shorter_path(self, monkeypatch): monkeypatch.setattr(pip._internal.req.req_uninstall, 'is_local', diff --git a/tests/unit/test_unit_outdated.py b/tests/unit/test_unit_outdated.py index 853531db5..1f20664e7 100644 --- a/tests/unit/test_unit_outdated.py +++ b/tests/unit/test_unit_outdated.py @@ -3,7 +3,6 @@ import os import sys from contextlib import contextmanager - import freezegun import pretend import pytest diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py index 1cfd4c409..e6b473dae 100644 --- a/tests/unit/test_vcs.py +++ b/tests/unit/test_vcs.py @@ -2,7 +2,7 @@ import pytest from mock import Mock from pip._vendor.packaging.version import parse as parse_version -from pip._internal.vcs import RevOptions, VersionControl +from pip._internal.vcs import RevOptions from pip._internal.vcs.bazaar import Bazaar from pip._internal.vcs.git import Git, looks_like_hash from pip._internal.vcs.mercurial import Mercurial @@ -122,14 +122,60 @@ def test_git_is_commit_id_equal(git, rev_name, result): assert git.is_commit_id_equal('/path', rev_name) is result -def test_translate_egg_surname(): - vc = VersionControl() - assert vc.translate_egg_surname("foo") == "foo" - assert vc.translate_egg_surname("foo/bar") == "foo_bar" - assert vc.translate_egg_surname("foo/1.2.3") == "foo_1.2.3" +# The non-SVN backends all use the same get_netloc_and_auth(), so only test +# Git as a representative. +@pytest.mark.parametrize('netloc, expected', [ + # Test a basic case. + ('example.com', ('example.com', (None, None))), + # Test with username and password. + ('user:pass@example.com', ('user:pass@example.com', (None, None))), +]) +def test_git__get_netloc_and_auth(netloc, expected): + """ + Test VersionControl.get_netloc_and_auth(). + """ + actual = Git().get_netloc_and_auth(netloc) + assert actual == expected -def test_bazaar_simple_urls(): +@pytest.mark.parametrize('netloc, expected', [ + # Test a basic case. + ('example.com', ('example.com', (None, None))), + # Test with username and no password. + ('user@example.com', ('example.com', ('user', None))), + # Test with username and password. + ('user:pass@example.com', ('example.com', ('user', 'pass'))), + # Test the password containing an @ symbol. + ('user:pass@word@example.com', ('example.com', ('user', 'pass@word'))), + # Test the password containing a : symbol. + ('user:pass:word@example.com', ('example.com', ('user', 'pass:word'))), +]) +def test_subversion__get_netloc_and_auth(netloc, expected): + """ + Test Subversion.get_netloc_and_auth(). + """ + actual = Subversion().get_netloc_and_auth(netloc) + assert actual == expected + + +def test_git__get_url_rev__idempotent(): + """ + Check that Git.get_url_rev_and_auth() is idempotent for what the code calls + "stub URLs" (i.e. URLs that don't contain "://"). + + Also check that it doesn't change self.url. + """ + url = 'git+git@git.example.com:MyProject#egg=MyProject' + vcs = Git(url) + result1 = vcs.get_url_rev_and_auth(url) + assert vcs.url == url + result2 = vcs.get_url_rev_and_auth(url) + expected = ('git@git.example.com:MyProject', None, (None, None)) + assert result1 == expected + assert result2 == expected + + +def test_bazaar__get_url_rev_and_auth(): """ Test bzr url support. @@ -154,23 +200,64 @@ def test_bazaar_simple_urls(): url='bzr+lp:MyLaunchpadProject#egg=MyLaunchpadProject' ) - assert http_bzr_repo.get_url_rev() == ( - 'http://bzr.myproject.org/MyProject/trunk/', None, + assert http_bzr_repo.get_url_rev_and_auth(http_bzr_repo.url) == ( + 'http://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert https_bzr_repo.get_url_rev() == ( - 'https://bzr.myproject.org/MyProject/trunk/', None, + assert https_bzr_repo.get_url_rev_and_auth(https_bzr_repo.url) == ( + 'https://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert ssh_bzr_repo.get_url_rev() == ( - 'bzr+ssh://bzr.myproject.org/MyProject/trunk/', None, + assert ssh_bzr_repo.get_url_rev_and_auth(ssh_bzr_repo.url) == ( + 'bzr+ssh://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert ftp_bzr_repo.get_url_rev() == ( - 'ftp://bzr.myproject.org/MyProject/trunk/', None, + assert ftp_bzr_repo.get_url_rev_and_auth(ftp_bzr_repo.url) == ( + 'ftp://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert sftp_bzr_repo.get_url_rev() == ( - 'sftp://bzr.myproject.org/MyProject/trunk/', None, + assert sftp_bzr_repo.get_url_rev_and_auth(sftp_bzr_repo.url) == ( + 'sftp://bzr.myproject.org/MyProject/trunk/', None, (None, None), ) - assert launchpad_bzr_repo.get_url_rev() == ( - 'lp:MyLaunchpadProject', None, + assert launchpad_bzr_repo.get_url_rev_and_auth(launchpad_bzr_repo.url) == ( + 'lp:MyLaunchpadProject', None, (None, None), + ) + + +# The non-SVN backends all use the same make_rev_args(), so only test +# Git as a representative. +@pytest.mark.parametrize('username, password, expected', [ + (None, None, []), + ('user', None, []), + ('user', 'pass', []), +]) +def test_git__make_rev_args(username, password, expected): + """ + Test VersionControl.make_rev_args(). + """ + actual = Git().make_rev_args(username, password) + assert actual == expected + + +@pytest.mark.parametrize('username, password, expected', [ + (None, None, []), + ('user', None, ['--username', 'user']), + ('user', 'pass', ['--username', 'user', '--password', 'pass']), +]) +def test_subversion__make_rev_args(username, password, expected): + """ + Test Subversion.make_rev_args(). + """ + actual = Subversion().make_rev_args(username, password) + assert actual == expected + + +def test_subversion__get_url_rev_options(): + """ + Test Subversion.get_url_rev_options(). + """ + url = 'svn+https://user:pass@svn.example.com/MyProject@v1.0#egg=MyProject' + url, rev_options = Subversion().get_url_rev_options(url) + assert url == 'https://svn.example.com/MyProject' + assert rev_options.rev == 'v1.0' + assert rev_options.extra_args == ( + ['--username', 'user', '--password', 'pass'] ) diff --git a/tests/unit/test_wheel.py b/tests/unit/test_wheel.py index 6d3c1c7a7..0fb907589 100644 --- a/tests/unit/test_wheel.py +++ b/tests/unit/test_wheel.py @@ -1,4 +1,5 @@ """Tests for wheel binary packages and .dist-info.""" +import logging import os import pytest @@ -367,7 +368,8 @@ class TestWheelBuilder(object): wb = wheel.WheelBuilder( finder=Mock(), preparer=Mock(), wheel_cache=None, ) - wb.build([wheel_req], session=Mock()) + with caplog.at_level(logging.INFO): + wb.build([wheel_req], session=Mock()) assert "due to already being wheel" in caplog.text assert mock_build_one.mock_calls == [] @@ -470,3 +472,16 @@ class TestMessageAboutScriptsNotOnPATH(object): scripts=[os.path.join('a', 'b', 'c')] ) assert retval is None + + def test_missing_PATH_env_treated_as_empty_PATH_env(self): + scripts = ['a/b/foo'] + + env = os.environ.copy() + del env['PATH'] + with patch.dict('os.environ', env, clear=True): + retval_missing = wheel.message_about_scripts_not_on_PATH(scripts) + + with patch.dict('os.environ', {'PATH': ''}): + retval_empty = wheel.message_about_scripts_not_on_PATH(scripts) + + assert retval_missing == retval_empty diff --git a/tools/lint-requirements.txt b/tools/lint-requirements.txt index a19517e78..92ed56ed6 100644 --- a/tools/lint-requirements.txt +++ b/tools/lint-requirements.txt @@ -1,2 +1,2 @@ -flake8 == 3.3.0 -isort == 4.2.5 +flake8 == 3.5.0 +isort == 4.3.4 diff --git a/tools/tests-requirements.txt b/tools/tests-requirements.txt index a49698b92..8c9bd7592 100644 --- a/tools/tests-requirements.txt +++ b/tools/tests-requirements.txt @@ -6,8 +6,6 @@ pytest-cov pytest-rerunfailures pytest-timeout pytest-xdist +pyyaml scripttest https://github.com/pypa/virtualenv/archive/master.zip#egg=virtualenv -# Revert the following hack after a PyYAML is released supporting Py3.7 -pyyaml; python_version < "3.7" -https://github.com/yaml/pyyaml/archive/master.zip#egg=pyyaml ; python_version >= "3.7" diff --git a/tox.ini b/tox.ini index ee8a3f118..c71dc2668 100644 --- a/tox.ini +++ b/tox.ini @@ -22,8 +22,8 @@ commands = pytest --timeout 300 --cov=pip --cov-report=term-missing --cov-report deps = -r{toxinidir}/tools/docs-requirements.txt basepython = python2.7 commands = - sphinx-build -W -b html -d {envtmpdir}/doctrees docs docs/build/html - sphinx-build -W -b man -d {envtmpdir}/doctrees docs docs/build/man + sphinx-build -W -d {envtmpdir}/doctrees -b html docs docs/build/html + sphinx-build -W -d {envtmpdir}/doctrees -b man docs docs/build/man [testenv:packaging] deps = @@ -36,8 +36,8 @@ commands = [lint] deps = -r{toxinidir}/tools/lint-requirements.txt commands = - flake8 . - isort --recursive --check-only --diff src/pip tests + flake8 + isort --check-only --diff [testenv:lint-py2] basepython = python2