From cc50bd37b8e04d158366e45bd4809a76dd381bea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= <mschoentgen@nuxeo.com>
Date: Thu, 16 Aug 2018 20:10:19 +0200
Subject: [PATCH 01/92] Fix 'logging.warn' method is deprecated, use 'warning'
 instead in wheel.py

---
 src/pip/_internal/wheel.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/_internal/wheel.py b/src/pip/_internal/wheel.py
index 60a0f295f..b1b1d8712 100644
--- a/src/pip/_internal/wheel.py
+++ b/src/pip/_internal/wheel.py
@@ -475,7 +475,7 @@ if __name__ == '__main__':
         if warn_script_location:
             msg = message_about_scripts_not_on_PATH(generated_console_scripts)
             if msg is not None:
-                logger.warn(msg)
+                logger.warning(msg)
 
     if len(gui) > 0:
         generated.extend(

From 860deed7dba7b4564ddcb84b30fa951478948d1c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= <mschoentgen@nuxeo.com>
Date: Thu, 16 Aug 2018 20:24:42 +0200
Subject: [PATCH 02/92] Fix DeprecationWarning: the imp module is deprecated in
 favour of importlib in pep425tags.py

---
 src/pip/_internal/pep425tags.py   |  8 ++++----
 src/pip/_internal/utils/compat.py | 13 +++++++++++++
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/src/pip/_internal/pep425tags.py b/src/pip/_internal/pep425tags.py
index 0b5c7832d..ab1a02985 100644
--- a/src/pip/_internal/pep425tags.py
+++ b/src/pip/_internal/pep425tags.py
@@ -11,6 +11,7 @@ import warnings
 from collections import OrderedDict
 
 import pip._internal.utils.glibc
+from pip._internal.utils.compat import get_extension_suffixes
 
 logger = logging.getLogger(__name__)
 
@@ -252,10 +253,9 @@ def get_supported(versions=None, noarch=False, platform=None,
         abis[0:0] = [abi]
 
     abi3s = set()
-    import imp
-    for suffix in imp.get_suffixes():
-        if suffix[0].startswith('.abi'):
-            abi3s.add(suffix[0].split('.', 2)[1])
+    for suffix in get_extension_suffixes():
+        if suffix.startswith('.abi'):
+            abi3s.add(suffix.split('.', 2)[1])
 
     abis.extend(sorted(list(abi3s)))
 
diff --git a/src/pip/_internal/utils/compat.py b/src/pip/_internal/utils/compat.py
index e6c008d3b..3114f2da4 100644
--- a/src/pip/_internal/utils/compat.py
+++ b/src/pip/_internal/utils/compat.py
@@ -25,6 +25,7 @@ except ImportError:
 __all__ = [
     "ipaddress", "uses_pycache", "console_to_str", "native_str",
     "get_path_uid", "stdlib_pkgs", "WINDOWS", "samefile", "get_terminal_size",
+    "get_extension_suffixes",
 ]
 
 
@@ -160,6 +161,18 @@ def get_path_uid(path):
     return file_uid
 
 
+if sys.version_info >= (3, 4):
+    from importlib.machinery import EXTENSION_SUFFIXES
+
+    def get_extension_suffixes():
+        return EXTENSION_SUFFIXES
+else:
+    from imp import get_suffixes
+
+    def get_extension_suffixes():
+        return [suffix[0] for suffix in get_suffixes()]
+
+
 def expanduser(path):
     """
     Expand ~ and ~user constructions.

From 3b1be840fc548e083131a149305a058f4fb3bc65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Micka=C3=ABl=20Schoentgen?= <mschoentgen@nuxeo.com>
Date: Thu, 16 Aug 2018 20:25:30 +0200
Subject: [PATCH 03/92] Fix one ResourceWarning: unclosed file in test_req.py

---
 tests/unit/test_req.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py
index fc3c0ad00..f41a89b92 100644
--- a/tests/unit/test_req.py
+++ b/tests/unit/test_req.py
@@ -64,7 +64,8 @@ class TestRequirementSet(object):
 
         build_dir = os.path.join(self.tempdir, 'build', 'simple')
         os.makedirs(build_dir)
-        open(os.path.join(build_dir, "setup.py"), 'w')
+        with open(os.path.join(build_dir, "setup.py"), 'w'):
+            pass
         reqset = RequirementSet()
         req = InstallRequirement.from_line('simple')
         req.is_direct = True

From 007dbf3ea6b86764fe28744f5636eb9927bb49a9 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Tue, 21 Aug 2018 01:51:06 -0700
Subject: [PATCH 04/92] Add two failing tests.

---
 tests/functional/test_install_vcs_git.py | 82 +++++++++++++++++++++++-
 1 file changed, 79 insertions(+), 3 deletions(-)

diff --git a/tests/functional/test_install_vcs_git.py b/tests/functional/test_install_vcs_git.py
index 6648e44fe..260f8c28d 100644
--- a/tests/functional/test_install_vcs_git.py
+++ b/tests/functional/test_install_vcs_git.py
@@ -10,15 +10,32 @@ from tests.lib.git_submodule_helpers import (
 from tests.lib.local_repos import local_checkout
 
 
+def _get_editable_repo_dir(script, package_name):
+    """
+    Return the repository directory for an editable install.
+    """
+    return script.venv_path / 'src' / package_name
+
+
 def _get_editable_branch(script, package_name):
     """
     Return the current branch of an editable install.
     """
-    repo_dir = script.venv_path / 'src' / package_name
+    repo_dir = _get_editable_repo_dir(script, package_name)
     result = script.run(
         'git', 'rev-parse', '--abbrev-ref', 'HEAD', cwd=repo_dir
     )
+    return result.stdout.strip()
 
+
+def _get_branch_remote(script, package_name, branch):
+    """
+
+    """
+    repo_dir = _get_editable_repo_dir(script, package_name)
+    result = script.run(
+        'git', 'config', 'branch.{}.remote'.format(branch), cwd=repo_dir
+    )
     return result.stdout.strip()
 
 
@@ -363,7 +380,65 @@ def test_git_works_with_editable_non_origin_repo(script):
     assert "version-pkg==0.1" in result.stdout
 
 
-def test_editable_non_master_default_branch(script):
+def test_editable__no_revision(script):
+    """
+    Test a basic install in editable mode specifying no revision.
+    """
+    version_pkg_path = _create_test_package(script)
+    _install_version_pkg_only(script, version_pkg_path)
+
+    branch = _get_editable_branch(script, 'version-pkg')
+    assert branch == 'master'
+
+    remote = _get_branch_remote(script, 'version-pkg', 'master')
+    assert remote == 'origin'
+
+
+def test_editable__branch_with_sha_same_as_default(script):
+    """
+    Test installing in editable mode a branch whose sha matches the sha
+    of the default branch.
+    """
+    version_pkg_path = _create_test_package(script)
+    # Create a second branch with the same SHA.
+    script.run(
+        'git', 'branch', 'develop', expect_stderr=True,
+        cwd=version_pkg_path,
+    )
+    _install_version_pkg_only(script, version_pkg_path, rev='develop')
+
+    branch = _get_editable_branch(script, 'version-pkg')
+    assert branch == 'develop'
+
+    remote = _get_branch_remote(script, 'version-pkg', 'develop')
+    assert remote == 'origin'
+
+
+def test_editable__branch_with_sha_different_from_default(script):
+    """
+    Test installing in editable mode a branch whose sha is different from
+    the sha of the default branch.
+    """
+    version_pkg_path = _create_test_package(script)
+    # Create a second branch.
+    script.run(
+        'git', 'branch', 'develop', expect_stderr=True,
+        cwd=version_pkg_path,
+    )
+    # Add another commit to the master branch to give it a different sha.
+    _change_test_package_version(script, version_pkg_path)
+
+    version = _install_version_pkg(script, version_pkg_path, rev='develop')
+    assert version == '0.1'
+
+    branch = _get_editable_branch(script, 'version-pkg')
+    assert branch == 'develop'
+
+    remote = _get_branch_remote(script, 'version-pkg', 'develop')
+    assert remote == 'origin'
+
+
+def test_editable__non_master_default_branch(script):
     """
     Test the branch you get after an editable install from a remote repo
     with a non-master default branch.
@@ -376,8 +451,9 @@ def test_editable_non_master_default_branch(script):
         cwd=version_pkg_path,
     )
     _install_version_pkg_only(script, version_pkg_path)
+
     branch = _get_editable_branch(script, 'version-pkg')
-    assert 'release' == branch
+    assert branch == 'release'
 
 
 def test_reinstalling_works_with_editable_non_master_branch(script):

From 8d6f7b56fa499f86abac9793ba9d0dad529d6063 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 19 Aug 2018 18:07:01 -0700
Subject: [PATCH 05/92] Add Git.get_branch().

---
 ...E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial |  0
 src/pip/_internal/vcs/git.py                  | 14 +++++++++++++
 tests/functional/test_vcs_git.py              | 21 +++++++++++++++++++
 3 files changed, 35 insertions(+)
 create mode 100644 news/88E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial

diff --git a/news/88E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial b/news/88E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index bacc037f7..aab58e098 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -77,6 +77,20 @@ class Git(VersionControl):
         version = '.'.join(version.split('.')[:3])
         return parse_version(version)
 
+    def get_branch(self, location):
+        """
+        Return the current branch, or None if HEAD isn't at a branch
+        (e.g. detached HEAD).
+        """
+        args = ['rev-parse', '--abbrev-ref', 'HEAD']
+        output = self.run_command(args, show_stdout=False, cwd=location)
+        branch = output.strip()
+
+        if branch == 'HEAD':
+            return None
+
+        return branch
+
     def export(self, location):
         """Export the Git repository at the url to the destination location"""
         if not location.endswith('/'):
diff --git a/tests/functional/test_vcs_git.py b/tests/functional/test_vcs_git.py
index 656cc33ed..65f4ebdb4 100644
--- a/tests/functional/test_vcs_git.py
+++ b/tests/functional/test_vcs_git.py
@@ -70,6 +70,27 @@ def test_git_work_tree_ignored():
         git.run_command(['status', temp_dir], extra_environ=env, cwd=temp_dir)
 
 
+def test_get_branch(script, tmpdir):
+    repo_dir = str(tmpdir)
+    script.run('git', 'init', cwd=repo_dir)
+    sha = do_commit(script, repo_dir)
+
+    git = Git()
+    assert git.get_branch(repo_dir) == 'master'
+
+    # Switch to a branch with the same SHA as "master" but whose name
+    # is alphabetically after.
+    script.run(
+        'git', 'checkout', '-b', 'release', cwd=repo_dir,
+        expect_stderr=True,
+    )
+    assert git.get_branch(repo_dir) == 'release'
+
+    # Also test the detached HEAD case.
+    script.run('git', 'checkout', sha, cwd=repo_dir, expect_stderr=True)
+    assert git.get_branch(repo_dir) is None
+
+
 def test_get_revision_sha(script):
     with TempDirectory(kind="testing") as temp:
         repo_dir = temp.path

From b11bf9e978cd6f872db98587759cebdf96c7904a Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Tue, 21 Aug 2018 02:02:07 -0700
Subject: [PATCH 06/92] Change get_revision_sha() to return is_branch.

---
 src/pip/_internal/vcs/git.py     | 14 ++++++++++----
 tests/functional/test_vcs_git.py | 12 ++++++------
 tests/unit/test_vcs.py           |  6 +++---
 3 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index aab58e098..2a80248e8 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -105,8 +105,8 @@ class Git(VersionControl):
 
     def get_revision_sha(self, dest, rev):
         """
-        Return a commit hash for the given revision if it names a remote
-        branch or tag.  Otherwise, return None.
+        Return (sha_or_none, is_branch), where sha_or_none is a commit hash
+        if the revision names a remote branch or tag, otherwise None.
 
         Args:
           dest: the repository directory.
@@ -129,7 +129,13 @@ class Git(VersionControl):
         branch_ref = 'refs/remotes/origin/{}'.format(rev)
         tag_ref = 'refs/tags/{}'.format(rev)
 
-        return refs.get(branch_ref) or refs.get(tag_ref)
+        sha = refs.get(branch_ref)
+        if sha is not None:
+            return (sha, True)
+
+        sha = refs.get(tag_ref)
+
+        return (sha, False)
 
     def resolve_revision(self, dest, url, rev_options):
         """
@@ -140,7 +146,7 @@ class Git(VersionControl):
           rev_options: a RevOptions object.
         """
         rev = rev_options.arg_rev
-        sha = self.get_revision_sha(dest, rev)
+        sha, is_branch = self.get_revision_sha(dest, rev)
 
         if sha is not None:
             return rev_options.make_new(sha)
diff --git a/tests/functional/test_vcs_git.py b/tests/functional/test_vcs_git.py
index 65f4ebdb4..2a54af58f 100644
--- a/tests/functional/test_vcs_git.py
+++ b/tests/functional/test_vcs_git.py
@@ -37,9 +37,9 @@ def add_commits(script, dest, count):
     return shas
 
 
-def check_rev(repo_dir, rev, expected_sha):
+def check_rev(repo_dir, rev, expected):
     git = Git()
-    assert git.get_revision_sha(repo_dir, rev) == expected_sha
+    assert git.get_revision_sha(repo_dir, rev) == expected
 
 
 def test_git_dir_ignored():
@@ -123,9 +123,9 @@ def test_get_revision_sha(script):
         script.run('git', 'tag', 'aaa/v1.0', head_sha, cwd=repo_dir)
         script.run('git', 'tag', 'zzz/v1.0', head_sha, cwd=repo_dir)
 
-        check_rev(repo_dir, 'v1.0', tag_sha)
-        check_rev(repo_dir, 'v2.0', tag_sha)
-        check_rev(repo_dir, 'origin-branch', origin_sha)
+        check_rev(repo_dir, 'v1.0', (tag_sha, False))
+        check_rev(repo_dir, 'v2.0', (tag_sha, False))
+        check_rev(repo_dir, 'origin-branch', (origin_sha, True))
 
         ignored_names = [
             # Local branches should be ignored.
@@ -143,7 +143,7 @@ def test_get_revision_sha(script):
             'does-not-exist',
         ]
         for name in ignored_names:
-            check_rev(repo_dir, name, None)
+            check_rev(repo_dir, name, (None, False))
 
 
 @pytest.mark.network
diff --git a/tests/unit/test_vcs.py b/tests/unit/test_vcs.py
index c9ad863cc..7e8934c76 100644
--- a/tests/unit/test_vcs.py
+++ b/tests/unit/test_vcs.py
@@ -109,7 +109,7 @@ def test_git_get_src_requirements(git, dist):
 
 @patch('pip._internal.vcs.git.Git.get_revision_sha')
 def test_git_resolve_revision_rev_exists(get_sha_mock):
-    get_sha_mock.return_value = '123456'
+    get_sha_mock.return_value = ('123456', False)
     git = Git()
     rev_options = git.make_rev_options('develop')
 
@@ -120,7 +120,7 @@ def test_git_resolve_revision_rev_exists(get_sha_mock):
 
 @patch('pip._internal.vcs.git.Git.get_revision_sha')
 def test_git_resolve_revision_rev_not_found(get_sha_mock):
-    get_sha_mock.return_value = None
+    get_sha_mock.return_value = (None, False)
     git = Git()
     rev_options = git.make_rev_options('develop')
 
@@ -131,7 +131,7 @@ def test_git_resolve_revision_rev_not_found(get_sha_mock):
 
 @patch('pip._internal.vcs.git.Git.get_revision_sha')
 def test_git_resolve_revision_not_found_warning(get_sha_mock, caplog):
-    get_sha_mock.return_value = None
+    get_sha_mock.return_value = (None, False)
     git = Git()
 
     url = 'git+https://git.example.com'

From 06f329059dbce717ee085010f88aa5ebd18a1642 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Tue, 21 Aug 2018 03:00:29 -0700
Subject: [PATCH 07/92] Update fetch_new() to checkout the correct branch.

---
 news/2037.bugfix             |  1 +
 src/pip/_internal/vcs/git.py | 23 ++++++++++++++++++-----
 2 files changed, 19 insertions(+), 5 deletions(-)
 create mode 100644 news/2037.bugfix

diff --git a/news/2037.bugfix b/news/2037.bugfix
new file mode 100644
index 000000000..aca18b07b
--- /dev/null
+++ b/news/2037.bugfix
@@ -0,0 +1 @@
+Checkout the correct branch when doing an editable Git install.
\ No newline at end of file
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index 2a80248e8..7169dc700 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -149,7 +149,10 @@ class Git(VersionControl):
         sha, is_branch = self.get_revision_sha(dest, rev)
 
         if sha is not None:
-            return rev_options.make_new(sha)
+            rev_options = rev_options.make_new(sha)
+            rev_options.branch_name = rev if is_branch else None
+
+            return rev_options
 
         # Do not show a warning for the common case of something that has
         # the form of a Git commit hash.
@@ -197,10 +200,20 @@ class Git(VersionControl):
         if rev_options.rev:
             # Then a specific revision was requested.
             rev_options = self.resolve_revision(dest, url, rev_options)
-            # Only do a checkout if the current commit id doesn't match
-            # the requested revision.
-            if not self.is_commit_id_equal(dest, rev_options.rev):
-                cmd_args = ['checkout', '-q'] + rev_options.to_args()
+            branch_name = getattr(rev_options, 'branch_name', None)
+            if branch_name is None:
+                # Only do a checkout if the current commit id doesn't match
+                # the requested revision.
+                if not self.is_commit_id_equal(dest, rev_options.rev):
+                    cmd_args = ['checkout', '-q'] + rev_options.to_args()
+                    self.run_command(cmd_args, cwd=dest)
+            elif self.get_branch(dest) != branch_name:
+                # Then a specific branch was requested, and that branch
+                # is not yet checked out.
+                track_branch = 'origin/{}'.format(branch_name)
+                cmd_args = [
+                    'checkout', '-b', branch_name, '--track', track_branch,
+                ]
                 self.run_command(cmd_args, cwd=dest)
 
         #: repo may contain submodules

From 0d817939c334a835f5c2c45394b8b4dba8352de8 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Tue, 21 Aug 2018 13:35:29 -0700
Subject: [PATCH 08/92] Fix tests on Python 3.4 (GROUP=2).

---
 tests/functional/test_install_vcs_git.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/functional/test_install_vcs_git.py b/tests/functional/test_install_vcs_git.py
index 260f8c28d..77296baf1 100644
--- a/tests/functional/test_install_vcs_git.py
+++ b/tests/functional/test_install_vcs_git.py
@@ -397,7 +397,7 @@ def test_editable__no_revision(script):
 def test_editable__branch_with_sha_same_as_default(script):
     """
     Test installing in editable mode a branch whose sha matches the sha
-    of the default branch.
+    of the default branch, but is different from the default branch.
     """
     version_pkg_path = _create_test_package(script)
     # Create a second branch with the same SHA.
@@ -405,7 +405,9 @@ def test_editable__branch_with_sha_same_as_default(script):
         'git', 'branch', 'develop', expect_stderr=True,
         cwd=version_pkg_path,
     )
-    _install_version_pkg_only(script, version_pkg_path, rev='develop')
+    _install_version_pkg_only(
+        script, version_pkg_path, rev='develop', expect_stderr=True
+    )
 
     branch = _get_editable_branch(script, 'version-pkg')
     assert branch == 'develop'
@@ -428,7 +430,9 @@ def test_editable__branch_with_sha_different_from_default(script):
     # Add another commit to the master branch to give it a different sha.
     _change_test_package_version(script, version_pkg_path)
 
-    version = _install_version_pkg(script, version_pkg_path, rev='develop')
+    version = _install_version_pkg(
+        script, version_pkg_path, rev='develop', expect_stderr=True
+    )
     assert version == '0.1'
 
     branch = _get_editable_branch(script, 'version-pkg')

From ccd030b40d7d212e35cc66d5d5b7d513f0739336 Mon Sep 17 00:00:00 2001
From: Benoit Pierre <benoit.pierre@gmail.com>
Date: Sun, 16 Sep 2018 21:02:00 +0200
Subject: [PATCH 09/92] appveyor: do not pass integration jobs on failed unit
 tests

---
 appveyor.yml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 7fe18eacb..cfad2a52f 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,6 +1,8 @@
 environment:
   matrix:
     # Unit and integration tests.
+    - PYTHON: "C:\\Python27"
+    - PYTHON: "C:\\Python36-x64"
     - PYTHON: "C:\\Python27"
       RUN_INTEGRATION_TESTS: "True"
     - PYTHON: "C:\\Python36-x64"
@@ -58,8 +60,10 @@ test_script:
             subst T: $env:TEMP
             $env:TEMP = "T:\"
             $env:TMP = "T:\"
-            tox -e py -- -m unit -n 3
             if ($env:RUN_INTEGRATION_TESTS -eq "True") {
                 tox -e py -- -m integration -n 3 --duration=5
             }
+            else {
+                tox -e py -- -m unit -n 3
+            }
         }

From 8136b410f75281b9e95ce134c97c29463be05d9a Mon Sep 17 00:00:00 2001
From: Jon Dufresne <jon.dufresne@gmail.com>
Date: Sun, 16 Sep 2018 13:21:09 -0700
Subject: [PATCH 10/92] Correct capitalization of PyPI

As spelled on https://pypi.org/.
---
 NEWS.rst                                          | 2 +-
 news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial | 0
 tests/data/packages3/dinner/index.html            | 4 ++--
 tests/data/packages3/index.html                   | 4 ++--
 tests/data/packages3/requiredinner/index.html     | 4 ++--
 tests/functional/test_download.py                 | 2 +-
 tests/functional/test_install.py                  | 2 +-
 tests/functional/test_search.py                   | 2 +-
 tests/unit/test_finder.py                         | 2 +-
 9 files changed, 11 insertions(+), 11 deletions(-)
 create mode 100644 news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial

diff --git a/NEWS.rst b/NEWS.rst
index b7fc3f7a2..7182ea1c5 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -1207,7 +1207,7 @@ Improved Documentation
 
 - **Dropped support for Python 2.4** The minimum supported Python version is
   now Python 2.5.
-- Fixed pypi mirror support being broken on some DNS responses. Thanks
+- Fixed PyPI mirror support being broken on some DNS responses. Thanks
   philwhin. (#605)
 - Fixed pip uninstall removing files it didn't install. Thanks pjdelport.
   (#355)
diff --git a/news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial b/news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/data/packages3/dinner/index.html b/tests/data/packages3/dinner/index.html
index 6afabea66..e258eb16b 100644
--- a/tests/data/packages3/dinner/index.html
+++ b/tests/data/packages3/dinner/index.html
@@ -1,6 +1,6 @@
-<html><head><title>PyPi Mirror</title></head>
+<html><head><title>PyPI Mirror</title></head>
 <body>
-  <h1>PyPi Mirror</h1>
+  <h1>PyPI Mirror</h1>
   <h2>For testing --index-url with a file:// url for the index</h2>
   <a href='Dinner-1.0.tar.gz'>Dinner-1.0.tar.gz</a><br />
   <a href='Dinner-2.0.tar.gz'>Dinner-2.0.tar.gz</a><br />
diff --git a/tests/data/packages3/index.html b/tests/data/packages3/index.html
index a59811ba0..d66e70ec6 100644
--- a/tests/data/packages3/index.html
+++ b/tests/data/packages3/index.html
@@ -1,6 +1,6 @@
-<html><head><title>PyPi Mirror</title></head>
+<html><head><title>PyPI Mirror</title></head>
 <body>
-  <h1>PyPi Mirror</h1>
+  <h1>PyPI Mirror</h1>
   <h2>For testing --index-url with a file:// url for the index</h2>
   <a href='requiredinner'>requiredinner</a><br />
   <a href='Dinner'>Dinner</a><br />
diff --git a/tests/data/packages3/requiredinner/index.html b/tests/data/packages3/requiredinner/index.html
index 52701cd5c..0981c9c72 100644
--- a/tests/data/packages3/requiredinner/index.html
+++ b/tests/data/packages3/requiredinner/index.html
@@ -1,6 +1,6 @@
-<html><head><title>PyPi Mirror</title></head>
+<html><head><title>PyPI Mirror</title></head>
 <body>
-  <h1>PyPi Mirror</h1>
+  <h1>PyPI Mirror</h1>
   <h2>For testing --index-url with a file:// url for the index</h2>
   <a href='requiredinner-1.0.tar.gz'>requiredinner=1.0.tar.gz</a><br />
 </body>
diff --git a/tests/functional/test_download.py b/tests/functional/test_download.py
index 5b03324be..40f370127 100644
--- a/tests/functional/test_download.py
+++ b/tests/functional/test_download.py
@@ -57,7 +57,7 @@ def test_download_wheel(script, data):
 @pytest.mark.network
 def test_single_download_from_requirements_file(script):
     """
-    It should support download (in the scratch path) from PyPi from a
+    It should support download (in the scratch path) from PyPI from a
     requirements file
     """
     script.scratch_path.join("test-req.txt").write(textwrap.dedent("""
diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py
index fa586c0ae..1be8e10e3 100644
--- a/tests/functional/test_install.py
+++ b/tests/functional/test_install.py
@@ -629,7 +629,7 @@ def test_install_folder_using_relative_path(script):
 @pytest.mark.network
 def test_install_package_which_contains_dev_in_name(script):
     """
-    Test installing package from pypi which contains 'dev' in name
+    Test installing package from PyPI which contains 'dev' in name
     """
     result = script.pip('install', 'django-devserver==0.0.4')
     devserver_folder = script.site_packages / 'devserver'
diff --git a/tests/functional/test_search.py b/tests/functional/test_search.py
index e65e72fb0..46e6a6647 100644
--- a/tests/functional/test_search.py
+++ b/tests/functional/test_search.py
@@ -25,7 +25,7 @@ def test_version_compare():
 
 def test_pypi_xml_transformation():
     """
-    Test transformation of data structures (pypi xmlrpc to custom list).
+    Test transformation of data structures (PyPI xmlrpc to custom list).
 
     """
     pypi_hits = [
diff --git a/tests/unit/test_finder.py b/tests/unit/test_finder.py
index e719b21f1..05ebed3b5 100644
--- a/tests/unit/test_finder.py
+++ b/tests/unit/test_finder.py
@@ -95,7 +95,7 @@ def test_finder_detects_latest_already_satisfied_find_links(data):
 def test_finder_detects_latest_already_satisfied_pypi_links():
     """Test PackageFinder detects latest already satisfied using pypi links"""
     req = install_req_from_line('initools', None)
-    # the latest initools on pypi is 0.3.1
+    # the latest initools on PyPI is 0.3.1
     latest_version = "0.3.1"
     satisfied_by = Mock(
         location="/path",

From 10911fe88194cb512d56185e34a0466c64e88616 Mon Sep 17 00:00:00 2001
From: Laurie Opperman <laurie@sitesee.com.au>
Date: Mon, 17 Sep 2018 21:56:46 +1000
Subject: [PATCH 11/92] Fix error when autocompleting after flag

When running autocomplete after any command-line option with unspecified
'metavar' attribute, a traceback for an 'AttributeError' would be
displayed. This is fixed for first checking if 'metavar' was defined.
Fixes #5751
---
 src/pip/_internal/cli/autocompletion.py |  3 ++-
 tests/functional/test_completion.py     | 24 ++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/pip/_internal/cli/autocompletion.py b/src/pip/_internal/cli/autocompletion.py
index de97463fa..0a04199e6 100644
--- a/src/pip/_internal/cli/autocompletion.py
+++ b/src/pip/_internal/cli/autocompletion.py
@@ -116,7 +116,8 @@ def get_path_completion_type(cwords, cword, opts):
             continue
         for o in str(opt).split('/'):
             if cwords[cword - 2].split('=')[0] == o:
-                if any(x in ('path', 'file', 'dir')
+                if not opt.metavar or any(
+                        x in ('path', 'file', 'dir')
                         for x in opt.metavar.split('/')):
                     return opt.metavar
 
diff --git a/tests/functional/test_completion.py b/tests/functional/test_completion.py
index bd7a5afc4..586a0a89b 100644
--- a/tests/functional/test_completion.py
+++ b/tests/functional/test_completion.py
@@ -204,6 +204,30 @@ def test_completion_not_files_after_option(script, data):
     )
 
 
+@pytest.mark.parametrize("cl_opts", ["-U", "--user", "-h"])
+def test_completion_not_files_after_nonexpecting_option(script, data, cl_opts):
+    """
+    Test not getting completion files after options which not applicable
+    (e.g. ``pip install``)
+    """
+    res, env = setup_completion(
+        script=script,
+        words=('pip install %s r' % cl_opts),
+        cword='2',
+        cwd=data.completion_paths,
+    )
+    assert not any(out in res.stdout for out in
+                   ('requirements.txt', 'readme.txt',)), (
+        "autocomplete function completed <file> when "
+        "it should not complete"
+    )
+    assert not any(os.path.join(out, '') in res.stdout
+                   for out in ('replay', 'resources')), (
+        "autocomplete function completed <dir> when "
+        "it should not complete"
+    )
+
+
 def test_completion_directories_after_option(script, data):
     """
     Test getting completion <dir> after options in command

From b441c15956f9f6f612f0d6675b1e097db6a76e32 Mon Sep 17 00:00:00 2001
From: Laurie Opperman <laurie@sitesee.com.au>
Date: Mon, 17 Sep 2018 22:10:04 +1000
Subject: [PATCH 12/92] Add news fragment

---
 news/5751.bugfix | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 news/5751.bugfix

diff --git a/news/5751.bugfix b/news/5751.bugfix
new file mode 100644
index 000000000..36926c9a6
--- /dev/null
+++ b/news/5751.bugfix
@@ -0,0 +1 @@
+Avoid traceback printing on autocomplete after flags in the CLI.

From 9a2f1aa8a34db826c9b3b61a456e619ba105ab19 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Tue, 18 Sep 2018 12:59:49 +0530
Subject: [PATCH 13/92] Pin to the older mypy version

---
 tools/mypy-requirements.txt | 1 +
 tox.ini                     | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 tools/mypy-requirements.txt

diff --git a/tools/mypy-requirements.txt b/tools/mypy-requirements.txt
new file mode 100644
index 000000000..47f2fcd86
--- /dev/null
+++ b/tools/mypy-requirements.txt
@@ -0,0 +1 @@
+mypy == 0.620
diff --git a/tox.ini b/tox.ini
index 082af73ef..4a3272955 100644
--- a/tox.ini
+++ b/tox.ini
@@ -60,7 +60,7 @@ commands = {[lint]commands}
 [testenv:mypy]
 skip_install = True
 basepython = python3
-deps = mypy
+deps = -r{toxinidir}/tools/mypy-requirements.txt
 commands =
     mypy src
     mypy src -2

From dcc3c16b5461b45c1c4e26a3f8342560260af204 Mon Sep 17 00:00:00 2001
From: Donald Stufft <donald@stufft.io>
Date: Mon, 17 Sep 2018 19:57:56 -0400
Subject: [PATCH 14/92] Only revalidate /simple/ pages instead of caching for
 10 minutes

---
 news/5670.bugfix           |  2 ++
 src/pip/_internal/index.py | 15 ++++++++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)
 create mode 100644 news/5670.bugfix

diff --git a/news/5670.bugfix b/news/5670.bugfix
new file mode 100644
index 000000000..9ffac6638
--- /dev/null
+++ b/news/5670.bugfix
@@ -0,0 +1,2 @@
+Always revalidate cached simple API pages instead of blindly caching them for up to 10
+minutes.
diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index ea8363e78..fd6609734 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -783,7 +783,20 @@ class HTMLPage(object):
                 url,
                 headers={
                     "Accept": "text/html",
-                    "Cache-Control": "max-age=600",
+                    # We don't want to blindly returned cached data for
+                    # /simple/, because authors generally expecting that
+                    # twine upload && pip install will function, but if
+                    # they've done a pip install in the last ~10 minutes
+                    # it won't. Thus by setting this to zero we will not
+                    # blindly use any cached data, however the benefit of
+                    # using max-age=0 instead of no-cache, is that we will
+                    # still support conditional requests, so we will still
+                    # minimize traffic sent in cases where the page hasn't
+                    # changed at all, we will just always incur the round
+                    # trip for the conditional GET now instead of only
+                    # once per 10 minutes.
+                    # For more information, please see pypa/pip#5670.
+                    "Cache-Control": "max-age=0",
                 },
             )
             resp.raise_for_status()

From c14c28f7583fcb7b3797f0cfc9bd932a19cb1851 Mon Sep 17 00:00:00 2001
From: Hugo <hugovk@users.noreply.github.com>
Date: Tue, 18 Sep 2018 21:20:41 +0300
Subject: [PATCH 15/92] Upgrade pyparsing to 2.2.1

From https://github.com/pyparsing/pyparsing/blob/pyparsing_2.2.1/pyparsing.py
---
 src/pip/_vendor/pyparsing.py | 50 ++++++++++++++++++++++++++----------
 1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/src/pip/_vendor/pyparsing.py b/src/pip/_vendor/pyparsing.py
index ba2619c23..4aa30ee6b 100644
--- a/src/pip/_vendor/pyparsing.py
+++ b/src/pip/_vendor/pyparsing.py
@@ -1,6 +1,6 @@
 # module pyparsing.py
 #
-# Copyright (c) 2003-2016  Paul T. McGuire
+# Copyright (c) 2003-2018  Paul T. McGuire
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -25,6 +25,7 @@
 __doc__ = \
 """
 pyparsing module - Classes and methods to define and execute parsing grammars
+=============================================================================
 
 The pyparsing module is an alternative approach to creating and executing simple grammars,
 vs. the traditional lex/yacc approach, or the use of regular expressions.  With pyparsing, you
@@ -36,7 +37,7 @@ C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And}
 (L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to
 L{Literal} expressions)::
 
-    from pip._vendor.pyparsing import Word, alphas
+    from pyparsing import Word, alphas
 
     # define grammar of a greeting
     greet = Word(alphas) + "," + Word(alphas) + "!"
@@ -58,10 +59,23 @@ The pyparsing module handles some of the problems that are typically vexing when
  - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello  ,  World  !", etc.)
  - quoted strings
  - embedded comments
+
+
+Getting Started -
+-----------------
+Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing
+classes inherit from. Use the docstrings for examples of how to:
+ - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes
+ - construct character word-group expressions using the L{Word} class
+ - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes
+ - use L{'+'<And>}, L{'|'<MatchFirst>}, L{'^'<Or>}, and L{'&'<Each>} operators to combine simple expressions into more complex ones
+ - associate names with your parsed results using L{ParserElement.setResultsName}
+ - find some helpful expression short-cuts like L{delimitedList} and L{oneOf}
+ - find more useful common expressions in the L{pyparsing_common} namespace class
 """
 
-__version__ = "2.2.0"
-__versionTime__ = "06 Mar 2017 02:06 UTC"
+__version__ = "2.2.1"
+__versionTime__ = "18 Sep 2018 00:49 UTC"
 __author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
 
 import string
@@ -82,6 +96,15 @@ try:
 except ImportError:
     from threading import RLock
 
+try:
+    # Python 3
+    from collections.abc import Iterable
+    from collections.abc import MutableMapping
+except ImportError:
+    # Python 2.7
+    from collections import Iterable
+    from collections import MutableMapping
+
 try:
     from collections import OrderedDict as _OrderedDict
 except ImportError:
@@ -940,7 +963,7 @@ class ParseResults(object):
     def __dir__(self):
         return (dir(type(self)) + list(self.keys()))
 
-collections.MutableMapping.register(ParseResults)
+MutableMapping.register(ParseResults)
 
 def col (loc,strg):
     """Returns current column within a string, counting newlines as line separators.
@@ -1025,11 +1048,11 @@ def _trim_arity(func, maxargs=2):
             # special handling for Python 3.5.0 - extra deep call stack by 1
             offset = -3 if system_version == (3,5,0) else -2
             frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset]
-            return [(frame_summary.filename, frame_summary.lineno)]
+            return [frame_summary[:2]]
         def extract_tb(tb, limit=0):
             frames = traceback.extract_tb(tb, limit=limit)
             frame_summary = frames[-1]
-            return [(frame_summary.filename, frame_summary.lineno)]
+            return [frame_summary[:2]]
     else:
         extract_stack = traceback.extract_stack
         extract_tb = traceback.extract_tb
@@ -1374,7 +1397,7 @@ class ParserElement(object):
             else:
                 preloc = loc
             tokensStart = preloc
-            if self.mayIndexError or loc >= len(instring):
+            if self.mayIndexError or preloc >= len(instring):
                 try:
                     loc,tokens = self.parseImpl( instring, preloc, doActions )
                 except IndexError:
@@ -1408,7 +1431,6 @@ class ParserElement(object):
                                                   self.resultsName,
                                                   asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                   modal=self.modalResults )
-
         if debugging:
             #~ print ("Matched",self,"->",retTokens.asList())
             if (self.debugActions[1] ):
@@ -1572,7 +1594,7 @@ class ParserElement(object):
            after importing pyparsing.
            
            Example::
-               from pip._vendor import pyparsing
+               import pyparsing
                pyparsing.ParserElement.enablePackrat()
         """
         if not ParserElement._packratEnabled:
@@ -3242,7 +3264,7 @@ class ParseExpression(ParserElement):
 
         if isinstance( exprs, basestring ):
             self.exprs = [ ParserElement._literalStringClass( exprs ) ]
-        elif isinstance( exprs, collections.Iterable ):
+        elif isinstance( exprs, Iterable ):
             exprs = list(exprs)
             # if sequence of strings provided, wrap with Literal
             if all(isinstance(expr, basestring) for expr in exprs):
@@ -4393,7 +4415,7 @@ def traceParseAction(f):
 
         @traceParseAction
         def remove_duplicate_chars(tokens):
-            return ''.join(sorted(set(''.join(tokens)))
+            return ''.join(sorted(set(''.join(tokens))))
 
         wds = OneOrMore(wd).setParseAction(remove_duplicate_chars)
         print(wds.parseString("slkdjs sld sldd sdlf sdljf"))
@@ -4583,7 +4605,7 @@ def oneOf( strs, caseless=False, useRegex=True ):
     symbols = []
     if isinstance(strs,basestring):
         symbols = strs.split()
-    elif isinstance(strs, collections.Iterable):
+    elif isinstance(strs, Iterable):
         symbols = list(strs)
     else:
         warnings.warn("Invalid argument to oneOf, expected string or iterable",
@@ -4734,7 +4756,7 @@ stringEnd   = StringEnd().setName("stringEnd")
 _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
 _escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
 _escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
-_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1) | Regex(r"\w", re.UNICODE)
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1)
 _charRange = Group(_singleChar + Suppress("-") + _singleChar)
 _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
 

From 9a97ac7147db4831820791ecbedd8603641e880f Mon Sep 17 00:00:00 2001
From: hugovk <hugovk@users.noreply.github.com>
Date: Tue, 18 Sep 2018 21:33:30 +0300
Subject: [PATCH 16/92] Add news file for pyparsing upgrade

---
 news/5013.feature | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 news/5013.feature

diff --git a/news/5013.feature b/news/5013.feature
new file mode 100644
index 000000000..2f3771fc1
--- /dev/null
+++ b/news/5013.feature
@@ -0,0 +1,2 @@
+Upgrade pyparsing to 2.2.1.
+

From d5b363be5bc773f6b7e2bf4b343d14061f018f0e Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Wed, 19 Sep 2018 09:19:08 +0530
Subject: [PATCH 17/92] Revert "Limit progress updates to avoid swamping the
 TTY"

---
 news/5124.trivial             |  1 -
 src/pip/_internal/utils/ui.py | 10 ----------
 2 files changed, 11 deletions(-)
 delete mode 100644 news/5124.trivial

diff --git a/news/5124.trivial b/news/5124.trivial
deleted file mode 100644
index bc0a19b87..000000000
--- a/news/5124.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Limit progress bar update interval to 200 ms.
diff --git a/src/pip/_internal/utils/ui.py b/src/pip/_internal/utils/ui.py
index 4a337241d..6bab904ab 100644
--- a/src/pip/_internal/utils/ui.py
+++ b/src/pip/_internal/utils/ui.py
@@ -137,7 +137,6 @@ class DownloadProgressMixin(object):
     def __init__(self, *args, **kwargs):
         super(DownloadProgressMixin, self).__init__(*args, **kwargs)
         self.message = (" " * (get_indentation() + 2)) + self.message
-        self.last_update = 0.0
 
     @property
     def downloaded(self):
@@ -162,15 +161,6 @@ class DownloadProgressMixin(object):
             self.next(n)
         self.finish()
 
-    def update(self):
-        # limit updates to avoid swamping the TTY
-        now = time.time()
-        if now < self.last_update + 0.2:
-            return
-        self.last_update = now
-
-        super(DownloadProgressMixin, self).update()
-
 
 class WindowsMixin(object):
 

From d90532f8dd29ddd5ea74b27603a93e069ec2de80 Mon Sep 17 00:00:00 2001
From: hugovk <hugovk@users.noreply.github.com>
Date: Wed, 19 Sep 2018 09:15:47 +0300
Subject: [PATCH 18/92] Upgrade pyparsing to 2.2.1

---
 src/pip/_vendor/vendor.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
index 48cbf8c6d..de239b6c8 100644
--- a/src/pip/_vendor/vendor.txt
+++ b/src/pip/_vendor/vendor.txt
@@ -11,7 +11,7 @@ progress==1.4
 ipaddress==1.0.22  # Only needed on 2.6 and 2.7
 packaging==17.1
 pep517==0.2
-pyparsing==2.2.0
+pyparsing==2.2.1
 pytoml==0.1.16
 retrying==1.3.3
 requests==2.19.1

From e938d439a21821dcd4ec70713c33bcc155eadcad Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Wed, 19 Sep 2018 20:00:38 +0530
Subject: [PATCH 19/92] Switch to a global variable for exit status codes

---
 src/pip/_internal/cli/parser.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/pip/_internal/cli/parser.py b/src/pip/_internal/cli/parser.py
index 269190b59..e1eaac420 100644
--- a/src/pip/_internal/cli/parser.py
+++ b/src/pip/_internal/cli/parser.py
@@ -9,6 +9,7 @@ from distutils.util import strtobool
 
 from pip._vendor.six import string_types
 
+from pip._internal.cli.status_codes import UNKNOWN_ERROR
 from pip._internal.configuration import Configuration, ConfigurationError
 from pip._internal.utils.compat import get_terminal_size
 
@@ -232,7 +233,7 @@ class ConfigOptionParser(CustomOptionParser):
         try:
             self.config.load()
         except ConfigurationError as err:
-            self.exit(2, err.args[0])
+            self.exit(UNKNOWN_ERROR, str(err))
 
         defaults = self._update_defaults(self.defaults.copy())  # ours
         for option in self._get_all_options():
@@ -244,7 +245,7 @@ class ConfigOptionParser(CustomOptionParser):
 
     def error(self, msg):
         self.print_usage(sys.stderr)
-        self.exit(2, "%s\n" % msg)
+        self.exit(UNKNOWN_ERROR, "%s\n" % msg)
 
 
 def invalid_config_error_message(action, key, val):

From 6a18c24803ef7c24b79d70fc2f4713f69a2fd091 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Wed, 19 Sep 2018 19:51:18 +0530
Subject: [PATCH 20/92] Show error messages when configuration-related errors
 occur

---
 news/5798.feature                  |  1 +
 src/pip/_internal/configuration.py | 19 +++++++++++++------
 src/pip/_internal/exceptions.py    | 19 +++++++++++++++++++
 tests/unit/test_configuration.py   | 14 ++++++++++++++
 4 files changed, 47 insertions(+), 6 deletions(-)
 create mode 100644 news/5798.feature

diff --git a/news/5798.feature b/news/5798.feature
new file mode 100644
index 000000000..10d598594
--- /dev/null
+++ b/news/5798.feature
@@ -0,0 +1 @@
+Malformed configuration files now show helpful error messages, instead of tracebacks.
diff --git a/src/pip/_internal/configuration.py b/src/pip/_internal/configuration.py
index 32133de01..fe6df9b75 100644
--- a/src/pip/_internal/configuration.py
+++ b/src/pip/_internal/configuration.py
@@ -18,7 +18,9 @@ import os
 from pip._vendor import six
 from pip._vendor.six.moves import configparser
 
-from pip._internal.exceptions import ConfigurationError
+from pip._internal.exceptions import (
+    ConfigurationError, ConfigurationFileCouldNotBeLoaded,
+)
 from pip._internal.locations import (
     legacy_config_file, new_config_file, running_under_virtualenv,
     site_config_files, venv_config_file,
@@ -289,11 +291,16 @@ class Configuration(object):
             try:
                 parser.read(fname)
             except UnicodeDecodeError:
-                raise ConfigurationError((
-                    "ERROR: "
-                    "Configuration file contains invalid %s characters.\n"
-                    "Please fix your configuration, located at %s\n"
-                ) % (locale.getpreferredencoding(False), fname))
+                # See https://github.com/pypa/pip/issues/4963
+                raise ConfigurationFileCouldNotBeLoaded(
+                    reason="contains invalid {} characters".format(
+                        locale.getpreferredencoding(False)
+                    ),
+                    fname=fname,
+                )
+            except configparser.Error as error:
+                # See https://github.com/pypa/pip/issues/4893
+                raise ConfigurationFileCouldNotBeLoaded(error=error)
         return parser
 
     def _load_environment_vars(self):
diff --git a/src/pip/_internal/exceptions.py b/src/pip/_internal/exceptions.py
index ad6f41253..f1ca6f36d 100644
--- a/src/pip/_internal/exceptions.py
+++ b/src/pip/_internal/exceptions.py
@@ -247,3 +247,22 @@ class HashMismatch(HashError):
 class UnsupportedPythonVersion(InstallationError):
     """Unsupported python version according to Requires-Python package
     metadata."""
+
+
+class ConfigurationFileCouldNotBeLoaded(ConfigurationError):
+    """When there are errors while loading a configuration file
+    """
+
+    def __init__(self, reason="could not be loaded", fname=None, error=None):
+        super(ConfigurationFileCouldNotBeLoaded, self).__init__(error)
+        self.reason = reason
+        self.fname = fname
+        self.error = error
+
+    def __str__(self):
+        if self.fname is not None:
+            message_part = " in {}.".format(self.fname)
+        else:
+            assert self.error is not None
+            message_part = ".\n{}\n".format(self.error.message)
+        return "Configuration file {}{}".format(self.reason, message_part)
diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py
index ef1ddf96a..a5a06e718 100644
--- a/tests/unit/test_configuration.py
+++ b/tests/unit/test_configuration.py
@@ -68,6 +68,20 @@ class TestConfigurationLoading(ConfigurationMixin):
         with pytest.raises(ConfigurationError):
             self.configuration.get_value(":env:.version")
 
+    def test_environment_config_errors_if_malformed(self):
+        contents = """
+            test]
+            hello = 4
+        """
+        with self.tmpfile(contents) as config_file:
+            os.environ["PIP_CONFIG_FILE"] = config_file
+            with pytest.raises(ConfigurationError) as err:
+                self.configuration.load()
+
+        assert "section header" in str(err.value)
+        assert "1" in str(err.value)
+        assert config_file in str(err.value)
+
 
 class TestConfigurationPrecedence(ConfigurationMixin):
     # Tests for methods to that determine the order of precedence of

From 59b8a80c9be8a71fa2e09f1cdc8c861f82bb28d6 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Thu, 20 Sep 2018 00:09:02 -0700
Subject: [PATCH 21/92] Remove two unreachable returns in
 VersionControl.get_src_requirement().

---
 news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial | 0
 src/pip/_internal/vcs/git.py                      | 2 --
 src/pip/_internal/vcs/mercurial.py                | 2 --
 3 files changed, 4 deletions(-)
 create mode 100644 news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial

diff --git a/news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial b/news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index 7169dc700..d8fd2df19 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -295,8 +295,6 @@ class Git(VersionControl):
         if not repo.lower().startswith('git:'):
             repo = 'git+' + repo
         egg_project_name = dist.egg_name().split('-', 1)[0]
-        if not repo:
-            return None
         current_rev = self.get_revision(location)
         req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name)
         subdirectory = self._get_subdirectory(location)
diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py
index 86d71ef2d..49039a912 100644
--- a/src/pip/_internal/vcs/mercurial.py
+++ b/src/pip/_internal/vcs/mercurial.py
@@ -89,8 +89,6 @@ class Mercurial(VersionControl):
         if not repo.lower().startswith('hg:'):
             repo = 'hg+' + repo
         egg_project_name = dist.egg_name().split('-', 1)[0]
-        if not repo:
-            return None
         current_rev_hash = self.get_revision_hash(location)
         return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name)
 

From 3065bea8f995f6678cec7fd1777197fcf5edf1fd Mon Sep 17 00:00:00 2001
From: Donald Stufft <donald@stufft.io>
Date: Mon, 24 Sep 2018 16:37:33 -0400
Subject: [PATCH 22/92] get-pip.py is mostly a giant base85'd blob and can't be
 reviewed

---
 docs/html/installing.rst | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/html/installing.rst b/docs/html/installing.rst
index 99eef7360..35ba05db9 100644
--- a/docs/html/installing.rst
+++ b/docs/html/installing.rst
@@ -23,8 +23,6 @@ To install pip, securely download `get-pip.py
 
  curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
 
-As when running any script downloaded from the web, ensure that you have
-reviewed the code and are happy that it works as you expect.
 Then run the following::
 
  python get-pip.py

From 520df5356ffd3c9a9b0da0f624a070699b1c7839 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Mon, 24 Sep 2018 14:53:39 -0700
Subject: [PATCH 23/92] Add misc.make_vcs_requirement_url().

---
 ...6F9BDF-53AF-4885-A966-6474A27A6D46.trivial |  0
 src/pip/_internal/operations/freeze.py        |  9 +++----
 src/pip/_internal/utils/misc.py               | 14 ++++++++++
 src/pip/_internal/vcs/bazaar.py               |  8 +++---
 src/pip/_internal/vcs/git.py                  | 12 ++++-----
 src/pip/_internal/vcs/mercurial.py            |  7 ++---
 src/pip/_internal/vcs/subversion.py           |  7 ++---
 tests/unit/test_utils.py                      | 26 +++++++++++++++++--
 8 files changed, 60 insertions(+), 23 deletions(-)
 create mode 100644 news/516F9BDF-53AF-4885-A966-6474A27A6D46.trivial

diff --git a/news/516F9BDF-53AF-4885-A966-6474A27A6D46.trivial b/news/516F9BDF-53AF-4885-A966-6474A27A6D46.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 1ceb7fedb..aeeb86910 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -16,7 +16,7 @@ from pip._internal.req.constructors import (
 from pip._internal.req.req_file import COMMENT_RE
 from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.misc import (
-    dist_is_editable, get_installed_distributions,
+    dist_is_editable, get_installed_distributions, make_vcs_requirement_url,
 )
 
 logger = logging.getLogger(__name__)
@@ -233,11 +233,8 @@ class FrozenRequirement(object):
                     else:
                         rev = '{%s}' % date_match.group(1)
                     editable = True
-                    req = '%s@%s#egg=%s' % (
-                        svn_location,
-                        rev,
-                        cls.egg_name(dist)
-                    )
+                    egg_name = cls.egg_name(dist)
+                    req = make_vcs_requirement_url(svn_location, rev, egg_name)
         return cls(dist.project_name, req, editable, comments)
 
     @staticmethod
diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py
index f7afd1df5..84a421fe4 100644
--- a/src/pip/_internal/utils/misc.py
+++ b/src/pip/_internal/utils/misc.py
@@ -856,6 +856,20 @@ def enum(*sequential, **named):
     return type('Enum', (), enums)
 
 
+def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None):
+    """
+    Return the URL for a VCS requirement.
+
+    Args:
+      repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+").
+    """
+    req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name)
+    if subdir:
+        req += '&subdirectory={}'.format(subdir)
+
+    return req
+
+
 def split_auth_from_netloc(netloc):
     """
     Parse out and remove the auth information from a netloc.
diff --git a/src/pip/_internal/vcs/bazaar.py b/src/pip/_internal/vcs/bazaar.py
index d5c6efaf5..3cc66c9dc 100644
--- a/src/pip/_internal/vcs/bazaar.py
+++ b/src/pip/_internal/vcs/bazaar.py
@@ -6,7 +6,9 @@ import os
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 
 from pip._internal.download import path_to_url
-from pip._internal.utils.misc import display_path, rmtree
+from pip._internal.utils.misc import (
+    display_path, make_vcs_requirement_url, rmtree,
+)
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.vcs import VersionControl, vcs
 
@@ -98,9 +100,9 @@ class Bazaar(VersionControl):
             return None
         if not repo.lower().startswith('bzr:'):
             repo = 'bzr+' + repo
-        egg_project_name = dist.egg_name().split('-', 1)[0]
         current_rev = self.get_revision(location)
-        return '%s@%s#egg=%s' % (repo, current_rev, egg_project_name)
+        egg_project_name = dist.egg_name().split('-', 1)[0]
+        return make_vcs_requirement_url(repo, current_rev, egg_project_name)
 
     def is_commit_id_equal(self, dest, name):
         """Always assume the versions don't match"""
diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py
index d8fd2df19..977853946 100644
--- a/src/pip/_internal/vcs/git.py
+++ b/src/pip/_internal/vcs/git.py
@@ -10,7 +10,7 @@ from pip._vendor.six.moves.urllib import request as urllib_request
 
 from pip._internal.exceptions import BadCommand
 from pip._internal.utils.compat import samefile
-from pip._internal.utils.misc import display_path
+from pip._internal.utils.misc import display_path, make_vcs_requirement_url
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.vcs import VersionControl, vcs
 
@@ -294,12 +294,12 @@ class Git(VersionControl):
         repo = self.get_url(location)
         if not repo.lower().startswith('git:'):
             repo = 'git+' + repo
-        egg_project_name = dist.egg_name().split('-', 1)[0]
         current_rev = self.get_revision(location)
-        req = '%s@%s#egg=%s' % (repo, current_rev, egg_project_name)
-        subdirectory = self._get_subdirectory(location)
-        if subdirectory:
-            req += '&subdirectory=' + subdirectory
+        egg_project_name = dist.egg_name().split('-', 1)[0]
+        subdir = self._get_subdirectory(location)
+        req = make_vcs_requirement_url(repo, current_rev, egg_project_name,
+                                       subdir=subdir)
+
         return req
 
     def get_url_rev_and_auth(self, url):
diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py
index 49039a912..17cfb67d1 100644
--- a/src/pip/_internal/vcs/mercurial.py
+++ b/src/pip/_internal/vcs/mercurial.py
@@ -6,7 +6,7 @@ import os
 from pip._vendor.six.moves import configparser
 
 from pip._internal.download import path_to_url
-from pip._internal.utils.misc import display_path
+from pip._internal.utils.misc import display_path, make_vcs_requirement_url
 from pip._internal.utils.temp_dir import TempDirectory
 from pip._internal.vcs import VersionControl, vcs
 
@@ -88,9 +88,10 @@ class Mercurial(VersionControl):
         repo = self.get_url(location)
         if not repo.lower().startswith('hg:'):
             repo = 'hg+' + repo
-        egg_project_name = dist.egg_name().split('-', 1)[0]
         current_rev_hash = self.get_revision_hash(location)
-        return '%s@%s#egg=%s' % (repo, current_rev_hash, egg_project_name)
+        egg_project_name = dist.egg_name().split('-', 1)[0]
+        return make_vcs_requirement_url(repo, current_rev_hash,
+                                        egg_project_name)
 
     def is_commit_id_equal(self, dest, name):
         """Always assume the versions don't match"""
diff --git a/src/pip/_internal/vcs/subversion.py b/src/pip/_internal/vcs/subversion.py
index 19e2e70dc..6f7cb5d94 100644
--- a/src/pip/_internal/vcs/subversion.py
+++ b/src/pip/_internal/vcs/subversion.py
@@ -7,7 +7,7 @@ import re
 from pip._internal.models.link import Link
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import (
-    display_path, rmtree, split_auth_from_netloc,
+    display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc,
 )
 from pip._internal.vcs import VersionControl, vcs
 
@@ -199,10 +199,11 @@ class Subversion(VersionControl):
         repo = self.get_url(location)
         if repo is None:
             return None
+        repo = 'svn+' + repo
+        rev = self.get_revision(location)
         # FIXME: why not project name?
         egg_project_name = dist.egg_name().split('-', 1)[0]
-        rev = self.get_revision(location)
-        return 'svn+%s@%s#egg=%s' % (repo, rev, egg_project_name)
+        return make_vcs_requirement_url(repo, rev, egg_project_name)
 
     def is_commit_id_equal(self, dest, name):
         """Always assume the versions don't match"""
diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index 947d25892..972de2e0e 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -24,8 +24,8 @@ from pip._internal.utils.glibc import check_glibc_version
 from pip._internal.utils.hashes import Hashes, MissingHashes
 from pip._internal.utils.misc import (
     call_subprocess, egg_link_path, ensure_dir, get_installed_distributions,
-    get_prog, normalize_path, remove_auth_from_url, rmtree,
-    split_auth_from_netloc, untar_file, unzip_file,
+    get_prog, make_vcs_requirement_url, normalize_path, remove_auth_from_url,
+    rmtree, split_auth_from_netloc, untar_file, unzip_file,
 )
 from pip._internal.utils.packaging import check_dist_requires_python
 from pip._internal.utils.temp_dir import TempDirectory
@@ -627,6 +627,28 @@ def test_call_subprocess_closes_stdin():
         call_subprocess([sys.executable, '-c', 'input()'])
 
 
+@pytest.mark.parametrize('args, expected', [
+    # Test without subdir.
+    (('git+https://example.com/pkg', 'dev', 'myproj'),
+     'git+https://example.com/pkg@dev#egg=myproj'),
+    # Test with subdir.
+    (('git+https://example.com/pkg', 'dev', 'myproj', 'sub/dir'),
+     'git+https://example.com/pkg@dev#egg=myproj&subdirectory=sub/dir'),
+    # Test with None subdir.
+    (('git+https://example.com/pkg', 'dev', 'myproj', None),
+     'git+https://example.com/pkg@dev#egg=myproj'),
+])
+def test_make_vcs_requirement_url(args, expected):
+    if len(args) == 3:
+        url, rev, egg_name = args
+        actual = make_vcs_requirement_url(url, rev, egg_name)
+    else:
+        url, rev, egg_name, subdir = args
+        actual = make_vcs_requirement_url(url, rev, egg_name, subdir=subdir)
+
+    assert actual == expected
+
+
 @pytest.mark.parametrize('netloc, expected', [
     # Test a basic case.
     ('example.com', ('example.com', (None, None))),

From 4afc1e3f69c81a149353e422bd39ae1a08641136 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Tue, 25 Sep 2018 02:23:32 -0700
Subject: [PATCH 24/92] Use *args in the test.

This incorporates a review suggestion of @pradyunsg.
---
 tests/unit/test_utils.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py
index 972de2e0e..591c8ab54 100644
--- a/tests/unit/test_utils.py
+++ b/tests/unit/test_utils.py
@@ -639,13 +639,7 @@ def test_call_subprocess_closes_stdin():
      'git+https://example.com/pkg@dev#egg=myproj'),
 ])
 def test_make_vcs_requirement_url(args, expected):
-    if len(args) == 3:
-        url, rev, egg_name = args
-        actual = make_vcs_requirement_url(url, rev, egg_name)
-    else:
-        url, rev, egg_name, subdir = args
-        actual = make_vcs_requirement_url(url, rev, egg_name, subdir=subdir)
-
+    actual = make_vcs_requirement_url(*args)
     assert actual == expected
 
 

From 309cb370916fd7da11e09f793f3d0b6dd2043ecc Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Wed, 26 Sep 2018 22:11:27 -0700
Subject: [PATCH 25/92] Add FrozenRequirement._init_args_from_dist() helper
 method.

---
 .../88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial |  0
 src/pip/_internal/operations/freeze.py           | 16 ++++++++++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)
 create mode 100644 news/88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial

diff --git a/news/88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial b/news/88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index aeeb86910..beb2feb87 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -168,7 +168,13 @@ class FrozenRequirement(object):
     _date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
 
     @classmethod
-    def from_dist(cls, dist, dependency_links):
+    def _init_args_from_dist(cls, dist, dependency_links):
+        """
+        Compute and return arguments (req, editable, comments) to pass to
+        FrozenRequirement.__init__().
+
+        This method is for use in FrozenRequirement.from_dist().
+        """
         location = os.path.normcase(os.path.abspath(dist.location))
         comments = []
         from pip._internal.vcs import vcs, get_src_requirement
@@ -235,7 +241,13 @@ class FrozenRequirement(object):
                     editable = True
                     egg_name = cls.egg_name(dist)
                     req = make_vcs_requirement_url(svn_location, rev, egg_name)
-        return cls(dist.project_name, req, editable, comments)
+
+        return (req, editable, comments)
+
+    @classmethod
+    def from_dist(cls, dist, dependency_links):
+        args = cls._init_args_from_dist(dist, dependency_links)
+        return cls(dist.project_name, *args)
 
     @staticmethod
     def egg_name(dist):

From 894f6558f16ee67a6f171387806a97107fbbca54 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 27 Sep 2018 15:41:02 +0800
Subject: [PATCH 26/92] Remove skip_archives argument

---
 src/pip/_internal/index.py | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index fd6609734..c0091aced 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -731,7 +731,7 @@ class HTMLPage(object):
         return self.url
 
     @classmethod
-    def get_page(cls, link, skip_archives=True, session=None):
+    def get_page(cls, link, session=None):
         if session is None:
             raise TypeError(
                 "get_page() missing 1 required keyword argument: 'session'"
@@ -748,22 +748,21 @@ class HTMLPage(object):
                 return None
 
         try:
-            if skip_archives:
-                filename = link.filename
-                for bad_ext in ARCHIVE_EXTENSIONS:
-                    if filename.endswith(bad_ext):
-                        content_type = cls._get_content_type(
-                            url, session=session,
+            filename = link.filename
+            for bad_ext in ARCHIVE_EXTENSIONS:
+                if filename.endswith(bad_ext):
+                    content_type = cls._get_content_type(
+                        url, session=session,
+                    )
+                    if content_type.lower().startswith('text/html'):
+                        break
+                    else:
+                        logger.debug(
+                            'Skipping page %s because of Content-Type: %s',
+                            link,
+                            content_type,
                         )
-                        if content_type.lower().startswith('text/html'):
-                            break
-                        else:
-                            logger.debug(
-                                'Skipping page %s because of Content-Type: %s',
-                                link,
-                                content_type,
-                            )
-                            return
+                        return
 
             logger.debug('Getting page %s', url)
 

From d71f3311dcb52f9608f0c2ff26a9f0e8a70e8e50 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 27 Sep 2018 15:41:07 +0800
Subject: [PATCH 27/92] Fix egg_info_matches version split logic

---
 src/pip/_internal/index.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index c0091aced..492e809ca 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -694,7 +694,7 @@ def egg_info_matches(
         return None
     if search_name is None:
         full_match = match.group(0)
-        return full_match[full_match.index('-'):]
+        return full_match.split('-', 1)[-1]
     name = match.group(0).lower()
     # To match the "safe" name that pkg_resources creates:
     name = name.replace('_', '-')

From 9d1c2dbe29f1685d4277c120028da3d86bda7add Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 27 Sep 2018 15:50:21 +0800
Subject: [PATCH 28/92] News

---
 news/5819.bugfix  | 1 +
 news/5819.trivial | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 news/5819.bugfix
 create mode 100644 news/5819.trivial

diff --git a/news/5819.bugfix b/news/5819.bugfix
new file mode 100644
index 000000000..9fea81279
--- /dev/null
+++ b/news/5819.bugfix
@@ -0,0 +1 @@
+Fix incorrect parsing of egg names if pip needs to guess the package name.
diff --git a/news/5819.trivial b/news/5819.trivial
new file mode 100644
index 000000000..9ffde7cb6
--- /dev/null
+++ b/news/5819.trivial
@@ -0,0 +1 @@
+Simplify always-true conditions in ``HTMLPage.get_page()``.

From 8c04e2ba7be573669cfb332f7454448010c69045 Mon Sep 17 00:00:00 2001
From: Benoit Pierre <benoit.pierre@gmail.com>
Date: Fri, 28 Sep 2018 00:29:12 +0200
Subject: [PATCH 29/92] travis: fix Python 3.8 jobs

---
 .travis.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 6ecfd9b64..c4a2eb9ed 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -57,8 +57,12 @@ jobs:
 
     - env: GROUP=1
       python: 3.8-dev
+      dist: xenial
+      sudo: required
     - env: GROUP=2
       python: 3.8-dev
+      dist: xenial
+      sudo: required
 
   # It's okay to fail on the in-development CPython version.
   fast_finish: true

From 96a3254a1f866a977972368fbc1bd642b3774c1a Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 14:49:40 +0800
Subject: [PATCH 30/92] Extract base URL parsing logic

---
 src/pip/_internal/index.py | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 492e809ca..c2d40e752 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -706,6 +706,30 @@ def egg_info_matches(
         return None
 
 
+def _parse_base_url(document, page_url):
+    """Get the base URL of this document.
+
+    This looks for a ``<base>`` tag in the HTML document. If present, its href
+    attribute denotes the base URL of anchor tags in the document. If there is
+    no such tag (or if it does not have a valid href attribute), the HTML
+    file's URL is used as the base URL.
+
+    :param document: An HTML document representation. The current
+        implementation expects the result of ``html5lib.parse()``.
+    :param page_url: The URL of the HTML document.
+    """
+    bases = [
+        x for x in document.findall(".//base")
+        if x.get("href") is not None
+    ]
+    if not bases:
+        return page_url
+    parsed_url = bases[0].get("href")
+    if parsed_url:
+        return parsed_url
+    return page_url
+
+
 class HTMLPage(object):
     """Represents one page, along with its URL"""
 
@@ -851,14 +875,7 @@ class HTMLPage(object):
 
     @cached_property
     def base_url(self):
-        bases = [
-            x for x in self.parsed.findall(".//base")
-            if x.get("href") is not None
-        ]
-        if bases and bases[0].get("href"):
-            return bases[0].get("href")
-        else:
-            return self.url
+        return _parse_base_url(self.parsed, self.url)
 
     @property
     def links(self):

From 40bf3688b4436bff9ce5243ca1559a84c4b92b18 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 14:56:34 +0800
Subject: [PATCH 31/92] Convert base_url property into local variable

This property is only used in HTMLPage.links, which is only called once
per instance in PackageFinder.find_all_candidates(). This would not
affect performance or behavior, but improves data locality.

The unit test of this property is and modified to test the underlying
function instead.
---
 src/pip/_internal/index.py |  9 +++------
 tests/unit/test_index.py   | 10 +++++++---
 2 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index c2d40e752..074f192b5 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -34,7 +34,7 @@ from pip._internal.utils.compat import ipaddress
 from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.logging import indent_log
 from pip._internal.utils.misc import (
-    ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, cached_property, normalize_path,
+    ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path,
     remove_auth_from_url,
 )
 from pip._internal.utils.packaging import check_requires_python
@@ -873,18 +873,15 @@ class HTMLPage(object):
 
         return resp.headers.get("Content-Type", "")
 
-    @cached_property
-    def base_url(self):
-        return _parse_base_url(self.parsed, self.url)
-
     @property
     def links(self):
         """Yields all links in the page"""
+        base_url = _parse_base_url(self.parsed, self.url)
         for anchor in self.parsed.findall(".//a"):
             if anchor.get("href"):
                 href = anchor.get("href")
                 url = self.clean_link(
-                    urllib_parse.urljoin(self.base_url, href)
+                    urllib_parse.urljoin(base_url, href)
                 )
                 pyrequire = anchor.get('data-requires-python')
                 pyrequire = unescape(pyrequire) if pyrequire else None
diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index 4ede29589..66fc0b5d0 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -1,9 +1,10 @@
 import os.path
 
 import pytest
+from pip._vendor import html5lib
 
 from pip._internal.download import PipSession
-from pip._internal.index import HTMLPage, Link, PackageFinder
+from pip._internal.index import Link, PackageFinder, _parse_base_url
 
 
 def test_sort_locations_file_expand_dir(data):
@@ -107,8 +108,11 @@ class TestLink(object):
         ),
     ],
 )
-def test_base_url(html, url, expected):
-    assert HTMLPage(html, url).base_url == expected
+def test_parse_base_url(html, url, expected):
+    document = html5lib.parse(
+        html, transport_encoding=None, namespaceHTMLElements=False,
+    )
+    assert _parse_base_url(document, url) == expected
 
 
 class MockLogger(object):

From 94d042892422f091e796b52ad03e582c2f93552e Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 15:00:58 +0800
Subject: [PATCH 32/92] Extract encoding detection into helper function

---
 src/pip/_internal/index.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 074f192b5..54a57abd7 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -730,22 +730,24 @@ def _parse_base_url(document, page_url):
     return page_url
 
 
+def _get_encoding_from_headers(headers):
+    """Determine if we have any encoding information in our headers.
+    """
+    if headers and "Content-Type" in headers:
+        content_type, params = cgi.parse_header(headers["Content-Type"])
+        if "charset" in params:
+            return params['charset']
+    return None
+
+
 class HTMLPage(object):
     """Represents one page, along with its URL"""
 
     def __init__(self, content, url, headers=None):
-        # Determine if we have any encoding information in our headers
-        encoding = None
-        if headers and "Content-Type" in headers:
-            content_type, params = cgi.parse_header(headers["Content-Type"])
-
-            if "charset" in params:
-                encoding = params['charset']
-
         self.content = content
         self.parsed = html5lib.parse(
             self.content,
-            transport_encoding=encoding,
+            transport_encoding=_get_encoding_from_headers(headers),
             namespaceHTMLElements=False,
         )
         self.url = url

From 28ef0c201437645d84c13ebc1b6943527c299a43 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 15:02:51 +0800
Subject: [PATCH 33/92] Convert HTMLPage.parsed into a local variable

This attribute is only used by HTMLPage.links, which is only used once
per instance in PackageFinder.find_all_candidates(), so this change does
not affect performance or behavior, but improves data locality.
---
 src/pip/_internal/index.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 54a57abd7..75766f5d2 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -745,11 +745,6 @@ class HTMLPage(object):
 
     def __init__(self, content, url, headers=None):
         self.content = content
-        self.parsed = html5lib.parse(
-            self.content,
-            transport_encoding=_get_encoding_from_headers(headers),
-            namespaceHTMLElements=False,
-        )
         self.url = url
         self.headers = headers
 
@@ -878,8 +873,13 @@ class HTMLPage(object):
     @property
     def links(self):
         """Yields all links in the page"""
-        base_url = _parse_base_url(self.parsed, self.url)
-        for anchor in self.parsed.findall(".//a"):
+        document = html5lib.parse(
+            self.content,
+            transport_encoding=_get_encoding_from_headers(self.headers),
+            namespaceHTMLElements=False,
+        )
+        base_url = _parse_base_url(document, self.url)
+        for anchor in document.findall(".//a"):
             if anchor.get("href"):
                 href = anchor.get("href")
                 url = self.clean_link(

From dc7e6b5950bea7421b014ce7302397f3e5135077 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 15:52:00 +0800
Subject: [PATCH 34/92] Refactor _parse_base_url to use early return

---
 src/pip/_internal/index.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 75766f5d2..573b8dda8 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -718,15 +718,10 @@ def _parse_base_url(document, page_url):
         implementation expects the result of ``html5lib.parse()``.
     :param page_url: The URL of the HTML document.
     """
-    bases = [
-        x for x in document.findall(".//base")
-        if x.get("href") is not None
-    ]
-    if not bases:
-        return page_url
-    parsed_url = bases[0].get("href")
-    if parsed_url:
-        return parsed_url
+    for base in document.findall(".//base"):
+        href = base.get("href")
+        if href is not None:
+            return href
     return page_url
 
 

From 69e1e182853d89c2eb620fb0efa9a0a5ae35c000 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 15:58:22 +0800
Subject: [PATCH 35/92] News fragment

---
 news/5826.trivial | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 news/5826.trivial

diff --git a/news/5826.trivial b/news/5826.trivial
new file mode 100644
index 000000000..6f4ec80d5
--- /dev/null
+++ b/news/5826.trivial
@@ -0,0 +1 @@
+Refactor HTMLPage to reduce attributes on it.

From aa83222bf002ddca78560ddfb734630e6a51d36b Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 16:48:08 +0800
Subject: [PATCH 36/92] Rename base URL detection function

---
 src/pip/_internal/index.py | 6 +++---
 tests/unit/test_index.py   | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 573b8dda8..2bf9c75a0 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -706,8 +706,8 @@ def egg_info_matches(
         return None
 
 
-def _parse_base_url(document, page_url):
-    """Get the base URL of this document.
+def _determine_base_url(document, page_url):
+    """Determine the HTML document's base URL.
 
     This looks for a ``<base>`` tag in the HTML document. If present, its href
     attribute denotes the base URL of anchor tags in the document. If there is
@@ -873,7 +873,7 @@ class HTMLPage(object):
             transport_encoding=_get_encoding_from_headers(self.headers),
             namespaceHTMLElements=False,
         )
-        base_url = _parse_base_url(document, self.url)
+        base_url = _determine_base_url(document, self.url)
         for anchor in document.findall(".//a"):
             if anchor.get("href"):
                 href = anchor.get("href")
diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index 66fc0b5d0..d67aa4d0d 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -4,7 +4,7 @@ import pytest
 from pip._vendor import html5lib
 
 from pip._internal.download import PipSession
-from pip._internal.index import Link, PackageFinder, _parse_base_url
+from pip._internal.index import Link, PackageFinder, _determine_base_url
 
 
 def test_sort_locations_file_expand_dir(data):
@@ -108,11 +108,11 @@ class TestLink(object):
         ),
     ],
 )
-def test_parse_base_url(html, url, expected):
+def test_determine_base_url(html, url, expected):
     document = html5lib.parse(
         html, transport_encoding=None, namespaceHTMLElements=False,
     )
-    assert _parse_base_url(document, url) == expected
+    assert _determine_base_url(document, url) == expected
 
 
 class MockLogger(object):

From 47ba191d3a7ca98c9349fcf8f02b2c2f5e5616e1 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 16:49:38 +0800
Subject: [PATCH 37/92] Rename HTMLPage.links to clarify its purpose

---
 src/pip/_internal/index.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 2bf9c75a0..6ae7a0ab4 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -415,7 +415,7 @@ class PackageFinder(object):
             logger.debug('Analyzing links from page %s', page.url)
             with indent_log():
                 page_versions.extend(
-                    self._package_versions(page.links, search)
+                    self._package_versions(page.iter_links(), search)
                 )
 
         dependency_versions = self._package_versions(
@@ -865,8 +865,7 @@ class HTMLPage(object):
 
         return resp.headers.get("Content-Type", "")
 
-    @property
-    def links(self):
+    def iter_links(self):
         """Yields all links in the page"""
         document = html5lib.parse(
             self.content,

From 6f85fcdd7d682c54363826934932be486f5abc8c Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 23:58:24 +0800
Subject: [PATCH 38/92] Move clean_link out of HTMLPage

---
 src/pip/_internal/index.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 6ae7a0ab4..9eb8e1207 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -735,6 +735,16 @@ def _get_encoding_from_headers(headers):
     return None
 
 
+_CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
+
+
+def _clean_link(url):
+    """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
+    the link, it will be rewritten to %20 (while not over-quoting
+    % or other characters)."""
+    return _CLEAN_LINK_RE.sub(lambda match: '%%%2x' % ord(match.group(0)), url)
+
+
 class HTMLPage(object):
     """Represents one page, along with its URL"""
 
@@ -876,22 +886,11 @@ class HTMLPage(object):
         for anchor in document.findall(".//a"):
             if anchor.get("href"):
                 href = anchor.get("href")
-                url = self.clean_link(
-                    urllib_parse.urljoin(base_url, href)
-                )
+                url = _clean_link(urllib_parse.urljoin(base_url, href))
                 pyrequire = anchor.get('data-requires-python')
                 pyrequire = unescape(pyrequire) if pyrequire else None
                 yield Link(url, self, requires_python=pyrequire)
 
-    _clean_re = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I)
-
-    def clean_link(self, url):
-        """Makes sure a link is fully encoded.  That is, if a ' ' shows up in
-        the link, it will be rewritten to %20 (while not over-quoting
-        % or other characters)."""
-        return self._clean_re.sub(
-            lambda match: '%%%2x' % ord(match.group(0)), url)
-
 
 Search = namedtuple('Search', 'supplied canonical formats')
 """Capture key aspects of a search.

From 9cec269d197424bbd2f2eb6d1667321bec163c25 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Sat, 29 Sep 2018 17:54:13 +0530
Subject: [PATCH 39/92] Upgrade pkg_resources to 40.4.3 (via setuptools)

---
 news/setuptools.vendor                      |  1 +
 src/pip/_vendor/pkg_resources/__init__.py   | 67 +++++++++++++--------
 src/pip/_vendor/pkg_resources/py31compat.py |  5 +-
 src/pip/_vendor/vendor.txt                  |  2 +-
 4 files changed, 47 insertions(+), 28 deletions(-)
 create mode 100644 news/setuptools.vendor

diff --git a/news/setuptools.vendor b/news/setuptools.vendor
new file mode 100644
index 000000000..2f3815833
--- /dev/null
+++ b/news/setuptools.vendor
@@ -0,0 +1 @@
+Upgrade pkg_resources to 40.4.3 (via setuptools)
diff --git a/src/pip/_vendor/pkg_resources/__init__.py b/src/pip/_vendor/pkg_resources/__init__.py
index f2815cc6b..0b432f689 100644
--- a/src/pip/_vendor/pkg_resources/__init__.py
+++ b/src/pip/_vendor/pkg_resources/__init__.py
@@ -47,6 +47,11 @@ except ImportError:
     # Python 3.2 compatibility
     import imp as _imp
 
+try:
+    FileExistsError
+except NameError:
+    FileExistsError = OSError
+
 from pip._vendor import six
 from pip._vendor.six.moves import urllib, map, filter
 
@@ -78,8 +83,11 @@ __import__('pip._vendor.packaging.requirements')
 __import__('pip._vendor.packaging.markers')
 
 
-if (3, 0) < sys.version_info < (3, 3):
-    raise RuntimeError("Python 3.3 or later is required")
+__metaclass__ = type
+
+
+if (3, 0) < sys.version_info < (3, 4):
+    raise RuntimeError("Python 3.4 or later is required")
 
 if six.PY2:
     # Those builtin exceptions are only defined in Python 3
@@ -537,7 +545,7 @@ class IResourceProvider(IMetadataProvider):
         """List of resource names in the directory (like ``os.listdir()``)"""
 
 
-class WorkingSet(object):
+class WorkingSet:
     """A collection of active distributions on sys.path (or a similar list)"""
 
     def __init__(self, entries=None):
@@ -637,13 +645,12 @@ class WorkingSet(object):
         distributions in the working set, otherwise only ones matching
         both `group` and `name` are yielded (in distribution order).
         """
-        for dist in self:
-            entries = dist.get_entry_map(group)
-            if name is None:
-                for ep in entries.values():
-                    yield ep
-            elif name in entries:
-                yield entries[name]
+        return (
+            entry
+            for dist in self
+            for entry in dist.get_entry_map(group).values()
+            if name is None or name == entry.name
+        )
 
     def run_script(self, requires, script_name):
         """Locate distribution for `requires` and run `script_name` script"""
@@ -944,7 +951,7 @@ class _ReqExtras(dict):
         return not req.marker or any(extra_evals)
 
 
-class Environment(object):
+class Environment:
     """Searchable snapshot of distributions on a search path"""
 
     def __init__(
@@ -959,7 +966,7 @@ class Environment(object):
         `platform` is an optional string specifying the name of the platform
         that platform-specific distributions must be compatible with.  If
         unspecified, it defaults to the current platform.  `python` is an
-        optional string naming the desired version of Python (e.g. ``'3.3'``);
+        optional string naming the desired version of Python (e.g. ``'3.6'``);
         it defaults to the current version.
 
         You may explicitly set `platform` (and/or `python`) to ``None`` if you
@@ -2087,7 +2094,12 @@ def _handle_ns(packageName, path_item):
     importer = get_importer(path_item)
     if importer is None:
         return None
-    loader = importer.find_module(packageName)
+
+    # capture warnings due to #1111
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        loader = importer.find_module(packageName)
+
     if loader is None:
         return None
     module = sys.modules.get(packageName)
@@ -2132,12 +2144,13 @@ def _rebuild_mod_path(orig_path, package_name, module):
         parts = path_parts[:-module_parts]
         return safe_sys_path_index(_normalize_cached(os.sep.join(parts)))
 
-    if not isinstance(orig_path, list):
-        # Is this behavior useful when module.__path__ is not a list?
-        return
+    new_path = sorted(orig_path, key=position_in_sys_path)
+    new_path = [_normalize_cached(p) for p in new_path]
 
-    orig_path.sort(key=position_in_sys_path)
-    module.__path__[:] = [_normalize_cached(p) for p in orig_path]
+    if isinstance(module.__path__, list):
+        module.__path__[:] = new_path
+    else:
+        module.__path__ = new_path
 
 
 def declare_namespace(packageName):
@@ -2148,9 +2161,10 @@ def declare_namespace(packageName):
         if packageName in _namespace_packages:
             return
 
-        path, parent = sys.path, None
-        if '.' in packageName:
-            parent = '.'.join(packageName.split('.')[:-1])
+        path = sys.path
+        parent, _, _ = packageName.rpartition('.')
+
+        if parent:
             declare_namespace(parent)
             if parent not in _namespace_packages:
                 __import__(parent)
@@ -2161,7 +2175,7 @@ def declare_namespace(packageName):
 
         # Track what packages are namespaces, so when new path items are added,
         # they can be updated
-        _namespace_packages.setdefault(parent, []).append(packageName)
+        _namespace_packages.setdefault(parent or None, []).append(packageName)
         _namespace_packages.setdefault(packageName, [])
 
         for path_item in path:
@@ -2279,7 +2293,7 @@ EGG_NAME = re.compile(
 ).match
 
 
-class EntryPoint(object):
+class EntryPoint:
     """Object representing an advertised importable object"""
 
     def __init__(self, name, module_name, attrs=(), extras=(), dist=None):
@@ -2433,7 +2447,7 @@ def _version_from_file(lines):
     return safe_version(value.strip()) or None
 
 
-class Distribution(object):
+class Distribution:
     """Wrap an actual or potential sys.path entry w/metadata"""
     PKG_INFO = 'PKG-INFO'
 
@@ -3027,7 +3041,10 @@ def _bypass_ensure_directory(path):
     dirname, filename = split(path)
     if dirname and filename and not isdir(dirname):
         _bypass_ensure_directory(dirname)
-        mkdir(dirname, 0o755)
+        try:
+            mkdir(dirname, 0o755)
+        except FileExistsError:
+            pass
 
 
 def split_sections(s):
diff --git a/src/pip/_vendor/pkg_resources/py31compat.py b/src/pip/_vendor/pkg_resources/py31compat.py
index 331a51bb0..a381c424f 100644
--- a/src/pip/_vendor/pkg_resources/py31compat.py
+++ b/src/pip/_vendor/pkg_resources/py31compat.py
@@ -2,6 +2,8 @@ import os
 import errno
 import sys
 
+from .extern import six
+
 
 def _makedirs_31(path, exist_ok=False):
     try:
@@ -15,8 +17,7 @@ def _makedirs_31(path, exist_ok=False):
 #  and exists_ok considerations are disentangled.
 # See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663
 needs_makedirs = (
-    sys.version_info < (3, 2, 5) or
-    (3, 3) <= sys.version_info < (3, 3, 6) or
+    six.PY2 or
     (3, 4) <= sys.version_info < (3, 4, 1)
 )
 makedirs = _makedirs_31 if needs_makedirs else os.makedirs
diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
index de239b6c8..258321f00 100644
--- a/src/pip/_vendor/vendor.txt
+++ b/src/pip/_vendor/vendor.txt
@@ -19,5 +19,5 @@ requests==2.19.1
     idna==2.7
     urllib3==1.23
     certifi==2018.4.16
-setuptools==39.2.0
+setuptools==40.4.3
 webencodings==0.5.1

From 619b91e33469f68f229465513dd2a0ed71bea38d Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Sat, 29 Sep 2018 17:56:02 +0530
Subject: [PATCH 40/92] Upgrade certifi to 2018.8.24

---
 news/certifi.vendor                 |   1 +
 src/pip/_vendor/certifi/__init__.py |   2 +-
 src/pip/_vendor/certifi/cacert.pem  | 226 ++++++++--------------------
 src/pip/_vendor/vendor.txt          |   2 +-
 4 files changed, 66 insertions(+), 165 deletions(-)
 create mode 100644 news/certifi.vendor

diff --git a/news/certifi.vendor b/news/certifi.vendor
new file mode 100644
index 000000000..eb401fd67
--- /dev/null
+++ b/news/certifi.vendor
@@ -0,0 +1 @@
+Upgrade certifi to 2018.8.24
diff --git a/src/pip/_vendor/certifi/__init__.py b/src/pip/_vendor/certifi/__init__.py
index 0c4963ef6..aa329fbb4 100644
--- a/src/pip/_vendor/certifi/__init__.py
+++ b/src/pip/_vendor/certifi/__init__.py
@@ -1,3 +1,3 @@
 from .core import where, old_where
 
-__version__ = "2018.04.16"
+__version__ = "2018.08.24"
diff --git a/src/pip/_vendor/certifi/cacert.pem b/src/pip/_vendor/certifi/cacert.pem
index 2713f541c..85de024e7 100644
--- a/src/pip/_vendor/certifi/cacert.pem
+++ b/src/pip/_vendor/certifi/cacert.pem
@@ -3692,169 +3692,6 @@ lSTAGiecMjvAwNW6qef4BENThe5SId6d9SWDPp5YSy/XZxMOIQIwBeF1Ad5o7Sof
 TUwJCA3sS61kFyjndc5FZXIhF8siQQ6ME5g4mlRtm8rifOoCWCKR
 -----END CERTIFICATE-----
 
-# Issuer: CN=Certplus Root CA G1 O=Certplus
-# Subject: CN=Certplus Root CA G1 O=Certplus
-# Label: "Certplus Root CA G1"
-# Serial: 1491911565779898356709731176965615564637713
-# MD5 Fingerprint: 7f:09:9c:f7:d9:b9:5c:69:69:56:d5:37:3e:14:0d:42
-# SHA1 Fingerprint: 22:fd:d0:b7:fd:a2:4e:0d:ac:49:2c:a0:ac:a6:7b:6a:1f:e3:f7:66
-# SHA256 Fingerprint: 15:2a:40:2b:fc:df:2c:d5:48:05:4d:22:75:b3:9c:7f:ca:3e:c0:97:80:78:b0:f0:ea:76:e5:61:a6:c7:43:3e
------BEGIN CERTIFICATE-----
-MIIFazCCA1OgAwIBAgISESBVg+QtPlRWhS2DN7cs3EYRMA0GCSqGSIb3DQEBDQUA
-MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy
-dHBsdXMgUm9vdCBDQSBHMTAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBa
-MD4xCzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2Vy
-dHBsdXMgUm9vdCBDQSBHMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB
-ANpQh7bauKk+nWT6VjOaVj0W5QOVsjQcmm1iBdTYj+eJZJ+622SLZOZ5KmHNr49a
-iZFluVj8tANfkT8tEBXgfs+8/H9DZ6itXjYj2JizTfNDnjl8KvzsiNWI7nC9hRYt
-6kuJPKNxQv4c/dMcLRC4hlTqQ7jbxofaqK6AJc96Jh2qkbBIb6613p7Y1/oA/caP
-0FG7Yn2ksYyy/yARujVjBYZHYEMzkPZHogNPlk2dT8Hq6pyi/jQu3rfKG3akt62f
-6ajUeD94/vI4CTYd0hYCyOwqaK/1jpTvLRN6HkJKHRUxrgwEV/xhc/MxVoYxgKDE
-EW4wduOU8F8ExKyHcomYxZ3MVwia9Az8fXoFOvpHgDm2z4QTd28n6v+WZxcIbekN
-1iNQMLAVdBM+5S//Ds3EC0pd8NgAM0lm66EYfFkuPSi5YXHLtaW6uOrc4nBvCGrc
-h2c0798wct3zyT8j/zXhviEpIDCB5BmlIOklynMxdCm+4kLV87ImZsdo/Rmz5yCT
-mehd4F6H50boJZwKKSTUzViGUkAksnsPmBIgJPaQbEfIDbsYIC7Z/fyL8inqh3SV
-4EJQeIQEQWGw9CEjjy3LKCHyamz0GqbFFLQ3ZU+V/YDI+HLlJWvEYLF7bY5KinPO
-WftwenMGE9nTdDckQQoRb5fc5+R+ob0V8rqHDz1oihYHAgMBAAGjYzBhMA4GA1Ud
-DwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBSowcCbkahDFXxd
-Bie0KlHYlwuBsTAfBgNVHSMEGDAWgBSowcCbkahDFXxdBie0KlHYlwuBsTANBgkq
-hkiG9w0BAQ0FAAOCAgEAnFZvAX7RvUz1isbwJh/k4DgYzDLDKTudQSk0YcbX8ACh
-66Ryj5QXvBMsdbRX7gp8CXrc1cqh0DQT+Hern+X+2B50ioUHj3/MeXrKls3N/U/7
-/SMNkPX0XtPGYX2eEeAC7gkE2Qfdpoq3DIMku4NQkv5gdRE+2J2winq14J2by5BS
-S7CTKtQ+FjPlnsZlFT5kOwQ/2wyPX1wdaR+v8+khjPPvl/aatxm2hHSco1S1cE5j
-2FddUyGbQJJD+tZ3VTNPZNX70Cxqjm0lpu+F6ALEUz65noe8zDUa3qHpimOHZR4R
-Kttjd5cUvpoUmRGywO6wT/gUITJDT5+rosuoD6o7BlXGEilXCNQ314cnrUlZp5Gr
-RHpejXDbl85IULFzk/bwg2D5zfHhMf1bfHEhYxQUqq/F3pN+aLHsIqKqkHWetUNy
-6mSjhEv9DKgma3GX7lZjZuhCVPnHHd/Qj1vfyDBviP4NxDMcU6ij/UgQ8uQKTuEV
-V/xuZDDCVRHc6qnNSlSsKWNEz0pAoNZoWRsz+e86i9sgktxChL8Bq4fA1SCC28a5
-g4VCXA9DO2pJNdWY9BW/+mGBDAkgGNLQFwzLSABQ6XaCjGTXOqAHVcweMcDvOrRl
-++O/QmueD6i9a5jc2NvLi6Td11n0bt3+qsOR0C5CB8AMTVPNJLFMWx5R9N/pkvo=
------END CERTIFICATE-----
-
-# Issuer: CN=Certplus Root CA G2 O=Certplus
-# Subject: CN=Certplus Root CA G2 O=Certplus
-# Label: "Certplus Root CA G2"
-# Serial: 1492087096131536844209563509228951875861589
-# MD5 Fingerprint: a7:ee:c4:78:2d:1b:ee:2d:b9:29:ce:d6:a7:96:32:31
-# SHA1 Fingerprint: 4f:65:8e:1f:e9:06:d8:28:02:e9:54:47:41:c9:54:25:5d:69:cc:1a
-# SHA256 Fingerprint: 6c:c0:50:41:e6:44:5e:74:69:6c:4c:fb:c9:f8:0f:54:3b:7e:ab:bb:44:b4:ce:6f:78:7c:6a:99:71:c4:2f:17
------BEGIN CERTIFICATE-----
-MIICHDCCAaKgAwIBAgISESDZkc6uo+jF5//pAq/Pc7xVMAoGCCqGSM49BAMDMD4x
-CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs
-dXMgUm9vdCBDQSBHMjAeFw0xNDA1MjYwMDAwMDBaFw0zODAxMTUwMDAwMDBaMD4x
-CzAJBgNVBAYTAkZSMREwDwYDVQQKDAhDZXJ0cGx1czEcMBoGA1UEAwwTQ2VydHBs
-dXMgUm9vdCBDQSBHMjB2MBAGByqGSM49AgEGBSuBBAAiA2IABM0PW1aC3/BFGtat
-93nwHcmsltaeTpwftEIRyoa/bfuFo8XlGVzX7qY/aWfYeOKmycTbLXku54uNAm8x
-Ik0G42ByRZ0OQneezs/lf4WbGOT8zC5y0xaTTsqZY1yhBSpsBqNjMGEwDgYDVR0P
-AQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFNqDYwJ5jtpMxjwj
-FNiPwyCrKGBZMB8GA1UdIwQYMBaAFNqDYwJ5jtpMxjwjFNiPwyCrKGBZMAoGCCqG
-SM49BAMDA2gAMGUCMHD+sAvZ94OX7PNVHdTcswYO/jOYnYs5kGuUIe22113WTNch
-p+e/IQ8rzfcq3IUHnQIxAIYUFuXcsGXCwI4Un78kFmjlvPl5adytRSv3tjFzzAal
-U5ORGpOucGpnutee5WEaXw==
------END CERTIFICATE-----
-
-# Issuer: CN=OpenTrust Root CA G1 O=OpenTrust
-# Subject: CN=OpenTrust Root CA G1 O=OpenTrust
-# Label: "OpenTrust Root CA G1"
-# Serial: 1492036577811947013770400127034825178844775
-# MD5 Fingerprint: 76:00:cc:81:29:cd:55:5e:88:6a:7a:2e:f7:4d:39:da
-# SHA1 Fingerprint: 79:91:e8:34:f7:e2:ee:dd:08:95:01:52:e9:55:2d:14:e9:58:d5:7e
-# SHA256 Fingerprint: 56:c7:71:28:d9:8c:18:d9:1b:4c:fd:ff:bc:25:ee:91:03:d4:75:8e:a2:ab:ad:82:6a:90:f3:45:7d:46:0e:b4
------BEGIN CERTIFICATE-----
-MIIFbzCCA1egAwIBAgISESCzkFU5fX82bWTCp59rY45nMA0GCSqGSIb3DQEBCwUA
-MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w
-ZW5UcnVzdCBSb290IENBIEcxMB4XDTE0MDUyNjA4NDU1MFoXDTM4MDExNTAwMDAw
-MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU
-T3BlblRydXN0IFJvb3QgQ0EgRzEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
-AoICAQD4eUbalsUwXopxAy1wpLuwxQjczeY1wICkES3d5oeuXT2R0odsN7faYp6b
-wiTXj/HbpqbfRm9RpnHLPhsxZ2L3EVs0J9V5ToybWL0iEA1cJwzdMOWo010hOHQX
-/uMftk87ay3bfWAfjH1MBcLrARYVmBSO0ZB3Ij/swjm4eTrwSSTilZHcYTSSjFR0
-77F9jAHiOH3BX2pfJLKOYheteSCtqx234LSWSE9mQxAGFiQD4eCcjsZGT44ameGP
-uY4zbGneWK2gDqdkVBFpRGZPTBKnjix9xNRbxQA0MMHZmf4yzgeEtE7NCv82TWLx
-p2NX5Ntqp66/K7nJ5rInieV+mhxNaMbBGN4zK1FGSxyO9z0M+Yo0FMT7MzUj8czx
-Kselu7Cizv5Ta01BG2Yospb6p64KTrk5M0ScdMGTHPjgniQlQ/GbI4Kq3ywgsNw2
-TgOzfALU5nsaqocTvz6hdLubDuHAk5/XpGbKuxs74zD0M1mKB3IDVedzagMxbm+W
-G+Oin6+Sx+31QrclTDsTBM8clq8cIqPQqwWyTBIjUtz9GVsnnB47ev1CI9sjgBPw
-vFEVVJSmdz7QdFG9URQIOTfLHzSpMJ1ShC5VkLG631UAC9hWLbFJSXKAqWLXwPYY
-EQRVzXR7z2FwefR7LFxckvzluFqrTJOVoSfupb7PcSNCupt2LQIDAQABo2MwYTAO
-BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUl0YhVyE1
-2jZVx/PxN3DlCPaTKbYwHwYDVR0jBBgwFoAUl0YhVyE12jZVx/PxN3DlCPaTKbYw
-DQYJKoZIhvcNAQELBQADggIBAB3dAmB84DWn5ph76kTOZ0BP8pNuZtQ5iSas000E
-PLuHIT839HEl2ku6q5aCgZG27dmxpGWX4m9kWaSW7mDKHyP7Rbr/jyTwyqkxf3kf
-gLMtMrpkZ2CvuVnN35pJ06iCsfmYlIrM4LvgBBuZYLFGZdwIorJGnkSI6pN+VxbS
-FXJfLkur1J1juONI5f6ELlgKn0Md/rcYkoZDSw6cMoYsYPXpSOqV7XAp8dUv/TW0
-V8/bhUiZucJvbI/NeJWsZCj9VrDDb8O+WVLhX4SPgPL0DTatdrOjteFkdjpY3H1P
-XlZs5VVZV6Xf8YpmMIzUUmI4d7S+KNfKNsSbBfD4Fdvb8e80nR14SohWZ25g/4/I
-i+GOvUKpMwpZQhISKvqxnUOOBZuZ2mKtVzazHbYNeS2WuOvyDEsMpZTGMKcmGS3t
-TAZQMPH9WD25SxdfGbRqhFS0OE85og2WaMMolP3tLR9Ka0OWLpABEPs4poEL0L91
-09S5zvE/bw4cHjdx5RiHdRk/ULlepEU0rbDK5uUTdg8xFKmOLZTW1YVNcxVPS/Ky
-Pu1svf0OnWZzsD2097+o4BGkxK51CUpjAEggpsadCwmKtODmzj7HPiY46SvepghJ
-AwSQiumPv+i2tCqjI40cHLI5kqiPAlxAOXXUc0ECd97N4EOH1uS6SsNsEn/+KuYj
-1oxx
------END CERTIFICATE-----
-
-# Issuer: CN=OpenTrust Root CA G2 O=OpenTrust
-# Subject: CN=OpenTrust Root CA G2 O=OpenTrust
-# Label: "OpenTrust Root CA G2"
-# Serial: 1492012448042702096986875987676935573415441
-# MD5 Fingerprint: 57:24:b6:59:24:6b:ae:c8:fe:1c:0c:20:f2:c0:4e:eb
-# SHA1 Fingerprint: 79:5f:88:60:c5:ab:7c:3d:92:e6:cb:f4:8d:e1:45:cd:11:ef:60:0b
-# SHA256 Fingerprint: 27:99:58:29:fe:6a:75:15:c1:bf:e8:48:f9:c4:76:1d:b1:6c:22:59:29:25:7b:f4:0d:08:94:f2:9e:a8:ba:f2
------BEGIN CERTIFICATE-----
-MIIFbzCCA1egAwIBAgISESChaRu/vbm9UpaPI+hIvyYRMA0GCSqGSIb3DQEBDQUA
-MEAxCzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9w
-ZW5UcnVzdCBSb290IENBIEcyMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAw
-MFowQDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwU
-T3BlblRydXN0IFJvb3QgQ0EgRzIwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK
-AoICAQDMtlelM5QQgTJT32F+D3Y5z1zCU3UdSXqWON2ic2rxb95eolq5cSG+Ntmh
-/LzubKh8NBpxGuga2F8ORAbtp+Dz0mEL4DKiltE48MLaARf85KxP6O6JHnSrT78e
-CbY2albz4e6WiWYkBuTNQjpK3eCasMSCRbP+yatcfD7J6xcvDH1urqWPyKwlCm/6
-1UWY0jUJ9gNDlP7ZvyCVeYCYitmJNbtRG6Q3ffyZO6v/v6wNj0OxmXsWEH4db0fE
-FY8ElggGQgT4hNYdvJGmQr5J1WqIP7wtUdGejeBSzFfdNTVY27SPJIjki9/ca1TS
-gSuyzpJLHB9G+h3Ykst2Z7UJmQnlrBcUVXDGPKBWCgOz3GIZ38i1MH/1PCZ1Eb3X
-G7OHngevZXHloM8apwkQHZOJZlvoPGIytbU6bumFAYueQ4xncyhZW+vj3CzMpSZy
-YhK05pyDRPZRpOLAeiRXyg6lPzq1O4vldu5w5pLeFlwoW5cZJ5L+epJUzpM5ChaH
-vGOz9bGTXOBut9Dq+WIyiET7vycotjCVXRIouZW+j1MY5aIYFuJWpLIsEPUdN6b4
-t/bQWVyJ98LVtZR00dX+G7bw5tYee9I8y6jj9RjzIR9u701oBnstXW5DiabA+aC/
-gh7PU3+06yzbXfZqfUAkBXKJOAGTy3HCOV0GEfZvePg3DTmEJwIDAQABo2MwYTAO
-BgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUajn6QiL3
-5okATV59M4PLuG53hq8wHwYDVR0jBBgwFoAUajn6QiL35okATV59M4PLuG53hq8w
-DQYJKoZIhvcNAQENBQADggIBAJjLq0A85TMCl38th6aP1F5Kr7ge57tx+4BkJamz
-Gj5oXScmp7oq4fBXgwpkTx4idBvpkF/wrM//T2h6OKQQbA2xx6R3gBi2oihEdqc0
-nXGEL8pZ0keImUEiyTCYYW49qKgFbdEfwFFEVn8nNQLdXpgKQuswv42hm1GqO+qT
-RmTFAHneIWv2V6CG1wZy7HBGS4tz3aAhdT7cHcCP009zHIXZ/n9iyJVvttN7jLpT
-wm+bREx50B1ws9efAvSyB7DH5fitIw6mVskpEndI2S9G/Tvw/HRwkqWOOAgfZDC2
-t0v7NqwQjqBSM2OdAzVWxWm9xiNaJ5T2pBL4LTM8oValX9YZ6e18CL13zSdkzJTa
-TkZQh+D5wVOAHrut+0dSixv9ovneDiK3PTNZbNTe9ZUGMg1RGUFcPk8G97krgCf2
-o6p6fAbhQ8MTOWIaNr3gKC6UAuQpLmBVrkA9sHSSXvAgZJY/X0VdiLWK2gKgW0VU
-3jg9CcCoSmVGFvyqv1ROTVu+OEO3KMqLM6oaJbolXCkvW0pujOotnCr2BXbgd5eA
-iN1nE28daCSLT7d0geX0YJ96Vdc+N9oWaz53rK4YcJUIeSkDiv7BO7M/Gg+kO14f
-WKGVyasvc0rQLW6aWQ9VGHgtPFGml4vmu7JwqkwR3v98KzfUetF3NI/n+UL3PIEM
-S1IK
------END CERTIFICATE-----
-
-# Issuer: CN=OpenTrust Root CA G3 O=OpenTrust
-# Subject: CN=OpenTrust Root CA G3 O=OpenTrust
-# Label: "OpenTrust Root CA G3"
-# Serial: 1492104908271485653071219941864171170455615
-# MD5 Fingerprint: 21:37:b4:17:16:92:7b:67:46:70:a9:96:d7:a8:13:24
-# SHA1 Fingerprint: 6e:26:64:f3:56:bf:34:55:bf:d1:93:3f:7c:01:de:d8:13:da:8a:a6
-# SHA256 Fingerprint: b7:c3:62:31:70:6e:81:07:8c:36:7c:b8:96:19:8f:1e:32:08:dd:92:69:49:dd:8f:57:09:a4:10:f7:5b:62:92
------BEGIN CERTIFICATE-----
-MIICITCCAaagAwIBAgISESDm+Ez8JLC+BUCs2oMbNGA/MAoGCCqGSM49BAMDMEAx
-CzAJBgNVBAYTAkZSMRIwEAYDVQQKDAlPcGVuVHJ1c3QxHTAbBgNVBAMMFE9wZW5U
-cnVzdCBSb290IENBIEczMB4XDTE0MDUyNjAwMDAwMFoXDTM4MDExNTAwMDAwMFow
-QDELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCU9wZW5UcnVzdDEdMBsGA1UEAwwUT3Bl
-blRydXN0IFJvb3QgQ0EgRzMwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARK7liuTcpm
-3gY6oxH84Bjwbhy6LTAMidnW7ptzg6kjFYwvWYpa3RTqnVkrQ7cG7DK2uu5Bta1d
-oYXM6h0UZqNnfkbilPPntlahFVmhTzeXuSIevRHr9LIfXsMUmuXZl5mjYzBhMA4G
-A1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBRHd8MUi2I5
-DMlv4VBN0BBY3JWIbTAfBgNVHSMEGDAWgBRHd8MUi2I5DMlv4VBN0BBY3JWIbTAK
-BggqhkjOPQQDAwNpADBmAjEAj6jcnboMBBf6Fek9LykBl7+BFjNAk2z8+e2AcG+q
-j9uEwov1NcoG3GRvaBbhj5G5AjEA2Euly8LQCGzpGPta3U1fJAuwACEl74+nBCZx
-4nxp5V2a+EEfOzmTk51V6s2N8fvB
------END CERTIFICATE-----
-
 # Issuer: CN=ISRG Root X1 O=Internet Security Research Group
 # Subject: CN=ISRG Root X1 O=Internet Security Research Group
 # Label: "ISRG Root X1"
@@ -4398,3 +4235,66 @@ MA8GA1UdEwEB/wQFMAMBAf8wHwYDVR0jBBgwFoAUW8pe5d7SgarNqC1kUbbZcpuX
 ytRrJPOwPYdGWBrssd9v+1a6cGvHOMzosYxPD/fxZ3YOg9AeUY8CMD32IygmTMZg
 h5Mmm7I1HrrW9zzRHM76JTymGoEVW/MSD2zuZYrJh6j5B+BimoxcSg==
 -----END CERTIFICATE-----
+
+# Issuer: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6
+# Subject: CN=GlobalSign O=GlobalSign OU=GlobalSign Root CA - R6
+# Label: "GlobalSign Root CA - R6"
+# Serial: 1417766617973444989252670301619537
+# MD5 Fingerprint: 4f:dd:07:e4:d4:22:64:39:1e:0c:37:42:ea:d1:c6:ae
+# SHA1 Fingerprint: 80:94:64:0e:b5:a7:a1:ca:11:9c:1f:dd:d5:9f:81:02:63:a7:fb:d1
+# SHA256 Fingerprint: 2c:ab:ea:fe:37:d0:6c:a2:2a:ba:73:91:c0:03:3d:25:98:29:52:c4:53:64:73:49:76:3a:3a:b5:ad:6c:cf:69
+-----BEGIN CERTIFICATE-----
+MIIFgzCCA2ugAwIBAgIORea7A4Mzw4VlSOb/RVEwDQYJKoZIhvcNAQEMBQAwTDEg
+MB4GA1UECxMXR2xvYmFsU2lnbiBSb290IENBIC0gUjYxEzARBgNVBAoTCkdsb2Jh
+bFNpZ24xEzARBgNVBAMTCkdsb2JhbFNpZ24wHhcNMTQxMjEwMDAwMDAwWhcNMzQx
+MjEwMDAwMDAwWjBMMSAwHgYDVQQLExdHbG9iYWxTaWduIFJvb3QgQ0EgLSBSNjET
+MBEGA1UEChMKR2xvYmFsU2lnbjETMBEGA1UEAxMKR2xvYmFsU2lnbjCCAiIwDQYJ
+KoZIhvcNAQEBBQADggIPADCCAgoCggIBAJUH6HPKZvnsFMp7PPcNCPG0RQssgrRI
+xutbPK6DuEGSMxSkb3/pKszGsIhrxbaJ0cay/xTOURQh7ErdG1rG1ofuTToVBu1k
+ZguSgMpE3nOUTvOniX9PeGMIyBJQbUJmL025eShNUhqKGoC3GYEOfsSKvGRMIRxD
+aNc9PIrFsmbVkJq3MQbFvuJtMgamHvm566qjuL++gmNQ0PAYid/kD3n16qIfKtJw
+LnvnvJO7bVPiSHyMEAc4/2ayd2F+4OqMPKq0pPbzlUoSB239jLKJz9CgYXfIWHSw
+1CM69106yqLbnQneXUQtkPGBzVeS+n68UARjNN9rkxi+azayOeSsJDa38O+2HBNX
+k7besvjihbdzorg1qkXy4J02oW9UivFyVm4uiMVRQkQVlO6jxTiWm05OWgtH8wY2
+SXcwvHE35absIQh1/OZhFj931dmRl4QKbNQCTXTAFO39OfuD8l4UoQSwC+n+7o/h
+bguyCLNhZglqsQY6ZZZZwPA1/cnaKI0aEYdwgQqomnUdnjqGBQCe24DWJfncBZ4n
+WUx2OVvq+aWh2IMP0f/fMBH5hc8zSPXKbWQULHpYT9NLCEnFlWQaYw55PfWzjMpY
+rZxCRXluDocZXFSxZba/jJvcE+kNb7gu3GduyYsRtYQUigAZcIN5kZeR1Bonvzce
+MgfYFGM8KEyvAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTAD
+AQH/MB0GA1UdDgQWBBSubAWjkxPioufi1xzWx/B/yGdToDAfBgNVHSMEGDAWgBSu
+bAWjkxPioufi1xzWx/B/yGdToDANBgkqhkiG9w0BAQwFAAOCAgEAgyXt6NH9lVLN
+nsAEoJFp5lzQhN7craJP6Ed41mWYqVuoPId8AorRbrcWc+ZfwFSY1XS+wc3iEZGt
+Ixg93eFyRJa0lV7Ae46ZeBZDE1ZXs6KzO7V33EByrKPrmzU+sQghoefEQzd5Mr61
+55wsTLxDKZmOMNOsIeDjHfrYBzN2VAAiKrlNIC5waNrlU/yDXNOd8v9EDERm8tLj
+vUYAGm0CuiVdjaExUd1URhxN25mW7xocBFymFe944Hn+Xds+qkxV/ZoVqW/hpvvf
+cDDpw+5CRu3CkwWJ+n1jez/QcYF8AOiYrg54NMMl+68KnyBr3TsTjxKM4kEaSHpz
+oHdpx7Zcf4LIHv5YGygrqGytXm3ABdJ7t+uA/iU3/gKbaKxCXcPu9czc8FB10jZp
+nOZ7BN9uBmm23goJSFmH63sUYHpkqmlD75HHTOwY3WzvUy2MmeFe8nI+z1TIvWfs
+pA9MRf/TuTAjB0yPEL+GltmZWrSZVxykzLsViVO6LAUP5MSeGbEYNNVMnbrt9x+v
+JJUEeKgDu+6B5dpffItKoZB0JaezPkvILFa9x8jvOOJckvB595yEunQtYQEgfn7R
+8k8HWV+LLUNS60YMlOH1Zkd5d9VUWx+tJDfLRVpOoERIyNiwmcUVhAn21klJwGW4
+5hpxbqCo8YLoRT5s1gLXCmeDBVrJpBA=
+-----END CERTIFICATE-----
+
+# Issuer: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed
+# Subject: CN=OISTE WISeKey Global Root GC CA O=WISeKey OU=OISTE Foundation Endorsed
+# Label: "OISTE WISeKey Global Root GC CA"
+# Serial: 44084345621038548146064804565436152554
+# MD5 Fingerprint: a9:d6:b9:2d:2f:93:64:f8:a5:69:ca:91:e9:68:07:23
+# SHA1 Fingerprint: e0:11:84:5e:34:de:be:88:81:b9:9c:f6:16:26:d1:96:1f:c3:b9:31
+# SHA256 Fingerprint: 85:60:f9:1c:36:24:da:ba:95:70:b5:fe:a0:db:e3:6f:f1:1a:83:23:be:94:86:85:4f:b3:f3:4a:55:71:19:8d
+-----BEGIN CERTIFICATE-----
+MIICaTCCAe+gAwIBAgIQISpWDK7aDKtARb8roi066jAKBggqhkjOPQQDAzBtMQsw
+CQYDVQQGEwJDSDEQMA4GA1UEChMHV0lTZUtleTEiMCAGA1UECxMZT0lTVEUgRm91
+bmRhdGlvbiBFbmRvcnNlZDEoMCYGA1UEAxMfT0lTVEUgV0lTZUtleSBHbG9iYWwg
+Um9vdCBHQyBDQTAeFw0xNzA1MDkwOTQ4MzRaFw00MjA1MDkwOTU4MzNaMG0xCzAJ
+BgNVBAYTAkNIMRAwDgYDVQQKEwdXSVNlS2V5MSIwIAYDVQQLExlPSVNURSBGb3Vu
+ZGF0aW9uIEVuZG9yc2VkMSgwJgYDVQQDEx9PSVNURSBXSVNlS2V5IEdsb2JhbCBS
+b290IEdDIENBMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAETOlQwMYPchi82PG6s4ni
+eUqjFqdrVCTbUf/q9Akkwwsin8tqJ4KBDdLArzHkdIJuyiXZjHWd8dvQmqJLIX4W
+p2OQ0jnUsYd4XxiWD1AbNTcPasbc2RNNpI6QN+a9WzGRo1QwUjAOBgNVHQ8BAf8E
+BAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUSIcUrOPDnpBgOtfKie7T
+rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV
+57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg
+Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9
+-----END CERTIFICATE-----
diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
index 258321f00..20aaa7c15 100644
--- a/src/pip/_vendor/vendor.txt
+++ b/src/pip/_vendor/vendor.txt
@@ -18,6 +18,6 @@ requests==2.19.1
     chardet==3.0.4
     idna==2.7
     urllib3==1.23
-    certifi==2018.4.16
+    certifi==2018.8.24
 setuptools==40.4.3
 webencodings==0.5.1

From 04c6e5b27ed6ac4e3ff5a0d1dfaaedf46002fff9 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Sat, 29 Sep 2018 17:57:45 +0530
Subject: [PATCH 41/92] Upgrade packaging to 18.0

---
 news/packaging.vendor                     | 1 +
 src/pip/_vendor/packaging/__about__.py    | 4 ++--
 src/pip/_vendor/packaging/requirements.py | 8 ++++----
 src/pip/_vendor/packaging/specifiers.py   | 2 +-
 src/pip/_vendor/vendor.txt                | 2 +-
 5 files changed, 9 insertions(+), 8 deletions(-)
 create mode 100644 news/packaging.vendor

diff --git a/news/packaging.vendor b/news/packaging.vendor
new file mode 100644
index 000000000..ca83b8d34
--- /dev/null
+++ b/news/packaging.vendor
@@ -0,0 +1 @@
+Upgrade packaging to 18.0
diff --git a/src/pip/_vendor/packaging/__about__.py b/src/pip/_vendor/packaging/__about__.py
index 4255c5b55..21fc6ce3e 100644
--- a/src/pip/_vendor/packaging/__about__.py
+++ b/src/pip/_vendor/packaging/__about__.py
@@ -12,10 +12,10 @@ __title__ = "packaging"
 __summary__ = "Core utilities for Python packages"
 __uri__ = "https://github.com/pypa/packaging"
 
-__version__ = "17.1"
+__version__ = "18.0"
 
 __author__ = "Donald Stufft and individual contributors"
 __email__ = "donald@stufft.io"
 
 __license__ = "BSD or Apache License, Version 2.0"
-__copyright__ = "Copyright 2014-2016 %s" % __author__
+__copyright__ = "Copyright 2014-2018 %s" % __author__
diff --git a/src/pip/_vendor/packaging/requirements.py b/src/pip/_vendor/packaging/requirements.py
index 2760483a7..d40bd8c5c 100644
--- a/src/pip/_vendor/packaging/requirements.py
+++ b/src/pip/_vendor/packaging/requirements.py
@@ -92,16 +92,16 @@ class Requirement(object):
         try:
             req = REQUIREMENT.parseString(requirement_string)
         except ParseException as e:
-            raise InvalidRequirement(
-                "Invalid requirement, parse error at \"{0!r}\"".format(
-                    requirement_string[e.loc:e.loc + 8]))
+            raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format(
+                requirement_string[e.loc:e.loc + 8], e.msg
+            ))
 
         self.name = req.name
         if req.url:
             parsed_url = urlparse.urlparse(req.url)
             if not (parsed_url.scheme and parsed_url.netloc) or (
                     not parsed_url.scheme and not parsed_url.netloc):
-                raise InvalidRequirement("Invalid URL given")
+                raise InvalidRequirement("Invalid URL: {0}".format(req.url))
             self.url = req.url
         else:
             self.url = None
diff --git a/src/pip/_vendor/packaging/specifiers.py b/src/pip/_vendor/packaging/specifiers.py
index 9b6353f05..4c798999d 100644
--- a/src/pip/_vendor/packaging/specifiers.py
+++ b/src/pip/_vendor/packaging/specifiers.py
@@ -503,7 +503,7 @@ class Specifier(_IndividualSpecifier):
                 return False
 
         # Ensure that we do not allow a local version of the version mentioned
-        # in the specifier, which is techincally greater than, to match.
+        # in the specifier, which is technically greater than, to match.
         if prospective.local is not None:
             if Version(prospective.base_version) == Version(spec.base_version):
                 return False
diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
index 20aaa7c15..e73cc30df 100644
--- a/src/pip/_vendor/vendor.txt
+++ b/src/pip/_vendor/vendor.txt
@@ -9,7 +9,7 @@ msgpack-python==0.5.6
 lockfile==0.12.2
 progress==1.4
 ipaddress==1.0.22  # Only needed on 2.6 and 2.7
-packaging==17.1
+packaging==18.0
 pep517==0.2
 pyparsing==2.2.1
 pytoml==0.1.16

From 50ab3bf51e6c9262b7aa987a24e18d102619b146 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Sat, 29 Sep 2018 18:00:58 +0530
Subject: [PATCH 42/92] Upgrade pytoml to 0.1.19

---
 news/pytoml.vendor               | 1 +
 src/pip/_vendor/pytoml/parser.py | 4 ++--
 src/pip/_vendor/vendor.txt       | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)
 create mode 100644 news/pytoml.vendor

diff --git a/news/pytoml.vendor b/news/pytoml.vendor
new file mode 100644
index 000000000..f1fab347b
--- /dev/null
+++ b/news/pytoml.vendor
@@ -0,0 +1 @@
+Upgrade pytoml to 0.1.19
diff --git a/src/pip/_vendor/pytoml/parser.py b/src/pip/_vendor/pytoml/parser.py
index e03a03fbd..9f94e9230 100644
--- a/src/pip/_vendor/pytoml/parser.py
+++ b/src/pip/_vendor/pytoml/parser.py
@@ -223,8 +223,8 @@ _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?
 _datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))')
 
 _basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*')
-_litstr_re = re.compile(r"[^'\000-\037]*")
-_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\011\013-\037]))*")
+_litstr_re = re.compile(r"[^'\000\010\012-\037]*")
+_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*")
 def _p_value(s, object_pairs_hook):
     pos = s.pos()
 
diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt
index e73cc30df..9389dd947 100644
--- a/src/pip/_vendor/vendor.txt
+++ b/src/pip/_vendor/vendor.txt
@@ -12,7 +12,7 @@ ipaddress==1.0.22  # Only needed on 2.6 and 2.7
 packaging==18.0
 pep517==0.2
 pyparsing==2.2.1
-pytoml==0.1.16
+pytoml==0.1.19
 retrying==1.3.3
 requests==2.19.1
     chardet==3.0.4

From 00c043d144661d30d6efd702588b0b54274d6f86 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 00:28:14 +0800
Subject: [PATCH 43/92] Add unit tests for egg_info_matches

---
 news/5821.trivial        |  1 +
 tests/unit/test_index.py | 34 +++++++++++++++++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)
 create mode 100644 news/5821.trivial

diff --git a/news/5821.trivial b/news/5821.trivial
new file mode 100644
index 000000000..25c6284e8
--- /dev/null
+++ b/news/5821.trivial
@@ -0,0 +1 @@
+Add unit tests for egg_info_matches.
diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index d67aa4d0d..bca09a24e 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -4,7 +4,9 @@ import pytest
 from pip._vendor import html5lib
 
 from pip._internal.download import PipSession
-from pip._internal.index import Link, PackageFinder, _determine_base_url
+from pip._internal.index import (
+    Link, PackageFinder, _determine_base_url, egg_info_matches,
+)
 
 
 def test_sort_locations_file_expand_dir(data):
@@ -158,3 +160,33 @@ def test_get_formatted_locations_basic_auth():
 
     result = finder.get_formatted_locations()
     assert 'user' not in result and 'pass' not in result
+
+
+@pytest.mark.parametrize(
+    ("egg_info", "search_name", "expected"),
+    [
+        # Trivial.
+        ("pip-18.0", "pip", "18.0"),
+        ("pip-18.0", None, "18.0"),
+
+        # Non-canonical names.
+        ("Jinja2-2.10", "jinja2", "2.10"),
+        ("jinja2-2.10", "Jinja2", "2.10"),
+
+        # Ambiguous names. Should be smart enough if the package name is
+        # provided, otherwise make a guess.
+        ("foo-2-2", "foo", "2-2"),
+        ("foo-2-2", "foo-2", "2"),
+        ("foo-2-2", None, "2-2"),
+        ("im-valid", None, "valid"),
+
+        # Invalid names.
+        ("invalid", None, None),
+        ("im_invalid", None, None),
+        ("the-package-name-8.19", "does-not-match", None),
+    ],
+)
+def test_egg_info_matches(egg_info, search_name, expected):
+    link = None     # Only used for reporting.
+    version = egg_info_matches(egg_info, search_name, link)
+    assert version == expected

From d2753607455589381b118a6b8d23a99b47ee8856 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Fri, 28 Sep 2018 15:08:12 +0800
Subject: [PATCH 44/92] Move static methods out of HTMLPage

Because the class is going away, but these functions will still be
needed by the thing replacing it.
---
 src/pip/_internal/index.py | 237 ++++++++++++++++++-------------------
 1 file changed, 117 insertions(+), 120 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 9eb8e1207..e0124c338 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -59,6 +59,122 @@ SECURE_ORIGINS = [
 logger = logging.getLogger(__name__)
 
 
+def _get_content_type(url, session):
+    """Get the Content-Type of the given url, using a HEAD request"""
+    scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
+    if scheme not in {'http', 'https'}:
+        # FIXME: some warning or something?
+        # assertion error?
+        return ''
+
+    resp = session.head(url, allow_redirects=True)
+    resp.raise_for_status()
+
+    return resp.headers.get("Content-Type", "")
+
+
+def _handle_get_page_fail(link, reason, url, meth=None):
+    if meth is None:
+        meth = logger.debug
+    meth("Could not fetch URL %s: %s - skipping", link, reason)
+
+
+def _get_html_page(link, session=None):
+    if session is None:
+        raise TypeError(
+            "_get_html_page() missing 1 required keyword argument: 'session'"
+        )
+
+    url = link.url
+    url = url.split('#', 1)[0]
+
+    # Check for VCS schemes that do not support lookup as web pages.
+    from pip._internal.vcs import VcsSupport
+    for scheme in VcsSupport.schemes:
+        if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
+            logger.debug('Cannot look at %s URL %s', scheme, link)
+            return None
+
+    try:
+        filename = link.filename
+        for bad_ext in ARCHIVE_EXTENSIONS:
+            if filename.endswith(bad_ext):
+                content_type = _get_content_type(url, session=session)
+                if content_type.lower().startswith('text/html'):
+                    break
+                else:
+                    logger.debug(
+                        'Skipping page %s because of Content-Type: %s',
+                        link,
+                        content_type,
+                    )
+                    return
+
+        logger.debug('Getting page %s', url)
+
+        # Tack index.html onto file:// URLs that point to directories
+        (scheme, netloc, path, params, query, fragment) = \
+            urllib_parse.urlparse(url)
+        if (scheme == 'file' and
+                os.path.isdir(urllib_request.url2pathname(path))):
+            # add trailing slash if not present so urljoin doesn't trim
+            # final segment
+            if not url.endswith('/'):
+                url += '/'
+            url = urllib_parse.urljoin(url, 'index.html')
+            logger.debug(' file: URL is directory, getting %s', url)
+
+        resp = session.get(
+            url,
+            headers={
+                "Accept": "text/html",
+                # We don't want to blindly returned cached data for
+                # /simple/, because authors generally expecting that
+                # twine upload && pip install will function, but if
+                # they've done a pip install in the last ~10 minutes
+                # it won't. Thus by setting this to zero we will not
+                # blindly use any cached data, however the benefit of
+                # using max-age=0 instead of no-cache, is that we will
+                # still support conditional requests, so we will still
+                # minimize traffic sent in cases where the page hasn't
+                # changed at all, we will just always incur the round
+                # trip for the conditional GET now instead of only
+                # once per 10 minutes.
+                # For more information, please see pypa/pip#5670.
+                "Cache-Control": "max-age=0",
+            },
+        )
+        resp.raise_for_status()
+
+        # The check for archives above only works if the url ends with
+        # something that looks like an archive. However that is not a
+        # requirement of an url. Unless we issue a HEAD request on every
+        # url we cannot know ahead of time for sure if something is HTML
+        # or not. However we can check after we've downloaded it.
+        content_type = resp.headers.get('Content-Type', 'unknown')
+        if not content_type.lower().startswith("text/html"):
+            logger.debug(
+                'Skipping page %s because of Content-Type: %s',
+                link,
+                content_type,
+            )
+            return
+
+        inst = HTMLPage(resp.content, resp.url, resp.headers)
+    except requests.HTTPError as exc:
+        _handle_get_page_fail(link, exc, url)
+    except SSLError as exc:
+        reason = "There was a problem confirming the ssl certificate: "
+        reason += str(exc)
+        _handle_get_page_fail(link, reason, url, meth=logger.info)
+    except requests.ConnectionError as exc:
+        _handle_get_page_fail(link, "connection error: %s" % exc, url)
+    except requests.Timeout:
+        _handle_get_page_fail(link, "timed out", url)
+    else:
+        return inst
+
+
 class PackageFinder(object):
     """This finds packages.
 
@@ -674,7 +790,7 @@ class PackageFinder(object):
         return InstallationCandidate(search.supplied, version, link)
 
     def _get_page(self, link):
-        return HTMLPage.get_page(link, session=self.session)
+        return _get_html_page(link, session=self.session)
 
 
 def egg_info_matches(
@@ -756,125 +872,6 @@ class HTMLPage(object):
     def __str__(self):
         return self.url
 
-    @classmethod
-    def get_page(cls, link, session=None):
-        if session is None:
-            raise TypeError(
-                "get_page() missing 1 required keyword argument: 'session'"
-            )
-
-        url = link.url
-        url = url.split('#', 1)[0]
-
-        # Check for VCS schemes that do not support lookup as web pages.
-        from pip._internal.vcs import VcsSupport
-        for scheme in VcsSupport.schemes:
-            if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
-                logger.debug('Cannot look at %s URL %s', scheme, link)
-                return None
-
-        try:
-            filename = link.filename
-            for bad_ext in ARCHIVE_EXTENSIONS:
-                if filename.endswith(bad_ext):
-                    content_type = cls._get_content_type(
-                        url, session=session,
-                    )
-                    if content_type.lower().startswith('text/html'):
-                        break
-                    else:
-                        logger.debug(
-                            'Skipping page %s because of Content-Type: %s',
-                            link,
-                            content_type,
-                        )
-                        return
-
-            logger.debug('Getting page %s', url)
-
-            # Tack index.html onto file:// URLs that point to directories
-            (scheme, netloc, path, params, query, fragment) = \
-                urllib_parse.urlparse(url)
-            if (scheme == 'file' and
-                    os.path.isdir(urllib_request.url2pathname(path))):
-                # add trailing slash if not present so urljoin doesn't trim
-                # final segment
-                if not url.endswith('/'):
-                    url += '/'
-                url = urllib_parse.urljoin(url, 'index.html')
-                logger.debug(' file: URL is directory, getting %s', url)
-
-            resp = session.get(
-                url,
-                headers={
-                    "Accept": "text/html",
-                    # We don't want to blindly returned cached data for
-                    # /simple/, because authors generally expecting that
-                    # twine upload && pip install will function, but if
-                    # they've done a pip install in the last ~10 minutes
-                    # it won't. Thus by setting this to zero we will not
-                    # blindly use any cached data, however the benefit of
-                    # using max-age=0 instead of no-cache, is that we will
-                    # still support conditional requests, so we will still
-                    # minimize traffic sent in cases where the page hasn't
-                    # changed at all, we will just always incur the round
-                    # trip for the conditional GET now instead of only
-                    # once per 10 minutes.
-                    # For more information, please see pypa/pip#5670.
-                    "Cache-Control": "max-age=0",
-                },
-            )
-            resp.raise_for_status()
-
-            # The check for archives above only works if the url ends with
-            # something that looks like an archive. However that is not a
-            # requirement of an url. Unless we issue a HEAD request on every
-            # url we cannot know ahead of time for sure if something is HTML
-            # or not. However we can check after we've downloaded it.
-            content_type = resp.headers.get('Content-Type', 'unknown')
-            if not content_type.lower().startswith("text/html"):
-                logger.debug(
-                    'Skipping page %s because of Content-Type: %s',
-                    link,
-                    content_type,
-                )
-                return
-
-            inst = cls(resp.content, resp.url, resp.headers)
-        except requests.HTTPError as exc:
-            cls._handle_fail(link, exc, url)
-        except SSLError as exc:
-            reason = "There was a problem confirming the ssl certificate: "
-            reason += str(exc)
-            cls._handle_fail(link, reason, url, meth=logger.info)
-        except requests.ConnectionError as exc:
-            cls._handle_fail(link, "connection error: %s" % exc, url)
-        except requests.Timeout:
-            cls._handle_fail(link, "timed out", url)
-        else:
-            return inst
-
-    @staticmethod
-    def _handle_fail(link, reason, url, meth=None):
-        if meth is None:
-            meth = logger.debug
-
-        meth("Could not fetch URL %s: %s - skipping", link, reason)
-
-    @staticmethod
-    def _get_content_type(url, session):
-        """Get the Content-Type of the given url, using a HEAD request"""
-        scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
-        if scheme not in {'http', 'https'}:
-            # FIXME: some warning or something?
-            # assertion error?
-            return ''
-
-        resp = session.head(url, allow_redirects=True)
-        resp.raise_for_status()
-
-        return resp.headers.get("Content-Type", "")
-
     def iter_links(self):
         """Yields all links in the page"""
         document = html5lib.parse(

From bd6a7ea6c658d4575c0179d8c732fee5010489b3 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Sun, 30 Sep 2018 13:46:04 +0800
Subject: [PATCH 45/92] Pass the URL instead of the page itself into Link

This argument (comes_from) is only used for reporting, and it is enough
to pass in only the URL since that's what is actually used.
---
 src/pip/_internal/index.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index e0124c338..8c2f24f1b 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -886,7 +886,7 @@ class HTMLPage(object):
                 url = _clean_link(urllib_parse.urljoin(base_url, href))
                 pyrequire = anchor.get('data-requires-python')
                 pyrequire = unescape(pyrequire) if pyrequire else None
-                yield Link(url, self, requires_python=pyrequire)
+                yield Link(url, self.url, requires_python=pyrequire)
 
 
 Search = namedtuple('Search', 'supplied canonical formats')

From 8f432f503e936e3f386964b13e215f693c680d19 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Sun, 30 Sep 2018 03:59:27 +0800
Subject: [PATCH 46/92] News

---
 news/5833.trivial | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 news/5833.trivial

diff --git a/news/5833.trivial b/news/5833.trivial
new file mode 100644
index 000000000..f01ca0023
--- /dev/null
+++ b/news/5833.trivial
@@ -0,0 +1 @@
+Move static and class methods out of HTMLPage for prepare for refactoring.

From caabd1f8b2a48bc6c16fbea58ea5f425a4fad1b6 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Sun, 30 Sep 2018 22:55:51 +0530
Subject: [PATCH 47/92] Fix the vendoring script to cover relative imports

---
 src/pip/_vendor/pkg_resources/py31compat.py | 2 +-
 src/pip/_vendor/pyparsing.py                | 4 ++--
 tasks/vendoring/__init__.py                 | 3 ++-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/pip/_vendor/pkg_resources/py31compat.py b/src/pip/_vendor/pkg_resources/py31compat.py
index a381c424f..a2d3007ce 100644
--- a/src/pip/_vendor/pkg_resources/py31compat.py
+++ b/src/pip/_vendor/pkg_resources/py31compat.py
@@ -2,7 +2,7 @@ import os
 import errno
 import sys
 
-from .extern import six
+from pip._vendor import six
 
 
 def _makedirs_31(path, exist_ok=False):
diff --git a/src/pip/_vendor/pyparsing.py b/src/pip/_vendor/pyparsing.py
index 4aa30ee6b..865152d7c 100644
--- a/src/pip/_vendor/pyparsing.py
+++ b/src/pip/_vendor/pyparsing.py
@@ -37,7 +37,7 @@ C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And}
 (L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to
 L{Literal} expressions)::
 
-    from pyparsing import Word, alphas
+    from pip._vendor.pyparsing import Word, alphas
 
     # define grammar of a greeting
     greet = Word(alphas) + "," + Word(alphas) + "!"
@@ -1594,7 +1594,7 @@ class ParserElement(object):
            after importing pyparsing.
            
            Example::
-               import pyparsing
+               from pip._vendor import pyparsing
                pyparsing.ParserElement.enablePackrat()
         """
         if not ParserElement._packratEnabled:
diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py
index 554c53541..8bcdfcfbd 100644
--- a/tasks/vendoring/__init__.py
+++ b/tasks/vendoring/__init__.py
@@ -75,7 +75,8 @@ def rewrite_file_imports(item, vendored_libs):
     """Rewrite 'import xxx' and 'from xxx import' for vendored_libs"""
     text = item.read_text(encoding='utf-8')
     # Revendor pkg_resources.extern first
-    text = re.sub(r'pkg_resources.extern', r'pip._vendor', text)
+    text = re.sub(r'pkg_resources\.extern', r'pip._vendor', text)
+    text = re.sub(r'from \.extern', r'from pip._vendor', text)
     for lib in vendored_libs:
         text = re.sub(
             r'(\n\s*|^)import %s(\n\s*)' % lib,

From 29b12fa80b10ae0123f6c22617ff8fb56249537f Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Sun, 30 Sep 2018 23:04:46 +0530
Subject: [PATCH 48/92] Fix test to exactly match output

---
 tests/unit/test_configuration.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py
index a5a06e718..3bad21ad2 100644
--- a/tests/unit/test_configuration.py
+++ b/tests/unit/test_configuration.py
@@ -78,9 +78,9 @@ class TestConfigurationLoading(ConfigurationMixin):
             with pytest.raises(ConfigurationError) as err:
                 self.configuration.load()
 
-        assert "section header" in str(err.value)
-        assert "1" in str(err.value)
-        assert config_file in str(err.value)
+        assert "section header" in str(err.value)  # error kind
+        assert "1" in str(err.value)  # line number
+        assert repr(config_file) in str(err.value)  # file name
 
 
 class TestConfigurationPrecedence(ConfigurationMixin):

From 8dcae5ba48bcff30c1eee05b5f17b19bf7562114 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 13:36:10 +0800
Subject: [PATCH 49/92] Warn when dropping index URL value to directory

---
 news/5827.feature          | 1 +
 src/pip/_internal/index.py | 5 +++++
 2 files changed, 6 insertions(+)
 create mode 100644 news/5827.feature

diff --git a/news/5827.feature b/news/5827.feature
new file mode 100644
index 000000000..2ef8d45be
--- /dev/null
+++ b/news/5827.feature
@@ -0,0 +1 @@
+A warning message is emitted when dropping an ``--[extra-]index-url`` value that points to an existing local directory.
diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 9eb8e1207..6d38e1240 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -216,6 +216,11 @@ class PackageFinder(object):
                             sort_path(os.path.join(path, item))
                     elif is_file_url:
                         urls.append(url)
+                    else:
+                        logger.warning(
+                            "Path '{0}' is ignored: "
+                            "it is a directory.".format(path),
+                        )
                 elif os.path.isfile(path):
                     sort_path(path)
                 else:

From 7a17a1f7dd05ac7a613ae659917a728e08a951be Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 15:54:04 +0800
Subject: [PATCH 50/92] Extract VCS scheme check into helper function

---
 src/pip/_internal/index.py | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 8c2f24f1b..87c0b0b42 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -59,6 +59,18 @@ SECURE_ORIGINS = [
 logger = logging.getLogger(__name__)
 
 
+def _match_vcs_scheme(url):
+    """Look for VCS schemes in the URL.
+
+    Returns the matched VCS scheme, or None if there's no match.
+    """
+    from pip._internal.vcs import VcsSupport
+    for scheme in VcsSupport.schemes:
+        if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
+            return scheme
+    return None
+
+
 def _get_content_type(url, session):
     """Get the Content-Type of the given url, using a HEAD request"""
     scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
@@ -89,11 +101,10 @@ def _get_html_page(link, session=None):
     url = url.split('#', 1)[0]
 
     # Check for VCS schemes that do not support lookup as web pages.
-    from pip._internal.vcs import VcsSupport
-    for scheme in VcsSupport.schemes:
-        if url.lower().startswith(scheme) and url[len(scheme)] in '+:':
-            logger.debug('Cannot look at %s URL %s', scheme, link)
-            return None
+    vcs_scheme = _match_vcs_scheme(url)
+    if vcs_scheme is not None:
+        logger.debug('Cannot look at %s URL %s', vcs_scheme, link)
+        return None
 
     try:
         filename = link.filename

From 72779c674e956bcb0bd7df4030215983d2fd999d Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 16:07:00 +0800
Subject: [PATCH 51/92] Extract request part of get_page into helper

Also switch to use exceptions as flow control for VCS and Content-Type
detection. Hopefully this makes the flow much clearer.
---
 src/pip/_internal/index.py | 168 +++++++++++++++++++++----------------
 1 file changed, 96 insertions(+), 72 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 87c0b0b42..23ddd93b0 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -85,6 +85,91 @@ def _get_content_type(url, session):
     return resp.headers.get("Content-Type", "")
 
 
+class _SchemeIsVCS(Exception):
+    pass
+
+
+class _NotHTML(Exception):
+    pass
+
+
+def _get_html_data(url, filename, session):
+    """Access an HTML page with GET to retrieve data.
+
+    Returns a 3-tuple `(content, url, headers)`. `content` should be binary
+    containing the body; `url` is the response URL, and `headers` a mapping
+    containing response headers.
+
+    This consists of five parts:
+
+    1. Check whether the scheme is supported (non-VCS). Raises `_SchemeIsVCS`
+       on failure.
+    2. Check whether the URL looks like an archive. If it does, make a HEAD
+       request to check the Content-Type header is indeed HTML, and raise
+       `_NotHTML` if it's not. Raise HTTP exceptions on network failures.
+    3. If URL scheme is file: and points to a directory, make it point to
+       index.html instead.
+    4. Actually perform the request. Raise HTTP exceptions on network failures.
+    5. Check whether Content-Type header to make sure the thing we got is HTML,
+       and raise `_NotHTML` if it's not.
+    """
+    # Check for VCS schemes that do not support lookup as web pages.
+    vcs_scheme = _match_vcs_scheme(url)
+    if vcs_scheme:
+        raise _SchemeIsVCS(vcs_scheme)
+
+    for bad_ext in ARCHIVE_EXTENSIONS:
+        if filename.endswith(bad_ext):
+            content_type = _get_content_type(url, session=session)
+            if not content_type.lower().startswith('text/html'):
+                raise _NotHTML(content_type)
+
+    logger.debug('Getting page %s', url)
+
+    # Tack index.html onto file:// URLs that point to directories
+    scheme, _, path, _, _, _ = urllib_parse.urlparse(url)
+    if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))):
+        # add trailing slash if not present so urljoin doesn't trim
+        # final segment
+        if not url.endswith('/'):
+            url += '/'
+        url = urllib_parse.urljoin(url, 'index.html')
+        logger.debug(' file: URL is directory, getting %s', url)
+
+    resp = session.get(
+        url,
+        headers={
+            "Accept": "text/html",
+            # We don't want to blindly returned cached data for
+            # /simple/, because authors generally expecting that
+            # twine upload && pip install will function, but if
+            # they've done a pip install in the last ~10 minutes
+            # it won't. Thus by setting this to zero we will not
+            # blindly use any cached data, however the benefit of
+            # using max-age=0 instead of no-cache, is that we will
+            # still support conditional requests, so we will still
+            # minimize traffic sent in cases where the page hasn't
+            # changed at all, we will just always incur the round
+            # trip for the conditional GET now instead of only
+            # once per 10 minutes.
+            # For more information, please see pypa/pip#5670.
+            "Cache-Control": "max-age=0",
+        },
+    )
+    resp.raise_for_status()
+
+    # The check for archives above only works if the url ends with
+    # something that looks like an archive. However that is not a
+    # requirement of an url. Unless we issue a HEAD request on every
+    # url we cannot know ahead of time for sure if something is HTML
+    # or not. However we can check after we've downloaded it.
+    content_type = resp.headers.get('Content-Type', 'unknown')
+    if not content_type.lower().startswith("text/html"):
+        raise _NotHTML(content_type)
+
+    return resp.content, resp.url, resp.headers
+
+
 def _handle_get_page_fail(link, reason, url, meth=None):
     if meth is None:
         meth = logger.debug
@@ -97,81 +182,20 @@ def _get_html_page(link, session=None):
             "_get_html_page() missing 1 required keyword argument: 'session'"
         )
 
-    url = link.url
-    url = url.split('#', 1)[0]
-
-    # Check for VCS schemes that do not support lookup as web pages.
-    vcs_scheme = _match_vcs_scheme(url)
-    if vcs_scheme is not None:
-        logger.debug('Cannot look at %s URL %s', vcs_scheme, link)
-        return None
+    url = link.url.split('#', 1)[0]
 
     try:
-        filename = link.filename
-        for bad_ext in ARCHIVE_EXTENSIONS:
-            if filename.endswith(bad_ext):
-                content_type = _get_content_type(url, session=session)
-                if content_type.lower().startswith('text/html'):
-                    break
-                else:
-                    logger.debug(
-                        'Skipping page %s because of Content-Type: %s',
-                        link,
-                        content_type,
-                    )
-                    return
-
-        logger.debug('Getting page %s', url)
-
-        # Tack index.html onto file:// URLs that point to directories
-        (scheme, netloc, path, params, query, fragment) = \
-            urllib_parse.urlparse(url)
-        if (scheme == 'file' and
-                os.path.isdir(urllib_request.url2pathname(path))):
-            # add trailing slash if not present so urljoin doesn't trim
-            # final segment
-            if not url.endswith('/'):
-                url += '/'
-            url = urllib_parse.urljoin(url, 'index.html')
-            logger.debug(' file: URL is directory, getting %s', url)
-
-        resp = session.get(
-            url,
-            headers={
-                "Accept": "text/html",
-                # We don't want to blindly returned cached data for
-                # /simple/, because authors generally expecting that
-                # twine upload && pip install will function, but if
-                # they've done a pip install in the last ~10 minutes
-                # it won't. Thus by setting this to zero we will not
-                # blindly use any cached data, however the benefit of
-                # using max-age=0 instead of no-cache, is that we will
-                # still support conditional requests, so we will still
-                # minimize traffic sent in cases where the page hasn't
-                # changed at all, we will just always incur the round
-                # trip for the conditional GET now instead of only
-                # once per 10 minutes.
-                # For more information, please see pypa/pip#5670.
-                "Cache-Control": "max-age=0",
-            },
+        content, resp_url, headers = _get_html_data(
+            url, link.filename, session=session,
+        )
+        inst = HTMLPage(content, resp_url, headers)
+    except _SchemeIsVCS as exc:
+        logger.debug('Cannot look at %s URL %s', exc, link)
+    except _NotHTML as exc:
+        logger.debug(
+            'Skipping page %s because of Content-Type: %s',
+            link, exc,
         )
-        resp.raise_for_status()
-
-        # The check for archives above only works if the url ends with
-        # something that looks like an archive. However that is not a
-        # requirement of an url. Unless we issue a HEAD request on every
-        # url we cannot know ahead of time for sure if something is HTML
-        # or not. However we can check after we've downloaded it.
-        content_type = resp.headers.get('Content-Type', 'unknown')
-        if not content_type.lower().startswith("text/html"):
-            logger.debug(
-                'Skipping page %s because of Content-Type: %s',
-                link,
-                content_type,
-            )
-            return
-
-        inst = HTMLPage(resp.content, resp.url, resp.headers)
     except requests.HTTPError as exc:
         _handle_get_page_fail(link, exc, url)
     except SSLError as exc:

From bb6dae6e71190f572926be94ae7f2337fc1d8911 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 16:07:41 +0800
Subject: [PATCH 52/92] Inline PackageFinder._get_page()

---
 src/pip/_internal/index.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 23ddd93b0..02daa5c54 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -707,7 +707,7 @@ class PackageFinder(object):
                 continue
             seen.add(location)
 
-            page = self._get_page(location)
+            page = _get_html_page(location, session=self.session)
             if page is None:
                 continue
 
@@ -824,9 +824,6 @@ class PackageFinder(object):
 
         return InstallationCandidate(search.supplied, version, link)
 
-    def _get_page(self, link):
-        return _get_html_page(link, session=self.session)
-
 
 def egg_info_matches(
         egg_info, search_name, link,

From 025dc45ad1305b4c8156b060b229e01d38b4b5b4 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 16:49:57 +0800
Subject: [PATCH 53/92] Don't need to unpack prematurely

---
 src/pip/_internal/index.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 02daa5c54..41c130760 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -93,12 +93,8 @@ class _NotHTML(Exception):
     pass
 
 
-def _get_html_data(url, filename, session):
-    """Access an HTML page with GET to retrieve data.
-
-    Returns a 3-tuple `(content, url, headers)`. `content` should be binary
-    containing the body; `url` is the response URL, and `headers` a mapping
-    containing response headers.
+def _get_html_response(url, filename, session):
+    """Access an HTML page with GET, and return the response.
 
     This consists of five parts:
 
@@ -167,7 +163,7 @@ def _get_html_data(url, filename, session):
     if not content_type.lower().startswith("text/html"):
         raise _NotHTML(content_type)
 
-    return resp.content, resp.url, resp.headers
+    return resp
 
 
 def _handle_get_page_fail(link, reason, url, meth=None):
@@ -185,10 +181,7 @@ def _get_html_page(link, session=None):
     url = link.url.split('#', 1)[0]
 
     try:
-        content, resp_url, headers = _get_html_data(
-            url, link.filename, session=session,
-        )
-        inst = HTMLPage(content, resp_url, headers)
+        resp = _get_html_response(url, link.filename, session=session)
     except _SchemeIsVCS as exc:
         logger.debug('Cannot look at %s URL %s', exc, link)
     except _NotHTML as exc:
@@ -207,7 +200,7 @@ def _get_html_page(link, session=None):
     except requests.Timeout:
         _handle_get_page_fail(link, "timed out", url)
     else:
-        return inst
+        return HTMLPage(resp.content, resp.url, resp.headers)
 
 
 class PackageFinder(object):

From b36d91c2a6f0990f470741d3bdd87731b7782d52 Mon Sep 17 00:00:00 2001
From: Alexey Popravka <a.popravka@smartweb.com.ua>
Date: Fri, 8 Jun 2018 13:33:08 +0300
Subject: [PATCH 54/92] add failing test for HTMLPage.get_page raising
 RetryError

---
 tests/unit/test_index.py | 24 ++++++++++++++++++++++--
 1 file changed, 22 insertions(+), 2 deletions(-)

diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index bca09a24e..19d8d8ef8 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -1,11 +1,12 @@
 import os.path
 
 import pytest
-from pip._vendor import html5lib
+from mock import Mock
+from pip._vendor import html5lib, requests
 
 from pip._internal.download import PipSession
 from pip._internal.index import (
-    Link, PackageFinder, _determine_base_url, egg_info_matches,
+    HTMLPage, Link, PackageFinder, _determine_base_url, egg_info_matches,
 )
 
 
@@ -190,3 +191,22 @@ def test_egg_info_matches(egg_info, search_name, expected):
     link = None     # Only used for reporting.
     version = egg_info_matches(egg_info, search_name, link)
     assert version == expected
+
+
+@pytest.mark.parametrize(
+    ('exception', 'message'),
+    [
+        (requests.HTTPError, 'Http error'),
+        (requests.exceptions.RetryError, 'Retry error'),
+    ],
+)
+def test_request_retries(exception, message, caplog):
+    link = Link('http://localhost')
+    session = Mock(PipSession)
+    session.get.return_value = resp = Mock()
+    resp.raise_for_status.side_effect = exception(message)
+    assert HTMLPage.get_page(link, session=session) is None
+    assert (
+        'Could not fetch URL http://localhost: %s - skipping' % message
+        in caplog.text
+    )

From 41a9d5e0038aa22daf0c844d465b26c83c7e30a1 Mon Sep 17 00:00:00 2001
From: Alexey Popravka <a.popravka@smartweb.com.ua>
Date: Fri, 8 Jun 2018 13:33:50 +0300
Subject: [PATCH 55/92] handle RetryError in HTMLPage/PackageFinder

---
 src/pip/_internal/index.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 8c2f24f1b..5d0392c3d 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -16,7 +16,7 @@ from pip._vendor.distlib.compat import unescape
 from pip._vendor.packaging import specifiers
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.packaging.version import parse as parse_version
-from pip._vendor.requests.exceptions import SSLError
+from pip._vendor.requests.exceptions import SSLError, RetryError
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
 
@@ -163,6 +163,8 @@ def _get_html_page(link, session=None):
         inst = HTMLPage(resp.content, resp.url, resp.headers)
     except requests.HTTPError as exc:
         _handle_get_page_fail(link, exc, url)
+    except RetryError as exc:
+        _handle_get_page_fail(link, exc, url)
     except SSLError as exc:
         reason = "There was a problem confirming the ssl certificate: "
         reason += str(exc)

From ad640916cf401cb42c7c91f2b7a47b389cdd89b9 Mon Sep 17 00:00:00 2001
From: Alexey Popravka <a.popravka@smartweb.com.ua>
Date: Fri, 8 Jun 2018 13:40:54 +0300
Subject: [PATCH 56/92] add news fragments

---
 news/5270.bugfix | 2 ++
 news/5483.bugfix | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 news/5270.bugfix
 create mode 100644 news/5483.bugfix

diff --git a/news/5270.bugfix b/news/5270.bugfix
new file mode 100644
index 000000000..9db8184c3
--- /dev/null
+++ b/news/5270.bugfix
@@ -0,0 +1,2 @@
+Handle `requests.exceptions.RetryError` raised in `PackageFinder` that was
+causing pip to fail silently when some indexes were unreachable.
diff --git a/news/5483.bugfix b/news/5483.bugfix
new file mode 100644
index 000000000..9db8184c3
--- /dev/null
+++ b/news/5483.bugfix
@@ -0,0 +1,2 @@
+Handle `requests.exceptions.RetryError` raised in `PackageFinder` that was
+causing pip to fail silently when some indexes were unreachable.

From a853ca02b8f0af0badcf2ce47ca97b2059b5a3cc Mon Sep 17 00:00:00 2001
From: Alexey Popravka <a.popravka@smartweb.com.ua>
Date: Mon, 11 Jun 2018 17:34:46 +0300
Subject: [PATCH 57/92] fix lint issues

---
 src/pip/_internal/index.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 5d0392c3d..8cc2465db 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -16,7 +16,7 @@ from pip._vendor.distlib.compat import unescape
 from pip._vendor.packaging import specifiers
 from pip._vendor.packaging.utils import canonicalize_name
 from pip._vendor.packaging.version import parse as parse_version
-from pip._vendor.requests.exceptions import SSLError, RetryError
+from pip._vendor.requests.exceptions import RetryError, SSLError
 from pip._vendor.six.moves.urllib import parse as urllib_parse
 from pip._vendor.six.moves.urllib import request as urllib_request
 

From 5295d8b0a6c50d5c8cc0755bb35a2ec0da139567 Mon Sep 17 00:00:00 2001
From: Alexey Popravka <alexey.popravka@horsedevel.com>
Date: Sat, 29 Sep 2018 22:47:17 +0300
Subject: [PATCH 58/92] test fix: setup logging level

---
 tests/unit/test_index.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index 19d8d8ef8..5acbbafad 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -1,3 +1,4 @@
+import logging
 import os.path
 
 import pytest
@@ -201,6 +202,7 @@ def test_egg_info_matches(egg_info, search_name, expected):
     ],
 )
 def test_request_retries(exception, message, caplog):
+    caplog.set_level(logging.DEBUG)
     link = Link('http://localhost')
     session = Mock(PipSession)
     session.get.return_value = resp = Mock()

From 290e3d592e9661daa9a84acf27476a780d07f798 Mon Sep 17 00:00:00 2001
From: Alexey Popravka <a.popravka@smartweb.com.ua>
Date: Tue, 2 Oct 2018 13:21:13 +0300
Subject: [PATCH 59/92] split test into two

---
 tests/unit/test_index.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index 5acbbafad..ce81a9deb 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -7,7 +7,7 @@ from pip._vendor import html5lib, requests
 
 from pip._internal.download import PipSession
 from pip._internal.index import (
-    HTMLPage, Link, PackageFinder, _determine_base_url, egg_info_matches,
+    Link, PackageFinder, _determine_base_url, _get_html_page, egg_info_matches,
 )
 
 
@@ -194,21 +194,26 @@ def test_egg_info_matches(egg_info, search_name, expected):
     assert version == expected
 
 
-@pytest.mark.parametrize(
-    ('exception', 'message'),
-    [
-        (requests.HTTPError, 'Http error'),
-        (requests.exceptions.RetryError, 'Retry error'),
-    ],
-)
-def test_request_retries(exception, message, caplog):
+def test_request_http_error(caplog):
     caplog.set_level(logging.DEBUG)
     link = Link('http://localhost')
     session = Mock(PipSession)
     session.get.return_value = resp = Mock()
-    resp.raise_for_status.side_effect = exception(message)
-    assert HTMLPage.get_page(link, session=session) is None
+    resp.raise_for_status.side_effect = requests.HTTPError('Http error')
+    assert _get_html_page(link, session=session) is None
     assert (
-        'Could not fetch URL http://localhost: %s - skipping' % message
+        'Could not fetch URL http://localhost: Http error - skipping'
+        in caplog.text
+    )
+
+
+def test_request_retries(caplog):
+    caplog.set_level(logging.DEBUG)
+    link = Link('http://localhost')
+    session = Mock(PipSession)
+    session.get.side_effect = requests.exceptions.RetryError('Retry error')
+    assert _get_html_page(link, session=session) is None
+    assert (
+        'Could not fetch URL http://localhost: Retry error - skipping'
         in caplog.text
     )

From 3350cc45161a3024634cb4e22cdc46bfb80dfa04 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Wed, 3 Oct 2018 20:23:40 +0530
Subject: [PATCH 60/92] bodge: Allow both str and repr to be used for
 configuration file name

This is because Python 2's and Python 3's configparser treat the two
differently and we're reusing that message.
---
 tests/unit/test_configuration.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py
index 3bad21ad2..9fe5bd98a 100644
--- a/tests/unit/test_configuration.py
+++ b/tests/unit/test_configuration.py
@@ -80,7 +80,10 @@ class TestConfigurationLoading(ConfigurationMixin):
 
         assert "section header" in str(err.value)  # error kind
         assert "1" in str(err.value)  # line number
-        assert repr(config_file) in str(err.value)  # file name
+        assert (  # file name
+            config_file in str(err.value) or
+            repr(config_file) in str(err.value)
+        )
 
 
 class TestConfigurationPrecedence(ConfigurationMixin):

From 2c0da26c1d899468a841aa220b9536f2c8f31c95 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Fri, 5 Oct 2018 01:51:22 +0530
Subject: [PATCH 61/92] Generate NEWS for 18.1

---
 NEWS.rst                                      | 47 +++++++++++++++++++
 ...3AE47B-6AC8-490A-B9F5-2F30022A6918.trivial |  0
 ...771DE2-0EE8-4776-84E3-5496E9C9C9CC.trivial |  0
 news/2037.bugfix                              |  1 -
 ...37AE5E-B9C6-4BCE-BD7A-D68DD4529386.trivial |  0
 ...49E95B-9AAF-4885-972B-268BEA9E0E5F.trivial |  0
 news/4187.feature                             |  5 --
 news/5013.feature                             |  2 -
 ...6F9BDF-53AF-4885-A966-6474A27A6D46.trivial |  0
 news/5355.feature                             |  1 -
 news/5375.feature                             |  1 -
 news/5433.bugfix                              |  2 -
 news/5525.feature                             |  1 -
 news/5561.feature                             |  1 -
 news/5624.bugfix                              |  1 -
 news/5644.bugfix                              |  1 -
 news/5670.bugfix                              |  2 -
 news/5679.bugfix                              |  1 -
 news/5748.trivial                             |  1 -
 news/5751.bugfix                              |  1 -
 news/5753.trivial                             |  1 -
 news/5798.feature                             |  1 -
 news/5819.bugfix                              |  1 -
 news/5819.trivial                             |  1 -
 news/5821.trivial                             |  1 -
 news/5826.trivial                             |  1 -
 news/5833.trivial                             |  1 -
 ...ACBF2F-4917-4C85-9F41-32C2F998E8AD.trivial |  0
 ...f41bbe-675e-4afa-a1ba-ac18208429ff.trivial |  0
 ...BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial |  0
 ...E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial |  0
 ...C94B14-B963-4645-BB9C-D63C6D854294.trivial |  0
 ...A012C5-DBB9-4909-A724-3E375CBF4D3A.trivial |  0
 ...C51B95-AFCC-4955-A416-5435650C8D15.trivial |  0
 ...B8EE86-500E-4C27-BC8E-136550E30A62.trivial |  0
 ...EFD11F-E9D5-4EFD-901F-D11BAF89808A.trivial |  0
 ...5B0FA7-1A6E-47F6-AF80-E26B679352A0.trivial |  0
 ...18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial |  0
 ...11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial |  0
 ...B6B89D-2437-428E-A94B-56A3210F1C6F.trivial |  0
 news/certifi.vendor                           |  1 -
 news/packaging.vendor                         |  1 -
 news/pep517.vendor                            |  1 -
 news/pytoml.vendor                            |  1 -
 news/setuptools.vendor                        |  1 -
 news/user_guide_fix_requirements_file_ref.doc |  1 -
 46 files changed, 47 insertions(+), 34 deletions(-)
 delete mode 100644 news/143AE47B-6AC8-490A-B9F5-2F30022A6918.trivial
 delete mode 100644 news/15771DE2-0EE8-4776-84E3-5496E9C9C9CC.trivial
 delete mode 100644 news/2037.bugfix
 delete mode 100644 news/3037AE5E-B9C6-4BCE-BD7A-D68DD4529386.trivial
 delete mode 100644 news/3249E95B-9AAF-4885-972B-268BEA9E0E5F.trivial
 delete mode 100644 news/4187.feature
 delete mode 100644 news/5013.feature
 delete mode 100644 news/516F9BDF-53AF-4885-A966-6474A27A6D46.trivial
 delete mode 100644 news/5355.feature
 delete mode 100644 news/5375.feature
 delete mode 100644 news/5433.bugfix
 delete mode 100644 news/5525.feature
 delete mode 100644 news/5561.feature
 delete mode 100644 news/5624.bugfix
 delete mode 100644 news/5644.bugfix
 delete mode 100644 news/5670.bugfix
 delete mode 100644 news/5679.bugfix
 delete mode 100644 news/5748.trivial
 delete mode 100644 news/5751.bugfix
 delete mode 100644 news/5753.trivial
 delete mode 100644 news/5798.feature
 delete mode 100644 news/5819.bugfix
 delete mode 100644 news/5819.trivial
 delete mode 100644 news/5821.trivial
 delete mode 100644 news/5826.trivial
 delete mode 100644 news/5833.trivial
 delete mode 100644 news/5CACBF2F-4917-4C85-9F41-32C2F998E8AD.trivial
 delete mode 100644 news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial
 delete mode 100644 news/88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial
 delete mode 100644 news/88E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial
 delete mode 100644 news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial
 delete mode 100644 news/8EA012C5-DBB9-4909-A724-3E375CBF4D3A.trivial
 delete mode 100644 news/8EC51B95-AFCC-4955-A416-5435650C8D15.trivial
 delete mode 100644 news/94B8EE86-500E-4C27-BC8E-136550E30A62.trivial
 delete mode 100644 news/ACEFD11F-E9D5-4EFD-901F-D11BAF89808A.trivial
 delete mode 100644 news/BE5B0FA7-1A6E-47F6-AF80-E26B679352A0.trivial
 delete mode 100644 news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial
 delete mode 100644 news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial
 delete mode 100644 news/D7B6B89D-2437-428E-A94B-56A3210F1C6F.trivial
 delete mode 100644 news/certifi.vendor
 delete mode 100644 news/packaging.vendor
 delete mode 100644 news/pep517.vendor
 delete mode 100644 news/pytoml.vendor
 delete mode 100644 news/setuptools.vendor
 delete mode 100644 news/user_guide_fix_requirements_file_ref.doc

diff --git a/NEWS.rst b/NEWS.rst
index 7182ea1c5..ceaff88c1 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -7,6 +7,53 @@
 
 .. towncrier release notes start
 
+18.1 (2018-10-05)
+=================
+
+Features
+--------
+
+- Allow PEP 508 URL requirements to be used as dependencies.
+
+  As a security measure, pip will raise an exception when installing packages from
+  PyPI if those packages depend on packages not also hosted on PyPI.
+  In the future, PyPI will block uploading packages with such external URL dependencies directly. (`#4187 <https://github.com/pypa/pip/issues/4187>`_)
+- Upgrade pyparsing to 2.2.1. (`#5013 <https://github.com/pypa/pip/issues/5013>`_)
+- Allows dist options (--abi, --python-version, --platform, --implementation) when installing with --target (`#5355 <https://github.com/pypa/pip/issues/5355>`_)
+- Support passing ``svn+ssh`` URLs with a username to ``pip install -e``. (`#5375 <https://github.com/pypa/pip/issues/5375>`_)
+- pip now ensures that the RECORD file is sorted when installing from a wheel file. (`#5525 <https://github.com/pypa/pip/issues/5525>`_)
+- Add support for Python 3.7. (`#5561 <https://github.com/pypa/pip/issues/5561>`_)
+- Malformed configuration files now show helpful error messages, instead of tracebacks. (`#5798 <https://github.com/pypa/pip/issues/5798>`_)
+
+Bug Fixes
+---------
+
+- Checkout the correct branch when doing an editable Git install. (`#2037 <https://github.com/pypa/pip/issues/2037>`_)
+- Run self-version-check only on commands that may access the index, instead of
+  trying on every run and failing to do so due to missing options. (`#5433 <https://github.com/pypa/pip/issues/5433>`_)
+- Allow a Git ref to be installed over an existing installation. (`#5624 <https://github.com/pypa/pip/issues/5624>`_)
+- Show a better error message when a configuration option has an invalid value. (`#5644 <https://github.com/pypa/pip/issues/5644>`_)
+- Always revalidate cached simple API pages instead of blindly caching them for up to 10
+  minutes. (`#5670 <https://github.com/pypa/pip/issues/5670>`_)
+- Avoid caching self-version-check information when cache is disabled. (`#5679 <https://github.com/pypa/pip/issues/5679>`_)
+- Avoid traceback printing on autocomplete after flags in the CLI. (`#5751 <https://github.com/pypa/pip/issues/5751>`_)
+- Fix incorrect parsing of egg names if pip needs to guess the package name. (`#5819 <https://github.com/pypa/pip/issues/5819>`_)
+
+Vendored Libraries
+------------------
+
+- Upgrade certifi to 2018.8.24
+- Upgrade packaging to 18.0
+- Add pep517 version 0.2
+- Upgrade pytoml to 0.1.19
+- Upgrade pkg_resources to 40.4.3 (via setuptools)
+
+Improved Documentation
+----------------------
+
+- Fix "Requirements Files" reference in User Guide (`#user_guide_fix_requirements_file_ref <https://github.com/pypa/pip/issues/user_guide_fix_requirements_file_ref>`_)
+
+
 18.0 (2018-07-22)
 =================
 
diff --git a/news/143AE47B-6AC8-490A-B9F5-2F30022A6918.trivial b/news/143AE47B-6AC8-490A-B9F5-2F30022A6918.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/15771DE2-0EE8-4776-84E3-5496E9C9C9CC.trivial b/news/15771DE2-0EE8-4776-84E3-5496E9C9C9CC.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/2037.bugfix b/news/2037.bugfix
deleted file mode 100644
index aca18b07b..000000000
--- a/news/2037.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Checkout the correct branch when doing an editable Git install.
\ No newline at end of file
diff --git a/news/3037AE5E-B9C6-4BCE-BD7A-D68DD4529386.trivial b/news/3037AE5E-B9C6-4BCE-BD7A-D68DD4529386.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/3249E95B-9AAF-4885-972B-268BEA9E0E5F.trivial b/news/3249E95B-9AAF-4885-972B-268BEA9E0E5F.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/4187.feature b/news/4187.feature
deleted file mode 100644
index 03a874bc3..000000000
--- a/news/4187.feature
+++ /dev/null
@@ -1,5 +0,0 @@
-Allow PEP 508 URL requirements to be used as dependencies.
-
-As a security measure, pip will raise an exception when installing packages from
-PyPI if those packages depend on packages not also hosted on PyPI.
-In the future, PyPI will block uploading packages with such external URL dependencies directly.
diff --git a/news/5013.feature b/news/5013.feature
deleted file mode 100644
index 2f3771fc1..000000000
--- a/news/5013.feature
+++ /dev/null
@@ -1,2 +0,0 @@
-Upgrade pyparsing to 2.2.1.
-
diff --git a/news/516F9BDF-53AF-4885-A966-6474A27A6D46.trivial b/news/516F9BDF-53AF-4885-A966-6474A27A6D46.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/5355.feature b/news/5355.feature
deleted file mode 100644
index fc2e108f2..000000000
--- a/news/5355.feature
+++ /dev/null
@@ -1 +0,0 @@
-Allows dist options (--abi, --python-version, --platform, --implementation) when installing with --target
diff --git a/news/5375.feature b/news/5375.feature
deleted file mode 100644
index 5e6460d3d..000000000
--- a/news/5375.feature
+++ /dev/null
@@ -1 +0,0 @@
-Support passing ``svn+ssh`` URLs with a username to ``pip install -e``.
diff --git a/news/5433.bugfix b/news/5433.bugfix
deleted file mode 100644
index bed41fce8..000000000
--- a/news/5433.bugfix
+++ /dev/null
@@ -1,2 +0,0 @@
-Run self-version-check only on commands that may access the index, instead of
-trying on every run and failing to do so due to missing options.
diff --git a/news/5525.feature b/news/5525.feature
deleted file mode 100644
index 1af8be6f9..000000000
--- a/news/5525.feature
+++ /dev/null
@@ -1 +0,0 @@
-pip now ensures that the RECORD file is sorted when installing from a wheel file.
diff --git a/news/5561.feature b/news/5561.feature
deleted file mode 100644
index 30e37cb99..000000000
--- a/news/5561.feature
+++ /dev/null
@@ -1 +0,0 @@
-Add support for Python 3.7.
diff --git a/news/5624.bugfix b/news/5624.bugfix
deleted file mode 100644
index fc6b58fe8..000000000
--- a/news/5624.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Allow a Git ref to be installed over an existing installation.
diff --git a/news/5644.bugfix b/news/5644.bugfix
deleted file mode 100644
index 39b720772..000000000
--- a/news/5644.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Show a better error message when a configuration option has an invalid value.
\ No newline at end of file
diff --git a/news/5670.bugfix b/news/5670.bugfix
deleted file mode 100644
index 9ffac6638..000000000
--- a/news/5670.bugfix
+++ /dev/null
@@ -1,2 +0,0 @@
-Always revalidate cached simple API pages instead of blindly caching them for up to 10
-minutes.
diff --git a/news/5679.bugfix b/news/5679.bugfix
deleted file mode 100644
index 250166925..000000000
--- a/news/5679.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Avoid caching self-version-check information when cache is disabled.
diff --git a/news/5748.trivial b/news/5748.trivial
deleted file mode 100644
index 6ca14b372..000000000
--- a/news/5748.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Remove the unmatched bracket in the --no-clean option's help text.
\ No newline at end of file
diff --git a/news/5751.bugfix b/news/5751.bugfix
deleted file mode 100644
index 36926c9a6..000000000
--- a/news/5751.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Avoid traceback printing on autocomplete after flags in the CLI.
diff --git a/news/5753.trivial b/news/5753.trivial
deleted file mode 100644
index 4e6860c24..000000000
--- a/news/5753.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Fix links to NEWS entry guidelines.
diff --git a/news/5798.feature b/news/5798.feature
deleted file mode 100644
index 10d598594..000000000
--- a/news/5798.feature
+++ /dev/null
@@ -1 +0,0 @@
-Malformed configuration files now show helpful error messages, instead of tracebacks.
diff --git a/news/5819.bugfix b/news/5819.bugfix
deleted file mode 100644
index 9fea81279..000000000
--- a/news/5819.bugfix
+++ /dev/null
@@ -1 +0,0 @@
-Fix incorrect parsing of egg names if pip needs to guess the package name.
diff --git a/news/5819.trivial b/news/5819.trivial
deleted file mode 100644
index 9ffde7cb6..000000000
--- a/news/5819.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Simplify always-true conditions in ``HTMLPage.get_page()``.
diff --git a/news/5821.trivial b/news/5821.trivial
deleted file mode 100644
index 25c6284e8..000000000
--- a/news/5821.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Add unit tests for egg_info_matches.
diff --git a/news/5826.trivial b/news/5826.trivial
deleted file mode 100644
index 6f4ec80d5..000000000
--- a/news/5826.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Refactor HTMLPage to reduce attributes on it.
diff --git a/news/5833.trivial b/news/5833.trivial
deleted file mode 100644
index f01ca0023..000000000
--- a/news/5833.trivial
+++ /dev/null
@@ -1 +0,0 @@
-Move static and class methods out of HTMLPage for prepare for refactoring.
diff --git a/news/5CACBF2F-4917-4C85-9F41-32C2F998E8AD.trivial b/news/5CACBF2F-4917-4C85-9F41-32C2F998E8AD.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial b/news/62f41bbe-675e-4afa-a1ba-ac18208429ff.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial b/news/88BB743A-BE4E-4012-A714-9FE3B36BD30A.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/88E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial b/news/88E800D4-2360-48F6-BD1D-9C6BAB0D059E.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial b/news/8CC94B14-B963-4645-BB9C-D63C6D854294.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/8EA012C5-DBB9-4909-A724-3E375CBF4D3A.trivial b/news/8EA012C5-DBB9-4909-A724-3E375CBF4D3A.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/8EC51B95-AFCC-4955-A416-5435650C8D15.trivial b/news/8EC51B95-AFCC-4955-A416-5435650C8D15.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/94B8EE86-500E-4C27-BC8E-136550E30A62.trivial b/news/94B8EE86-500E-4C27-BC8E-136550E30A62.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/ACEFD11F-E9D5-4EFD-901F-D11BAF89808A.trivial b/news/ACEFD11F-E9D5-4EFD-901F-D11BAF89808A.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/BE5B0FA7-1A6E-47F6-AF80-E26B679352A0.trivial b/news/BE5B0FA7-1A6E-47F6-AF80-E26B679352A0.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial b/news/CD18D0D2-2C80-43CD-9A26-ED2535E2E840.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial b/news/CF11FF8C-C348-4523-9DFC-A7FEBCADF154.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/D7B6B89D-2437-428E-A94B-56A3210F1C6F.trivial b/news/D7B6B89D-2437-428E-A94B-56A3210F1C6F.trivial
deleted file mode 100644
index e69de29bb..000000000
diff --git a/news/certifi.vendor b/news/certifi.vendor
deleted file mode 100644
index eb401fd67..000000000
--- a/news/certifi.vendor
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade certifi to 2018.8.24
diff --git a/news/packaging.vendor b/news/packaging.vendor
deleted file mode 100644
index ca83b8d34..000000000
--- a/news/packaging.vendor
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade packaging to 18.0
diff --git a/news/pep517.vendor b/news/pep517.vendor
deleted file mode 100644
index 53f5c85d4..000000000
--- a/news/pep517.vendor
+++ /dev/null
@@ -1 +0,0 @@
-Add pep517 version 0.2
diff --git a/news/pytoml.vendor b/news/pytoml.vendor
deleted file mode 100644
index f1fab347b..000000000
--- a/news/pytoml.vendor
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade pytoml to 0.1.19
diff --git a/news/setuptools.vendor b/news/setuptools.vendor
deleted file mode 100644
index 2f3815833..000000000
--- a/news/setuptools.vendor
+++ /dev/null
@@ -1 +0,0 @@
-Upgrade pkg_resources to 40.4.3 (via setuptools)
diff --git a/news/user_guide_fix_requirements_file_ref.doc b/news/user_guide_fix_requirements_file_ref.doc
deleted file mode 100644
index 07112d375..000000000
--- a/news/user_guide_fix_requirements_file_ref.doc
+++ /dev/null
@@ -1 +0,0 @@
-Fix "Requirements Files" reference in User Guide

From d80c90240b874af72f30f253375314b4ac4e96a0 Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Fri, 5 Oct 2018 01:52:49 +0530
Subject: [PATCH 62/92] Generate AUTHORS

---
 AUTHORS.txt | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/AUTHORS.txt b/AUTHORS.txt
index f02621e55..e845ac715 100644
--- a/AUTHORS.txt
+++ b/AUTHORS.txt
@@ -12,6 +12,7 @@ Alexandre Conrad <alexandre.conrad@gmail.com>
 Alli <alzeih@users.noreply.github.com>
 Anatoly Techtonik <techtonik@gmail.com>
 Andrei Geacar <andrei.geacar@gmail.com>
+Andrew Gaul <andrew@gaul.org>
 Andrey Bulgakov <mail@andreiko.ru>
 Andrés Delfino <34587441+andresdelfino@users.noreply.github.com>
 Andrés Delfino <adelfino@gmail.com>
@@ -36,6 +37,8 @@ Ashley Manton <ajd.manton@googlemail.com>
 Atsushi Odagiri <aodagx@gmail.com>
 Avner Cohen <israbirding@gmail.com>
 Baptiste Mispelon <bmispelon@gmail.com>
+Barney Gale <barney.gale@gmail.com>
+barneygale <barney.gale@gmail.com>
 Bartek Ogryczak <b.ogryczak@gmail.com>
 Bastian Venthur <mail@venthur.de>
 Ben Darnell <ben@bendarnell.com>
@@ -46,6 +49,7 @@ Benjamin VanEvery <ben@simondata.com>
 Benoit Pierre <benoit.pierre@gmail.com>
 Berker Peksag <berker.peksag@gmail.com>
 Bernardo B. Marques <bernardo.fire@gmail.com>
+Bernhard M. Wiedemann <bwiedemann@suse.de>
 Bogdan Opanchuk <bogdan@opanchuk.net>
 Brad Erickson <eosrei@gmail.com>
 Bradley Ayers <bradley.ayers@gmail.com>
@@ -55,6 +59,7 @@ Brian Rosner <brosner@gmail.com>
 BrownTruck <BrownTruck@users.noreply.github.com>
 Bruno Oliveira <nicoddemus@gmail.com>
 Bruno Renié <brutasse@gmail.com>
+Bstrdsmkr <bstrdsmkr@gmail.com>
 Buck Golemon <buck@yelp.com>
 burrows <burrows@preveil.com>
 Bussonnier Matthias <bussonniermatthias@gmail.com>
@@ -157,6 +162,7 @@ Herbert Pfennig <herbert@albinen.com>
 Hsiaoming Yang <lepture@me.com>
 Hugo <hugovk@users.noreply.github.com>
 Hugo Lopes Tavares <hltbra@gmail.com>
+hugovk <hugovk@users.noreply.github.com>
 Hynek Schlawack <hs@ox.cx>
 Ian Bicking <ianb@colorstudy.com>
 Ian Cordasco <graffatcolmingov@gmail.com>
@@ -182,6 +188,7 @@ Jannis Leidel <jannis@leidel.info>
 jarondl <me@jarondl.net>
 Jason R. Coombs <jaraco@jaraco.com>
 Jay Graves <jay@skabber.com>
+Jean-Christophe Fillion-Robin <jchris.fillionr@kitware.com>
 Jeff Barber <jbarber@computer.org>
 Jeff Dairiki <dairiki@dairiki.org>
 Jeremy Stanley <fungi@yuggoth.org>
@@ -194,6 +201,7 @@ Jon Dufresne <jon.dufresne@gmail.com>
 Jon Parise <jon@indelible.org>
 Jon Wayne Parrott <jjramone13@gmail.com>
 Jonas Nockert <jonasnockert@gmail.com>
+Jonathan Herbert <foohyfooh@gmail.com>
 Joost Molenaar <j.j.molenaar@gmail.com>
 Jorge Niedbalski <niedbalski@gmail.com>
 Joseph Long <jdl@fastmail.fm>
@@ -219,10 +227,12 @@ kpinc <kop@meme.com>
 Kumar McMillan <kumar.mcmillan@gmail.com>
 Kyle Persohn <kyle.persohn@gmail.com>
 Laurent Bristiel <laurent@bristiel.com>
+Laurie Opperman <laurie@sitesee.com.au>
 Leon Sasson <leonsassonha@gmail.com>
 Lev Givon <lev@columbia.edu>
 Lincoln de Sousa <lincoln@comum.org>
 Lipis <lipiridis@gmail.com>
+Loren Carvalho <lcarvalho@linkedin.com>
 Lucas Cimon <lucas.cimon@gmail.com>
 Ludovic Gasc <gmludo@gmail.com>
 Luke Macken <lmacken@redhat.com>
@@ -259,6 +269,7 @@ Michael E. Karpeles <michael.karpeles@gmail.com>
 Michael Klich <michal@michalklich.com>
 Michael Williamson <mike@zwobble.org>
 michaelpacer <michaelpacer@gmail.com>
+Mickaël Schoentgen <mschoentgen@nuxeo.com>
 Miguel Araujo Perez <miguel.araujo.perez@gmail.com>
 Mihir Singh <git.service@mihirsingh.com>
 Min RK <benjaminrk@gmail.com>
@@ -272,6 +283,7 @@ Nehal J Wani <nehaljw.kkd1@gmail.com>
 Nick Coghlan <ncoghlan@gmail.com>
 Nick Stenning <nick@whiteink.com>
 Nikhil Benesch <nikhil.benesch@gmail.com>
+Nitesh Sharma <nbsharma@outlook.com>
 Nowell Strite <nowell@strite.org>
 nvdv <modestdev@gmail.com>
 Ofekmeister <ofekmeister@gmail.com>
@@ -380,6 +392,7 @@ Tomer Chachamu <tomer.chachamu@gmail.com>
 Tony Zhaocheng Tan <tony@tonytan.io>
 Toshio Kuratomi <toshio@fedoraproject.org>
 Travis Swicegood <development@domain51.com>
+Tzu-ping Chung <uranusjr@gmail.com>
 Valentin Haenel <valentin.haenel@gmx.de>
 Victor Stinner <victor.stinner@gmail.com>
 Viktor Szépe <viktor@szepe.net>

From 40d1fe382c440c78853ee4b6c508c213e5382dcd Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Fri, 5 Oct 2018 01:53:28 +0530
Subject: [PATCH 63/92] Bump version to 18.1

---
 src/pip/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/__init__.py b/src/pip/__init__.py
index 2e8c4e43d..ae265fa7d 100644
--- a/src/pip/__init__.py
+++ b/src/pip/__init__.py
@@ -1 +1 @@
-__version__ = "18.1.dev0"
+__version__ = "18.1"

From c86b74279a5d9837903712b612e7dbb74c5a79dc Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Fri, 5 Oct 2018 17:16:33 +0530
Subject: [PATCH 64/92] Bump version to reopen for development

---
 src/pip/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/pip/__init__.py b/src/pip/__init__.py
index ae265fa7d..98e17e8d3 100644
--- a/src/pip/__init__.py
+++ b/src/pip/__init__.py
@@ -1 +1 @@
-__version__ = "18.1"
+__version__ = "19.0.dev0"

From c9e47c9903b1e1a6e2f0d7d59eb1592bce755324 Mon Sep 17 00:00:00 2001
From: Benoit Pierre <benoit.pierre@gmail.com>
Date: Mon, 1 Oct 2018 05:22:08 +0200
Subject: [PATCH 65/92] tox: fix environment setup

Ensure a stable version of pip is used for installing
dependencies, and not the version about to be tested.
---
 tools/tox_pip.py | 28 ++++++++++++++++++++++++++++
 tox.ini          |  8 +++++++-
 2 files changed, 35 insertions(+), 1 deletion(-)
 create mode 100644 tools/tox_pip.py

diff --git a/tools/tox_pip.py b/tools/tox_pip.py
new file mode 100644
index 000000000..8d91fbf56
--- /dev/null
+++ b/tools/tox_pip.py
@@ -0,0 +1,28 @@
+import os
+import shutil
+import subprocess
+import sys
+from glob import glob
+
+VIRTUAL_ENV = os.environ['VIRTUAL_ENV']
+TOX_PIP_DIR = os.path.join(VIRTUAL_ENV, 'pip')
+
+
+def pip(args):
+    # First things first, get a recent (stable) version of pip.
+    if not os.path.exists(TOX_PIP_DIR):
+        subprocess.check_call([sys.executable, '-m', 'pip',
+                               '--disable-pip-version-check',
+                               'install', '-t', TOX_PIP_DIR,
+                               'pip'])
+        shutil.rmtree(glob(os.path.join(TOX_PIP_DIR, 'pip-*.dist-info'))[0])
+    # And use that version.
+    pypath = os.environ.get('PYTHONPATH')
+    pypath = pypath.split(os.pathsep) if pypath is not None else []
+    pypath.insert(0, TOX_PIP_DIR)
+    os.environ['PYTHONPATH'] = os.pathsep.join(pypath)
+    subprocess.check_call([sys.executable, '-m', 'pip'] + args)
+
+
+if __name__ == '__main__':
+    pip(sys.argv[1:])
diff --git a/tox.ini b/tox.ini
index 4a3272955..c48d695de 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,6 +3,11 @@ envlist =
     docs, packaging, lint-py2, lint-py3, mypy,
     py27, py34, py35, py36, py37, py38, pypy, pypy3
 
+[helpers]
+# Wrapper for calls to pip that make sure the version being used is the
+# original virtualenv (stable) version, and not the code being tested.
+pip = python {toxinidir}/tools/tox_pip.py
+
 [testenv]
 passenv = CI GIT_SSL_CAINFO
 setenv =
@@ -11,7 +16,8 @@ setenv =
     LC_CTYPE = en_US.UTF-8
 deps = -r{toxinidir}/tools/tests-requirements.txt
 commands = pytest --timeout 300 []
-install_command = python -m pip install {opts} {packages}
+install_command = {[helpers]pip} install {opts} {packages}
+list_dependencies_command = {[helpers]pip} freeze --all
 usedevelop = True
 
 [testenv:coverage-py3]

From 0a58159571a47d4839216ff96487bd23aa427da1 Mon Sep 17 00:00:00 2001
From: Benoit Pierre <benoit.pierre@gmail.com>
Date: Fri, 5 Oct 2018 16:52:06 +0200
Subject: [PATCH 66/92] drop workaround for Python<=2.5

---
 src/pip/_internal/utils/misc.py    |   4 ----
 tests/data/packages/README.txt     |   4 ----
 tests/data/packages/paxpkg.tar.bz2 | Bin 1094 -> 0 bytes
 tests/functional/test_install.py   |   7 -------
 4 files changed, 15 deletions(-)
 delete mode 100644 tests/data/packages/paxpkg.tar.bz2

diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py
index 84a421fe4..6998e16f0 100644
--- a/src/pip/_internal/utils/misc.py
+++ b/src/pip/_internal/utils/misc.py
@@ -520,15 +520,11 @@ def untar_file(filename, location):
         mode = 'r:*'
     tar = tarfile.open(filename, mode)
     try:
-        # note: python<=2.5 doesn't seem to know about pax headers, filter them
         leading = has_leading_dir([
             member.name for member in tar.getmembers()
-            if member.name != 'pax_global_header'
         ])
         for member in tar.getmembers():
             fn = member.name
-            if fn == 'pax_global_header':
-                continue
             if leading:
                 fn = split_leading_dir(fn)[1]
             path = os.path.join(location, fn)
diff --git a/tests/data/packages/README.txt b/tests/data/packages/README.txt
index b2f61b513..a7fccc5bb 100644
--- a/tests/data/packages/README.txt
+++ b/tests/data/packages/README.txt
@@ -59,10 +59,6 @@ parent/child-0.1.tar.gz
 The parent-0.1.tar.gz and child-0.1.tar.gz packages are used by
 test_uninstall:test_uninstall_overlapping_package.
 
-paxpkg.tar.bz2
---------------
-tar with pax headers
-
 pkgwithmpkg-1.0.tar.gz; pkgwithmpkg-1.0-py2.7-macosx10.7.mpkg.zip
 -----------------------------------------------------------------
 used for osx test case (tests.test_finder:test_no_mpkg)
diff --git a/tests/data/packages/paxpkg.tar.bz2 b/tests/data/packages/paxpkg.tar.bz2
deleted file mode 100644
index d4fe6f4a96ed983d459dbc61d414454689e03ebc..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1094
zcmV-M1iAY{T4*^jL0KkKS<w$%Z2$#6f0o~n06<#z|MLIu-;V$9|L_C=05AYR0Ekco
z9xiKcInGnNJ1``n8c5Le(s~dLsKfvUKp7eU0qOt-P!nkyqclT7pa2a320@?z001R4
zNYr|oquP<_p{9}QXwU=oPf4H+00Z?+H1vQ2jF={dLn9+5hJZ8~7(g@{G{|Ba03?t^
z!%&zqL8SE5dWWcK=q84Z0004yH2R2LEWIto(G0(DY#wcc*fXZB3UHaW2{V#Ofre=$
z02`+7Q#~X!RD}d+V3K%*^egCA6VU13OPXtrrV(-#Nebm-gp$e#fP|TnhCCJijJ@fn
zv_qR|%t{6em)4F3U3rvu*gl;fcRptHK<dI3WX#O$F>T1Xwiz0)Sy~!M+??^tQ`!*P
z=gKNs0mEd&m|a?)7<YW~J2^NyQ^!o&5b-*l^gujHnfCTYdZDt%J_yB=pyG}JRxcT5
ztO!(2T4K&TiW3+@-;5>4BUij}`7jz8GdLjv5x&mOZ7^MR*u;=;<Ks9ILECqmI-Tpi
zJhtuk_DY3$35WoxEQnm(e6F@JNpKl$r3!>7lk+)skeY-h9E8l(vDu8$2}1+qL2yj=
zIxX^{ux}cIGQ+N<R0xySi-`sxo2s2e81q%EXfT>(t((!I^<LS-oI#}#l6%MtU|isZ
zxr>y<9BJNaJy0s)nzAmt;VlzrQ*CzGHXUnN;+UCPR95)6>f5c+Gfi|BUoiKM{?nG;
zp=sbNNhFd>azpt0WJi>x<q1MvJ|gpop<U$r<`Wp~%~a0-_(+max|C(X&{A;<jz{p~
z-{-tcMqO;pvGZlo6B*=M!&B_T^+H4{I9NJuhzq+;he<T|2e36s;F`@Ul)@A#++rHi
zt(!IR@0_NvBXU=C;!`}6oHYNT=IIDgZ5u~D4Vb7Ht{C??T?n(MvJE#z6fGY^Xf<KK
zLAEYJ?S)ptJZ3>ezA{RzdnbsEF5Zg3Ku{=ZPjf}nDM<?ZJHl7rthC|JzboTC!>mGK
z3Cvxp4iVfE%?6Ssu-T4$k}WDU_X6g5jpQj{Vv;?yRExmZLs-D584Q%9VgM(G`6Wb_
zi{WM}u8#o(978Gys9A-Hb0Y}lfnqnJq2k#}$7U_j6^KF^q?qRr6@w3M@X5f<XqFrZ
zxg(zR>r*8@S%fAvd)cNB*z{LjViw6zVoWf@6sHgyFl?)HX~a$(gyn-AYjaP%6XJa@
z);#`u|Eh(lOfik9t7HU&B~}os2GS==I))|CYDRO7hWW7AnWa8NSejiI>u)@g=7rFZ
zX-ZRDrn9yT1Zl~aGKkJoVFtKs76h?nVYfnvsW>r+T%cAJ%>s%@)J8+Z6bnRakZD^@
zuw+;1rs^D+U4?WFz%m$5CrRq84P7W+gBeg+#WU~NO!9y@G6m&e$;H5Fmg!ERB){VB
MNT&)C8X@bApnJ9Se*gdg

diff --git a/tests/functional/test_install.py b/tests/functional/test_install.py
index 1be8e10e3..3cbc2fe96 100644
--- a/tests/functional/test_install.py
+++ b/tests/functional/test_install.py
@@ -522,13 +522,6 @@ def test_install_global_option(script):
     assert '0.1\n' in result.stdout
 
 
-def test_install_with_pax_header(script, data):
-    """
-    test installing from a tarball with pax header for python<2.6
-    """
-    script.pip('install', 'paxpkg.tar.bz2', cwd=data.packages)
-
-
 def test_install_with_hacked_egg_info(script, data):
     """
     test installing a package which defines its own egg_info class

From 9e26002fcde8a2f16ce92295a44896173f53422d Mon Sep 17 00:00:00 2001
From: Pradyun Gedam <pradyunsg@gmail.com>
Date: Mon, 8 Oct 2018 20:46:17 +0530
Subject: [PATCH 67/92] Bump deprecation version to unlock workflow

---
 src/pip/_internal/index.py             | 2 +-
 src/pip/_internal/operations/freeze.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 8c2f24f1b..29d01c486 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -293,7 +293,7 @@ class PackageFinder(object):
                 "Dependency Links processing has been deprecated and will be "
                 "removed in a future release.",
                 replacement="PEP 508 URL dependencies",
-                gone_in="18.2",
+                gone_in="19.0",
                 issue=4187,
             )
             self.dependency_links.extend(links)
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index beb2feb87..5fdb245bc 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -227,7 +227,7 @@ class FrozenRequirement(object):
                         "SVN editable detection based on dependency links "
                         "will be dropped in the future.",
                         replacement=None,
-                        gone_in="18.2",
+                        gone_in="19.0",
                         issue=4187,
                     )
                     comments.append(

From 16333ef1da6abba9673e33397140e830566a509b Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 02:33:49 -0700
Subject: [PATCH 68/92] Add first guard clause.

---
 ...4EEB42-0067-4C7E-B02A-01FAE49D1D98.trivial |  0
 src/pip/_internal/operations/freeze.py        | 88 ++++++++++---------
 2 files changed, 45 insertions(+), 43 deletions(-)
 create mode 100644 news/E74EEB42-0067-4C7E-B02A-01FAE49D1D98.trivial

diff --git a/news/E74EEB42-0067-4C7E-B02A-01FAE49D1D98.trivial b/news/E74EEB42-0067-4C7E-B02A-01FAE49D1D98.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 5fdb245bc..d14641ce2 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -197,50 +197,52 @@ class FrozenRequirement(object):
                 )
                 req = dist.as_requirement()
                 editable = False
-        else:
-            editable = False
-            req = dist.as_requirement()
-            specs = req.specs
-            assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
-                'Expected 1 spec with == or ===; specs = %r; dist = %r' % \
-                (specs, dist)
-            version = specs[0][1]
-            ver_match = cls._rev_re.search(version)
-            date_match = cls._date_re.search(version)
-            if ver_match or date_match:
-                svn_backend = vcs.get_backend('svn')
-                if svn_backend:
-                    svn_location = svn_backend().get_location(
-                        dist,
-                        dependency_links,
-                    )
-                if not svn_location:
-                    logger.warning(
-                        'Warning: cannot find svn location for %s', req,
-                    )
-                    comments.append(
-                        '## FIXME: could not find svn URL in dependency_links '
-                        'for this package:'
-                    )
+
+            return (req, editable, comments)
+
+        editable = False
+        req = dist.as_requirement()
+        specs = req.specs
+        assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
+            'Expected 1 spec with == or ===; specs = %r; dist = %r' % \
+            (specs, dist)
+        version = specs[0][1]
+        ver_match = cls._rev_re.search(version)
+        date_match = cls._date_re.search(version)
+        if ver_match or date_match:
+            svn_backend = vcs.get_backend('svn')
+            if svn_backend:
+                svn_location = svn_backend().get_location(
+                    dist,
+                    dependency_links,
+                )
+            if not svn_location:
+                logger.warning(
+                    'Warning: cannot find svn location for %s', req,
+                )
+                comments.append(
+                    '## FIXME: could not find svn URL in dependency_links '
+                    'for this package:'
+                )
+            else:
+                deprecated(
+                    "SVN editable detection based on dependency links "
+                    "will be dropped in the future.",
+                    replacement=None,
+                    gone_in="19.0",
+                    issue=4187,
+                )
+                comments.append(
+                    '# Installing as editable to satisfy requirement %s:' %
+                    req
+                )
+                if ver_match:
+                    rev = ver_match.group(1)
                 else:
-                    deprecated(
-                        "SVN editable detection based on dependency links "
-                        "will be dropped in the future.",
-                        replacement=None,
-                        gone_in="19.0",
-                        issue=4187,
-                    )
-                    comments.append(
-                        '# Installing as editable to satisfy requirement %s:' %
-                        req
-                    )
-                    if ver_match:
-                        rev = ver_match.group(1)
-                    else:
-                        rev = '{%s}' % date_match.group(1)
-                    editable = True
-                    egg_name = cls.egg_name(dist)
-                    req = make_vcs_requirement_url(svn_location, rev, egg_name)
+                    rev = '{%s}' % date_match.group(1)
+                editable = True
+                egg_name = cls.egg_name(dist)
+                req = make_vcs_requirement_url(svn_location, rev, egg_name)
 
         return (req, editable, comments)
 

From e4ce62a5de2306a508ce450fa9aa38fa9d2e80e5 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 02:36:26 -0700
Subject: [PATCH 69/92] Add second guard clause.

---
 src/pip/_internal/operations/freeze.py | 68 +++++++++++++-------------
 1 file changed, 35 insertions(+), 33 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index d14641ce2..c56ba8018 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -209,40 +209,42 @@ class FrozenRequirement(object):
         version = specs[0][1]
         ver_match = cls._rev_re.search(version)
         date_match = cls._date_re.search(version)
-        if ver_match or date_match:
-            svn_backend = vcs.get_backend('svn')
-            if svn_backend:
-                svn_location = svn_backend().get_location(
-                    dist,
-                    dependency_links,
-                )
-            if not svn_location:
-                logger.warning(
-                    'Warning: cannot find svn location for %s', req,
-                )
-                comments.append(
-                    '## FIXME: could not find svn URL in dependency_links '
-                    'for this package:'
-                )
+        if not (ver_match or date_match):
+            return (req, editable, comments)
+
+        svn_backend = vcs.get_backend('svn')
+        if svn_backend:
+            svn_location = svn_backend().get_location(
+                dist,
+                dependency_links,
+            )
+        if not svn_location:
+            logger.warning(
+                'Warning: cannot find svn location for %s', req,
+            )
+            comments.append(
+                '## FIXME: could not find svn URL in dependency_links '
+                'for this package:'
+            )
+        else:
+            deprecated(
+                "SVN editable detection based on dependency links "
+                "will be dropped in the future.",
+                replacement=None,
+                gone_in="19.0",
+                issue=4187,
+            )
+            comments.append(
+                '# Installing as editable to satisfy requirement %s:' %
+                req
+            )
+            if ver_match:
+                rev = ver_match.group(1)
             else:
-                deprecated(
-                    "SVN editable detection based on dependency links "
-                    "will be dropped in the future.",
-                    replacement=None,
-                    gone_in="19.0",
-                    issue=4187,
-                )
-                comments.append(
-                    '# Installing as editable to satisfy requirement %s:' %
-                    req
-                )
-                if ver_match:
-                    rev = ver_match.group(1)
-                else:
-                    rev = '{%s}' % date_match.group(1)
-                editable = True
-                egg_name = cls.egg_name(dist)
-                req = make_vcs_requirement_url(svn_location, rev, egg_name)
+                rev = '{%s}' % date_match.group(1)
+            editable = True
+            egg_name = cls.egg_name(dist)
+            req = make_vcs_requirement_url(svn_location, rev, egg_name)
 
         return (req, editable, comments)
 

From 6d84b4747ab8997d882edfb6c921cedb9cb158f4 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 02:39:20 -0700
Subject: [PATCH 70/92] Add third guard clause.

---
 src/pip/_internal/operations/freeze.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index c56ba8018..813a5aa28 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -188,15 +188,17 @@ class FrozenRequirement(object):
                     "falling back to uneditable format", exc
                 )
                 req = None
-            if req is None:
-                logger.warning(
-                    'Could not determine repository location of %s', location
-                )
-                comments.append(
-                    '## !! Could not determine repository location'
-                )
-                req = dist.as_requirement()
-                editable = False
+            if req is not None:
+                return (req, editable, comments)
+
+            logger.warning(
+                'Could not determine repository location of %s', location
+            )
+            comments.append(
+                '## !! Could not determine repository location'
+            )
+            req = dist.as_requirement()
+            editable = False
 
             return (req, editable, comments)
 

From 7aa7bf0295657b43fc9f2f9c7545609d041b5f6a Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 02:41:17 -0700
Subject: [PATCH 71/92] Add fourth guard clause.

---
 src/pip/_internal/operations/freeze.py | 37 +++++++++++++-------------
 1 file changed, 19 insertions(+), 18 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 813a5aa28..7f0295e26 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -228,25 +228,26 @@ class FrozenRequirement(object):
                 '## FIXME: could not find svn URL in dependency_links '
                 'for this package:'
             )
+            return (req, editable, comments)
+
+        deprecated(
+            "SVN editable detection based on dependency links "
+            "will be dropped in the future.",
+            replacement=None,
+            gone_in="19.0",
+            issue=4187,
+        )
+        comments.append(
+            '# Installing as editable to satisfy requirement %s:' %
+            req
+        )
+        if ver_match:
+            rev = ver_match.group(1)
         else:
-            deprecated(
-                "SVN editable detection based on dependency links "
-                "will be dropped in the future.",
-                replacement=None,
-                gone_in="19.0",
-                issue=4187,
-            )
-            comments.append(
-                '# Installing as editable to satisfy requirement %s:' %
-                req
-            )
-            if ver_match:
-                rev = ver_match.group(1)
-            else:
-                rev = '{%s}' % date_match.group(1)
-            editable = True
-            egg_name = cls.egg_name(dist)
-            req = make_vcs_requirement_url(svn_location, rev, egg_name)
+            rev = '{%s}' % date_match.group(1)
+        editable = True
+        egg_name = cls.egg_name(dist)
+        req = make_vcs_requirement_url(svn_location, rev, egg_name)
 
         return (req, editable, comments)
 

From f5026e7db9463dac256de4a1bc32847d7cb4a317 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 02:47:46 -0700
Subject: [PATCH 72/92] Remove unneeded boolean flag variables.

---
 src/pip/_internal/operations/freeze.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 7f0295e26..f602c665b 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -179,7 +179,6 @@ class FrozenRequirement(object):
         comments = []
         from pip._internal.vcs import vcs, get_src_requirement
         if dist_is_editable(dist) and vcs.get_backend_name(location):
-            editable = True
             try:
                 req = get_src_requirement(dist, location)
             except InstallationError as exc:
@@ -189,7 +188,7 @@ class FrozenRequirement(object):
                 )
                 req = None
             if req is not None:
-                return (req, editable, comments)
+                return (req, True, comments)
 
             logger.warning(
                 'Could not determine repository location of %s', location
@@ -198,11 +197,9 @@ class FrozenRequirement(object):
                 '## !! Could not determine repository location'
             )
             req = dist.as_requirement()
-            editable = False
 
-            return (req, editable, comments)
+            return (req, False, comments)
 
-        editable = False
         req = dist.as_requirement()
         specs = req.specs
         assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
@@ -212,7 +209,7 @@ class FrozenRequirement(object):
         ver_match = cls._rev_re.search(version)
         date_match = cls._date_re.search(version)
         if not (ver_match or date_match):
-            return (req, editable, comments)
+            return (req, False, comments)
 
         svn_backend = vcs.get_backend('svn')
         if svn_backend:
@@ -228,7 +225,7 @@ class FrozenRequirement(object):
                 '## FIXME: could not find svn URL in dependency_links '
                 'for this package:'
             )
-            return (req, editable, comments)
+            return (req, False, comments)
 
         deprecated(
             "SVN editable detection based on dependency links "
@@ -245,11 +242,10 @@ class FrozenRequirement(object):
             rev = ver_match.group(1)
         else:
             rev = '{%s}' % date_match.group(1)
-        editable = True
         egg_name = cls.egg_name(dist)
         req = make_vcs_requirement_url(svn_location, rev, egg_name)
 
-        return (req, editable, comments)
+        return (req, True, comments)
 
     @classmethod
     def from_dist(cls, dist, dependency_links):

From 39b413352d742d26bb59ab3d0a3f4b23a81f9826 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 03:03:22 -0700
Subject: [PATCH 73/92] Combine some lines.

---
 src/pip/_internal/operations/freeze.py | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index f602c665b..bcc2a854b 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -193,9 +193,7 @@ class FrozenRequirement(object):
             logger.warning(
                 'Could not determine repository location of %s', location
             )
-            comments.append(
-                '## !! Could not determine repository location'
-            )
+            comments.append('## !! Could not determine repository location')
             req = dist.as_requirement()
 
             return (req, False, comments)
@@ -213,14 +211,9 @@ class FrozenRequirement(object):
 
         svn_backend = vcs.get_backend('svn')
         if svn_backend:
-            svn_location = svn_backend().get_location(
-                dist,
-                dependency_links,
-            )
+            svn_location = svn_backend().get_location(dist, dependency_links)
         if not svn_location:
-            logger.warning(
-                'Warning: cannot find svn location for %s', req,
-            )
+            logger.warning('Warning: cannot find svn location for %s', req)
             comments.append(
                 '## FIXME: could not find svn URL in dependency_links '
                 'for this package:'
@@ -235,8 +228,7 @@ class FrozenRequirement(object):
             issue=4187,
         )
         comments.append(
-            '# Installing as editable to satisfy requirement %s:' %
-            req
+            '# Installing as editable to satisfy requirement %s:' % req
         )
         if ver_match:
             rev = ver_match.group(1)

From e80a90e96cc28f6a3536ac89c59ee2caddc1c9e6 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 03:06:04 -0700
Subject: [PATCH 74/92] Add try-else clause.

---
 src/pip/_internal/operations/freeze.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index bcc2a854b..371b50754 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -186,9 +186,9 @@ class FrozenRequirement(object):
                     "Error when trying to get requirement for VCS system %s, "
                     "falling back to uneditable format", exc
                 )
-                req = None
-            if req is not None:
-                return (req, True, comments)
+            else:
+                if req is not None:
+                    return (req, True, comments)
 
             logger.warning(
                 'Could not determine repository location of %s', location

From b3ac98eff450b35b8831600437ed92664a4f6d0e Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Sun, 30 Sep 2018 03:10:40 -0700
Subject: [PATCH 75/92] Only define comments when needed.

---
 src/pip/_internal/operations/freeze.py | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 371b50754..79ec88bf1 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -176,7 +176,6 @@ class FrozenRequirement(object):
         This method is for use in FrozenRequirement.from_dist().
         """
         location = os.path.normcase(os.path.abspath(dist.location))
-        comments = []
         from pip._internal.vcs import vcs, get_src_requirement
         if dist_is_editable(dist) and vcs.get_backend_name(location):
             try:
@@ -188,12 +187,12 @@ class FrozenRequirement(object):
                 )
             else:
                 if req is not None:
-                    return (req, True, comments)
+                    return (req, True, [])
 
             logger.warning(
                 'Could not determine repository location of %s', location
             )
-            comments.append('## !! Could not determine repository location')
+            comments = ['## !! Could not determine repository location']
             req = dist.as_requirement()
 
             return (req, False, comments)
@@ -207,17 +206,17 @@ class FrozenRequirement(object):
         ver_match = cls._rev_re.search(version)
         date_match = cls._date_re.search(version)
         if not (ver_match or date_match):
-            return (req, False, comments)
+            return (req, False, [])
 
         svn_backend = vcs.get_backend('svn')
         if svn_backend:
             svn_location = svn_backend().get_location(dist, dependency_links)
         if not svn_location:
             logger.warning('Warning: cannot find svn location for %s', req)
-            comments.append(
+            comments = [
                 '## FIXME: could not find svn URL in dependency_links '
                 'for this package:'
-            )
+            ]
             return (req, False, comments)
 
         deprecated(
@@ -227,9 +226,9 @@ class FrozenRequirement(object):
             gone_in="19.0",
             issue=4187,
         )
-        comments.append(
+        comments = [
             '# Installing as editable to satisfy requirement %s:' % req
-        )
+        ]
         if ver_match:
             rev = ver_match.group(1)
         else:

From 500598287d4be3a2e181148ed3139cf415e08a4f Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Mon, 8 Oct 2018 22:19:00 -0700
Subject: [PATCH 76/92] Remove deprecated SVN-specific freeze code.

---
 news/5866.removal                      |  2 +
 src/pip/_internal/operations/freeze.py | 53 +-------------------------
 2 files changed, 4 insertions(+), 51 deletions(-)
 create mode 100644 news/5866.removal

diff --git a/news/5866.removal b/news/5866.removal
new file mode 100644
index 000000000..f9bbd0549
--- /dev/null
+++ b/news/5866.removal
@@ -0,0 +1,2 @@
+Remove the deprecated SVN editable detection based on dependency links
+during freeze.
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 79ec88bf1..31d6b46c0 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -14,9 +14,8 @@ from pip._internal.req.constructors import (
     install_req_from_editable, install_req_from_line,
 )
 from pip._internal.req.req_file import COMMENT_RE
-from pip._internal.utils.deprecation import deprecated
 from pip._internal.utils.misc import (
-    dist_is_editable, get_installed_distributions, make_vcs_requirement_url,
+    dist_is_editable, get_installed_distributions,
 )
 
 logger = logging.getLogger(__name__)
@@ -164,9 +163,6 @@ class FrozenRequirement(object):
         self.editable = editable
         self.comments = comments
 
-    _rev_re = re.compile(r'-r(\d+)$')
-    _date_re = re.compile(r'-(20\d\d\d\d\d\d)$')
-
     @classmethod
     def _init_args_from_dist(cls, dist, dependency_links):
         """
@@ -198,59 +194,14 @@ class FrozenRequirement(object):
             return (req, False, comments)
 
         req = dist.as_requirement()
-        specs = req.specs
-        assert len(specs) == 1 and specs[0][0] in ["==", "==="], \
-            'Expected 1 spec with == or ===; specs = %r; dist = %r' % \
-            (specs, dist)
-        version = specs[0][1]
-        ver_match = cls._rev_re.search(version)
-        date_match = cls._date_re.search(version)
-        if not (ver_match or date_match):
-            return (req, False, [])
 
-        svn_backend = vcs.get_backend('svn')
-        if svn_backend:
-            svn_location = svn_backend().get_location(dist, dependency_links)
-        if not svn_location:
-            logger.warning('Warning: cannot find svn location for %s', req)
-            comments = [
-                '## FIXME: could not find svn URL in dependency_links '
-                'for this package:'
-            ]
-            return (req, False, comments)
-
-        deprecated(
-            "SVN editable detection based on dependency links "
-            "will be dropped in the future.",
-            replacement=None,
-            gone_in="19.0",
-            issue=4187,
-        )
-        comments = [
-            '# Installing as editable to satisfy requirement %s:' % req
-        ]
-        if ver_match:
-            rev = ver_match.group(1)
-        else:
-            rev = '{%s}' % date_match.group(1)
-        egg_name = cls.egg_name(dist)
-        req = make_vcs_requirement_url(svn_location, rev, egg_name)
-
-        return (req, True, comments)
+        return (req, False, [])
 
     @classmethod
     def from_dist(cls, dist, dependency_links):
         args = cls._init_args_from_dist(dist, dependency_links)
         return cls(dist.project_name, *args)
 
-    @staticmethod
-    def egg_name(dist):
-        name = dist.egg_name()
-        match = re.search(r'-py\d\.\d$', name)
-        if match:
-            name = name[:match.start()]
-        return name
-
     def __str__(self):
         req = self.req
         if self.editable:

From faff436100d8ffa83c94a30ed1271ce49b7ba1ae Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Mon, 8 Oct 2018 22:21:45 -0700
Subject: [PATCH 77/92] Add guard clause.

---
 src/pip/_internal/operations/freeze.py | 35 +++++++++++++-------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 31d6b46c0..b173875bf 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -173,29 +173,28 @@ class FrozenRequirement(object):
         """
         location = os.path.normcase(os.path.abspath(dist.location))
         from pip._internal.vcs import vcs, get_src_requirement
-        if dist_is_editable(dist) and vcs.get_backend_name(location):
-            try:
-                req = get_src_requirement(dist, location)
-            except InstallationError as exc:
-                logger.warning(
-                    "Error when trying to get requirement for VCS system %s, "
-                    "falling back to uneditable format", exc
-                )
-            else:
-                if req is not None:
-                    return (req, True, [])
-
-            logger.warning(
-                'Could not determine repository location of %s', location
-            )
-            comments = ['## !! Could not determine repository location']
+        if not dist_is_editable(dist) or not vcs.get_backend_name(location):
             req = dist.as_requirement()
+            return (req, False, [])
 
-            return (req, False, comments)
+        try:
+            req = get_src_requirement(dist, location)
+        except InstallationError as exc:
+            logger.warning(
+                "Error when trying to get requirement for VCS system %s, "
+                "falling back to uneditable format", exc
+            )
+        else:
+            if req is not None:
+                return (req, True, [])
 
+        logger.warning(
+            'Could not determine repository location of %s', location
+        )
+        comments = ['## !! Could not determine repository location']
         req = dist.as_requirement()
 
-        return (req, False, [])
+        return (req, False, comments)
 
     @classmethod
     def from_dist(cls, dist, dependency_links):

From f3d82aa741a91c7c47c1ad49ac0315e7f1b00678 Mon Sep 17 00:00:00 2001
From: Felix Yan <felixonmars@archlinux.org>
Date: Tue, 9 Oct 2018 22:58:50 +0800
Subject: [PATCH 78/92] Fix a typo in tests/lib/configuration_helpers.py
 (#5858)

---
 tests/lib/configuration_helpers.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/lib/configuration_helpers.py b/tests/lib/configuration_helpers.py
index 1164be8a7..bd9ab79b5 100644
--- a/tests/lib/configuration_helpers.py
+++ b/tests/lib/configuration_helpers.py
@@ -34,13 +34,13 @@ class ConfigurationMixin(object):
         old = self.configuration._load_config_files
 
         @functools.wraps(old)
-        def overidden():
+        def overridden():
             # Manual Overload
             self.configuration._config[variant].update(di)
             self.configuration._parsers[variant].append((None, None))
             return old()
 
-        self.configuration._load_config_files = overidden
+        self.configuration._load_config_files = overridden
 
     @contextlib.contextmanager
     def tmpfile(self, contents):

From 4f18dc898453b8321a6ef78773f9d8ef642a5152 Mon Sep 17 00:00:00 2001
From: Benoit Pierre <benoit.pierre@gmail.com>
Date: Tue, 9 Oct 2018 18:25:00 +0200
Subject: [PATCH 79/92] travis: preemptively switch to VM-based jobs (#5864)

Rational: https://blog.travis-ci.com/2018-10-04-combining-linux-infrastructures
---
 .travis.yml | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index c4a2eb9ed..058209841 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,4 @@
 language: python
-sudo: false
 cache: pip
 dist: trusty
 
@@ -41,11 +40,9 @@ jobs:
     - env: GROUP=1
       python: 3.7
       dist: xenial
-      sudo: required
     - env: GROUP=2
       python: 3.7
       dist: xenial
-      sudo: required
     - env: GROUP=1
       python: 3.5
     - env: GROUP=2
@@ -58,11 +55,9 @@ jobs:
     - env: GROUP=1
       python: 3.8-dev
       dist: xenial
-      sudo: required
     - env: GROUP=2
       python: 3.8-dev
       dist: xenial
-      sudo: required
 
   # It's okay to fail on the in-development CPython version.
   fast_finish: true

From 8e3d30307b37abcc599c712d72059ef9c421ba7e Mon Sep 17 00:00:00 2001
From: Sean <me@sean.taipei>
Date: Wed, 10 Oct 2018 10:35:03 +0800
Subject: [PATCH 80/92] Fix Typo

---
 NEWS.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/NEWS.rst b/NEWS.rst
index ceaff88c1..da44faec5 100644
--- a/NEWS.rst
+++ b/NEWS.rst
@@ -64,7 +64,7 @@ Process
 - Formally document our deprecation process as a minimum of 6 months of deprecation
   warnings.
 - Adopt and document NEWS fragment writing style.
-- Switch to releasing a new, non bug fix version of pip every 3 months.
+- Switch to releasing a new, non-bug fix version of pip every 3 months.
 
 Deprecations and Removals
 -------------------------
@@ -165,7 +165,7 @@ Bug Fixes
 ---------
 
 - Prevent false-positive installation warnings due to incomplete name
-  normalizaton. (#5134)
+  normalization. (#5134)
 - Fix issue where installing from Git with a short SHA would fail. (#5140)
 - Accept pre-release versions when checking for conflicts with pip check or pip
   install. (#5141)

From 2f16b88fb39e2ac269262b804aa2ce2cbeb91421 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Wed, 10 Oct 2018 02:16:00 -0700
Subject: [PATCH 81/92] Refactor away VcsSupport.get_backend_from_location().

---
 news/D9A540D9-7665-48A5-A1C8-5141ED49E404.trivial |  0
 src/pip/_internal/operations/freeze.py            | 10 ++++++++--
 src/pip/_internal/vcs/__init__.py                 | 10 ++--------
 3 files changed, 10 insertions(+), 10 deletions(-)
 create mode 100644 news/D9A540D9-7665-48A5-A1C8-5141ED49E404.trivial

diff --git a/news/D9A540D9-7665-48A5-A1C8-5141ED49E404.trivial b/news/D9A540D9-7665-48A5-A1C8-5141ED49E404.trivial
new file mode 100644
index 000000000..e69de29bb
diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index b173875bf..3911a0fcc 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -173,12 +173,18 @@ class FrozenRequirement(object):
         """
         location = os.path.normcase(os.path.abspath(dist.location))
         from pip._internal.vcs import vcs, get_src_requirement
-        if not dist_is_editable(dist) or not vcs.get_backend_name(location):
+        if not dist_is_editable(dist):
+            req = dist.as_requirement()
+            return (req, False, [])
+
+        vc_name = vcs.get_backend_name(location)
+
+        if not vc_name:
             req = dist.as_requirement()
             return (req, False, [])
 
         try:
-            req = get_src_requirement(dist, location)
+            req = get_src_requirement(vc_name, dist, location)
         except InstallationError as exc:
             logger.warning(
                 "Error when trying to get requirement for VCS system %s, "
diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py
index 794b35d65..4249a7c0e 100644
--- a/src/pip/_internal/vcs/__init__.py
+++ b/src/pip/_internal/vcs/__init__.py
@@ -150,12 +150,6 @@ class VcsSupport(object):
         if name in self._registry:
             return self._registry[name]
 
-    def get_backend_from_location(self, location):
-        vc_type = self.get_backend_name(location)
-        if vc_type:
-            return self.get_backend(vc_type)
-        return None
-
 
 vcs = VcsSupport()
 
@@ -487,8 +481,8 @@ class VersionControl(object):
         return cls.is_repository_directory(location)
 
 
-def get_src_requirement(dist, location):
-    version_control = vcs.get_backend_from_location(location)
+def get_src_requirement(vc_name, dist, location):
+    version_control = vcs.get_backend(vc_name)
     if version_control:
         try:
             return version_control().get_src_requirement(dist,

From 00a2ff198e178b440793820de78e06031c378252 Mon Sep 17 00:00:00 2001
From: Chris Jerdonek <chris.jerdonek@gmail.com>
Date: Wed, 10 Oct 2018 02:34:40 -0700
Subject: [PATCH 82/92] Remove unneeded if block in vcs.get_src_requirement().

---
 src/pip/_internal/operations/freeze.py |  6 ++--
 src/pip/_internal/vcs/__init__.py      | 39 ++++++++++----------------
 2 files changed, 18 insertions(+), 27 deletions(-)

diff --git a/src/pip/_internal/operations/freeze.py b/src/pip/_internal/operations/freeze.py
index 3911a0fcc..af484f2c1 100644
--- a/src/pip/_internal/operations/freeze.py
+++ b/src/pip/_internal/operations/freeze.py
@@ -177,14 +177,14 @@ class FrozenRequirement(object):
             req = dist.as_requirement()
             return (req, False, [])
 
-        vc_name = vcs.get_backend_name(location)
+        vc_type = vcs.get_backend_type(location)
 
-        if not vc_name:
+        if not vc_type:
             req = dist.as_requirement()
             return (req, False, [])
 
         try:
-            req = get_src_requirement(vc_name, dist, location)
+            req = get_src_requirement(vc_type, dist, location)
         except InstallationError as exc:
             logger.warning(
                 "Error when trying to get requirement for VCS system %s, "
diff --git a/src/pip/_internal/vcs/__init__.py b/src/pip/_internal/vcs/__init__.py
index 4249a7c0e..d34c8be0b 100644
--- a/src/pip/_internal/vcs/__init__.py
+++ b/src/pip/_internal/vcs/__init__.py
@@ -133,16 +133,16 @@ class VcsSupport(object):
         else:
             logger.warning('Cannot unregister because no class or name given')
 
-    def get_backend_name(self, location):
+    def get_backend_type(self, location):
         """
-        Return the name of the version control backend if found at given
-        location, e.g. vcs.get_backend_name('/path/to/vcs/checkout')
+        Return the type of the version control backend if found at given
+        location, e.g. vcs.get_backend_type('/path/to/vcs/checkout')
         """
         for vc_type in self._registry.values():
             if vc_type.controls_location(location):
                 logger.debug('Determine that %s uses VCS: %s',
                              location, vc_type.name)
-                return vc_type.name
+                return vc_type
         return None
 
     def get_backend(self, name):
@@ -481,23 +481,14 @@ class VersionControl(object):
         return cls.is_repository_directory(location)
 
 
-def get_src_requirement(vc_name, dist, location):
-    version_control = vcs.get_backend(vc_name)
-    if version_control:
-        try:
-            return version_control().get_src_requirement(dist,
-                                                         location)
-        except BadCommand:
-            logger.warning(
-                'cannot determine version of editable source in %s '
-                '(%s command not found in path)',
-                location,
-                version_control.name,
-            )
-            return dist.as_requirement()
-    logger.warning(
-        'cannot determine version of editable source in %s (is not SVN '
-        'checkout, Git clone, Mercurial clone or Bazaar branch)',
-        location,
-    )
-    return dist.as_requirement()
+def get_src_requirement(vc_type, dist, location):
+    try:
+        return vc_type().get_src_requirement(dist, location)
+    except BadCommand:
+        logger.warning(
+            'cannot determine version of editable source in %s '
+            '(%s command not found in path)',
+            location,
+            vc_type.name,
+        )
+        return dist.as_requirement()

From 44713cafe5affbe80df6b5c890db6ad2aa780119 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 18:24:38 +0800
Subject: [PATCH 83/92] Move VCS scheme check back out

---
 src/pip/_internal/index.py | 27 ++++++++++-----------------
 1 file changed, 10 insertions(+), 17 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 41c130760..9d6528848 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -85,10 +85,6 @@ def _get_content_type(url, session):
     return resp.headers.get("Content-Type", "")
 
 
-class _SchemeIsVCS(Exception):
-    pass
-
-
 class _NotHTML(Exception):
     pass
 
@@ -98,22 +94,15 @@ def _get_html_response(url, filename, session):
 
     This consists of five parts:
 
-    1. Check whether the scheme is supported (non-VCS). Raises `_SchemeIsVCS`
-       on failure.
-    2. Check whether the URL looks like an archive. If it does, make a HEAD
+    1. Check whether the URL looks like an archive. If it does, make a HEAD
        request to check the Content-Type header is indeed HTML, and raise
        `_NotHTML` if it's not. Raise HTTP exceptions on network failures.
-    3. If URL scheme is file: and points to a directory, make it point to
+    2. If URL scheme is file: and points to a directory, make it point to
        index.html instead.
-    4. Actually perform the request. Raise HTTP exceptions on network failures.
-    5. Check whether Content-Type header to make sure the thing we got is HTML,
+    3. Actually perform the request. Raise HTTP exceptions on network failures.
+    4. Check whether Content-Type header to make sure the thing we got is HTML,
        and raise `_NotHTML` if it's not.
     """
-    # Check for VCS schemes that do not support lookup as web pages.
-    vcs_scheme = _match_vcs_scheme(url)
-    if vcs_scheme:
-        raise _SchemeIsVCS(vcs_scheme)
-
     for bad_ext in ARCHIVE_EXTENSIONS:
         if filename.endswith(bad_ext):
             content_type = _get_content_type(url, session=session)
@@ -180,10 +169,14 @@ def _get_html_page(link, session=None):
 
     url = link.url.split('#', 1)[0]
 
+    # Check for VCS schemes that do not support lookup as web pages.
+    vcs_scheme = _match_vcs_scheme(url)
+    if vcs_scheme:
+        logger.debug('Cannot look at %s URL %s', vcs_scheme, link)
+        return None
+
     try:
         resp = _get_html_response(url, link.filename, session=session)
-    except _SchemeIsVCS as exc:
-        logger.debug('Cannot look at %s URL %s', exc, link)
     except _NotHTML as exc:
         logger.debug(
             'Skipping page %s because of Content-Type: %s',

From bad04d85491a31297d113738baba00a3f11ac898 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Mon, 1 Oct 2018 18:53:51 +0800
Subject: [PATCH 84/92] Extract content type detection logic

---
 src/pip/_internal/index.py | 59 ++++++++++++++++++++++----------------
 1 file changed, 34 insertions(+), 25 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 9d6528848..1dea748b7 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -71,8 +71,28 @@ def _match_vcs_scheme(url):
     return None
 
 
-def _get_content_type(url, session):
-    """Get the Content-Type of the given url, using a HEAD request"""
+def _is_filename_like_archive(filename):
+    """Check whether the filename looks like an archive.
+    """
+    for bad_ext in ARCHIVE_EXTENSIONS:
+        if filename.endswith(bad_ext):
+            return True
+    return False
+
+
+class _NotHTML(Exception):
+    pass
+
+
+def _ensure_html_header(headers):
+    content_type = headers.get("Content-Type", "")
+    if not content_type.lower().startswith("text/html"):
+        raise _NotHTML(content_type)
+
+
+def _ensure_html_response(url, session):
+    """Send a HEAD request to the URL, and ensure the Content-Type is HTML.
+    """
     scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
     if scheme not in {'http', 'https'}:
         # FIXME: some warning or something?
@@ -82,33 +102,20 @@ def _get_content_type(url, session):
     resp = session.head(url, allow_redirects=True)
     resp.raise_for_status()
 
-    return resp.headers.get("Content-Type", "")
+    _ensure_html_header(resp.headers)
 
 
-class _NotHTML(Exception):
-    pass
-
-
-def _get_html_response(url, filename, session):
+def _get_html_response(url, session):
     """Access an HTML page with GET, and return the response.
 
-    This consists of five parts:
+    This consists of three parts:
 
-    1. Check whether the URL looks like an archive. If it does, make a HEAD
-       request to check the Content-Type header is indeed HTML, and raise
-       `_NotHTML` if it's not. Raise HTTP exceptions on network failures.
-    2. If URL scheme is file: and points to a directory, make it point to
+    1. If URL scheme is file: and points to a directory, make it point to
        index.html instead.
-    3. Actually perform the request. Raise HTTP exceptions on network failures.
-    4. Check whether Content-Type header to make sure the thing we got is HTML,
+    2. Actually perform the request. Raise HTTP exceptions on network failures.
+    3. Check whether Content-Type header to make sure the thing we got is HTML,
        and raise `_NotHTML` if it's not.
     """
-    for bad_ext in ARCHIVE_EXTENSIONS:
-        if filename.endswith(bad_ext):
-            content_type = _get_content_type(url, session=session)
-            if not content_type.lower().startswith('text/html'):
-                raise _NotHTML(content_type)
-
     logger.debug('Getting page %s', url)
 
     # Tack index.html onto file:// URLs that point to directories
@@ -148,9 +155,7 @@ def _get_html_response(url, filename, session):
     # requirement of an url. Unless we issue a HEAD request on every
     # url we cannot know ahead of time for sure if something is HTML
     # or not. However we can check after we've downloaded it.
-    content_type = resp.headers.get('Content-Type', 'unknown')
-    if not content_type.lower().startswith("text/html"):
-        raise _NotHTML(content_type)
+    _ensure_html_header(resp.headers)
 
     return resp
 
@@ -176,7 +181,11 @@ def _get_html_page(link, session=None):
         return None
 
     try:
-        resp = _get_html_response(url, link.filename, session=session)
+        # If the URL looks suspiciously like an archive, send a HEAD first to
+        # check the Content-Type is HTML, to avoid downloading a large file.
+        if _is_filename_like_archive(link.filename):
+            _ensure_html_response(url, session=session)
+        resp = _get_html_response(url, session=session)
     except _NotHTML as exc:
         logger.debug(
             'Skipping page %s because of Content-Type: %s',

From 231e6165dd819acec08fb12179dfd1734a481419 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Tue, 2 Oct 2018 15:01:04 +0800
Subject: [PATCH 85/92] Group HEAD optimization logic into HTML getter, add
 docs

---
 src/pip/_internal/index.py | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 1dea748b7..858277889 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -71,9 +71,10 @@ def _match_vcs_scheme(url):
     return None
 
 
-def _is_filename_like_archive(filename):
-    """Check whether the filename looks like an archive.
+def _is_url_like_archive(url):
+    """Check whether the URL looks like an archive.
     """
+    filename = Link(url).filename
     for bad_ext in ARCHIVE_EXTENSIONS:
         if filename.endswith(bad_ext):
             return True
@@ -85,13 +86,19 @@ class _NotHTML(Exception):
 
 
 def _ensure_html_header(headers):
+    """Check the Content-Type header to ensure the response contains HTML.
+
+    Raises `_NotHTML` is the content type is not text/html.
+    """
     content_type = headers.get("Content-Type", "")
     if not content_type.lower().startswith("text/html"):
         raise _NotHTML(content_type)
 
 
 def _ensure_html_response(url, session):
-    """Send a HEAD request to the URL, and ensure the Content-Type is HTML.
+    """Send a HEAD request to the URL, and ensure the response contains HTML.
+
+    Raises `_NotHTML` is the content type is not text/html.
     """
     scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
     if scheme not in {'http', 'https'}:
@@ -110,12 +117,17 @@ def _get_html_response(url, session):
 
     This consists of three parts:
 
-    1. If URL scheme is file: and points to a directory, make it point to
+    1. If the URL looks suspiciously like an archive, send a HEAD first to
+       check the Content-Type is HTML, to avoid downloading a large file.
+    2. If URL scheme is file: and points to a directory, make it point to
        index.html instead.
-    2. Actually perform the request. Raise HTTP exceptions on network failures.
-    3. Check whether Content-Type header to make sure the thing we got is HTML,
+    3. Actually perform the request. Raise HTTP exceptions on network failures.
+    4. Check whether Content-Type header to make sure the thing we got is HTML,
        and raise `_NotHTML` if it's not.
     """
+    if _is_url_like_archive(url):
+        _ensure_html_response(url, session=session)
+
     logger.debug('Getting page %s', url)
 
     # Tack index.html onto file:// URLs that point to directories
@@ -181,10 +193,6 @@ def _get_html_page(link, session=None):
         return None
 
     try:
-        # If the URL looks suspiciously like an archive, send a HEAD first to
-        # check the Content-Type is HTML, to avoid downloading a large file.
-        if _is_filename_like_archive(link.filename):
-            _ensure_html_response(url, session=session)
         resp = _get_html_response(url, session=session)
     except _NotHTML as exc:
         logger.debug(

From d4da76ceaa53476eed16cb03429c392bc837b480 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Tue, 2 Oct 2018 18:16:02 +0800
Subject: [PATCH 86/92] Move directory check out of _get_html_response()

This also "fixes" an edge case where a directory is dropped if it is
named like an archive.
---
 news/5838.bugfix           |  1 +
 src/pip/_internal/index.py | 20 ++++++++++----------
 2 files changed, 11 insertions(+), 10 deletions(-)
 create mode 100644 news/5838.bugfix

diff --git a/news/5838.bugfix b/news/5838.bugfix
new file mode 100644
index 000000000..b83a9fa91
--- /dev/null
+++ b/news/5838.bugfix
@@ -0,0 +1 @@
+Fix content type detection if a directory named like an archive is used as a package source.
diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 858277889..e564b7a25 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -130,16 +130,6 @@ def _get_html_response(url, session):
 
     logger.debug('Getting page %s', url)
 
-    # Tack index.html onto file:// URLs that point to directories
-    scheme, _, path, _, _, _ = urllib_parse.urlparse(url)
-    if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))):
-        # add trailing slash if not present so urljoin doesn't trim
-        # final segment
-        if not url.endswith('/'):
-            url += '/'
-        url = urllib_parse.urljoin(url, 'index.html')
-        logger.debug(' file: URL is directory, getting %s', url)
-
     resp = session.get(
         url,
         headers={
@@ -192,6 +182,16 @@ def _get_html_page(link, session=None):
         logger.debug('Cannot look at %s URL %s', vcs_scheme, link)
         return None
 
+    # Tack index.html onto file:// URLs that point to directories
+    scheme, _, path, _, _, _ = urllib_parse.urlparse(url)
+    if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))):
+        # add trailing slash if not present so urljoin doesn't trim
+        # final segment
+        if not url.endswith('/'):
+            url += '/'
+        url = urllib_parse.urljoin(url, 'index.html')
+        logger.debug(' file: URL is directory, getting %s', url)
+
     try:
         resp = _get_html_response(url, session=session)
     except _NotHTML as exc:

From a9ebb795ed64c7eb1807603eb0fc650e2354f57a Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Tue, 2 Oct 2018 18:27:14 +0800
Subject: [PATCH 87/92] More detailed information on Content-Type error, fix
 typos

---
 src/pip/_internal/index.py | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index e564b7a25..2da3cfdf3 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -82,23 +82,26 @@ def _is_url_like_archive(url):
 
 
 class _NotHTML(Exception):
-    pass
+    def __init__(self, content_type, response):
+        super(_NotHTML, self).__init__(content_type, response)
+        self.content_type = content_type
+        self.response = response
 
 
-def _ensure_html_header(headers):
+def _ensure_html_header(response):
     """Check the Content-Type header to ensure the response contains HTML.
 
-    Raises `_NotHTML` is the content type is not text/html.
+    Raises `_NotHTML` if the content type is not text/html.
     """
-    content_type = headers.get("Content-Type", "")
+    content_type = response.headers.get("Content-Type", "")
     if not content_type.lower().startswith("text/html"):
-        raise _NotHTML(content_type)
+        raise _NotHTML(content_type, response)
 
 
 def _ensure_html_response(url, session):
     """Send a HEAD request to the URL, and ensure the response contains HTML.
 
-    Raises `_NotHTML` is the content type is not text/html.
+    Raises `_NotHTML` if the content type is not text/html.
     """
     scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
     if scheme not in {'http', 'https'}:
@@ -109,7 +112,7 @@ def _ensure_html_response(url, session):
     resp = session.head(url, allow_redirects=True)
     resp.raise_for_status()
 
-    _ensure_html_header(resp.headers)
+    _ensure_html_header(resp)
 
 
 def _get_html_response(url, session):
@@ -119,10 +122,9 @@ def _get_html_response(url, session):
 
     1. If the URL looks suspiciously like an archive, send a HEAD first to
        check the Content-Type is HTML, to avoid downloading a large file.
-    2. If URL scheme is file: and points to a directory, make it point to
-       index.html instead.
-    3. Actually perform the request. Raise HTTP exceptions on network failures.
-    4. Check whether Content-Type header to make sure the thing we got is HTML,
+       Raise `_NotHTML` if it is not.
+    2. Actually perform the request. Raise HTTP exceptions on network failures.
+    3. Check whether Content-Type header to make sure the thing we got is HTML,
        and raise `_NotHTML` if it's not.
     """
     if _is_url_like_archive(url):
@@ -157,7 +159,7 @@ def _get_html_response(url, session):
     # requirement of an url. Unless we issue a HEAD request on every
     # url we cannot know ahead of time for sure if something is HTML
     # or not. However we can check after we've downloaded it.
-    _ensure_html_header(resp.headers)
+    _ensure_html_header(resp)
 
     return resp
 
@@ -196,8 +198,8 @@ def _get_html_page(link, session=None):
         resp = _get_html_response(url, session=session)
     except _NotHTML as exc:
         logger.debug(
-            'Skipping page %s because of Content-Type: %s',
-            link, exc,
+            'Skipping page %s because the %s request got Content-Type: %s',
+            link, exc.response.request.method, exc.content_type,
         )
     except requests.HTTPError as exc:
         _handle_get_page_fail(link, exc, url)

From 68cf77ac0bc86dfdc5b847179e43a30a0baaec78 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Tue, 2 Oct 2018 22:09:23 +0800
Subject: [PATCH 88/92] Only store what we need in _NotHTML

---
 src/pip/_internal/index.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 2da3cfdf3..85fca6498 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -82,10 +82,10 @@ def _is_url_like_archive(url):
 
 
 class _NotHTML(Exception):
-    def __init__(self, content_type, response):
-        super(_NotHTML, self).__init__(content_type, response)
+    def __init__(self, content_type, request_desc):
+        super(_NotHTML, self).__init__(content_type, request_desc)
         self.content_type = content_type
-        self.response = response
+        self.request_desc = request_desc
 
 
 def _ensure_html_header(response):
@@ -95,7 +95,7 @@ def _ensure_html_header(response):
     """
     content_type = response.headers.get("Content-Type", "")
     if not content_type.lower().startswith("text/html"):
-        raise _NotHTML(content_type, response)
+        raise _NotHTML(content_type, response.request.method)
 
 
 def _ensure_html_response(url, session):
@@ -199,7 +199,7 @@ def _get_html_page(link, session=None):
     except _NotHTML as exc:
         logger.debug(
             'Skipping page %s because the %s request got Content-Type: %s',
-            link, exc.response.request.method, exc.content_type,
+            link, exc.request_desc, exc.content_type,
         )
     except requests.HTTPError as exc:
         _handle_get_page_fail(link, exc, url)

From 65260d044982e0de902914d1fd24754d064c7c19 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Tue, 2 Oct 2018 22:21:34 +0800
Subject: [PATCH 89/92] Fix behavior if archive-like URL is not HTTP(S)

The original implementation erros out when this happens, and the new one
follows. The debug logging message is however changed to be clearer about
what happened.

Also fixed some minor typos.
---
 src/pip/_internal/index.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index 85fca6498..e7d260a22 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -98,16 +98,19 @@ def _ensure_html_header(response):
         raise _NotHTML(content_type, response.request.method)
 
 
+class _NotHTTP(Exception):
+    pass
+
+
 def _ensure_html_response(url, session):
     """Send a HEAD request to the URL, and ensure the response contains HTML.
 
-    Raises `_NotHTML` if the content type is not text/html.
+    Raises `_NotHTTP` if the URL is not available for a HEAD request, or
+    `_NotHTML` if the content type is not text/html.
     """
     scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url)
     if scheme not in {'http', 'https'}:
-        # FIXME: some warning or something?
-        # assertion error?
-        return ''
+        raise _NotHTTP()
 
     resp = session.head(url, allow_redirects=True)
     resp.raise_for_status()
@@ -122,7 +125,8 @@ def _get_html_response(url, session):
 
     1. If the URL looks suspiciously like an archive, send a HEAD first to
        check the Content-Type is HTML, to avoid downloading a large file.
-       Raise `_NotHTML` if it is not.
+       Raise `_NotHTTP` if the content type cannot be determined, or
+       `_NotHTML` if it is not HTML.
     2. Actually perform the request. Raise HTTP exceptions on network failures.
     3. Check whether Content-Type header to make sure the thing we got is HTML,
        and raise `_NotHTML` if it's not.
@@ -196,6 +200,11 @@ def _get_html_page(link, session=None):
 
     try:
         resp = _get_html_response(url, session=session)
+    except _NotHTTP as exc:
+        logger.debug(
+            'Skipping page %s because it looks like an archive, and cannot '
+            'be checked by HEAD.', link,
+        )
     except _NotHTML as exc:
         logger.debug(
             'Skipping page %s because the %s request got Content-Type: %s',

From 5441fc7289fc09eee948208ee45a283bd658f6cb Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 4 Oct 2018 01:58:10 +0800
Subject: [PATCH 90/92] Some tests for error and input correction

---
 tests/unit/test_index.py | 88 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 87 insertions(+), 1 deletion(-)

diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index bca09a24e..803c17eb6 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -1,11 +1,15 @@
+import logging
 import os.path
 
 import pytest
+from mock import Mock, call, patch
 from pip._vendor import html5lib
+from pip._vendor.six.moves.urllib import request as urllib_request
 
 from pip._internal.download import PipSession
 from pip._internal.index import (
-    Link, PackageFinder, _determine_base_url, egg_info_matches,
+    Link, PackageFinder, _determine_base_url, _get_html_page,
+    _get_html_response, _NotHTML, _NotHTTP, egg_info_matches,
 )
 
 
@@ -190,3 +194,85 @@ def test_egg_info_matches(egg_info, search_name, expected):
     link = None     # Only used for reporting.
     version = egg_info_matches(egg_info, search_name, link)
     assert version == expected
+
+
+@pytest.mark.parametrize(
+    "url, vcs_scheme",
+    [
+        ("svn+http://pypi.org/something", "svn"),
+        ("git+https://github.com/pypa/pip.git", "git"),
+    ],
+)
+def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme):
+    """`_get_html_page()` Should error an invalid scheme is given.
+
+    Only file:, http:, https:, and ftp: are allowed.
+    """
+    with caplog.at_level(logging.DEBUG):
+        page = _get_html_page(Link(url), session=Mock(PipSession))
+
+    assert page is None
+    assert caplog.record_tuples == [
+        (
+            "pip._internal.index",
+            logging.DEBUG,
+            "Cannot look at {} URL {}".format(vcs_scheme, url),
+        ),
+    ]
+
+
+def test_get_html_page_directory_append_index(tmpdir):
+    """`_get_html_page()` Should append "index.html" to a directory URL.
+    """
+    dirpath = tmpdir.mkdir("something")
+    dir_url = "file:{}".format(urllib_request.pathname2url(dirpath))
+
+    session = Mock(PipSession)
+    with patch("pip._internal.index._get_html_response") as mock_func:
+        _get_html_page(Link(dir_url), session=session)
+        assert mock_func.mock_calls == [
+            call("{}/index.html".format(dir_url.rstrip("/")), session=session)
+        ]
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "ftp://python.org/python-3.7.1.zip",
+        "file:///opt/data/pip-18.0.tar.gz",
+    ],
+)
+def test_get_html_response_archive_to_naive_scheme(url):
+    """
+    `_get_html_response()` should error on archive-like URL if the scheme
+    does not allow "poking" without getting data.
+    """
+    with pytest.raises(_NotHTTP):
+        _get_html_response(url, session=Mock(PipSession))
+
+
+@pytest.mark.parametrize(
+    "url, content_type",
+    [
+        ("http://python.org/python-3.7.1.zip", "application/zip"),
+        ("https://pypi.org/pip-18.0.tar.gz", "application/gzip"),
+    ],
+)
+def test_get_html_response_archive_to_http_scheme(url, content_type):
+    """
+    `_get_html_response()` should send a HEAD request on archive-like URL if
+    the scheme supports it.
+    """
+    session = Mock(PipSession)
+    session.head.return_value = Mock(**{
+        "request.method": "HEAD",
+        "headers": {"Content-Type": content_type},
+    })
+
+    with pytest.raises(_NotHTML) as ctx:
+        _get_html_response(url, session=session)
+
+    session.assert_has_calls([
+        call.head(url, allow_redirects=True),
+    ])
+    assert ctx.value.args == (content_type, "HEAD")

From 200f126584b36c3c7a1ec7fbe9b653d68bec7af2 Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 4 Oct 2018 16:24:04 +0800
Subject: [PATCH 91/92] Move HTML page tests to their own file, fix typos

---
 tests/unit/test_index.py           | 88 +--------------------------
 tests/unit/test_index_html_page.py | 98 ++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+), 87 deletions(-)
 create mode 100644 tests/unit/test_index_html_page.py

diff --git a/tests/unit/test_index.py b/tests/unit/test_index.py
index 803c17eb6..bca09a24e 100644
--- a/tests/unit/test_index.py
+++ b/tests/unit/test_index.py
@@ -1,15 +1,11 @@
-import logging
 import os.path
 
 import pytest
-from mock import Mock, call, patch
 from pip._vendor import html5lib
-from pip._vendor.six.moves.urllib import request as urllib_request
 
 from pip._internal.download import PipSession
 from pip._internal.index import (
-    Link, PackageFinder, _determine_base_url, _get_html_page,
-    _get_html_response, _NotHTML, _NotHTTP, egg_info_matches,
+    Link, PackageFinder, _determine_base_url, egg_info_matches,
 )
 
 
@@ -194,85 +190,3 @@ def test_egg_info_matches(egg_info, search_name, expected):
     link = None     # Only used for reporting.
     version = egg_info_matches(egg_info, search_name, link)
     assert version == expected
-
-
-@pytest.mark.parametrize(
-    "url, vcs_scheme",
-    [
-        ("svn+http://pypi.org/something", "svn"),
-        ("git+https://github.com/pypa/pip.git", "git"),
-    ],
-)
-def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme):
-    """`_get_html_page()` Should error an invalid scheme is given.
-
-    Only file:, http:, https:, and ftp: are allowed.
-    """
-    with caplog.at_level(logging.DEBUG):
-        page = _get_html_page(Link(url), session=Mock(PipSession))
-
-    assert page is None
-    assert caplog.record_tuples == [
-        (
-            "pip._internal.index",
-            logging.DEBUG,
-            "Cannot look at {} URL {}".format(vcs_scheme, url),
-        ),
-    ]
-
-
-def test_get_html_page_directory_append_index(tmpdir):
-    """`_get_html_page()` Should append "index.html" to a directory URL.
-    """
-    dirpath = tmpdir.mkdir("something")
-    dir_url = "file:{}".format(urllib_request.pathname2url(dirpath))
-
-    session = Mock(PipSession)
-    with patch("pip._internal.index._get_html_response") as mock_func:
-        _get_html_page(Link(dir_url), session=session)
-        assert mock_func.mock_calls == [
-            call("{}/index.html".format(dir_url.rstrip("/")), session=session)
-        ]
-
-
-@pytest.mark.parametrize(
-    "url",
-    [
-        "ftp://python.org/python-3.7.1.zip",
-        "file:///opt/data/pip-18.0.tar.gz",
-    ],
-)
-def test_get_html_response_archive_to_naive_scheme(url):
-    """
-    `_get_html_response()` should error on archive-like URL if the scheme
-    does not allow "poking" without getting data.
-    """
-    with pytest.raises(_NotHTTP):
-        _get_html_response(url, session=Mock(PipSession))
-
-
-@pytest.mark.parametrize(
-    "url, content_type",
-    [
-        ("http://python.org/python-3.7.1.zip", "application/zip"),
-        ("https://pypi.org/pip-18.0.tar.gz", "application/gzip"),
-    ],
-)
-def test_get_html_response_archive_to_http_scheme(url, content_type):
-    """
-    `_get_html_response()` should send a HEAD request on archive-like URL if
-    the scheme supports it.
-    """
-    session = Mock(PipSession)
-    session.head.return_value = Mock(**{
-        "request.method": "HEAD",
-        "headers": {"Content-Type": content_type},
-    })
-
-    with pytest.raises(_NotHTML) as ctx:
-        _get_html_response(url, session=session)
-
-    session.assert_has_calls([
-        call.head(url, allow_redirects=True),
-    ])
-    assert ctx.value.args == (content_type, "HEAD")
diff --git a/tests/unit/test_index_html_page.py b/tests/unit/test_index_html_page.py
new file mode 100644
index 000000000..70fae13e3
--- /dev/null
+++ b/tests/unit/test_index_html_page.py
@@ -0,0 +1,98 @@
+import logging
+
+import mock
+import pytest
+from pip._vendor.six.moves.urllib import request as urllib_request
+
+from pip._internal.download import PipSession
+from pip._internal.index import (
+    Link, PackageFinder, _determine_base_url, _get_html_page,
+    _get_html_response, _NotHTML, _NotHTTP, egg_info_matches,
+)
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "ftp://python.org/python-3.7.1.zip",
+        "file:///opt/data/pip-18.0.tar.gz",
+    ],
+)
+def test_get_html_response_archive_to_naive_scheme(url):
+    """
+    `_get_html_response()` should error on an archive-like URL if the scheme
+    does not allow "poking" without getting data.
+    """
+    with pytest.raises(_NotHTTP):
+        _get_html_response(url, session=mock.Mock(PipSession))
+
+
+@pytest.mark.parametrize(
+    "url, content_type",
+    [
+        ("http://python.org/python-3.7.1.zip", "application/zip"),
+        ("https://pypi.org/pip-18.0.tar.gz", "application/gzip"),
+    ],
+)
+def test_get_html_response_archive_to_http_scheme(url, content_type):
+    """
+    `_get_html_response()` should send a HEAD request on and archive-like URL
+    if the scheme supports it.
+    """
+    session = mock.Mock(PipSession)
+    session.head.return_value = mock.Mock(**{
+        "request.method": "HEAD",
+        "headers": {"Content-Type": content_type},
+    })
+
+    with pytest.raises(_NotHTML) as ctx:
+        _get_html_response(url, session=session)
+
+    session.assert_has_calls([
+        mock.call.head(url, allow_redirects=True),
+    ])
+    assert ctx.value.args == (content_type, "HEAD")
+
+
+@pytest.mark.parametrize(
+    "url, vcs_scheme",
+    [
+        ("svn+http://pypi.org/something", "svn"),
+        ("git+https://github.com/pypa/pip.git", "git"),
+    ],
+)
+def test_get_html_page_invalid_scheme(caplog, url, vcs_scheme):
+    """`_get_html_page()` should error if an invalid scheme is given.
+
+    Only file:, http:, https:, and ftp: are allowed.
+    """
+    with caplog.at_level(logging.DEBUG):
+        page = _get_html_page(Link(url), session=mock.Mock(PipSession))
+
+    assert page is None
+    assert caplog.record_tuples == [
+        (
+            "pip._internal.index",
+            logging.DEBUG,
+            "Cannot look at {} URL {}".format(vcs_scheme, url),
+        ),
+    ]
+
+
+def test_get_html_page_directory_append_index(tmpdir):
+    """`_get_html_page()` should append "index.html" to a directory URL.
+    """
+    dirpath = tmpdir.mkdir("something")
+    dir_url = "file:///{}".format(
+        urllib_request.pathname2url(dirpath).lstrip("/"),
+    )
+
+    session = mock.Mock(PipSession)
+    with mock.patch("pip._internal.index._get_html_response") as mock_func:
+        _get_html_page(Link(dir_url), session=session)
+        assert mock_func.mock_calls == [
+            mock.call(
+                "{}/index.html".format(dir_url.rstrip("/")),
+                session=session,
+            ),
+        ]

From fc53f711b934a2d7b9791cd6bbafa00221edc83b Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <uranusjr@gmail.com>
Date: Thu, 4 Oct 2018 17:08:39 +0800
Subject: [PATCH 92/92] More tests on successful scenarios, fix docstrings

---
 src/pip/_internal/index.py         |  6 +--
 tests/unit/test_index_html_page.py | 72 ++++++++++++++++++++++++++++--
 2 files changed, 71 insertions(+), 7 deletions(-)

diff --git a/src/pip/_internal/index.py b/src/pip/_internal/index.py
index e7d260a22..9f303a229 100644
--- a/src/pip/_internal/index.py
+++ b/src/pip/_internal/index.py
@@ -72,7 +72,7 @@ def _match_vcs_scheme(url):
 
 
 def _is_url_like_archive(url):
-    """Check whether the URL looks like an archive.
+    """Return whether the URL looks like an archive.
     """
     filename = Link(url).filename
     for bad_ext in ARCHIVE_EXTENSIONS:
@@ -128,8 +128,8 @@ def _get_html_response(url, session):
        Raise `_NotHTTP` if the content type cannot be determined, or
        `_NotHTML` if it is not HTML.
     2. Actually perform the request. Raise HTTP exceptions on network failures.
-    3. Check whether Content-Type header to make sure the thing we got is HTML,
-       and raise `_NotHTML` if it's not.
+    3. Check the Content-Type header to make sure we got HTML, and raise
+       `_NotHTML` otherwise.
     """
     if _is_url_like_archive(url):
         _ensure_html_response(url, session=session)
diff --git a/tests/unit/test_index_html_page.py b/tests/unit/test_index_html_page.py
index 70fae13e3..c872ad065 100644
--- a/tests/unit/test_index_html_page.py
+++ b/tests/unit/test_index_html_page.py
@@ -6,8 +6,7 @@ from pip._vendor.six.moves.urllib import request as urllib_request
 
 from pip._internal.download import PipSession
 from pip._internal.index import (
-    Link, PackageFinder, _determine_base_url, _get_html_page,
-    _get_html_response, _NotHTML, _NotHTTP, egg_info_matches,
+    Link, _get_html_page, _get_html_response, _NotHTML, _NotHTTP,
 )
 
 
@@ -36,8 +35,8 @@ def test_get_html_response_archive_to_naive_scheme(url):
 )
 def test_get_html_response_archive_to_http_scheme(url, content_type):
     """
-    `_get_html_response()` should send a HEAD request on and archive-like URL
-    if the scheme supports it.
+    `_get_html_response()` should send a HEAD request on an archive-like URL
+    if the scheme supports it, and raise `_NotHTML` if the response isn't HTML.
     """
     session = mock.Mock(PipSession)
     session.head.return_value = mock.Mock(**{
@@ -54,6 +53,71 @@ def test_get_html_response_archive_to_http_scheme(url, content_type):
     assert ctx.value.args == (content_type, "HEAD")
 
 
+@pytest.mark.parametrize(
+    "url",
+    [
+        "http://python.org/python-3.7.1.zip",
+        "https://pypi.org/pip-18.0.tar.gz",
+    ],
+)
+def test_get_html_response_archive_to_http_scheme_is_html(url):
+    """
+    `_get_html_response()` should work with archive-like URLs if the HEAD
+    request is responded with text/html.
+    """
+    session = mock.Mock(PipSession)
+    session.head.return_value = mock.Mock(**{
+        "request.method": "HEAD",
+        "headers": {"Content-Type": "text/html"},
+    })
+    session.get.return_value = mock.Mock(headers={"Content-Type": "text/html"})
+
+    resp = _get_html_response(url, session=session)
+
+    assert resp is not None
+    assert session.mock_calls == [
+        mock.call.head(url, allow_redirects=True),
+        mock.call.head().raise_for_status(),
+        mock.call.get(url, headers={
+            "Accept": "text/html", "Cache-Control": "max-age=0",
+        }),
+        mock.call.get().raise_for_status(),
+    ]
+
+
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://pypi.org/simple/pip",
+        "https://pypi.org/simple/pip/",
+        "https://python.org/sitemap.xml",
+    ],
+)
+def test_get_html_response_no_head(url):
+    """
+    `_get_html_response()` shouldn't send a HEAD request if the URL does not
+    look like an archive, only the GET request that retrieves data.
+    """
+    session = mock.Mock(PipSession)
+
+    # Mock the headers dict to ensure it is accessed.
+    session.get.return_value = mock.Mock(headers=mock.Mock(**{
+        "get.return_value": "text/html",
+    }))
+
+    resp = _get_html_response(url, session=session)
+
+    assert resp is not None
+    assert session.head.call_count == 0
+    assert session.get.mock_calls == [
+        mock.call(url, headers={
+            "Accept": "text/html", "Cache-Control": "max-age=0",
+        }),
+        mock.call().raise_for_status(),
+        mock.call().headers.get("Content-Type", ""),
+    ]
+
+
 @pytest.mark.parametrize(
     "url, vcs_scheme",
     [