From c0809f4183cbf40e3fd69245095537456d74b97f Mon Sep 17 00:00:00 2001 From: Tony Beswick Date: Tue, 24 Sep 2019 11:46:02 +1200 Subject: [PATCH 01/16] Added a `controls_location` implementation to the `Mercurial` class, so it porperly detects that subdirectories are under mercurial control. --- src/pip/_internal/vcs/mercurial.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 21697ff15..5af9c121e 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -9,6 +9,7 @@ from pip._internal.utils.misc import display_path, make_command, path_to_url from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs.versioncontrol import VersionControl, vcs +from pip._internal.exceptions import BadCommand if MYPY_CHECK_RUNNING: from pip._internal.utils.misc import HiddenText @@ -111,5 +112,16 @@ class Mercurial(VersionControl): """Always assume the versions don't match""" return False + @classmethod + def controls_location(cls, location): + if super(Mercurial, cls).controls_location(location): + return True + try: + r = cls.run_command(['identify'], cwd=location, show_stdout=False, extra_ok_returncodes=[255]) + return not r.startswith('abort:') + except BadCommand: + logger.debug("could not determine if %s is under hg control " + "because hg is not available", location) + return False vcs.register(Mercurial) From 063b899c3bffa00c6ab69901ad8477d6a28a2bcf Mon Sep 17 00:00:00 2001 From: Tony Beswick Date: Tue, 24 Sep 2019 12:20:53 +1200 Subject: [PATCH 02/16] Adding `get_subdirdctory` method to `Mercurial` class so it can detect a setup.py in a repo subdirectory. --- src/pip/_internal/vcs/mercurial.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 5af9c121e..7aa9b4ce1 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -9,6 +9,7 @@ from pip._internal.utils.misc import display_path, make_command, path_to_url from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs.versioncontrol import VersionControl, vcs +from pip._internal.utils.compat import samefile from pip._internal.exceptions import BadCommand if MYPY_CHECK_RUNNING: @@ -112,6 +113,32 @@ class Mercurial(VersionControl): """Always assume the versions don't match""" return False + @classmethod + def get_subdirectory(cls, location): + # find the repo root + root_dir = cls.run_command(['root'], + show_stdout=False, cwd=location).strip() + if not os.path.isabs(root_dir): + root_dir = os.path.join(location, root_dir) + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + # relative path of setup.py to repo root + if samefile(root_dir, location): + return None + return os.path.relpath(location, root_dir) + @classmethod def controls_location(cls, location): if super(Mercurial, cls).controls_location(location): From 73a6342d55d22d2e1086008981d5e84aa57a999f Mon Sep 17 00:00:00 2001 From: Tony Beswick Date: Tue, 24 Sep 2019 12:28:40 +1200 Subject: [PATCH 03/16] Added unit test for Mercurial to check for correct behaviour when either the source or setup.py is located in a subdirectory of the repo root. --- tests/functional/test_freeze.py | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 77f83796a..a06e81b7a 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -321,6 +321,47 @@ def test_freeze_git_clone_srcdir(script, tmpdir): ).strip() _check_output(result.stdout, expected) +@need_mercurial +def test_freeze_mrecurial_clone_srcdir(script, tmpdir): + """ + Test freezing a Mercurial clone where setup.py is in a subdirectory + relative to the repo root and the source code is in a subdirectory + relative to setup.py. + """ + # Returns path to a generated package called "version_pkg" + pkg_version = _create_test_package_with_srcdir(script, vcs='hg') + + result = script.run( + 'hg', 'clone', pkg_version, 'pip-test-package', + expect_stderr=True, + ) + repo_dir = script.scratch_path / 'pip-test-package' + result = script.run( + 'python', 'setup.py', 'develop', + cwd=repo_dir / 'subdir', + expect_stderr=True, + ) + result = script.pip('freeze', expect_stderr=True) + expected = textwrap.dedent( + """ + ...-e hg+...#egg=version_pkg&subdirectory=subdir + ... + """ + ).strip() + _check_output(result.stdout, expected) + + result = script.pip( + 'freeze', '-f', '%s#egg=pip_test_package' % repo_dir, + expect_stderr=True, + ) + expected = textwrap.dedent( + """ + -f %(repo)s#egg=pip_test_package... + -e hg+...#egg=version_pkg&subdirectory=subdir + ... + """ % {'repo': repo_dir}, + ).strip() + _check_output(result.stdout, expected) @pytest.mark.git def test_freeze_git_remote(script, tmpdir): From 356c6d20598aed809c1e3fa3582fec343a5e0a26 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Tue, 24 Sep 2019 16:33:39 +1200 Subject: [PATCH 04/16] fixed typo --- tests/functional/test_freeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index a06e81b7a..39e7661e7 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -322,7 +322,7 @@ def test_freeze_git_clone_srcdir(script, tmpdir): _check_output(result.stdout, expected) @need_mercurial -def test_freeze_mrecurial_clone_srcdir(script, tmpdir): +def test_freeze_mercurial_clone_srcdir(script, tmpdir): """ Test freezing a Mercurial clone where setup.py is in a subdirectory relative to the repo root and the source code is in a subdirectory From b95a16e310be3caa18c24a430ad94161f486e733 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Tue, 24 Sep 2019 16:50:05 +1200 Subject: [PATCH 05/16] Added news --- news/7071.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/7071.bugfix diff --git a/news/7071.bugfix b/news/7071.bugfix new file mode 100644 index 000000000..e4fa847d7 --- /dev/null +++ b/news/7071.bugfix @@ -0,0 +1 @@ +Fix `pip freeze` not showing correct entry for mercurial packages that use subdirectories. \ No newline at end of file From 657a7cb0c287043809ecacefba67cf4294feafb6 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Tue, 24 Sep 2019 21:05:30 +1200 Subject: [PATCH 06/16] - Added missing argument to run_command: on_returncode='ignore' - fixed lint errors. --- src/pip/_internal/vcs/mercurial.py | 13 +++++++++---- tests/functional/test_freeze.py | 6 ++++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 7aa9b4ce1..2b13a04cb 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -5,12 +5,12 @@ import os from pip._vendor.six.moves import configparser +from pip._internal.exceptions import BadCommand +from pip._internal.utils.compat import samefile from pip._internal.utils.misc import display_path, make_command, path_to_url from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs.versioncontrol import VersionControl, vcs -from pip._internal.utils.compat import samefile -from pip._internal.exceptions import BadCommand if MYPY_CHECK_RUNNING: from pip._internal.utils.misc import HiddenText @@ -117,7 +117,7 @@ class Mercurial(VersionControl): def get_subdirectory(cls, location): # find the repo root root_dir = cls.run_command(['root'], - show_stdout=False, cwd=location).strip() + show_stdout=False, cwd=location).strip() if not os.path.isabs(root_dir): root_dir = os.path.join(location, root_dir) # find setup.py @@ -144,11 +144,16 @@ class Mercurial(VersionControl): if super(Mercurial, cls).controls_location(location): return True try: - r = cls.run_command(['identify'], cwd=location, show_stdout=False, extra_ok_returncodes=[255]) + r = cls.run_command(['identify'], + cwd=location, + show_stdout=False, + on_returncode='ignore', + extra_ok_returncodes=[255]) return not r.startswith('abort:') except BadCommand: logger.debug("could not determine if %s is under hg control " "because hg is not available", location) return False + vcs.register(Mercurial) diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index 39e7661e7..b3e06bfdc 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -321,6 +321,7 @@ def test_freeze_git_clone_srcdir(script, tmpdir): ).strip() _check_output(result.stdout, expected) + @need_mercurial def test_freeze_mercurial_clone_srcdir(script, tmpdir): """ @@ -330,7 +331,7 @@ def test_freeze_mercurial_clone_srcdir(script, tmpdir): """ # Returns path to a generated package called "version_pkg" pkg_version = _create_test_package_with_srcdir(script, vcs='hg') - + result = script.run( 'hg', 'clone', pkg_version, 'pip-test-package', expect_stderr=True, @@ -349,7 +350,7 @@ def test_freeze_mercurial_clone_srcdir(script, tmpdir): """ ).strip() _check_output(result.stdout, expected) - + result = script.pip( 'freeze', '-f', '%s#egg=pip_test_package' % repo_dir, expect_stderr=True, @@ -363,6 +364,7 @@ def test_freeze_mercurial_clone_srcdir(script, tmpdir): ).strip() _check_output(result.stdout, expected) + @pytest.mark.git def test_freeze_git_remote(script, tmpdir): """ From b2bae6264120ed09526b5e56df10bd24c8a34e58 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Wed, 25 Sep 2019 11:40:38 +1200 Subject: [PATCH 07/16] Fixed problem where vcs commands were being called even if they're not installed, causing errors to be logged. --- src/pip/_internal/vcs/git.py | 4 ++-- src/pip/_internal/vcs/mercurial.py | 4 ++-- src/pip/_internal/vcs/versioncontrol.py | 17 ++++++++++++----- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 65069af7b..e92af2d05 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -362,8 +362,8 @@ class Git(VersionControl): @classmethod def controls_location(cls, location): - if super(Git, cls).controls_location(location): - return True + if not super(Git, cls).controls_location(location): + return False try: r = cls.run_command(['rev-parse'], cwd=location, diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index 2b13a04cb..de412a612 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -141,8 +141,8 @@ class Mercurial(VersionControl): @classmethod def controls_location(cls, location): - if super(Mercurial, cls).controls_location(location): - return True + if not super(Mercurial, cls).controls_location(location): + return False try: r = cls.run_command(['identify'], cwd=location, diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 27610602f..610fb5a43 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -621,10 +621,17 @@ class VersionControl(object): # type: (str) -> bool """ Check if a location is controlled by the vcs. - It is meant to be overridden to implement smarter detection - mechanisms for specific vcs. - This can do more than is_repository_directory() alone. For example, - the Git override checks that Git is actually available. + Searches up the filesystem and checks is_repository_directory(). + + It is meant to be extended to add smarter detection mechanisms for + specific vcs. For example, the Git override checks that Git is + actually available. """ - return cls.is_repository_directory(location) + while not cls.is_repository_directory(location): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem. + return False + return True From c6906f1a38cc98667777438a747395599e247c04 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Tue, 8 Oct 2019 18:15:22 +1300 Subject: [PATCH 08/16] - Abstracted out common `get_subdirectory()` code in `Git` and `Mercurial`, adding `get_repo_root_dir()` for the vcs specific code. - Reverted behaviour of `Git.controls_location()` and `Mercurial.controls_location()` to call the vcs command if the base `VersionControl.controls_location()` doesn't detect the vcs directory. - Added `log_failed_cmd` argument `VcsSupport.run_command()` to allow vcs commands to be tried without logging errors if they aren't present. - Corrected indentation. - Removed `expect_stderr=True` in `test_freeze_mercurial_clone_srcdir` as its not required. --- src/pip/_internal/utils/misc.py | 13 +++--- src/pip/_internal/vcs/git.py | 30 +++----------- src/pip/_internal/vcs/mercurial.py | 49 +++++++--------------- src/pip/_internal/vcs/versioncontrol.py | 54 +++++++++++++++++++++++-- tests/functional/test_freeze.py | 11 ++--- 5 files changed, 82 insertions(+), 75 deletions(-) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index 5f13f975c..915d925e6 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -628,7 +628,8 @@ def call_subprocess( command_desc=None, # type: Optional[str] extra_environ=None, # type: Optional[Mapping[str, Any]] unset_environ=None, # type: Optional[Iterable[str]] - spinner=None # type: Optional[SpinnerInterface] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True # type: Optional[bool] ): # type: (...) -> Text """ @@ -639,6 +640,7 @@ def call_subprocess( acceptable, in addition to 0. Defaults to None, which means []. unset_environ: an iterable of environment variable names to unset prior to calling subprocess.Popen(). + log_failed_cmd: if false, failed commands are not logged, only raised. """ if extra_ok_returncodes is None: extra_ok_returncodes = [] @@ -694,9 +696,10 @@ def call_subprocess( ) proc.stdin.close() except Exception as exc: - subprocess_logger.critical( - "Error %s while executing command %s", exc, command_desc, - ) + if log_failed_cmd: + subprocess_logger.critical( + "Error %s while executing command %s", exc, command_desc, + ) raise all_output = [] while True: @@ -727,7 +730,7 @@ def call_subprocess( spinner.finish("done") if proc_had_error: if on_returncode == 'raise': - if not showing_subprocess: + if not showing_subprocess and log_failed_cmd: # Then the subprocess streams haven't been logged to the # console yet. msg = make_subprocess_output_error( diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index e92af2d05..c6e0f548e 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -9,7 +9,6 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves.urllib import request as urllib_request from pip._internal.exceptions import BadCommand -from pip._internal.utils.compat import samefile from pip._internal.utils.misc import display_path, make_command from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -290,31 +289,13 @@ class Git(VersionControl): return current_rev.strip() @classmethod - def get_subdirectory(cls, location): - # find the repo root + def get_repo_root_dir(cls, location): git_dir = cls.run_command(['rev-parse', '--git-dir'], show_stdout=False, cwd=location).strip() if not os.path.isabs(git_dir): git_dir = os.path.join(location, git_dir) root_dir = os.path.join(git_dir, '..') - # find setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without - # finding setup.py - logger.warning( - "Could not find setup.py for directory %s (tried all " - "parent directories)", - orig_location, - ) - return None - # relative path of setup.py to repo root - if samefile(root_dir, location): - return None - return os.path.relpath(location, root_dir) + return os.path.abspath(root_dir) @classmethod def get_url_rev_and_auth(cls, url): @@ -362,13 +343,14 @@ class Git(VersionControl): @classmethod def controls_location(cls, location): - if not super(Git, cls).controls_location(location): - return False + if super(Git, cls).controls_location(location): + return True try: r = cls.run_command(['rev-parse'], cwd=location, show_stdout=False, - on_returncode='ignore') + on_returncode='ignore', + log_failed_cmd=False) return not r except BadCommand: logger.debug("could not determine if %s is under git control " diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index de412a612..e82fdb6f6 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -5,8 +5,7 @@ import os from pip._vendor.six.moves import configparser -from pip._internal.exceptions import BadCommand -from pip._internal.utils.compat import samefile +from pip._internal.exceptions import BadCommand, InstallationError from pip._internal.utils.misc import display_path, make_command, path_to_url from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING @@ -114,45 +113,25 @@ class Mercurial(VersionControl): return False @classmethod - def get_subdirectory(cls, location): - # find the repo root - root_dir = cls.run_command(['root'], - show_stdout=False, cwd=location).strip() + def get_repo_root_dir(cls, location): + root_dir = cls.run_command( + ['root'], show_stdout=False, cwd=location).strip() if not os.path.isabs(root_dir): root_dir = os.path.join(location, root_dir) - # find setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without - # finding setup.py - logger.warning( - "Could not find setup.py for directory %s (tried all " - "parent directories)", - orig_location, - ) - return None - # relative path of setup.py to repo root - if samefile(root_dir, location): - return None - return os.path.relpath(location, root_dir) + return os.path.abspath(root_dir) @classmethod def controls_location(cls, location): - if not super(Mercurial, cls).controls_location(location): - return False + if super(Mercurial, cls).controls_location(location): + return True try: - r = cls.run_command(['identify'], - cwd=location, - show_stdout=False, - on_returncode='ignore', - extra_ok_returncodes=[255]) - return not r.startswith('abort:') - except BadCommand: - logger.debug("could not determine if %s is under hg control " - "because hg is not available", location) + cls.run_command( + ['identify'], + cwd=location, + show_stdout=False, + on_returncode='raise', + log_failed_cmd=False) + except (BadCommand, InstallationError): return False diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 610fb5a43..ac5711286 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -11,6 +11,7 @@ from pip._vendor import pkg_resources from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._internal.exceptions import BadCommand +from pip._internal.utils.compat import samefile from pip._internal.utils.misc import ( ask_path_exists, backup_dir, @@ -236,11 +237,54 @@ class VersionControl(object): return not remote_url.lower().startswith('{}:'.format(cls.name)) @classmethod - def get_subdirectory(cls, repo_dir): + def get_subdirectory(cls, location): """ Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. """ - return None + # find the repo root + root_dir = cls.get_repo_root_dir(location) + if root_dir is None: + logger.warning( + "Repo root could not be detected for %s, " + "assuming it is the root.", + location) + return None + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + # relative path of setup.py to repo root + if samefile(root_dir, location): + return None + return os.path.relpath(location, root_dir) + + @classmethod + def get_repo_root_dir(cls, location): + """ + Return the absolute path to the repo root directory. + + Return None if not found. + This can be overridden by subclasses to interrogate the vcs tool to + find the repo root. + """ + while not cls.is_repository_directory(location): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem. + return None + return os.path.abspath(location) @classmethod def get_requirement_revision(cls, repo_dir): @@ -578,7 +622,8 @@ class VersionControl(object): extra_ok_returncodes=None, # type: Optional[Iterable[int]] command_desc=None, # type: Optional[str] extra_environ=None, # type: Optional[Mapping[str, Any]] - spinner=None # type: Optional[SpinnerInterface] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True ): # type: (...) -> Text """ @@ -594,7 +639,8 @@ class VersionControl(object): command_desc=command_desc, extra_environ=extra_environ, unset_environ=cls.unset_environ, - spinner=spinner) + spinner=spinner, + log_failed_cmd=log_failed_cmd) except OSError as e: # errno.ENOENT = no such file or directory # In other words, the VCS executable isn't available diff --git a/tests/functional/test_freeze.py b/tests/functional/test_freeze.py index b3e06bfdc..546a4828d 100644 --- a/tests/functional/test_freeze.py +++ b/tests/functional/test_freeze.py @@ -333,16 +333,14 @@ def test_freeze_mercurial_clone_srcdir(script, tmpdir): pkg_version = _create_test_package_with_srcdir(script, vcs='hg') result = script.run( - 'hg', 'clone', pkg_version, 'pip-test-package', - expect_stderr=True, + 'hg', 'clone', pkg_version, 'pip-test-package' ) repo_dir = script.scratch_path / 'pip-test-package' result = script.run( 'python', 'setup.py', 'develop', - cwd=repo_dir / 'subdir', - expect_stderr=True, + cwd=repo_dir / 'subdir' ) - result = script.pip('freeze', expect_stderr=True) + result = script.pip('freeze') expected = textwrap.dedent( """ ...-e hg+...#egg=version_pkg&subdirectory=subdir @@ -352,8 +350,7 @@ def test_freeze_mercurial_clone_srcdir(script, tmpdir): _check_output(result.stdout, expected) result = script.pip( - 'freeze', '-f', '%s#egg=pip_test_package' % repo_dir, - expect_stderr=True, + 'freeze', '-f', '%s#egg=pip_test_package' % repo_dir ) expected = textwrap.dedent( """ From a0cbe4c44087d95cff36e5fbfaf4da629464a866 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Wed, 9 Oct 2019 09:11:58 +1300 Subject: [PATCH 09/16] Fixing bad merge. --- src/pip/_internal/utils/misc.py | 225 -------------------------- src/pip/_internal/utils/subprocess.py | 13 +- src/pip/_internal/vcs/git.py | 3 +- 3 files changed, 10 insertions(+), 231 deletions(-) diff --git a/src/pip/_internal/utils/misc.py b/src/pip/_internal/utils/misc.py index c0c9120f2..563399510 100644 --- a/src/pip/_internal/utils/misc.py +++ b/src/pip/_internal/utils/misc.py @@ -525,231 +525,6 @@ def dist_location(dist): return normalize_path(dist.location) -def make_command(*args): - # type: (Union[str, HiddenText, CommandArgs]) -> CommandArgs - """ - Create a CommandArgs object. - """ - command_args = [] # type: CommandArgs - for arg in args: - # Check for list instead of CommandArgs since CommandArgs is - # only known during type-checking. - if isinstance(arg, list): - command_args.extend(arg) - else: - # Otherwise, arg is str or HiddenText. - command_args.append(arg) - - return command_args - - -def format_command_args(args): - # type: (Union[List[str], CommandArgs]) -> str - """ - Format command arguments for display. - """ - # For HiddenText arguments, display the redacted form by calling str(). - # Also, we don't apply str() to arguments that aren't HiddenText since - # this can trigger a UnicodeDecodeError in Python 2 if the argument - # has type unicode and includes a non-ascii character. (The type - # checker doesn't ensure the annotations are correct in all cases.) - return ' '.join( - shlex_quote(str(arg)) if isinstance(arg, HiddenText) - else shlex_quote(arg) for arg in args - ) - - -def reveal_command_args(args): - # type: (Union[List[str], CommandArgs]) -> List[str] - """ - Return the arguments in their raw, unredacted form. - """ - return [ - arg.secret if isinstance(arg, HiddenText) else arg for arg in args - ] - - -def make_subprocess_output_error( - cmd_args, # type: Union[List[str], CommandArgs] - cwd, # type: Optional[str] - lines, # type: List[Text] - exit_status, # type: int -): - # type: (...) -> Text - """ - Create and return the error message to use to log a subprocess error - with command output. - - :param lines: A list of lines, each ending with a newline. - """ - command = format_command_args(cmd_args) - # Convert `command` and `cwd` to text (unicode in Python 2) so we can use - # them as arguments in the unicode format string below. This avoids - # "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2 - # if either contains a non-ascii character. - command_display = str_to_display(command, desc='command bytes') - cwd_display = path_to_display(cwd) - - # We know the joined output value ends in a newline. - output = ''.join(lines) - msg = ( - # Use a unicode string to avoid "UnicodeEncodeError: 'ascii' - # codec can't encode character ..." in Python 2 when a format - # argument (e.g. `output`) has a non-ascii character. - u'Command errored out with exit status {exit_status}:\n' - ' command: {command_display}\n' - ' cwd: {cwd_display}\n' - 'Complete output ({line_count} lines):\n{output}{divider}' - ).format( - exit_status=exit_status, - command_display=command_display, - cwd_display=cwd_display, - line_count=len(lines), - output=output, - divider=LOG_DIVIDER, - ) - return msg - - -def call_subprocess( - cmd, # type: Union[List[str], CommandArgs] - show_stdout=False, # type: bool - cwd=None, # type: Optional[str] - on_returncode='raise', # type: str - extra_ok_returncodes=None, # type: Optional[Iterable[int]] - command_desc=None, # type: Optional[str] - extra_environ=None, # type: Optional[Mapping[str, Any]] - unset_environ=None, # type: Optional[Iterable[str]] - spinner=None, # type: Optional[SpinnerInterface] - log_failed_cmd=True # type: Optional[bool] -): - # type: (...) -> Text - """ - Args: - show_stdout: if true, use INFO to log the subprocess's stderr and - stdout streams. Otherwise, use DEBUG. Defaults to False. - extra_ok_returncodes: an iterable of integer return codes that are - acceptable, in addition to 0. Defaults to None, which means []. - unset_environ: an iterable of environment variable names to unset - prior to calling subprocess.Popen(). - log_failed_cmd: if false, failed commands are not logged, only raised. - """ - if extra_ok_returncodes is None: - extra_ok_returncodes = [] - if unset_environ is None: - unset_environ = [] - # Most places in pip use show_stdout=False. What this means is-- - # - # - We connect the child's output (combined stderr and stdout) to a - # single pipe, which we read. - # - We log this output to stderr at DEBUG level as it is received. - # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't - # requested), then we show a spinner so the user can still see the - # subprocess is in progress. - # - If the subprocess exits with an error, we log the output to stderr - # at ERROR level if it hasn't already been displayed to the console - # (e.g. if --verbose logging wasn't enabled). This way we don't log - # the output to the console twice. - # - # If show_stdout=True, then the above is still done, but with DEBUG - # replaced by INFO. - if show_stdout: - # Then log the subprocess output at INFO level. - log_subprocess = subprocess_logger.info - used_level = logging.INFO - else: - # Then log the subprocess output using DEBUG. This also ensures - # it will be logged to the log file (aka user_log), if enabled. - log_subprocess = subprocess_logger.debug - used_level = logging.DEBUG - - # Whether the subprocess will be visible in the console. - showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level - - # Only use the spinner if we're not showing the subprocess output - # and we have a spinner. - use_spinner = not showing_subprocess and spinner is not None - - if command_desc is None: - command_desc = format_command_args(cmd) - - log_subprocess("Running command %s", command_desc) - env = os.environ.copy() - if extra_environ: - env.update(extra_environ) - for name in unset_environ: - env.pop(name, None) - try: - proc = subprocess.Popen( - # Convert HiddenText objects to the underlying str. - reveal_command_args(cmd), - stderr=subprocess.STDOUT, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, cwd=cwd, env=env, - ) - proc.stdin.close() - except Exception as exc: - if log_failed_cmd: - subprocess_logger.critical( - "Error %s while executing command %s", exc, command_desc, - ) - raise - all_output = [] - while True: - # The "line" value is a unicode string in Python 2. - line = console_to_str(proc.stdout.readline()) - if not line: - break - line = line.rstrip() - all_output.append(line + '\n') - - # Show the line immediately. - log_subprocess(line) - # Update the spinner. - if use_spinner: - spinner.spin() - try: - proc.wait() - finally: - if proc.stdout: - proc.stdout.close() - proc_had_error = ( - proc.returncode and proc.returncode not in extra_ok_returncodes - ) - if use_spinner: - if proc_had_error: - spinner.finish("error") - else: - spinner.finish("done") - if proc_had_error: - if on_returncode == 'raise': - if not showing_subprocess and log_failed_cmd: - # Then the subprocess streams haven't been logged to the - # console yet. - msg = make_subprocess_output_error( - cmd_args=cmd, - cwd=cwd, - lines=all_output, - exit_status=proc.returncode, - ) - subprocess_logger.error(msg) - exc_msg = ( - 'Command errored out with exit status {}: {} ' - 'Check the logs for full command output.' - ).format(proc.returncode, command_desc) - raise InstallationError(exc_msg) - elif on_returncode == 'warn': - subprocess_logger.warning( - 'Command "%s" had error code %s in %s', - command_desc, proc.returncode, cwd, - ) - elif on_returncode == 'ignore': - pass - else: - raise ValueError('Invalid value: on_returncode=%s' % - repr(on_returncode)) - return ''.join(all_output) - - def write_output(msg, *args): # type: (str, str) -> None logger.info(msg, *args) diff --git a/src/pip/_internal/utils/subprocess.py b/src/pip/_internal/utils/subprocess.py index ddb418d24..2a0c5d1a6 100644 --- a/src/pip/_internal/utils/subprocess.py +++ b/src/pip/_internal/utils/subprocess.py @@ -123,7 +123,8 @@ def call_subprocess( command_desc=None, # type: Optional[str] extra_environ=None, # type: Optional[Mapping[str, Any]] unset_environ=None, # type: Optional[Iterable[str]] - spinner=None # type: Optional[SpinnerInterface] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True # type: Optional[bool] ): # type: (...) -> Text """ @@ -134,6 +135,7 @@ def call_subprocess( acceptable, in addition to 0. Defaults to None, which means []. unset_environ: an iterable of environment variable names to unset prior to calling subprocess.Popen(). + log_failed_cmd: if false, failed commands are not logged, only raised. """ if extra_ok_returncodes is None: extra_ok_returncodes = [] @@ -189,9 +191,10 @@ def call_subprocess( ) proc.stdin.close() except Exception as exc: - subprocess_logger.critical( - "Error %s while executing command %s", exc, command_desc, - ) + if log_failed_cmd: + subprocess_logger.critical( + "Error %s while executing command %s", exc, command_desc, + ) raise all_output = [] while True: @@ -222,7 +225,7 @@ def call_subprocess( spinner.finish("done") if proc_had_error: if on_returncode == 'raise': - if not showing_subprocess: + if not showing_subprocess and log_failed_cmd: # Then the subprocess streams haven't been logged to the # console yet. msg = make_subprocess_output_error( diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 7f5407ddc..d94a207fb 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -12,8 +12,9 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves.urllib import request as urllib_request from pip._internal.exceptions import BadCommand -from pip._internal.utils.misc import display_path, make_command from pip._internal.utils.compat import samefile +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command from pip._internal.utils.temp_dir import TempDirectory from pip._internal.utils.typing import MYPY_CHECK_RUNNING from pip._internal.vcs.versioncontrol import ( From 5ad4291b06899c6b2ebbbb22c5c69990e5ad7950 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Wed, 9 Oct 2019 09:40:59 +1300 Subject: [PATCH 10/16] Added newline to end of news file. --- news/7071.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/7071.bugfix b/news/7071.bugfix index e4fa847d7..f56ea39ff 100644 --- a/news/7071.bugfix +++ b/news/7071.bugfix @@ -1 +1 @@ -Fix `pip freeze` not showing correct entry for mercurial packages that use subdirectories. \ No newline at end of file +Fix `pip freeze` not showing correct entry for mercurial packages that use subdirectories. From f7b44a82fec455c27d21f7719c3a76523ebf4468 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Wed, 9 Oct 2019 09:45:41 +1300 Subject: [PATCH 11/16] Removed unused import. --- src/pip/_internal/vcs/git.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index d94a207fb..65ff69877 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -12,7 +12,6 @@ from pip._vendor.six.moves.urllib import parse as urllib_parse from pip._vendor.six.moves.urllib import request as urllib_request from pip._internal.exceptions import BadCommand -from pip._internal.utils.compat import samefile from pip._internal.utils.misc import display_path from pip._internal.utils.subprocess import make_command from pip._internal.utils.temp_dir import TempDirectory From 1028e307824c26a8704ea3a929ae57de2f242053 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Wed, 9 Oct 2019 10:12:38 +1300 Subject: [PATCH 12/16] Fixed lint error in news. --- news/7071.bugfix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/news/7071.bugfix b/news/7071.bugfix index f56ea39ff..f0463ce3c 100644 --- a/news/7071.bugfix +++ b/news/7071.bugfix @@ -1 +1 @@ -Fix `pip freeze` not showing correct entry for mercurial packages that use subdirectories. +Fix ``pip freeze`` not showing correct entry for mercurial packages that use subdirectories. From f81f3c902c181b416cf15a78c9bdc1466d732253 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Fri, 11 Oct 2019 12:27:17 +1300 Subject: [PATCH 13/16] Removed `get_repo_root_dir()`, this functionality is now included in `Git.get_subdirectory()` and `Mercurial.get_subdirectory()`. Added `find_path_to_setup_from_repo_root()` function to perform the common parts of `get_subdirectory()`. --- src/pip/_internal/vcs/git.py | 737 ++++++------ src/pip/_internal/vcs/mercurial.py | 299 ++--- src/pip/_internal/vcs/versioncontrol.py | 1359 +++++++++++------------ 3 files changed, 1198 insertions(+), 1197 deletions(-) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 65ff69877..6b8041140 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -1,365 +1,372 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import logging -import os.path -import re - -from pip._vendor.packaging.version import parse as parse_version -from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._vendor.six.moves.urllib import request as urllib_request - -from pip._internal.exceptions import BadCommand -from pip._internal.utils.misc import display_path -from pip._internal.utils.subprocess import make_command -from pip._internal.utils.temp_dir import TempDirectory -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.vcs.versioncontrol import ( - RemoteNotFoundError, - VersionControl, - vcs, -) - -if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple - from pip._internal.utils.misc import HiddenText - from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions - - -urlsplit = urllib_parse.urlsplit -urlunsplit = urllib_parse.urlunsplit - - -logger = logging.getLogger(__name__) - - -HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') - - -def looks_like_hash(sha): - return bool(HASH_REGEX.match(sha)) - - -class Git(VersionControl): - name = 'git' - dirname = '.git' - repo_name = 'clone' - schemes = ( - 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', - ) - # Prevent the user's environment variables from interfering with pip: - # https://github.com/pypa/pip/issues/1130 - unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') - default_arg_rev = 'HEAD' - - @staticmethod - def get_base_rev_args(rev): - return [rev] - - def get_git_version(self): - VERSION_PFX = 'git version ' - version = self.run_command(['version'], show_stdout=False) - if version.startswith(VERSION_PFX): - version = version[len(VERSION_PFX):].split()[0] - else: - version = '' - # get first 3 positions of the git version because - # on windows it is x.y.z.windows.t, and this parses as - # LegacyVersion which always smaller than a Version. - version = '.'.join(version.split('.')[:3]) - return parse_version(version) - - @classmethod - def get_current_branch(cls, location): - """ - Return the current branch, or None if HEAD isn't at a branch - (e.g. detached HEAD). - """ - # git-symbolic-ref exits with empty stdout if "HEAD" is a detached - # HEAD rather than a symbolic ref. In addition, the -q causes the - # command to exit with status code 1 instead of 128 in this case - # and to suppress the message to stderr. - args = ['symbolic-ref', '-q', 'HEAD'] - output = cls.run_command( - args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, - ) - ref = output.strip() - - if ref.startswith('refs/heads/'): - return ref[len('refs/heads/'):] - - return None - - def export(self, location, url): - # type: (str, HiddenText) -> None - """Export the Git repository at the url to the destination location""" - if not location.endswith('/'): - location = location + '/' - - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path, url=url) - self.run_command( - ['checkout-index', '-a', '-f', '--prefix', location], - show_stdout=False, cwd=temp_dir.path - ) - - @classmethod - def get_revision_sha(cls, dest, rev): - """ - Return (sha_or_none, is_branch), where sha_or_none is a commit hash - if the revision names a remote branch or tag, otherwise None. - - Args: - dest: the repository directory. - rev: the revision name. - """ - # Pass rev to pre-filter the list. - output = cls.run_command(['show-ref', rev], cwd=dest, - show_stdout=False, on_returncode='ignore') - refs = {} - for line in output.strip().splitlines(): - try: - sha, ref = line.split() - except ValueError: - # Include the offending line to simplify troubleshooting if - # this error ever occurs. - raise ValueError('unexpected show-ref line: {!r}'.format(line)) - - refs[ref] = sha - - branch_ref = 'refs/remotes/origin/{}'.format(rev) - tag_ref = 'refs/tags/{}'.format(rev) - - sha = refs.get(branch_ref) - if sha is not None: - return (sha, True) - - sha = refs.get(tag_ref) - - return (sha, False) - - @classmethod - def resolve_revision(cls, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> RevOptions - """ - Resolve a revision to a new RevOptions object with the SHA1 of the - branch, tag, or ref if found. - - Args: - rev_options: a RevOptions object. - """ - rev = rev_options.arg_rev - # The arg_rev property's implementation for Git ensures that the - # rev return value is always non-None. - assert rev is not None - - sha, is_branch = cls.get_revision_sha(dest, rev) - - if sha is not None: - rev_options = rev_options.make_new(sha) - rev_options.branch_name = rev if is_branch else None - - return rev_options - - # Do not show a warning for the common case of something that has - # the form of a Git commit hash. - if not looks_like_hash(rev): - logger.warning( - "Did not find branch or tag '%s', assuming revision or ref.", - rev, - ) - - if not rev.startswith('refs/'): - return rev_options - - # If it looks like a ref, we have to fetch it explicitly. - cls.run_command( - make_command('fetch', '-q', url, rev_options.to_args()), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - sha = cls.get_revision(dest, rev='FETCH_HEAD') - rev_options = rev_options.make_new(sha) - - return rev_options - - @classmethod - def is_commit_id_equal(cls, dest, name): - """ - Return whether the current commit hash equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - if not name: - # Then avoid an unnecessary subprocess call. - return False - - return cls.get_revision(dest) == name - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - rev_display = rev_options.to_display() - logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) - self.run_command(make_command('clone', '-q', url, dest)) - - if rev_options.rev: - # Then a specific revision was requested. - rev_options = self.resolve_revision(dest, url, rev_options) - branch_name = getattr(rev_options, 'branch_name', None) - if branch_name is None: - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - cmd_args = make_command( - 'checkout', '-q', rev_options.to_args(), - ) - self.run_command(cmd_args, cwd=dest) - elif self.get_current_branch(dest) != branch_name: - # Then a specific branch was requested, and that branch - # is not yet checked out. - track_branch = 'origin/{}'.format(branch_name) - cmd_args = [ - 'checkout', '-b', branch_name, '--track', track_branch, - ] - self.run_command(cmd_args, cwd=dest) - - #: repo may contain submodules - self.update_submodules(dest) - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - self.run_command( - make_command('config', 'remote.origin.url', url), - cwd=dest, - ) - cmd_args = make_command('checkout', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - self.update_submodules(dest) - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - # First fetch changes from the default remote - if self.get_git_version() >= parse_version('1.9.0'): - # fetch tags in addition to everything else - self.run_command(['fetch', '-q', '--tags'], cwd=dest) - else: - self.run_command(['fetch', '-q'], cwd=dest) - # Then reset to wanted revision (maybe even origin/master) - rev_options = self.resolve_revision(dest, url, rev_options) - cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - #: update submodules - self.update_submodules(dest) - - @classmethod - def get_remote_url(cls, location): - """ - Return URL of the first remote encountered. - - Raises RemoteNotFoundError if the repository does not have a remote - url configured. - """ - # We need to pass 1 for extra_ok_returncodes since the command - # exits with return code 1 if there are no matching lines. - stdout = cls.run_command( - ['config', '--get-regexp', r'remote\..*\.url'], - extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, - ) - remotes = stdout.splitlines() - try: - found_remote = remotes[0] - except IndexError: - raise RemoteNotFoundError - - for remote in remotes: - if remote.startswith('remote.origin.url '): - found_remote = remote - break - url = found_remote.split(' ')[1] - return url.strip() - - @classmethod - def get_revision(cls, location, rev=None): - if rev is None: - rev = 'HEAD' - current_rev = cls.run_command( - ['rev-parse', rev], show_stdout=False, cwd=location, - ) - return current_rev.strip() - - @classmethod - def get_repo_root_dir(cls, location): - git_dir = cls.run_command(['rev-parse', '--git-dir'], - show_stdout=False, cwd=location).strip() - if not os.path.isabs(git_dir): - git_dir = os.path.join(location, git_dir) - root_dir = os.path.join(git_dir, '..') - return os.path.abspath(root_dir) - - @classmethod - def get_url_rev_and_auth(cls, url): - # type: (str) -> Tuple[str, Optional[str], AuthInfo] - """ - Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. - That's required because although they use SSH they sometimes don't - work with a ssh:// scheme (e.g. GitHub). But we need a scheme for - parsing. Hence we remove it again afterwards and return it as a stub. - """ - # Works around an apparent Git bug - # (see https://article.gmane.org/gmane.comp.version-control.git/146500) - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme.endswith('file'): - initial_slashes = path[:-len(path.lstrip('/'))] - newpath = ( - initial_slashes + - urllib_request.url2pathname(path) - .replace('\\', '/').lstrip('/') - ) - url = urlunsplit((scheme, netloc, newpath, query, fragment)) - after_plus = scheme.find('+') + 1 - url = scheme[:after_plus] + urlunsplit( - (scheme[after_plus:], netloc, newpath, query, fragment), - ) - - if '://' not in url: - assert 'file:' not in url - url = url.replace('git+', 'git+ssh://') - url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) - url = url.replace('ssh://', '') - else: - url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) - - return url, rev, user_pass - - @classmethod - def update_submodules(cls, location): - if not os.path.exists(os.path.join(location, '.gitmodules')): - return - cls.run_command( - ['submodule', 'update', '--init', '--recursive', '-q'], - cwd=location, - ) - - @classmethod - def controls_location(cls, location): - if super(Git, cls).controls_location(location): - return True - try: - r = cls.run_command(['rev-parse'], - cwd=location, - show_stdout=False, - on_returncode='ignore', - log_failed_cmd=False) - return not r - except BadCommand: - logger.debug("could not determine if %s is under git control " - "because git is not available", location) - return False - - -vcs.register(Git) +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os.path +import re + +from pip._vendor.packaging.version import parse as parse_version +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.vcs.versioncontrol import ( + RemoteNotFoundError, + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + +urlsplit = urllib_parse.urlsplit +urlunsplit = urllib_parse.urlunsplit + + +logger = logging.getLogger(__name__) + + +HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') + + +def looks_like_hash(sha): + return bool(HASH_REGEX.match(sha)) + + +class Git(VersionControl): + name = 'git' + dirname = '.git' + repo_name = 'clone' + schemes = ( + 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', + ) + # Prevent the user's environment variables from interfering with pip: + # https://github.com/pypa/pip/issues/1130 + unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') + default_arg_rev = 'HEAD' + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def get_git_version(self): + VERSION_PFX = 'git version ' + version = self.run_command(['version'], show_stdout=False) + if version.startswith(VERSION_PFX): + version = version[len(VERSION_PFX):].split()[0] + else: + version = '' + # get first 3 positions of the git version because + # on windows it is x.y.z.windows.t, and this parses as + # LegacyVersion which always smaller than a Version. + version = '.'.join(version.split('.')[:3]) + return parse_version(version) + + @classmethod + def get_current_branch(cls, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = cls.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() + + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] + + return None + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Git repository at the url to the destination location""" + if not location.endswith('/'): + location = location + '/' + + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + self.run_command( + ['checkout-index', '-a', '-f', '--prefix', location], + show_stdout=False, cwd=temp_dir.path + ) + + @classmethod + def get_revision_sha(cls, dest, rev): + """ + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. + + Args: + dest: the repository directory. + rev: the revision name. + """ + # Pass rev to pre-filter the list. + output = cls.run_command(['show-ref', rev], cwd=dest, + show_stdout=False, on_returncode='ignore') + refs = {} + for line in output.strip().splitlines(): + try: + sha, ref = line.split() + except ValueError: + # Include the offending line to simplify troubleshooting if + # this error ever occurs. + raise ValueError('unexpected show-ref line: {!r}'.format(line)) + + refs[ref] = sha + + branch_ref = 'refs/remotes/origin/{}'.format(rev) + tag_ref = 'refs/tags/{}'.format(rev) + + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) + + sha = refs.get(tag_ref) + + return (sha, False) + + @classmethod + def resolve_revision(cls, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> RevOptions + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. + + Args: + rev_options: a RevOptions object. + """ + rev = rev_options.arg_rev + # The arg_rev property's implementation for Git ensures that the + # rev return value is always non-None. + assert rev is not None + + sha, is_branch = cls.get_revision_sha(dest, rev) + + if sha is not None: + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options + + # Do not show a warning for the common case of something that has + # the form of a Git commit hash. + if not looks_like_hash(rev): + logger.warning( + "Did not find branch or tag '%s', assuming revision or ref.", + rev, + ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + cls.run_command( + make_command('fetch', '-q', url, rev_options.to_args()), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = cls.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + + return rev_options + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the current commit hash equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + if not name: + # Then avoid an unnecessary subprocess call. + return False + + return cls.get_revision(dest) == name + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) + self.run_command(make_command('clone', '-q', url, dest)) + + if rev_options.rev: + # Then a specific revision was requested. + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = make_command( + 'checkout', '-q', rev_options.to_args(), + ) + self.run_command(cmd_args, cwd=dest) + elif self.get_current_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) + + #: repo may contain submodules + self.update_submodules(dest) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command( + make_command('config', 'remote.origin.url', url), + cwd=dest, + ) + cmd_args = make_command('checkout', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + self.update_submodules(dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + # First fetch changes from the default remote + if self.get_git_version() >= parse_version('1.9.0'): + # fetch tags in addition to everything else + self.run_command(['fetch', '-q', '--tags'], cwd=dest) + else: + self.run_command(['fetch', '-q'], cwd=dest) + # Then reset to wanted revision (maybe even origin/master) + rev_options = self.resolve_revision(dest, url, rev_options) + cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + #: update submodules + self.update_submodules(dest) + + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( + ['config', '--get-regexp', r'remote\..*\.url'], + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + + for remote in remotes: + if remote.startswith('remote.origin.url '): + found_remote = remote + break + url = found_remote.split(' ')[1] + return url.strip() + + @classmethod + def get_revision(cls, location, rev=None): + if rev is None: + rev = 'HEAD' + current_rev = cls.run_command( + ['rev-parse', rev], show_stdout=False, cwd=location, + ) + return current_rev.strip() + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + git_dir = cls.run_command( + ['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() + if not os.path.isabs(git_dir): + git_dir = os.path.join(location, git_dir) + repo_root = os.path.abspath(os.path.join(git_dir, '..')) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. + That's required because although they use SSH they sometimes don't + work with a ssh:// scheme (e.g. GitHub). But we need a scheme for + parsing. Hence we remove it again afterwards and return it as a stub. + """ + # Works around an apparent Git bug + # (see https://article.gmane.org/gmane.comp.version-control.git/146500) + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = ( + initial_slashes + + urllib_request.url2pathname(path) + .replace('\\', '/').lstrip('/') + ) + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+') + 1 + url = scheme[:after_plus] + urlunsplit( + (scheme[after_plus:], netloc, newpath, query, fragment), + ) + + if '://' not in url: + assert 'file:' not in url + url = url.replace('git+', 'git+ssh://') + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + url = url.replace('ssh://', '') + else: + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + + return url, rev, user_pass + + @classmethod + def update_submodules(cls, location): + if not os.path.exists(os.path.join(location, '.gitmodules')): + return + cls.run_command( + ['submodule', 'update', '--init', '--recursive', '-q'], + cwd=location, + ) + + @classmethod + def controls_location(cls, location): + if super(Git, cls).controls_location(location): + return True + try: + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore', + log_failed_cmd=False) + return not r + except BadCommand: + logger.debug("could not determine if %s is under git control " + "because git is not available", location) + return False + + +vcs.register(Git) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index d1036d699..dd04b8bc6 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -1,145 +1,154 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import logging -import os - -from pip._vendor.six.moves import configparser - -from pip._internal.exceptions import BadCommand, InstallationError -from pip._internal.utils.misc import display_path -from pip._internal.utils.subprocess import make_command -from pip._internal.utils.temp_dir import TempDirectory -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import path_to_url -from pip._internal.vcs.versioncontrol import VersionControl, vcs - -if MYPY_CHECK_RUNNING: - from pip._internal.utils.misc import HiddenText - from pip._internal.vcs.versioncontrol import RevOptions - - -logger = logging.getLogger(__name__) - - -class Mercurial(VersionControl): - name = 'hg' - dirname = '.hg' - repo_name = 'clone' - schemes = ( - 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', - ) - - @staticmethod - def get_base_rev_args(rev): - return [rev] - - def export(self, location, url): - # type: (str, HiddenText) -> None - """Export the Hg repository at the url to the destination location""" - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path, url=url) - - self.run_command( - ['archive', location], show_stdout=False, cwd=temp_dir.path - ) - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - rev_display = rev_options.to_display() - logger.info( - 'Cloning hg %s%s to %s', - url, - rev_display, - display_path(dest), - ) - self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) - self.run_command( - make_command('update', '-q', rev_options.to_args()), - cwd=dest, - ) - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - repo_config = os.path.join(dest, self.dirname, 'hgrc') - config = configparser.RawConfigParser() - try: - config.read(repo_config) - config.set('paths', 'default', url.secret) - with open(repo_config, 'w') as config_file: - config.write(config_file) - except (OSError, configparser.NoSectionError) as exc: - logger.warning( - 'Could not switch Mercurial repository to %s: %s', url, exc, - ) - else: - cmd_args = make_command('update', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - self.run_command(['pull', '-q'], cwd=dest) - cmd_args = make_command('update', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - @classmethod - def get_remote_url(cls, location): - url = cls.run_command( - ['showconfig', 'paths.default'], - show_stdout=False, cwd=location).strip() - if cls._is_local_repository(url): - url = path_to_url(url) - return url.strip() - - @classmethod - def get_revision(cls, location): - """ - Return the repository-local changeset revision number, as an integer. - """ - current_revision = cls.run_command( - ['parents', '--template={rev}'], - show_stdout=False, cwd=location).strip() - return current_revision - - @classmethod - def get_requirement_revision(cls, location): - """ - Return the changeset identification hash, as a 40-character - hexadecimal string - """ - current_rev_hash = cls.run_command( - ['parents', '--template={node}'], - show_stdout=False, cwd=location).strip() - return current_rev_hash - - @classmethod - def is_commit_id_equal(cls, dest, name): - """Always assume the versions don't match""" - return False - - @classmethod - def get_repo_root_dir(cls, location): - root_dir = cls.run_command( - ['root'], show_stdout=False, cwd=location).strip() - if not os.path.isabs(root_dir): - root_dir = os.path.join(location, root_dir) - return os.path.abspath(root_dir) - - @classmethod - def controls_location(cls, location): - if super(Mercurial, cls).controls_location(location): - return True - try: - cls.run_command( - ['identify'], - cwd=location, - show_stdout=False, - on_returncode='raise', - log_failed_cmd=False) - except (BadCommand, InstallationError): - return False - - -vcs.register(Mercurial) +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os + +from pip._vendor.six.moves import configparser + +from pip._internal.exceptions import BadCommand, InstallationError +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs.versioncontrol import ( + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import RevOptions + + +logger = logging.getLogger(__name__) + + +class Mercurial(VersionControl): + name = 'hg' + dirname = '.hg' + repo_name = 'clone' + schemes = ( + 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', + ) + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Hg repository at the url to the destination location""" + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + + self.run_command( + ['archive', location], show_stdout=False, cwd=temp_dir.path + ) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Cloning hg %s%s to %s', + url, + rev_display, + display_path(dest), + ) + self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) + self.run_command( + make_command('update', '-q', rev_options.to_args()), + cwd=dest, + ) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + repo_config = os.path.join(dest, self.dirname, 'hgrc') + config = configparser.RawConfigParser() + try: + config.read(repo_config) + config.set('paths', 'default', url.secret) + with open(repo_config, 'w') as config_file: + config.write(config_file) + except (OSError, configparser.NoSectionError) as exc: + logger.warning( + 'Could not switch Mercurial repository to %s: %s', url, exc, + ) + else: + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command(['pull', '-q'], cwd=dest) + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( + ['showconfig', 'paths.default'], + show_stdout=False, cwd=location).strip() + if cls._is_local_repository(url): + url = path_to_url(url) + return url.strip() + + @classmethod + def get_revision(cls, location): + """ + Return the repository-local changeset revision number, as an integer. + """ + current_revision = cls.run_command( + ['parents', '--template={rev}'], + show_stdout=False, cwd=location).strip() + return current_revision + + @classmethod + def get_requirement_revision(cls, location): + """ + Return the changeset identification hash, as a 40-character + hexadecimal string + """ + current_rev_hash = cls.run_command( + ['parents', '--template={node}'], + show_stdout=False, cwd=location).strip() + return current_rev_hash + + @classmethod + def is_commit_id_equal(cls, dest, name): + """Always assume the versions don't match""" + return False + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + repo_root = cls.run_command( + ['root'], show_stdout=False, cwd=location).strip() + if not os.path.isabs(repo_root): + repo_root = os.path.abspath(os.path.join(location, repo_root)) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def controls_location(cls, location): + if super(Mercurial, cls).controls_location(location): + return True + try: + cls.run_command( + ['identify'], + cwd=location, + show_stdout=False, + on_returncode='raise', + log_failed_cmd=False) + except (BadCommand, InstallationError): + return False + + +vcs.register(Mercurial) diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 03e69f598..4519f29f7 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -1,687 +1,672 @@ -"""Handles all VCS (version control) support""" - -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import errno -import logging -import os -import shutil -import sys - -from pip._vendor import pkg_resources -from pip._vendor.six.moves.urllib import parse as urllib_parse - -from pip._internal.exceptions import BadCommand -from pip._internal.utils.compat import samefile -from pip._internal.utils.misc import ( - ask_path_exists, - backup_dir, - display_path, - hide_url, - hide_value, - rmtree, -) -from pip._internal.utils.subprocess import call_subprocess, make_command -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import get_url_scheme - -if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union - ) - from pip._internal.utils.ui import SpinnerInterface - from pip._internal.utils.misc import HiddenText - from pip._internal.utils.subprocess import CommandArgs - - AuthInfo = Tuple[Optional[str], Optional[str]] - - -__all__ = ['vcs'] - - -logger = logging.getLogger(__name__) - - -def is_url(name): - # type: (Union[str, Text]) -> bool - """ - Return true if the name looks like a URL. - """ - scheme = get_url_scheme(name) - if scheme is None: - return False - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): - """ - Return the URL for a VCS requirement. - - Args: - repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). - project_name: the (unescaped) project name. - """ - egg_project_name = pkg_resources.to_filename(project_name) - req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) - if subdir: - req += '&subdirectory={}'.format(subdir) - - return req - - -class RemoteNotFoundError(Exception): - pass - - -class RevOptions(object): - - """ - Encapsulates a VCS-specific revision to install, along with any VCS - install options. - - Instances of this class should be treated as if immutable. - """ - - def __init__( - self, - vc_class, # type: Type[VersionControl] - rev=None, # type: Optional[str] - extra_args=None, # type: Optional[CommandArgs] - ): - # type: (...) -> None - """ - Args: - vc_class: a VersionControl subclass. - rev: the name of the revision to install. - extra_args: a list of extra options. - """ - if extra_args is None: - extra_args = [] - - self.extra_args = extra_args - self.rev = rev - self.vc_class = vc_class - self.branch_name = None # type: Optional[str] - - def __repr__(self): - return ''.format(self.vc_class.name, self.rev) - - @property - def arg_rev(self): - # type: () -> Optional[str] - if self.rev is None: - return self.vc_class.default_arg_rev - - return self.rev - - def to_args(self): - # type: () -> CommandArgs - """ - Return the VCS-specific command arguments. - """ - args = [] # type: CommandArgs - rev = self.arg_rev - if rev is not None: - args += self.vc_class.get_base_rev_args(rev) - args += self.extra_args - - return args - - def to_display(self): - # type: () -> str - if not self.rev: - return '' - - return ' (to revision {})'.format(self.rev) - - def make_new(self, rev): - # type: (str) -> RevOptions - """ - Make a copy of the current instance, but with a new rev. - - Args: - rev: the name of the revision for the new object. - """ - return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) - - -class VcsSupport(object): - _registry = {} # type: Dict[str, VersionControl] - schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] - - def __init__(self): - # type: () -> None - # Register more schemes with urlparse for various version control - # systems - urllib_parse.uses_netloc.extend(self.schemes) - # Python >= 2.7.4, 3.3 doesn't have uses_fragment - if getattr(urllib_parse, 'uses_fragment', None): - urllib_parse.uses_fragment.extend(self.schemes) - super(VcsSupport, self).__init__() - - def __iter__(self): - return self._registry.__iter__() - - @property - def backends(self): - # type: () -> List[VersionControl] - return list(self._registry.values()) - - @property - def dirnames(self): - # type: () -> List[str] - return [backend.dirname for backend in self.backends] - - @property - def all_schemes(self): - # type: () -> List[str] - schemes = [] # type: List[str] - for backend in self.backends: - schemes.extend(backend.schemes) - return schemes - - def register(self, cls): - # type: (Type[VersionControl]) -> None - if not hasattr(cls, 'name'): - logger.warning('Cannot register VCS %s', cls.__name__) - return - if cls.name not in self._registry: - self._registry[cls.name] = cls() - logger.debug('Registered VCS backend: %s', cls.name) - - def unregister(self, name): - # type: (str) -> None - if name in self._registry: - del self._registry[name] - - def get_backend_for_dir(self, location): - # type: (str) -> Optional[VersionControl] - """ - Return a VersionControl object if a repository of that type is found - at the given directory. - """ - for vcs_backend in self._registry.values(): - if vcs_backend.controls_location(location): - logger.debug('Determine that %s uses VCS: %s', - location, vcs_backend.name) - return vcs_backend - return None - - def get_backend(self, name): - # type: (str) -> Optional[VersionControl] - """ - Return a VersionControl object or None. - """ - name = name.lower() - return self._registry.get(name) - - -vcs = VcsSupport() - - -class VersionControl(object): - name = '' - dirname = '' - repo_name = '' - # List of supported schemes for this Version Control - schemes = () # type: Tuple[str, ...] - # Iterable of environment variable names to pass to call_subprocess(). - unset_environ = () # type: Tuple[str, ...] - default_arg_rev = None # type: Optional[str] - - @classmethod - def should_add_vcs_url_prefix(cls, remote_url): - """ - Return whether the vcs prefix (e.g. "git+") should be added to a - repository's remote url when used in a requirement. - """ - return not remote_url.lower().startswith('{}:'.format(cls.name)) - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - # find the repo root - root_dir = cls.get_repo_root_dir(location) - if root_dir is None: - logger.warning( - "Repo root could not be detected for %s, " - "assuming it is the root.", - location) - return None - # find setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without - # finding setup.py - logger.warning( - "Could not find setup.py for directory %s (tried all " - "parent directories)", - orig_location, - ) - return None - # relative path of setup.py to repo root - if samefile(root_dir, location): - return None - return os.path.relpath(location, root_dir) - - @classmethod - def get_repo_root_dir(cls, location): - """ - Return the absolute path to the repo root directory. - - Return None if not found. - This can be overridden by subclasses to interrogate the vcs tool to - find the repo root. - """ - while not cls.is_repository_directory(location): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem. - return None - return os.path.abspath(location) - - @classmethod - def get_requirement_revision(cls, repo_dir): - """ - Return the revision string that should be used in a requirement. - """ - return cls.get_revision(repo_dir) - - @classmethod - def get_src_requirement(cls, repo_dir, project_name): - """ - Return the requirement string to use to redownload the files - currently at the given repository directory. - - Args: - project_name: the (unescaped) project name. - - The return value has a form similar to the following: - - {repository_url}@{revision}#egg={project_name} - """ - repo_url = cls.get_remote_url(repo_dir) - if repo_url is None: - return None - - if cls.should_add_vcs_url_prefix(repo_url): - repo_url = '{}+{}'.format(cls.name, repo_url) - - revision = cls.get_requirement_revision(repo_dir) - subdir = cls.get_subdirectory(repo_dir) - req = make_vcs_requirement_url(repo_url, revision, project_name, - subdir=subdir) - - return req - - @staticmethod - def get_base_rev_args(rev): - """ - Return the base revision arguments for a vcs command. - - Args: - rev: the name of a revision to install. Cannot be None. - """ - raise NotImplementedError - - @classmethod - def make_rev_options(cls, rev=None, extra_args=None): - # type: (Optional[str], Optional[CommandArgs]) -> RevOptions - """ - Return a RevOptions object. - - Args: - rev: the name of a revision to install. - extra_args: a list of extra options. - """ - return RevOptions(cls, rev, extra_args=extra_args) - - @classmethod - def _is_local_repository(cls, repo): - # type: (str) -> bool - """ - posix absolute paths start with os.path.sep, - win32 ones start with drive (like c:\\folder) - """ - drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or bool(drive) - - def export(self, location, url): - # type: (str, HiddenText) -> None - """ - Export the repository at the url to the destination location - i.e. only download the files, without vcs informations - - :param url: the repository URL starting with a vcs prefix. - """ - raise NotImplementedError - - @classmethod - def get_netloc_and_auth(cls, netloc, scheme): - """ - Parse the repository URL's netloc, and return the new netloc to use - along with auth information. - - Args: - netloc: the original repository URL netloc. - scheme: the repository URL's scheme without the vcs prefix. - - This is mainly for the Subversion class to override, so that auth - information can be provided via the --username and --password options - instead of through the URL. For other subclasses like Git without - such an option, auth information must stay in the URL. - - Returns: (netloc, (username, password)). - """ - return netloc, (None, None) - - @classmethod - def get_url_rev_and_auth(cls, url): - # type: (str) -> Tuple[str, Optional[str], AuthInfo] - """ - Parse the repository URL to use, and return the URL, revision, - and auth info to use. - - Returns: (url, rev, (username, password)). - """ - scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) - if '+' not in scheme: - raise ValueError( - "Sorry, {!r} is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) - ) - # Remove the vcs prefix. - scheme = scheme.split('+', 1)[1] - netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev, user_pass - - @staticmethod - def make_rev_args(username, password): - # type: (Optional[str], Optional[HiddenText]) -> CommandArgs - """ - Return the RevOptions "extra arguments" to use in obtain(). - """ - return [] - - def get_url_rev_options(self, url): - # type: (HiddenText) -> Tuple[HiddenText, RevOptions] - """ - Return the URL and RevOptions object to use in obtain() and in - some cases export(), as a tuple (url, rev_options). - """ - secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) - username, secret_password = user_pass - password = None # type: Optional[HiddenText] - if secret_password is not None: - password = hide_value(secret_password) - extra_args = self.make_rev_args(username, password) - rev_options = self.make_rev_options(rev, extra_args=extra_args) - - return hide_url(secret_url), rev_options - - @staticmethod - def normalize_url(url): - # type: (str) -> str - """ - Normalize a URL for comparison by unquoting it and removing any - trailing slash. - """ - return urllib_parse.unquote(url).rstrip('/') - - @classmethod - def compare_urls(cls, url1, url2): - # type: (str, str) -> bool - """ - Compare two repo URLs for identity, ignoring incidental differences. - """ - return (cls.normalize_url(url1) == cls.normalize_url(url2)) - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Fetch a revision from a repository, in the case that this is the - first fetch from the repository. - - Args: - dest: the directory to fetch the repository to. - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Switch the repo at ``dest`` to point to ``URL``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Update an already-existing repo to the given ``rev_options``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - @classmethod - def is_commit_id_equal(cls, dest, name): - """ - Return whether the id of the current commit equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - raise NotImplementedError - - def obtain(self, dest, url): - # type: (str, HiddenText) -> None - """ - Install or update in editable mode the package represented by this - VersionControl object. - - :param dest: the repository directory in which to install or update. - :param url: the repository URL starting with a vcs prefix. - """ - url, rev_options = self.get_url_rev_options(url) - - if not os.path.exists(dest): - self.fetch_new(dest, url, rev_options) - return - - rev_display = rev_options.to_display() - if self.is_repository_directory(dest): - existing_url = self.get_remote_url(dest) - if self.compare_urls(existing_url, url.secret): - logger.debug( - '%s in %s exists, and has correct URL (%s)', - self.repo_name.title(), - display_path(dest), - url, - ) - if not self.is_commit_id_equal(dest, rev_options.rev): - logger.info( - 'Updating %s %s%s', - display_path(dest), - self.repo_name, - rev_display, - ) - self.update(dest, url, rev_options) - else: - logger.info('Skipping because already up-to-date.') - return - - logger.warning( - '%s %s in %s exists with URL %s', - self.name, - self.repo_name, - display_path(dest), - existing_url, - ) - prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', - ('s', 'i', 'w', 'b')) - else: - logger.warning( - 'Directory %s already exists, and is not a %s %s.', - dest, - self.name, - self.repo_name, - ) - # https://github.com/python/mypy/issues/1174 - prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore - ('i', 'w', 'b')) - - logger.warning( - 'The plan is to install the %s repository %s', - self.name, - url, - ) - response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) - - if response == 'a': - sys.exit(-1) - - if response == 'w': - logger.warning('Deleting %s', display_path(dest)) - rmtree(dest) - self.fetch_new(dest, url, rev_options) - return - - if response == 'b': - dest_dir = backup_dir(dest) - logger.warning( - 'Backing up %s to %s', display_path(dest), dest_dir, - ) - shutil.move(dest, dest_dir) - self.fetch_new(dest, url, rev_options) - return - - # Do nothing if the response is "i". - if response == 's': - logger.info( - 'Switching %s %s to %s%s', - self.repo_name, - display_path(dest), - url, - rev_display, - ) - self.switch(dest, url, rev_options) - - def unpack(self, location, url): - # type: (str, HiddenText) -> None - """ - Clean up current location and download the url repository - (and vcs infos) into location - - :param url: the repository URL starting with a vcs prefix. - """ - if os.path.exists(location): - rmtree(location) - self.obtain(location, url=url) - - @classmethod - def get_remote_url(cls, location): - """ - Return the url used at location - - Raises RemoteNotFoundError if the repository does not have a remote - url configured. - """ - raise NotImplementedError - - @classmethod - def get_revision(cls, location): - """ - Return the current commit id of the files at the given location. - """ - raise NotImplementedError - - @classmethod - def run_command( - cls, - cmd, # type: Union[List[str], CommandArgs] - show_stdout=True, # type: bool - cwd=None, # type: Optional[str] - on_returncode='raise', # type: str - extra_ok_returncodes=None, # type: Optional[Iterable[int]] - command_desc=None, # type: Optional[str] - extra_environ=None, # type: Optional[Mapping[str, Any]] - spinner=None, # type: Optional[SpinnerInterface] - log_failed_cmd=True - ): - # type: (...) -> Text - """ - Run a VCS subcommand - This is simply a wrapper around call_subprocess that adds the VCS - command name, and checks that the VCS is available - """ - cmd = make_command(cls.name, *cmd) - try: - return call_subprocess(cmd, show_stdout, cwd, - on_returncode=on_returncode, - extra_ok_returncodes=extra_ok_returncodes, - command_desc=command_desc, - extra_environ=extra_environ, - unset_environ=cls.unset_environ, - spinner=spinner, - log_failed_cmd=log_failed_cmd) - except OSError as e: - # errno.ENOENT = no such file or directory - # In other words, the VCS executable isn't available - if e.errno == errno.ENOENT: - raise BadCommand( - 'Cannot find command %r - do you have ' - '%r installed and in your ' - 'PATH?' % (cls.name, cls.name)) - else: - raise # re-raise exception if a different error occurred - - @classmethod - def is_repository_directory(cls, path): - # type: (str) -> bool - """ - Return whether a directory path is a repository directory. - """ - logger.debug('Checking in %s for %s (%s)...', - path, cls.dirname, cls.name) - return os.path.exists(os.path.join(path, cls.dirname)) - - @classmethod - def controls_location(cls, location): - # type: (str) -> bool - """ - Check if a location is controlled by the vcs. - - Searches up the filesystem and checks is_repository_directory(). - - It is meant to be extended to add smarter detection mechanisms for - specific vcs. For example, the Git override checks that Git is - actually available. - """ - while not cls.is_repository_directory(location): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem. - return False - return True +"""Handles all VCS (version control) support""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import errno +import logging +import os +import shutil +import sys + +from pip._vendor import pkg_resources +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.compat import samefile +from pip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + hide_url, + hide_value, + rmtree, +) +from pip._internal.utils.subprocess import call_subprocess, make_command +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union + ) + from pip._internal.utils.ui import SpinnerInterface + from pip._internal.utils.misc import HiddenText + from pip._internal.utils.subprocess import CommandArgs + + AuthInfo = Tuple[Optional[str], Optional[str]] + + +__all__ = ['vcs'] + + +logger = logging.getLogger(__name__) + + +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. + """ + egg_project_name = pkg_resources.to_filename(project_name) + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def find_path_to_setup_from_repo_root(location, repo_root): + """ + Find the path to `setup.py` by searching up the filesystem from `location`. + Return the path to `setup.py` relative to `repo_root`. + Return None if `setup.py` is in `repo_root` or cannot be found. + """ + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + + if samefile(repo_root, location): + return None + + return os.path.relpath(location, repo_root) + + +class RemoteNotFoundError(Exception): + pass + + +class RevOptions(object): + + """ + Encapsulates a VCS-specific revision to install, along with any VCS + install options. + + Instances of this class should be treated as if immutable. + """ + + def __init__( + self, + vc_class, # type: Type[VersionControl] + rev=None, # type: Optional[str] + extra_args=None, # type: Optional[CommandArgs] + ): + # type: (...) -> None + """ + Args: + vc_class: a VersionControl subclass. + rev: the name of the revision to install. + extra_args: a list of extra options. + """ + if extra_args is None: + extra_args = [] + + self.extra_args = extra_args + self.rev = rev + self.vc_class = vc_class + self.branch_name = None # type: Optional[str] + + def __repr__(self): + return ''.format(self.vc_class.name, self.rev) + + @property + def arg_rev(self): + # type: () -> Optional[str] + if self.rev is None: + return self.vc_class.default_arg_rev + + return self.rev + + def to_args(self): + # type: () -> CommandArgs + """ + Return the VCS-specific command arguments. + """ + args = [] # type: CommandArgs + rev = self.arg_rev + if rev is not None: + args += self.vc_class.get_base_rev_args(rev) + args += self.extra_args + + return args + + def to_display(self): + # type: () -> str + if not self.rev: + return '' + + return ' (to revision {})'.format(self.rev) + + def make_new(self, rev): + # type: (str) -> RevOptions + """ + Make a copy of the current instance, but with a new rev. + + Args: + rev: the name of the revision for the new object. + """ + return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) + + +class VcsSupport(object): + _registry = {} # type: Dict[str, VersionControl] + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] + + def __init__(self): + # type: () -> None + # Register more schemes with urlparse for various version control + # systems + urllib_parse.uses_netloc.extend(self.schemes) + # Python >= 2.7.4, 3.3 doesn't have uses_fragment + if getattr(urllib_parse, 'uses_fragment', None): + urllib_parse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + return self._registry.__iter__() + + @property + def backends(self): + # type: () -> List[VersionControl] + return list(self._registry.values()) + + @property + def dirnames(self): + # type: () -> List[str] + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + # type: () -> List[str] + schemes = [] # type: List[str] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + # type: (Type[VersionControl]) -> None + if not hasattr(cls, 'name'): + logger.warning('Cannot register VCS %s', cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls() + logger.debug('Registered VCS backend: %s', cls.name) + + def unregister(self, name): + # type: (str) -> None + if name in self._registry: + del self._registry[name] + + def get_backend_for_dir(self, location): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object if a repository of that type is found + at the given directory. + """ + for vcs_backend in self._registry.values(): + if vcs_backend.controls_location(location): + logger.debug('Determine that %s uses VCS: %s', + location, vcs_backend.name) + return vcs_backend + return None + + def get_backend(self, name): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object or None. + """ + name = name.lower() + return self._registry.get(name) + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + repo_name = '' + # List of supported schemes for this Version Control + schemes = () # type: Tuple[str, ...] + # Iterable of environment variable names to pass to call_subprocess(). + unset_environ = () # type: Tuple[str, ...] + default_arg_rev = None # type: Optional[str] + + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + """ + Return whether the vcs prefix (e.g. "git+") should be added to a + repository's remote url when used in a requirement. + """ + return not remote_url.lower().startswith('{}:'.format(cls.name)) + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + return None + + @classmethod + def get_requirement_revision(cls, repo_dir): + """ + Return the revision string that should be used in a requirement. + """ + return cls.get_revision(repo_dir) + + @classmethod + def get_src_requirement(cls, repo_dir, project_name): + """ + Return the requirement string to use to redownload the files + currently at the given repository directory. + + Args: + project_name: the (unescaped) project name. + + The return value has a form similar to the following: + + {repository_url}@{revision}#egg={project_name} + """ + repo_url = cls.get_remote_url(repo_dir) + if repo_url is None: + return None + + if cls.should_add_vcs_url_prefix(repo_url): + repo_url = '{}+{}'.format(cls.name, repo_url) + + revision = cls.get_requirement_revision(repo_dir) + subdir = cls.get_subdirectory(repo_dir) + req = make_vcs_requirement_url(repo_url, revision, project_name, + subdir=subdir) + + return req + + @staticmethod + def get_base_rev_args(rev): + """ + Return the base revision arguments for a vcs command. + + Args: + rev: the name of a revision to install. Cannot be None. + """ + raise NotImplementedError + + @classmethod + def make_rev_options(cls, rev=None, extra_args=None): + # type: (Optional[str], Optional[CommandArgs]) -> RevOptions + """ + Return a RevOptions object. + + Args: + rev: the name of a revision to install. + extra_args: a list of extra options. + """ + return RevOptions(cls, rev, extra_args=extra_args) + + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool + """ + posix absolute paths start with os.path.sep, + win32 ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or bool(drive) + + def export(self, location, url): + # type: (str, HiddenText) -> None + """ + Export the repository at the url to the destination location + i.e. only download the files, without vcs informations + + :param url: the repository URL starting with a vcs prefix. + """ + raise NotImplementedError + + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). + """ + scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev, user_pass + + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + """ + Return the RevOptions "extra arguments" to use in obtain(). + """ + return [] + + def get_url_rev_options(self, url): + # type: (HiddenText) -> Tuple[HiddenText, RevOptions] + """ + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). + """ + secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) + username, secret_password = user_pass + password = None # type: Optional[HiddenText] + if secret_password is not None: + password = hide_value(secret_password) + extra_args = self.make_rev_args(username, password) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return hide_url(secret_url), rev_options + + @staticmethod + def normalize_url(url): + # type: (str) -> str + """ + Normalize a URL for comparison by unquoting it and removing any + trailing slash. + """ + return urllib_parse.unquote(url).rstrip('/') + + @classmethod + def compare_urls(cls, url1, url2): + # type: (str, str) -> bool + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (cls.normalize_url(url1) == cls.normalize_url(url2)) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Switch the repo at ``dest`` to point to ``URL``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Update an already-existing repo to the given ``rev_options``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the id of the current commit equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + raise NotImplementedError + + def obtain(self, dest, url): + # type: (str, HiddenText) -> None + """ + Install or update in editable mode the package represented by this + VersionControl object. + + :param dest: the repository directory in which to install or update. + :param url: the repository URL starting with a vcs prefix. + """ + url, rev_options = self.get_url_rev_options(url) + + if not os.path.exists(dest): + self.fetch_new(dest, url, rev_options) + return + + rev_display = rev_options.to_display() + if self.is_repository_directory(dest): + existing_url = self.get_remote_url(dest) + if self.compare_urls(existing_url, url.secret): + logger.debug( + '%s in %s exists, and has correct URL (%s)', + self.repo_name.title(), + display_path(dest), + url, + ) + if not self.is_commit_id_equal(dest, rev_options.rev): + logger.info( + 'Updating %s %s%s', + display_path(dest), + self.repo_name, + rev_display, + ) + self.update(dest, url, rev_options) + else: + logger.info('Skipping because already up-to-date.') + return + + logger.warning( + '%s %s in %s exists with URL %s', + self.name, + self.repo_name, + display_path(dest), + existing_url, + ) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', + ('s', 'i', 'w', 'b')) + else: + logger.warning( + 'Directory %s already exists, and is not a %s %s.', + dest, + self.name, + self.repo_name, + ) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) + + logger.warning( + 'The plan is to install the %s repository %s', + self.name, + url, + ) + response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) + + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + self.fetch_new(dest, url, rev_options) + return + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + self.fetch_new(dest, url, rev_options) + return + + # Do nothing if the response is "i". + if response == 's': + logger.info( + 'Switching %s %s to %s%s', + self.repo_name, + display_path(dest), + url, + rev_display, + ) + self.switch(dest, url, rev_options) + + def unpack(self, location, url): + # type: (str, HiddenText) -> None + """ + Clean up current location and download the url repository + (and vcs infos) into location + + :param url: the repository URL starting with a vcs prefix. + """ + if os.path.exists(location): + rmtree(location) + self.obtain(location, url=url) + + @classmethod + def get_remote_url(cls, location): + """ + Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + raise NotImplementedError + + @classmethod + def get_revision(cls, location): + """ + Return the current commit id of the files at the given location. + """ + raise NotImplementedError + + @classmethod + def run_command( + cls, + cmd, # type: Union[List[str], CommandArgs] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True + ): + # type: (...) -> Text + """ + Run a VCS subcommand + This is simply a wrapper around call_subprocess that adds the VCS + command name, and checks that the VCS is available + """ + cmd = make_command(cls.name, *cmd) + try: + return call_subprocess(cmd, show_stdout, cwd, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, + spinner=spinner, + log_failed_cmd=log_failed_cmd) + except OSError as e: + # errno.ENOENT = no such file or directory + # In other words, the VCS executable isn't available + if e.errno == errno.ENOENT: + raise BadCommand( + 'Cannot find command %r - do you have ' + '%r installed and in your ' + 'PATH?' % (cls.name, cls.name)) + else: + raise # re-raise exception if a different error occurred + + @classmethod + def is_repository_directory(cls, path): + # type: (str) -> bool + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + + @classmethod + def controls_location(cls, location): + # type: (str) -> bool + """ + Check if a location is controlled by the vcs. + + Searches up the filesystem and checks is_repository_directory(). + + It is meant to be extended to add smarter detection mechanisms for + specific vcs. For example, the Git override checks that Git is + actually available. + """ + while not cls.is_repository_directory(location): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem. + return False + return True From f197479dc8b496e236b1db5321a79be97a37bfdf Mon Sep 17 00:00:00 2001 From: tbeswick Date: Fri, 11 Oct 2019 14:30:30 +1300 Subject: [PATCH 14/16] Fixed LF getting converted to CRLF in last commit. --- src/pip/_internal/vcs/git.py | 744 ++++++------- src/pip/_internal/vcs/mercurial.py | 308 +++--- src/pip/_internal/vcs/versioncontrol.py | 1344 +++++++++++------------ 3 files changed, 1198 insertions(+), 1198 deletions(-) diff --git a/src/pip/_internal/vcs/git.py b/src/pip/_internal/vcs/git.py index 6b8041140..92b845714 100644 --- a/src/pip/_internal/vcs/git.py +++ b/src/pip/_internal/vcs/git.py @@ -1,372 +1,372 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import logging -import os.path -import re - -from pip._vendor.packaging.version import parse as parse_version -from pip._vendor.six.moves.urllib import parse as urllib_parse -from pip._vendor.six.moves.urllib import request as urllib_request - -from pip._internal.exceptions import BadCommand -from pip._internal.utils.misc import display_path -from pip._internal.utils.subprocess import make_command -from pip._internal.utils.temp_dir import TempDirectory -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.vcs.versioncontrol import ( - RemoteNotFoundError, - VersionControl, - find_path_to_setup_from_repo_root, - vcs, -) - -if MYPY_CHECK_RUNNING: - from typing import Optional, Tuple - from pip._internal.utils.misc import HiddenText - from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions - - -urlsplit = urllib_parse.urlsplit -urlunsplit = urllib_parse.urlunsplit - - -logger = logging.getLogger(__name__) - - -HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') - - -def looks_like_hash(sha): - return bool(HASH_REGEX.match(sha)) - - -class Git(VersionControl): - name = 'git' - dirname = '.git' - repo_name = 'clone' - schemes = ( - 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', - ) - # Prevent the user's environment variables from interfering with pip: - # https://github.com/pypa/pip/issues/1130 - unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') - default_arg_rev = 'HEAD' - - @staticmethod - def get_base_rev_args(rev): - return [rev] - - def get_git_version(self): - VERSION_PFX = 'git version ' - version = self.run_command(['version'], show_stdout=False) - if version.startswith(VERSION_PFX): - version = version[len(VERSION_PFX):].split()[0] - else: - version = '' - # get first 3 positions of the git version because - # on windows it is x.y.z.windows.t, and this parses as - # LegacyVersion which always smaller than a Version. - version = '.'.join(version.split('.')[:3]) - return parse_version(version) - - @classmethod - def get_current_branch(cls, location): - """ - Return the current branch, or None if HEAD isn't at a branch - (e.g. detached HEAD). - """ - # git-symbolic-ref exits with empty stdout if "HEAD" is a detached - # HEAD rather than a symbolic ref. In addition, the -q causes the - # command to exit with status code 1 instead of 128 in this case - # and to suppress the message to stderr. - args = ['symbolic-ref', '-q', 'HEAD'] - output = cls.run_command( - args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, - ) - ref = output.strip() - - if ref.startswith('refs/heads/'): - return ref[len('refs/heads/'):] - - return None - - def export(self, location, url): - # type: (str, HiddenText) -> None - """Export the Git repository at the url to the destination location""" - if not location.endswith('/'): - location = location + '/' - - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path, url=url) - self.run_command( - ['checkout-index', '-a', '-f', '--prefix', location], - show_stdout=False, cwd=temp_dir.path - ) - - @classmethod - def get_revision_sha(cls, dest, rev): - """ - Return (sha_or_none, is_branch), where sha_or_none is a commit hash - if the revision names a remote branch or tag, otherwise None. - - Args: - dest: the repository directory. - rev: the revision name. - """ - # Pass rev to pre-filter the list. - output = cls.run_command(['show-ref', rev], cwd=dest, - show_stdout=False, on_returncode='ignore') - refs = {} - for line in output.strip().splitlines(): - try: - sha, ref = line.split() - except ValueError: - # Include the offending line to simplify troubleshooting if - # this error ever occurs. - raise ValueError('unexpected show-ref line: {!r}'.format(line)) - - refs[ref] = sha - - branch_ref = 'refs/remotes/origin/{}'.format(rev) - tag_ref = 'refs/tags/{}'.format(rev) - - sha = refs.get(branch_ref) - if sha is not None: - return (sha, True) - - sha = refs.get(tag_ref) - - return (sha, False) - - @classmethod - def resolve_revision(cls, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> RevOptions - """ - Resolve a revision to a new RevOptions object with the SHA1 of the - branch, tag, or ref if found. - - Args: - rev_options: a RevOptions object. - """ - rev = rev_options.arg_rev - # The arg_rev property's implementation for Git ensures that the - # rev return value is always non-None. - assert rev is not None - - sha, is_branch = cls.get_revision_sha(dest, rev) - - if sha is not None: - rev_options = rev_options.make_new(sha) - rev_options.branch_name = rev if is_branch else None - - return rev_options - - # Do not show a warning for the common case of something that has - # the form of a Git commit hash. - if not looks_like_hash(rev): - logger.warning( - "Did not find branch or tag '%s', assuming revision or ref.", - rev, - ) - - if not rev.startswith('refs/'): - return rev_options - - # If it looks like a ref, we have to fetch it explicitly. - cls.run_command( - make_command('fetch', '-q', url, rev_options.to_args()), - cwd=dest, - ) - # Change the revision to the SHA of the ref we fetched - sha = cls.get_revision(dest, rev='FETCH_HEAD') - rev_options = rev_options.make_new(sha) - - return rev_options - - @classmethod - def is_commit_id_equal(cls, dest, name): - """ - Return whether the current commit hash equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - if not name: - # Then avoid an unnecessary subprocess call. - return False - - return cls.get_revision(dest) == name - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - rev_display = rev_options.to_display() - logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) - self.run_command(make_command('clone', '-q', url, dest)) - - if rev_options.rev: - # Then a specific revision was requested. - rev_options = self.resolve_revision(dest, url, rev_options) - branch_name = getattr(rev_options, 'branch_name', None) - if branch_name is None: - # Only do a checkout if the current commit id doesn't match - # the requested revision. - if not self.is_commit_id_equal(dest, rev_options.rev): - cmd_args = make_command( - 'checkout', '-q', rev_options.to_args(), - ) - self.run_command(cmd_args, cwd=dest) - elif self.get_current_branch(dest) != branch_name: - # Then a specific branch was requested, and that branch - # is not yet checked out. - track_branch = 'origin/{}'.format(branch_name) - cmd_args = [ - 'checkout', '-b', branch_name, '--track', track_branch, - ] - self.run_command(cmd_args, cwd=dest) - - #: repo may contain submodules - self.update_submodules(dest) - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - self.run_command( - make_command('config', 'remote.origin.url', url), - cwd=dest, - ) - cmd_args = make_command('checkout', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - self.update_submodules(dest) - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - # First fetch changes from the default remote - if self.get_git_version() >= parse_version('1.9.0'): - # fetch tags in addition to everything else - self.run_command(['fetch', '-q', '--tags'], cwd=dest) - else: - self.run_command(['fetch', '-q'], cwd=dest) - # Then reset to wanted revision (maybe even origin/master) - rev_options = self.resolve_revision(dest, url, rev_options) - cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - #: update submodules - self.update_submodules(dest) - - @classmethod - def get_remote_url(cls, location): - """ - Return URL of the first remote encountered. - - Raises RemoteNotFoundError if the repository does not have a remote - url configured. - """ - # We need to pass 1 for extra_ok_returncodes since the command - # exits with return code 1 if there are no matching lines. - stdout = cls.run_command( - ['config', '--get-regexp', r'remote\..*\.url'], - extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, - ) - remotes = stdout.splitlines() - try: - found_remote = remotes[0] - except IndexError: - raise RemoteNotFoundError - - for remote in remotes: - if remote.startswith('remote.origin.url '): - found_remote = remote - break - url = found_remote.split(' ')[1] - return url.strip() - - @classmethod - def get_revision(cls, location, rev=None): - if rev is None: - rev = 'HEAD' - current_rev = cls.run_command( - ['rev-parse', rev], show_stdout=False, cwd=location, - ) - return current_rev.strip() - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - # find the repo root - git_dir = cls.run_command( - ['rev-parse', '--git-dir'], - show_stdout=False, cwd=location).strip() - if not os.path.isabs(git_dir): - git_dir = os.path.join(location, git_dir) - repo_root = os.path.abspath(os.path.join(git_dir, '..')) - return find_path_to_setup_from_repo_root(location, repo_root) - - @classmethod - def get_url_rev_and_auth(cls, url): - # type: (str) -> Tuple[str, Optional[str], AuthInfo] - """ - Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. - That's required because although they use SSH they sometimes don't - work with a ssh:// scheme (e.g. GitHub). But we need a scheme for - parsing. Hence we remove it again afterwards and return it as a stub. - """ - # Works around an apparent Git bug - # (see https://article.gmane.org/gmane.comp.version-control.git/146500) - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme.endswith('file'): - initial_slashes = path[:-len(path.lstrip('/'))] - newpath = ( - initial_slashes + - urllib_request.url2pathname(path) - .replace('\\', '/').lstrip('/') - ) - url = urlunsplit((scheme, netloc, newpath, query, fragment)) - after_plus = scheme.find('+') + 1 - url = scheme[:after_plus] + urlunsplit( - (scheme[after_plus:], netloc, newpath, query, fragment), - ) - - if '://' not in url: - assert 'file:' not in url - url = url.replace('git+', 'git+ssh://') - url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) - url = url.replace('ssh://', '') - else: - url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) - - return url, rev, user_pass - - @classmethod - def update_submodules(cls, location): - if not os.path.exists(os.path.join(location, '.gitmodules')): - return - cls.run_command( - ['submodule', 'update', '--init', '--recursive', '-q'], - cwd=location, - ) - - @classmethod - def controls_location(cls, location): - if super(Git, cls).controls_location(location): - return True - try: - r = cls.run_command(['rev-parse'], - cwd=location, - show_stdout=False, - on_returncode='ignore', - log_failed_cmd=False) - return not r - except BadCommand: - logger.debug("could not determine if %s is under git control " - "because git is not available", location) - return False - - -vcs.register(Git) +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os.path +import re + +from pip._vendor.packaging.version import parse as parse_version +from pip._vendor.six.moves.urllib import parse as urllib_parse +from pip._vendor.six.moves.urllib import request as urllib_request + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.vcs.versioncontrol import ( + RemoteNotFoundError, + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + +urlsplit = urllib_parse.urlsplit +urlunsplit = urllib_parse.urlunsplit + + +logger = logging.getLogger(__name__) + + +HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') + + +def looks_like_hash(sha): + return bool(HASH_REGEX.match(sha)) + + +class Git(VersionControl): + name = 'git' + dirname = '.git' + repo_name = 'clone' + schemes = ( + 'git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file', + ) + # Prevent the user's environment variables from interfering with pip: + # https://github.com/pypa/pip/issues/1130 + unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') + default_arg_rev = 'HEAD' + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def get_git_version(self): + VERSION_PFX = 'git version ' + version = self.run_command(['version'], show_stdout=False) + if version.startswith(VERSION_PFX): + version = version[len(VERSION_PFX):].split()[0] + else: + version = '' + # get first 3 positions of the git version because + # on windows it is x.y.z.windows.t, and this parses as + # LegacyVersion which always smaller than a Version. + version = '.'.join(version.split('.')[:3]) + return parse_version(version) + + @classmethod + def get_current_branch(cls, location): + """ + Return the current branch, or None if HEAD isn't at a branch + (e.g. detached HEAD). + """ + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = cls.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() + + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] + + return None + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Git repository at the url to the destination location""" + if not location.endswith('/'): + location = location + '/' + + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + self.run_command( + ['checkout-index', '-a', '-f', '--prefix', location], + show_stdout=False, cwd=temp_dir.path + ) + + @classmethod + def get_revision_sha(cls, dest, rev): + """ + Return (sha_or_none, is_branch), where sha_or_none is a commit hash + if the revision names a remote branch or tag, otherwise None. + + Args: + dest: the repository directory. + rev: the revision name. + """ + # Pass rev to pre-filter the list. + output = cls.run_command(['show-ref', rev], cwd=dest, + show_stdout=False, on_returncode='ignore') + refs = {} + for line in output.strip().splitlines(): + try: + sha, ref = line.split() + except ValueError: + # Include the offending line to simplify troubleshooting if + # this error ever occurs. + raise ValueError('unexpected show-ref line: {!r}'.format(line)) + + refs[ref] = sha + + branch_ref = 'refs/remotes/origin/{}'.format(rev) + tag_ref = 'refs/tags/{}'.format(rev) + + sha = refs.get(branch_ref) + if sha is not None: + return (sha, True) + + sha = refs.get(tag_ref) + + return (sha, False) + + @classmethod + def resolve_revision(cls, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> RevOptions + """ + Resolve a revision to a new RevOptions object with the SHA1 of the + branch, tag, or ref if found. + + Args: + rev_options: a RevOptions object. + """ + rev = rev_options.arg_rev + # The arg_rev property's implementation for Git ensures that the + # rev return value is always non-None. + assert rev is not None + + sha, is_branch = cls.get_revision_sha(dest, rev) + + if sha is not None: + rev_options = rev_options.make_new(sha) + rev_options.branch_name = rev if is_branch else None + + return rev_options + + # Do not show a warning for the common case of something that has + # the form of a Git commit hash. + if not looks_like_hash(rev): + logger.warning( + "Did not find branch or tag '%s', assuming revision or ref.", + rev, + ) + + if not rev.startswith('refs/'): + return rev_options + + # If it looks like a ref, we have to fetch it explicitly. + cls.run_command( + make_command('fetch', '-q', url, rev_options.to_args()), + cwd=dest, + ) + # Change the revision to the SHA of the ref we fetched + sha = cls.get_revision(dest, rev='FETCH_HEAD') + rev_options = rev_options.make_new(sha) + + return rev_options + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the current commit hash equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + if not name: + # Then avoid an unnecessary subprocess call. + return False + + return cls.get_revision(dest) == name + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) + self.run_command(make_command('clone', '-q', url, dest)) + + if rev_options.rev: + # Then a specific revision was requested. + rev_options = self.resolve_revision(dest, url, rev_options) + branch_name = getattr(rev_options, 'branch_name', None) + if branch_name is None: + # Only do a checkout if the current commit id doesn't match + # the requested revision. + if not self.is_commit_id_equal(dest, rev_options.rev): + cmd_args = make_command( + 'checkout', '-q', rev_options.to_args(), + ) + self.run_command(cmd_args, cwd=dest) + elif self.get_current_branch(dest) != branch_name: + # Then a specific branch was requested, and that branch + # is not yet checked out. + track_branch = 'origin/{}'.format(branch_name) + cmd_args = [ + 'checkout', '-b', branch_name, '--track', track_branch, + ] + self.run_command(cmd_args, cwd=dest) + + #: repo may contain submodules + self.update_submodules(dest) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command( + make_command('config', 'remote.origin.url', url), + cwd=dest, + ) + cmd_args = make_command('checkout', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + self.update_submodules(dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + # First fetch changes from the default remote + if self.get_git_version() >= parse_version('1.9.0'): + # fetch tags in addition to everything else + self.run_command(['fetch', '-q', '--tags'], cwd=dest) + else: + self.run_command(['fetch', '-q'], cwd=dest) + # Then reset to wanted revision (maybe even origin/master) + rev_options = self.resolve_revision(dest, url, rev_options) + cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + #: update submodules + self.update_submodules(dest) + + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( + ['config', '--get-regexp', r'remote\..*\.url'], + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + + for remote in remotes: + if remote.startswith('remote.origin.url '): + found_remote = remote + break + url = found_remote.split(' ')[1] + return url.strip() + + @classmethod + def get_revision(cls, location, rev=None): + if rev is None: + rev = 'HEAD' + current_rev = cls.run_command( + ['rev-parse', rev], show_stdout=False, cwd=location, + ) + return current_rev.strip() + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + git_dir = cls.run_command( + ['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() + if not os.path.isabs(git_dir): + git_dir = os.path.join(location, git_dir) + repo_root = os.path.abspath(os.path.join(git_dir, '..')) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. + That's required because although they use SSH they sometimes don't + work with a ssh:// scheme (e.g. GitHub). But we need a scheme for + parsing. Hence we remove it again afterwards and return it as a stub. + """ + # Works around an apparent Git bug + # (see https://article.gmane.org/gmane.comp.version-control.git/146500) + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = ( + initial_slashes + + urllib_request.url2pathname(path) + .replace('\\', '/').lstrip('/') + ) + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+') + 1 + url = scheme[:after_plus] + urlunsplit( + (scheme[after_plus:], netloc, newpath, query, fragment), + ) + + if '://' not in url: + assert 'file:' not in url + url = url.replace('git+', 'git+ssh://') + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + url = url.replace('ssh://', '') + else: + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) + + return url, rev, user_pass + + @classmethod + def update_submodules(cls, location): + if not os.path.exists(os.path.join(location, '.gitmodules')): + return + cls.run_command( + ['submodule', 'update', '--init', '--recursive', '-q'], + cwd=location, + ) + + @classmethod + def controls_location(cls, location): + if super(Git, cls).controls_location(location): + return True + try: + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore', + log_failed_cmd=False) + return not r + except BadCommand: + logger.debug("could not determine if %s is under git control " + "because git is not available", location) + return False + + +vcs.register(Git) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index dd04b8bc6..bc4b19c72 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -1,154 +1,154 @@ -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import logging -import os - -from pip._vendor.six.moves import configparser - -from pip._internal.exceptions import BadCommand, InstallationError -from pip._internal.utils.misc import display_path -from pip._internal.utils.subprocess import make_command -from pip._internal.utils.temp_dir import TempDirectory -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import path_to_url -from pip._internal.vcs.versioncontrol import ( - VersionControl, - find_path_to_setup_from_repo_root, - vcs, -) - -if MYPY_CHECK_RUNNING: - from pip._internal.utils.misc import HiddenText - from pip._internal.vcs.versioncontrol import RevOptions - - -logger = logging.getLogger(__name__) - - -class Mercurial(VersionControl): - name = 'hg' - dirname = '.hg' - repo_name = 'clone' - schemes = ( - 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', - ) - - @staticmethod - def get_base_rev_args(rev): - return [rev] - - def export(self, location, url): - # type: (str, HiddenText) -> None - """Export the Hg repository at the url to the destination location""" - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path, url=url) - - self.run_command( - ['archive', location], show_stdout=False, cwd=temp_dir.path - ) - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - rev_display = rev_options.to_display() - logger.info( - 'Cloning hg %s%s to %s', - url, - rev_display, - display_path(dest), - ) - self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) - self.run_command( - make_command('update', '-q', rev_options.to_args()), - cwd=dest, - ) - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - repo_config = os.path.join(dest, self.dirname, 'hgrc') - config = configparser.RawConfigParser() - try: - config.read(repo_config) - config.set('paths', 'default', url.secret) - with open(repo_config, 'w') as config_file: - config.write(config_file) - except (OSError, configparser.NoSectionError) as exc: - logger.warning( - 'Could not switch Mercurial repository to %s: %s', url, exc, - ) - else: - cmd_args = make_command('update', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - self.run_command(['pull', '-q'], cwd=dest) - cmd_args = make_command('update', '-q', rev_options.to_args()) - self.run_command(cmd_args, cwd=dest) - - @classmethod - def get_remote_url(cls, location): - url = cls.run_command( - ['showconfig', 'paths.default'], - show_stdout=False, cwd=location).strip() - if cls._is_local_repository(url): - url = path_to_url(url) - return url.strip() - - @classmethod - def get_revision(cls, location): - """ - Return the repository-local changeset revision number, as an integer. - """ - current_revision = cls.run_command( - ['parents', '--template={rev}'], - show_stdout=False, cwd=location).strip() - return current_revision - - @classmethod - def get_requirement_revision(cls, location): - """ - Return the changeset identification hash, as a 40-character - hexadecimal string - """ - current_rev_hash = cls.run_command( - ['parents', '--template={node}'], - show_stdout=False, cwd=location).strip() - return current_rev_hash - - @classmethod - def is_commit_id_equal(cls, dest, name): - """Always assume the versions don't match""" - return False - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - # find the repo root - repo_root = cls.run_command( - ['root'], show_stdout=False, cwd=location).strip() - if not os.path.isabs(repo_root): - repo_root = os.path.abspath(os.path.join(location, repo_root)) - return find_path_to_setup_from_repo_root(location, repo_root) - - @classmethod - def controls_location(cls, location): - if super(Mercurial, cls).controls_location(location): - return True - try: - cls.run_command( - ['identify'], - cwd=location, - show_stdout=False, - on_returncode='raise', - log_failed_cmd=False) - except (BadCommand, InstallationError): - return False - - -vcs.register(Mercurial) +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os + +from pip._vendor.six.moves import configparser + +from pip._internal.exceptions import BadCommand, InstallationError +from pip._internal.utils.misc import display_path +from pip._internal.utils.subprocess import make_command +from pip._internal.utils.temp_dir import TempDirectory +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import path_to_url +from pip._internal.vcs.versioncontrol import ( + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from pip._internal.utils.misc import HiddenText + from pip._internal.vcs.versioncontrol import RevOptions + + +logger = logging.getLogger(__name__) + + +class Mercurial(VersionControl): + name = 'hg' + dirname = '.hg' + repo_name = 'clone' + schemes = ( + 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', + ) + + @staticmethod + def get_base_rev_args(rev): + return [rev] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the Hg repository at the url to the destination location""" + with TempDirectory(kind="export") as temp_dir: + self.unpack(temp_dir.path, url=url) + + self.run_command( + ['archive', location], show_stdout=False, cwd=temp_dir.path + ) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Cloning hg %s%s to %s', + url, + rev_display, + display_path(dest), + ) + self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) + self.run_command( + make_command('update', '-q', rev_options.to_args()), + cwd=dest, + ) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + repo_config = os.path.join(dest, self.dirname, 'hgrc') + config = configparser.RawConfigParser() + try: + config.read(repo_config) + config.set('paths', 'default', url.secret) + with open(repo_config, 'w') as config_file: + config.write(config_file) + except (OSError, configparser.NoSectionError) as exc: + logger.warning( + 'Could not switch Mercurial repository to %s: %s', url, exc, + ) + else: + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + self.run_command(['pull', '-q'], cwd=dest) + cmd_args = make_command('update', '-q', rev_options.to_args()) + self.run_command(cmd_args, cwd=dest) + + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( + ['showconfig', 'paths.default'], + show_stdout=False, cwd=location).strip() + if cls._is_local_repository(url): + url = path_to_url(url) + return url.strip() + + @classmethod + def get_revision(cls, location): + """ + Return the repository-local changeset revision number, as an integer. + """ + current_revision = cls.run_command( + ['parents', '--template={rev}'], + show_stdout=False, cwd=location).strip() + return current_revision + + @classmethod + def get_requirement_revision(cls, location): + """ + Return the changeset identification hash, as a 40-character + hexadecimal string + """ + current_rev_hash = cls.run_command( + ['parents', '--template={node}'], + show_stdout=False, cwd=location).strip() + return current_rev_hash + + @classmethod + def is_commit_id_equal(cls, dest, name): + """Always assume the versions don't match""" + return False + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + repo_root = cls.run_command( + ['root'], show_stdout=False, cwd=location).strip() + if not os.path.isabs(repo_root): + repo_root = os.path.abspath(os.path.join(location, repo_root)) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def controls_location(cls, location): + if super(Mercurial, cls).controls_location(location): + return True + try: + cls.run_command( + ['identify'], + cwd=location, + show_stdout=False, + on_returncode='raise', + log_failed_cmd=False) + except (BadCommand, InstallationError): + return False + + +vcs.register(Mercurial) diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 4519f29f7..55afa908c 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -1,672 +1,672 @@ -"""Handles all VCS (version control) support""" - -# The following comment should be removed at some point in the future. -# mypy: disallow-untyped-defs=False - -from __future__ import absolute_import - -import errno -import logging -import os -import shutil -import sys - -from pip._vendor import pkg_resources -from pip._vendor.six.moves.urllib import parse as urllib_parse - -from pip._internal.exceptions import BadCommand -from pip._internal.utils.compat import samefile -from pip._internal.utils.misc import ( - ask_path_exists, - backup_dir, - display_path, - hide_url, - hide_value, - rmtree, -) -from pip._internal.utils.subprocess import call_subprocess, make_command -from pip._internal.utils.typing import MYPY_CHECK_RUNNING -from pip._internal.utils.urls import get_url_scheme - -if MYPY_CHECK_RUNNING: - from typing import ( - Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union - ) - from pip._internal.utils.ui import SpinnerInterface - from pip._internal.utils.misc import HiddenText - from pip._internal.utils.subprocess import CommandArgs - - AuthInfo = Tuple[Optional[str], Optional[str]] - - -__all__ = ['vcs'] - - -logger = logging.getLogger(__name__) - - -def is_url(name): - # type: (Union[str, Text]) -> bool - """ - Return true if the name looks like a URL. - """ - scheme = get_url_scheme(name) - if scheme is None: - return False - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): - """ - Return the URL for a VCS requirement. - - Args: - repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). - project_name: the (unescaped) project name. - """ - egg_project_name = pkg_resources.to_filename(project_name) - req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) - if subdir: - req += '&subdirectory={}'.format(subdir) - - return req - - -def find_path_to_setup_from_repo_root(location, repo_root): - """ - Find the path to `setup.py` by searching up the filesystem from `location`. - Return the path to `setup.py` relative to `repo_root`. - Return None if `setup.py` is in `repo_root` or cannot be found. - """ - # find setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without - # finding setup.py - logger.warning( - "Could not find setup.py for directory %s (tried all " - "parent directories)", - orig_location, - ) - return None - - if samefile(repo_root, location): - return None - - return os.path.relpath(location, repo_root) - - -class RemoteNotFoundError(Exception): - pass - - -class RevOptions(object): - - """ - Encapsulates a VCS-specific revision to install, along with any VCS - install options. - - Instances of this class should be treated as if immutable. - """ - - def __init__( - self, - vc_class, # type: Type[VersionControl] - rev=None, # type: Optional[str] - extra_args=None, # type: Optional[CommandArgs] - ): - # type: (...) -> None - """ - Args: - vc_class: a VersionControl subclass. - rev: the name of the revision to install. - extra_args: a list of extra options. - """ - if extra_args is None: - extra_args = [] - - self.extra_args = extra_args - self.rev = rev - self.vc_class = vc_class - self.branch_name = None # type: Optional[str] - - def __repr__(self): - return ''.format(self.vc_class.name, self.rev) - - @property - def arg_rev(self): - # type: () -> Optional[str] - if self.rev is None: - return self.vc_class.default_arg_rev - - return self.rev - - def to_args(self): - # type: () -> CommandArgs - """ - Return the VCS-specific command arguments. - """ - args = [] # type: CommandArgs - rev = self.arg_rev - if rev is not None: - args += self.vc_class.get_base_rev_args(rev) - args += self.extra_args - - return args - - def to_display(self): - # type: () -> str - if not self.rev: - return '' - - return ' (to revision {})'.format(self.rev) - - def make_new(self, rev): - # type: (str) -> RevOptions - """ - Make a copy of the current instance, but with a new rev. - - Args: - rev: the name of the revision for the new object. - """ - return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) - - -class VcsSupport(object): - _registry = {} # type: Dict[str, VersionControl] - schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] - - def __init__(self): - # type: () -> None - # Register more schemes with urlparse for various version control - # systems - urllib_parse.uses_netloc.extend(self.schemes) - # Python >= 2.7.4, 3.3 doesn't have uses_fragment - if getattr(urllib_parse, 'uses_fragment', None): - urllib_parse.uses_fragment.extend(self.schemes) - super(VcsSupport, self).__init__() - - def __iter__(self): - return self._registry.__iter__() - - @property - def backends(self): - # type: () -> List[VersionControl] - return list(self._registry.values()) - - @property - def dirnames(self): - # type: () -> List[str] - return [backend.dirname for backend in self.backends] - - @property - def all_schemes(self): - # type: () -> List[str] - schemes = [] # type: List[str] - for backend in self.backends: - schemes.extend(backend.schemes) - return schemes - - def register(self, cls): - # type: (Type[VersionControl]) -> None - if not hasattr(cls, 'name'): - logger.warning('Cannot register VCS %s', cls.__name__) - return - if cls.name not in self._registry: - self._registry[cls.name] = cls() - logger.debug('Registered VCS backend: %s', cls.name) - - def unregister(self, name): - # type: (str) -> None - if name in self._registry: - del self._registry[name] - - def get_backend_for_dir(self, location): - # type: (str) -> Optional[VersionControl] - """ - Return a VersionControl object if a repository of that type is found - at the given directory. - """ - for vcs_backend in self._registry.values(): - if vcs_backend.controls_location(location): - logger.debug('Determine that %s uses VCS: %s', - location, vcs_backend.name) - return vcs_backend - return None - - def get_backend(self, name): - # type: (str) -> Optional[VersionControl] - """ - Return a VersionControl object or None. - """ - name = name.lower() - return self._registry.get(name) - - -vcs = VcsSupport() - - -class VersionControl(object): - name = '' - dirname = '' - repo_name = '' - # List of supported schemes for this Version Control - schemes = () # type: Tuple[str, ...] - # Iterable of environment variable names to pass to call_subprocess(). - unset_environ = () # type: Tuple[str, ...] - default_arg_rev = None # type: Optional[str] - - @classmethod - def should_add_vcs_url_prefix(cls, remote_url): - """ - Return whether the vcs prefix (e.g. "git+") should be added to a - repository's remote url when used in a requirement. - """ - return not remote_url.lower().startswith('{}:'.format(cls.name)) - - @classmethod - def get_subdirectory(cls, location): - """ - Return the path to setup.py, relative to the repo root. - Return None if setup.py is in the repo root. - """ - return None - - @classmethod - def get_requirement_revision(cls, repo_dir): - """ - Return the revision string that should be used in a requirement. - """ - return cls.get_revision(repo_dir) - - @classmethod - def get_src_requirement(cls, repo_dir, project_name): - """ - Return the requirement string to use to redownload the files - currently at the given repository directory. - - Args: - project_name: the (unescaped) project name. - - The return value has a form similar to the following: - - {repository_url}@{revision}#egg={project_name} - """ - repo_url = cls.get_remote_url(repo_dir) - if repo_url is None: - return None - - if cls.should_add_vcs_url_prefix(repo_url): - repo_url = '{}+{}'.format(cls.name, repo_url) - - revision = cls.get_requirement_revision(repo_dir) - subdir = cls.get_subdirectory(repo_dir) - req = make_vcs_requirement_url(repo_url, revision, project_name, - subdir=subdir) - - return req - - @staticmethod - def get_base_rev_args(rev): - """ - Return the base revision arguments for a vcs command. - - Args: - rev: the name of a revision to install. Cannot be None. - """ - raise NotImplementedError - - @classmethod - def make_rev_options(cls, rev=None, extra_args=None): - # type: (Optional[str], Optional[CommandArgs]) -> RevOptions - """ - Return a RevOptions object. - - Args: - rev: the name of a revision to install. - extra_args: a list of extra options. - """ - return RevOptions(cls, rev, extra_args=extra_args) - - @classmethod - def _is_local_repository(cls, repo): - # type: (str) -> bool - """ - posix absolute paths start with os.path.sep, - win32 ones start with drive (like c:\\folder) - """ - drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or bool(drive) - - def export(self, location, url): - # type: (str, HiddenText) -> None - """ - Export the repository at the url to the destination location - i.e. only download the files, without vcs informations - - :param url: the repository URL starting with a vcs prefix. - """ - raise NotImplementedError - - @classmethod - def get_netloc_and_auth(cls, netloc, scheme): - """ - Parse the repository URL's netloc, and return the new netloc to use - along with auth information. - - Args: - netloc: the original repository URL netloc. - scheme: the repository URL's scheme without the vcs prefix. - - This is mainly for the Subversion class to override, so that auth - information can be provided via the --username and --password options - instead of through the URL. For other subclasses like Git without - such an option, auth information must stay in the URL. - - Returns: (netloc, (username, password)). - """ - return netloc, (None, None) - - @classmethod - def get_url_rev_and_auth(cls, url): - # type: (str) -> Tuple[str, Optional[str], AuthInfo] - """ - Parse the repository URL to use, and return the URL, revision, - and auth info to use. - - Returns: (url, rev, (username, password)). - """ - scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) - if '+' not in scheme: - raise ValueError( - "Sorry, {!r} is a malformed VCS url. " - "The format is +://, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) - ) - # Remove the vcs prefix. - scheme = scheme.split('+', 1)[1] - netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev, user_pass - - @staticmethod - def make_rev_args(username, password): - # type: (Optional[str], Optional[HiddenText]) -> CommandArgs - """ - Return the RevOptions "extra arguments" to use in obtain(). - """ - return [] - - def get_url_rev_options(self, url): - # type: (HiddenText) -> Tuple[HiddenText, RevOptions] - """ - Return the URL and RevOptions object to use in obtain() and in - some cases export(), as a tuple (url, rev_options). - """ - secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) - username, secret_password = user_pass - password = None # type: Optional[HiddenText] - if secret_password is not None: - password = hide_value(secret_password) - extra_args = self.make_rev_args(username, password) - rev_options = self.make_rev_options(rev, extra_args=extra_args) - - return hide_url(secret_url), rev_options - - @staticmethod - def normalize_url(url): - # type: (str) -> str - """ - Normalize a URL for comparison by unquoting it and removing any - trailing slash. - """ - return urllib_parse.unquote(url).rstrip('/') - - @classmethod - def compare_urls(cls, url1, url2): - # type: (str, str) -> bool - """ - Compare two repo URLs for identity, ignoring incidental differences. - """ - return (cls.normalize_url(url1) == cls.normalize_url(url2)) - - def fetch_new(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Fetch a revision from a repository, in the case that this is the - first fetch from the repository. - - Args: - dest: the directory to fetch the repository to. - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def switch(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Switch the repo at ``dest`` to point to ``URL``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def update(self, dest, url, rev_options): - # type: (str, HiddenText, RevOptions) -> None - """ - Update an already-existing repo to the given ``rev_options``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - @classmethod - def is_commit_id_equal(cls, dest, name): - """ - Return whether the id of the current commit equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - raise NotImplementedError - - def obtain(self, dest, url): - # type: (str, HiddenText) -> None - """ - Install or update in editable mode the package represented by this - VersionControl object. - - :param dest: the repository directory in which to install or update. - :param url: the repository URL starting with a vcs prefix. - """ - url, rev_options = self.get_url_rev_options(url) - - if not os.path.exists(dest): - self.fetch_new(dest, url, rev_options) - return - - rev_display = rev_options.to_display() - if self.is_repository_directory(dest): - existing_url = self.get_remote_url(dest) - if self.compare_urls(existing_url, url.secret): - logger.debug( - '%s in %s exists, and has correct URL (%s)', - self.repo_name.title(), - display_path(dest), - url, - ) - if not self.is_commit_id_equal(dest, rev_options.rev): - logger.info( - 'Updating %s %s%s', - display_path(dest), - self.repo_name, - rev_display, - ) - self.update(dest, url, rev_options) - else: - logger.info('Skipping because already up-to-date.') - return - - logger.warning( - '%s %s in %s exists with URL %s', - self.name, - self.repo_name, - display_path(dest), - existing_url, - ) - prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', - ('s', 'i', 'w', 'b')) - else: - logger.warning( - 'Directory %s already exists, and is not a %s %s.', - dest, - self.name, - self.repo_name, - ) - # https://github.com/python/mypy/issues/1174 - prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore - ('i', 'w', 'b')) - - logger.warning( - 'The plan is to install the %s repository %s', - self.name, - url, - ) - response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) - - if response == 'a': - sys.exit(-1) - - if response == 'w': - logger.warning('Deleting %s', display_path(dest)) - rmtree(dest) - self.fetch_new(dest, url, rev_options) - return - - if response == 'b': - dest_dir = backup_dir(dest) - logger.warning( - 'Backing up %s to %s', display_path(dest), dest_dir, - ) - shutil.move(dest, dest_dir) - self.fetch_new(dest, url, rev_options) - return - - # Do nothing if the response is "i". - if response == 's': - logger.info( - 'Switching %s %s to %s%s', - self.repo_name, - display_path(dest), - url, - rev_display, - ) - self.switch(dest, url, rev_options) - - def unpack(self, location, url): - # type: (str, HiddenText) -> None - """ - Clean up current location and download the url repository - (and vcs infos) into location - - :param url: the repository URL starting with a vcs prefix. - """ - if os.path.exists(location): - rmtree(location) - self.obtain(location, url=url) - - @classmethod - def get_remote_url(cls, location): - """ - Return the url used at location - - Raises RemoteNotFoundError if the repository does not have a remote - url configured. - """ - raise NotImplementedError - - @classmethod - def get_revision(cls, location): - """ - Return the current commit id of the files at the given location. - """ - raise NotImplementedError - - @classmethod - def run_command( - cls, - cmd, # type: Union[List[str], CommandArgs] - show_stdout=True, # type: bool - cwd=None, # type: Optional[str] - on_returncode='raise', # type: str - extra_ok_returncodes=None, # type: Optional[Iterable[int]] - command_desc=None, # type: Optional[str] - extra_environ=None, # type: Optional[Mapping[str, Any]] - spinner=None, # type: Optional[SpinnerInterface] - log_failed_cmd=True - ): - # type: (...) -> Text - """ - Run a VCS subcommand - This is simply a wrapper around call_subprocess that adds the VCS - command name, and checks that the VCS is available - """ - cmd = make_command(cls.name, *cmd) - try: - return call_subprocess(cmd, show_stdout, cwd, - on_returncode=on_returncode, - extra_ok_returncodes=extra_ok_returncodes, - command_desc=command_desc, - extra_environ=extra_environ, - unset_environ=cls.unset_environ, - spinner=spinner, - log_failed_cmd=log_failed_cmd) - except OSError as e: - # errno.ENOENT = no such file or directory - # In other words, the VCS executable isn't available - if e.errno == errno.ENOENT: - raise BadCommand( - 'Cannot find command %r - do you have ' - '%r installed and in your ' - 'PATH?' % (cls.name, cls.name)) - else: - raise # re-raise exception if a different error occurred - - @classmethod - def is_repository_directory(cls, path): - # type: (str) -> bool - """ - Return whether a directory path is a repository directory. - """ - logger.debug('Checking in %s for %s (%s)...', - path, cls.dirname, cls.name) - return os.path.exists(os.path.join(path, cls.dirname)) - - @classmethod - def controls_location(cls, location): - # type: (str) -> bool - """ - Check if a location is controlled by the vcs. - - Searches up the filesystem and checks is_repository_directory(). - - It is meant to be extended to add smarter detection mechanisms for - specific vcs. For example, the Git override checks that Git is - actually available. - """ - while not cls.is_repository_directory(location): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem. - return False - return True +"""Handles all VCS (version control) support""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import errno +import logging +import os +import shutil +import sys + +from pip._vendor import pkg_resources +from pip._vendor.six.moves.urllib import parse as urllib_parse + +from pip._internal.exceptions import BadCommand +from pip._internal.utils.compat import samefile +from pip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + hide_url, + hide_value, + rmtree, +) +from pip._internal.utils.subprocess import call_subprocess, make_command +from pip._internal.utils.typing import MYPY_CHECK_RUNNING +from pip._internal.utils.urls import get_url_scheme + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union + ) + from pip._internal.utils.ui import SpinnerInterface + from pip._internal.utils.misc import HiddenText + from pip._internal.utils.subprocess import CommandArgs + + AuthInfo = Tuple[Optional[str], Optional[str]] + + +__all__ = ['vcs'] + + +logger = logging.getLogger(__name__) + + +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. + """ + egg_project_name = pkg_resources.to_filename(project_name) + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def find_path_to_setup_from_repo_root(location, repo_root): + """ + Find the path to `setup.py` by searching up the filesystem from `location`. + Return the path to `setup.py` relative to `repo_root`. + Return None if `setup.py` is in `repo_root` or cannot be found. + """ + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + + if samefile(repo_root, location): + return None + + return os.path.relpath(location, repo_root) + + +class RemoteNotFoundError(Exception): + pass + + +class RevOptions(object): + + """ + Encapsulates a VCS-specific revision to install, along with any VCS + install options. + + Instances of this class should be treated as if immutable. + """ + + def __init__( + self, + vc_class, # type: Type[VersionControl] + rev=None, # type: Optional[str] + extra_args=None, # type: Optional[CommandArgs] + ): + # type: (...) -> None + """ + Args: + vc_class: a VersionControl subclass. + rev: the name of the revision to install. + extra_args: a list of extra options. + """ + if extra_args is None: + extra_args = [] + + self.extra_args = extra_args + self.rev = rev + self.vc_class = vc_class + self.branch_name = None # type: Optional[str] + + def __repr__(self): + return ''.format(self.vc_class.name, self.rev) + + @property + def arg_rev(self): + # type: () -> Optional[str] + if self.rev is None: + return self.vc_class.default_arg_rev + + return self.rev + + def to_args(self): + # type: () -> CommandArgs + """ + Return the VCS-specific command arguments. + """ + args = [] # type: CommandArgs + rev = self.arg_rev + if rev is not None: + args += self.vc_class.get_base_rev_args(rev) + args += self.extra_args + + return args + + def to_display(self): + # type: () -> str + if not self.rev: + return '' + + return ' (to revision {})'.format(self.rev) + + def make_new(self, rev): + # type: (str) -> RevOptions + """ + Make a copy of the current instance, but with a new rev. + + Args: + rev: the name of the revision for the new object. + """ + return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) + + +class VcsSupport(object): + _registry = {} # type: Dict[str, VersionControl] + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] + + def __init__(self): + # type: () -> None + # Register more schemes with urlparse for various version control + # systems + urllib_parse.uses_netloc.extend(self.schemes) + # Python >= 2.7.4, 3.3 doesn't have uses_fragment + if getattr(urllib_parse, 'uses_fragment', None): + urllib_parse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + return self._registry.__iter__() + + @property + def backends(self): + # type: () -> List[VersionControl] + return list(self._registry.values()) + + @property + def dirnames(self): + # type: () -> List[str] + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + # type: () -> List[str] + schemes = [] # type: List[str] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + # type: (Type[VersionControl]) -> None + if not hasattr(cls, 'name'): + logger.warning('Cannot register VCS %s', cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls() + logger.debug('Registered VCS backend: %s', cls.name) + + def unregister(self, name): + # type: (str) -> None + if name in self._registry: + del self._registry[name] + + def get_backend_for_dir(self, location): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object if a repository of that type is found + at the given directory. + """ + for vcs_backend in self._registry.values(): + if vcs_backend.controls_location(location): + logger.debug('Determine that %s uses VCS: %s', + location, vcs_backend.name) + return vcs_backend + return None + + def get_backend(self, name): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object or None. + """ + name = name.lower() + return self._registry.get(name) + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + repo_name = '' + # List of supported schemes for this Version Control + schemes = () # type: Tuple[str, ...] + # Iterable of environment variable names to pass to call_subprocess(). + unset_environ = () # type: Tuple[str, ...] + default_arg_rev = None # type: Optional[str] + + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + """ + Return whether the vcs prefix (e.g. "git+") should be added to a + repository's remote url when used in a requirement. + """ + return not remote_url.lower().startswith('{}:'.format(cls.name)) + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + return None + + @classmethod + def get_requirement_revision(cls, repo_dir): + """ + Return the revision string that should be used in a requirement. + """ + return cls.get_revision(repo_dir) + + @classmethod + def get_src_requirement(cls, repo_dir, project_name): + """ + Return the requirement string to use to redownload the files + currently at the given repository directory. + + Args: + project_name: the (unescaped) project name. + + The return value has a form similar to the following: + + {repository_url}@{revision}#egg={project_name} + """ + repo_url = cls.get_remote_url(repo_dir) + if repo_url is None: + return None + + if cls.should_add_vcs_url_prefix(repo_url): + repo_url = '{}+{}'.format(cls.name, repo_url) + + revision = cls.get_requirement_revision(repo_dir) + subdir = cls.get_subdirectory(repo_dir) + req = make_vcs_requirement_url(repo_url, revision, project_name, + subdir=subdir) + + return req + + @staticmethod + def get_base_rev_args(rev): + """ + Return the base revision arguments for a vcs command. + + Args: + rev: the name of a revision to install. Cannot be None. + """ + raise NotImplementedError + + @classmethod + def make_rev_options(cls, rev=None, extra_args=None): + # type: (Optional[str], Optional[CommandArgs]) -> RevOptions + """ + Return a RevOptions object. + + Args: + rev: the name of a revision to install. + extra_args: a list of extra options. + """ + return RevOptions(cls, rev, extra_args=extra_args) + + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool + """ + posix absolute paths start with os.path.sep, + win32 ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or bool(drive) + + def export(self, location, url): + # type: (str, HiddenText) -> None + """ + Export the repository at the url to the destination location + i.e. only download the files, without vcs informations + + :param url: the repository URL starting with a vcs prefix. + """ + raise NotImplementedError + + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). + """ + scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is +://, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev, user_pass + + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + """ + Return the RevOptions "extra arguments" to use in obtain(). + """ + return [] + + def get_url_rev_options(self, url): + # type: (HiddenText) -> Tuple[HiddenText, RevOptions] + """ + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). + """ + secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) + username, secret_password = user_pass + password = None # type: Optional[HiddenText] + if secret_password is not None: + password = hide_value(secret_password) + extra_args = self.make_rev_args(username, password) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return hide_url(secret_url), rev_options + + @staticmethod + def normalize_url(url): + # type: (str) -> str + """ + Normalize a URL for comparison by unquoting it and removing any + trailing slash. + """ + return urllib_parse.unquote(url).rstrip('/') + + @classmethod + def compare_urls(cls, url1, url2): + # type: (str, str) -> bool + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (cls.normalize_url(url1) == cls.normalize_url(url2)) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Switch the repo at ``dest`` to point to ``URL``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Update an already-existing repo to the given ``rev_options``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the id of the current commit equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + raise NotImplementedError + + def obtain(self, dest, url): + # type: (str, HiddenText) -> None + """ + Install or update in editable mode the package represented by this + VersionControl object. + + :param dest: the repository directory in which to install or update. + :param url: the repository URL starting with a vcs prefix. + """ + url, rev_options = self.get_url_rev_options(url) + + if not os.path.exists(dest): + self.fetch_new(dest, url, rev_options) + return + + rev_display = rev_options.to_display() + if self.is_repository_directory(dest): + existing_url = self.get_remote_url(dest) + if self.compare_urls(existing_url, url.secret): + logger.debug( + '%s in %s exists, and has correct URL (%s)', + self.repo_name.title(), + display_path(dest), + url, + ) + if not self.is_commit_id_equal(dest, rev_options.rev): + logger.info( + 'Updating %s %s%s', + display_path(dest), + self.repo_name, + rev_display, + ) + self.update(dest, url, rev_options) + else: + logger.info('Skipping because already up-to-date.') + return + + logger.warning( + '%s %s in %s exists with URL %s', + self.name, + self.repo_name, + display_path(dest), + existing_url, + ) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', + ('s', 'i', 'w', 'b')) + else: + logger.warning( + 'Directory %s already exists, and is not a %s %s.', + dest, + self.name, + self.repo_name, + ) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) + + logger.warning( + 'The plan is to install the %s repository %s', + self.name, + url, + ) + response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) + + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + self.fetch_new(dest, url, rev_options) + return + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + self.fetch_new(dest, url, rev_options) + return + + # Do nothing if the response is "i". + if response == 's': + logger.info( + 'Switching %s %s to %s%s', + self.repo_name, + display_path(dest), + url, + rev_display, + ) + self.switch(dest, url, rev_options) + + def unpack(self, location, url): + # type: (str, HiddenText) -> None + """ + Clean up current location and download the url repository + (and vcs infos) into location + + :param url: the repository URL starting with a vcs prefix. + """ + if os.path.exists(location): + rmtree(location) + self.obtain(location, url=url) + + @classmethod + def get_remote_url(cls, location): + """ + Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + raise NotImplementedError + + @classmethod + def get_revision(cls, location): + """ + Return the current commit id of the files at the given location. + """ + raise NotImplementedError + + @classmethod + def run_command( + cls, + cmd, # type: Union[List[str], CommandArgs] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True + ): + # type: (...) -> Text + """ + Run a VCS subcommand + This is simply a wrapper around call_subprocess that adds the VCS + command name, and checks that the VCS is available + """ + cmd = make_command(cls.name, *cmd) + try: + return call_subprocess(cmd, show_stdout, cwd, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, + spinner=spinner, + log_failed_cmd=log_failed_cmd) + except OSError as e: + # errno.ENOENT = no such file or directory + # In other words, the VCS executable isn't available + if e.errno == errno.ENOENT: + raise BadCommand( + 'Cannot find command %r - do you have ' + '%r installed and in your ' + 'PATH?' % (cls.name, cls.name)) + else: + raise # re-raise exception if a different error occurred + + @classmethod + def is_repository_directory(cls, path): + # type: (str) -> bool + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + + @classmethod + def controls_location(cls, location): + # type: (str) -> bool + """ + Check if a location is controlled by the vcs. + + Searches up the filesystem and checks is_repository_directory(). + + It is meant to be extended to add smarter detection mechanisms for + specific vcs. For example, the Git override checks that Git is + actually available. + """ + while not cls.is_repository_directory(location): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem. + return False + return True From 24a2be8af684885720ac376477fc864584e95bb2 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Fri, 11 Oct 2019 16:44:45 +1300 Subject: [PATCH 15/16] Reverting `VersionControl.controls_location()` to pre PR state. Its an optimization that belongs in another PR. --- src/pip/_internal/vcs/versioncontrol.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/pip/_internal/vcs/versioncontrol.py b/src/pip/_internal/vcs/versioncontrol.py index 55afa908c..9038ace80 100644 --- a/src/pip/_internal/vcs/versioncontrol.py +++ b/src/pip/_internal/vcs/versioncontrol.py @@ -656,17 +656,10 @@ class VersionControl(object): # type: (str) -> bool """ Check if a location is controlled by the vcs. + It is meant to be overridden to implement smarter detection + mechanisms for specific vcs. - Searches up the filesystem and checks is_repository_directory(). - - It is meant to be extended to add smarter detection mechanisms for - specific vcs. For example, the Git override checks that Git is - actually available. + This can do more than is_repository_directory() alone. For example, + the Git override checks that Git is actually available. """ - while not cls.is_repository_directory(location): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem. - return False - return True + return cls.is_repository_directory(location) From 7ebc54171c9612b78e59d8012c0d8c1b70f78431 Mon Sep 17 00:00:00 2001 From: tbeswick Date: Fri, 11 Oct 2019 17:15:35 +1300 Subject: [PATCH 16/16] Fixed missing return statement in `Mercurial.controls_location()`, it only got found after reverting `VersionControl.controls_location()` --- src/pip/_internal/vcs/mercurial.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/pip/_internal/vcs/mercurial.py b/src/pip/_internal/vcs/mercurial.py index bc4b19c72..d9b58cfe9 100644 --- a/src/pip/_internal/vcs/mercurial.py +++ b/src/pip/_internal/vcs/mercurial.py @@ -147,6 +147,7 @@ class Mercurial(VersionControl): show_stdout=False, on_returncode='raise', log_failed_cmd=False) + return True except (BadCommand, InstallationError): return False