mirror of https://github.com/pypa/pip
Automatic vendoring (#4093)
This commit is contained in:
parent
bb796adbda
commit
2c9735437a
|
@ -15,7 +15,8 @@ Policy
|
|||
pure Python.
|
||||
|
||||
* Any modifications made to libraries **MUST** be noted in
|
||||
``pip/_vendor/README.rst``.
|
||||
``pip/_vendor/README.rst`` and their corresponding patches **MUST** be
|
||||
included ``tasks/vendoring/patches``.
|
||||
|
||||
|
||||
Rationale
|
||||
|
@ -92,16 +93,11 @@ situation that we expect pip to be used and not mandate some external mechanism
|
|||
such as OS packages.
|
||||
|
||||
|
||||
pkg_resources
|
||||
-------------
|
||||
|
||||
pkg_resources has been pulled in from setuptools 28.8.0
|
||||
|
||||
|
||||
Modifications
|
||||
-------------
|
||||
|
||||
* html5lib has been modified to import six from pip._vendor
|
||||
* setuptools is completely stripped to only keep pkg_resources
|
||||
* pkg_resources has been modified to import its externs from pip._vendor
|
||||
* CacheControl has been modified to import its dependencies from pip._vendor
|
||||
* packaging has been modified to import its dependencies from pip._vendor
|
||||
|
@ -109,6 +105,16 @@ Modifications
|
|||
* Modified distro to delay importing argparse to avoid errors on 2.6
|
||||
|
||||
|
||||
Automatic Vendoring
|
||||
-------------------
|
||||
|
||||
Vendoring is automated via the ``vendoring.update`` task (defined in
|
||||
``tasks/vendoring/__init__.py``) from the content of
|
||||
``pip/_vendor/vendor.txt`` and the different patches in
|
||||
``tasks/vendoring/patches/``.
|
||||
Launch it via ``invoke vendoring.update`` (you will need ``invoke>=0.13.0``).
|
||||
|
||||
|
||||
Debundling
|
||||
----------
|
||||
|
||||
|
|
|
@ -1,34 +0,0 @@
|
|||
import os
|
||||
import sys
|
||||
import pip
|
||||
import glob
|
||||
import shutil
|
||||
|
||||
here = os.path.abspath(os.path.dirname(__file__))
|
||||
|
||||
def usage():
|
||||
print("Usage: re-vendor.py [clean|vendor]")
|
||||
sys.exit(1)
|
||||
|
||||
def clean():
|
||||
for fn in os.listdir(here):
|
||||
dirname = os.path.join(here, fn)
|
||||
if os.path.isdir(dirname):
|
||||
shutil.rmtree(dirname)
|
||||
# six is a single file, not a package
|
||||
os.unlink(os.path.join(here, 'six.py'))
|
||||
|
||||
def vendor():
|
||||
pip.main(['install', '-t', here, '-r', 'vendor.txt'])
|
||||
for dirname in glob.glob('*.egg-info'):
|
||||
shutil.rmtree(dirname)
|
||||
|
||||
if __name__ == '__main__':
|
||||
if len(sys.argv) != 2:
|
||||
usage()
|
||||
if sys.argv[1] == 'clean':
|
||||
clean()
|
||||
elif sys.argv[1] == 'vendor':
|
||||
vendor()
|
||||
else:
|
||||
usage()
|
|
@ -13,4 +13,5 @@ ipaddress==1.0.17 # Only needed on 2.6 and 2.7
|
|||
packaging==16.8
|
||||
pyparsing==2.1.10
|
||||
retrying==1.3.3
|
||||
setuptools==28.8.0
|
||||
webencodings==0.5
|
||||
|
|
|
@ -1,5 +1,6 @@
|
|||
import invoke
|
||||
|
||||
from . import generate
|
||||
from . import vendoring
|
||||
|
||||
ns = invoke.Collection(generate)
|
||||
ns = invoke.Collection(generate, vendoring)
|
||||
|
|
|
@ -4,12 +4,12 @@ import invoke
|
|||
|
||||
|
||||
@invoke.task
|
||||
def authors():
|
||||
def authors(ctx):
|
||||
print("[generate.authors] Generating AUTHORS")
|
||||
|
||||
# Get our list of authors
|
||||
print("[generate.authors] Collecting author names")
|
||||
r = invoke.run("git log --use-mailmap --format'=%aN <%aE>'", hide=True)
|
||||
r = ctx.run("git log --use-mailmap --format'=%aN <%aE>'", hide=True)
|
||||
authors = []
|
||||
seen_authors = set()
|
||||
for author in r.stdout.splitlines():
|
||||
|
|
|
@ -0,0 +1,129 @@
|
|||
""""Vendoring script, python 3.5 needed"""
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
|
||||
import invoke
|
||||
|
||||
TASK_NAME = 'update'
|
||||
|
||||
FILE_WHITE_LIST = (
|
||||
'Makefile',
|
||||
'vendor.txt',
|
||||
'__init__.py',
|
||||
'README.rst',
|
||||
)
|
||||
|
||||
|
||||
def drop_dir(path):
|
||||
shutil.rmtree(str(path))
|
||||
|
||||
|
||||
def remove_all(paths):
|
||||
for path in paths:
|
||||
if path.is_dir():
|
||||
drop_dir(path)
|
||||
else:
|
||||
path.unlink()
|
||||
|
||||
|
||||
def log(msg):
|
||||
print('[vendoring.%s] %s' % (TASK_NAME, msg))
|
||||
|
||||
|
||||
def clean_vendor(ctx, vendor_dir):
|
||||
# Old _vendor cleanup
|
||||
remove_all(vendor_dir.glob('*.pyc'))
|
||||
log('Cleaning %s' % vendor_dir)
|
||||
for item in vendor_dir.iterdir():
|
||||
if item.is_dir():
|
||||
shutil.rmtree(str(item))
|
||||
elif item.name not in FILE_WHITE_LIST:
|
||||
item.unlink()
|
||||
else:
|
||||
log('Skipping %s' % item)
|
||||
|
||||
|
||||
def rewrite_imports(package_dir, vendored_libs):
|
||||
for item in package_dir.iterdir():
|
||||
if item.is_dir():
|
||||
rewrite_imports(item, vendored_libs)
|
||||
elif item.name.endswith('.py'):
|
||||
rewrite_file_imports(item, vendored_libs)
|
||||
|
||||
|
||||
def rewrite_file_imports(item, vendored_libs):
|
||||
"""Rewrite 'import xxx' and 'from xxx import' for vendored_libs"""
|
||||
text = item.read_text()
|
||||
# Revendor pkg_resources.extern first
|
||||
text = re.sub(r'pkg_resources.extern', r'pip._vendor', text)
|
||||
for lib in vendored_libs:
|
||||
text = re.sub(
|
||||
r'(\n\s*)import %s' % lib,
|
||||
r'\1from pip._vendor import %s' % lib,
|
||||
text,
|
||||
)
|
||||
text = re.sub(
|
||||
r'(\n\s*)from %s' % lib,
|
||||
r'\1from pip._vendor.%s' % lib,
|
||||
text,
|
||||
)
|
||||
item.write_text(text)
|
||||
|
||||
|
||||
def apply_patch(ctx, patch_file_path):
|
||||
log('Applying patch %s' % patch_file_path.name)
|
||||
ctx.run('git apply %s' % patch_file_path)
|
||||
|
||||
|
||||
def vendor(ctx, vendor_dir):
|
||||
log('Reinstalling vendored libraries')
|
||||
ctx.run(
|
||||
'pip install -t {0} -r {0}/vendor.txt --no-compile'.format(
|
||||
str(vendor_dir),
|
||||
)
|
||||
)
|
||||
remove_all(vendor_dir.glob('*.dist-info'))
|
||||
remove_all(vendor_dir.glob('*.egg-info'))
|
||||
|
||||
# Cleanup setuptools unneeded parts
|
||||
(vendor_dir / 'easy_install.py').unlink()
|
||||
drop_dir(vendor_dir / 'setuptools')
|
||||
drop_dir(vendor_dir / 'pkg_resources' / '_vendor')
|
||||
drop_dir(vendor_dir / 'pkg_resources' / 'extern')
|
||||
|
||||
# Detect the vendored packages/modules
|
||||
vendored_libs = []
|
||||
for item in vendor_dir.iterdir():
|
||||
if item.is_dir():
|
||||
vendored_libs.append(item.name)
|
||||
elif item.name not in FILE_WHITE_LIST:
|
||||
vendored_libs.append(item.name[:-3])
|
||||
log("Detected vendored libraries: %s" % ", ".join(vendored_libs))
|
||||
|
||||
# Global import rewrites
|
||||
log("Rewriting all imports related to vendored libs")
|
||||
for item in vendor_dir.iterdir():
|
||||
if item.is_dir():
|
||||
rewrite_imports(item, vendored_libs)
|
||||
elif item.name not in FILE_WHITE_LIST:
|
||||
rewrite_file_imports(item, vendored_libs)
|
||||
|
||||
# Special cases: apply stored patches
|
||||
log("Apply patches")
|
||||
patch_dir = Path(__file__).parent / 'patches'
|
||||
for patch in patch_dir.glob('*.patch'):
|
||||
apply_patch(ctx, patch)
|
||||
|
||||
|
||||
@invoke.task(name=TASK_NAME)
|
||||
def main(ctx):
|
||||
git_root = Path(
|
||||
ctx.run('git rev-parse --show-toplevel', hide=True).stdout.strip()
|
||||
)
|
||||
vendor_dir = git_root / 'pip' / '_vendor'
|
||||
log('Using vendor dir: %s' % vendor_dir)
|
||||
clean_vendor(ctx, vendor_dir)
|
||||
vendor(ctx, vendor_dir)
|
||||
log('Revendoring complete')
|
|
@ -0,0 +1,24 @@
|
|||
diff --git a/pip/_vendor/cachecontrol/compat.py b/pip/_vendor/cachecontrol/compat.py
|
||||
index 54ec211..018e6ac 100644
|
||||
--- a/pip/_vendor/cachecontrol/compat.py
|
||||
+++ b/pip/_vendor/cachecontrol/compat.py
|
||||
@@ -10,17 +10,8 @@ except ImportError:
|
||||
import pickle
|
||||
|
||||
|
||||
-# Handle the case where the requests module has been patched to not have
|
||||
-# urllib3 bundled as part of its source.
|
||||
-try:
|
||||
- from pip._vendor.requests.packages.urllib3.response import HTTPResponse
|
||||
-except ImportError:
|
||||
- from urllib3.response import HTTPResponse
|
||||
-
|
||||
-try:
|
||||
- from pip._vendor.requests.packages.urllib3.util import is_fp_closed
|
||||
-except ImportError:
|
||||
- from urllib3.util import is_fp_closed
|
||||
+from pip._vendor.requests.packages.urllib3.response import HTTPResponse
|
||||
+from pip._vendor.requests.packages.urllib3.util import is_fp_closed
|
||||
|
||||
# Replicate some six behaviour
|
||||
try:
|
|
@ -0,0 +1,21 @@
|
|||
diff --git a/pip/_vendor/distro.py b/pip/_vendor/distro.py
|
||||
index 0afa527..9e7daad 100644
|
||||
--- a/pip/_vendor/distro.py
|
||||
+++ b/pip/_vendor/distro.py
|
||||
@@ -34,7 +34,6 @@ import sys
|
||||
import json
|
||||
import shlex
|
||||
import logging
|
||||
-import argparse
|
||||
import subprocess
|
||||
|
||||
|
||||
@@ -1052,6 +1051,8 @@ _distro = LinuxDistribution()
|
||||
|
||||
|
||||
def main():
|
||||
+ import argparse
|
||||
+
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.addHandler(logging.StreamHandler(sys.stdout))
|
|
@ -0,0 +1,48 @@
|
|||
diff --git a/pip/_vendor/requests/__init__.py b/pip/_vendor/requests/__init__.py
|
||||
index 9c3b769..44f6836 100644
|
||||
--- a/pip/_vendor/requests/__init__.py
|
||||
+++ b/pip/_vendor/requests/__init__.py
|
||||
@@ -48,11 +48,13 @@ __license__ = 'Apache 2.0'
|
||||
__copyright__ = 'Copyright 2016 Kenneth Reitz'
|
||||
|
||||
# Attempt to enable urllib3's SNI support, if possible
|
||||
-try:
|
||||
- from .packages.urllib3.contrib import pyopenssl
|
||||
- pyopenssl.inject_into_urllib3()
|
||||
-except ImportError:
|
||||
- pass
|
||||
+# Note: Patched by pip to prevent using the PyOpenSSL module. On Windows this
|
||||
+# prevents upgrading cryptography.
|
||||
+# try:
|
||||
+# from .packages.urllib3.contrib import pyopenssl
|
||||
+# pyopenssl.inject_into_urllib3()
|
||||
+# except ImportError:
|
||||
+# pass
|
||||
|
||||
import warnings
|
||||
|
||||
diff --git a/pip/_vendor/requests/compat.py b/pip/_vendor/requests/compat.py
|
||||
index eb6530d..353ec29 100644
|
||||
--- a/pip/_vendor/requests/compat.py
|
||||
+++ b/pip/_vendor/requests/compat.py
|
||||
@@ -25,12 +25,14 @@ is_py2 = (_ver[0] == 2)
|
||||
#: Python 3.x?
|
||||
is_py3 = (_ver[0] == 3)
|
||||
|
||||
-try:
|
||||
- import simplejson as json
|
||||
-except (ImportError, SyntaxError):
|
||||
- # simplejson does not support Python 3.2, it throws a SyntaxError
|
||||
- # because of u'...' Unicode literals.
|
||||
- import json
|
||||
+# Note: We've patched out simplejson support in pip because it prevents
|
||||
+# upgrading simplejson on Windows.
|
||||
+# try:
|
||||
+# import simplejson as json
|
||||
+# except (ImportError, SyntaxError):
|
||||
+# # simplejson does not support Python 3.2, it throws a SyntaxError
|
||||
+# # because of u'...' Unicode literals.
|
||||
+import json
|
||||
|
||||
# ---------
|
||||
# Specifics
|
Loading…
Reference in New Issue