mirror of https://github.com/pypa/pip
Automatic vendoring (#4093)
This commit is contained in:
parent
bb796adbda
commit
2c9735437a
|
@ -15,7 +15,8 @@ Policy
|
||||||
pure Python.
|
pure Python.
|
||||||
|
|
||||||
* Any modifications made to libraries **MUST** be noted in
|
* Any modifications made to libraries **MUST** be noted in
|
||||||
``pip/_vendor/README.rst``.
|
``pip/_vendor/README.rst`` and their corresponding patches **MUST** be
|
||||||
|
included ``tasks/vendoring/patches``.
|
||||||
|
|
||||||
|
|
||||||
Rationale
|
Rationale
|
||||||
|
@ -92,16 +93,11 @@ situation that we expect pip to be used and not mandate some external mechanism
|
||||||
such as OS packages.
|
such as OS packages.
|
||||||
|
|
||||||
|
|
||||||
pkg_resources
|
|
||||||
-------------
|
|
||||||
|
|
||||||
pkg_resources has been pulled in from setuptools 28.8.0
|
|
||||||
|
|
||||||
|
|
||||||
Modifications
|
Modifications
|
||||||
-------------
|
-------------
|
||||||
|
|
||||||
* html5lib has been modified to import six from pip._vendor
|
* html5lib has been modified to import six from pip._vendor
|
||||||
|
* setuptools is completely stripped to only keep pkg_resources
|
||||||
* pkg_resources has been modified to import its externs from pip._vendor
|
* pkg_resources has been modified to import its externs from pip._vendor
|
||||||
* CacheControl has been modified to import its dependencies from pip._vendor
|
* CacheControl has been modified to import its dependencies from pip._vendor
|
||||||
* packaging has been modified to import its dependencies from pip._vendor
|
* packaging has been modified to import its dependencies from pip._vendor
|
||||||
|
@ -109,6 +105,16 @@ Modifications
|
||||||
* Modified distro to delay importing argparse to avoid errors on 2.6
|
* Modified distro to delay importing argparse to avoid errors on 2.6
|
||||||
|
|
||||||
|
|
||||||
|
Automatic Vendoring
|
||||||
|
-------------------
|
||||||
|
|
||||||
|
Vendoring is automated via the ``vendoring.update`` task (defined in
|
||||||
|
``tasks/vendoring/__init__.py``) from the content of
|
||||||
|
``pip/_vendor/vendor.txt`` and the different patches in
|
||||||
|
``tasks/vendoring/patches/``.
|
||||||
|
Launch it via ``invoke vendoring.update`` (you will need ``invoke>=0.13.0``).
|
||||||
|
|
||||||
|
|
||||||
Debundling
|
Debundling
|
||||||
----------
|
----------
|
||||||
|
|
||||||
|
|
|
@ -1,34 +0,0 @@
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import pip
|
|
||||||
import glob
|
|
||||||
import shutil
|
|
||||||
|
|
||||||
here = os.path.abspath(os.path.dirname(__file__))
|
|
||||||
|
|
||||||
def usage():
|
|
||||||
print("Usage: re-vendor.py [clean|vendor]")
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
def clean():
|
|
||||||
for fn in os.listdir(here):
|
|
||||||
dirname = os.path.join(here, fn)
|
|
||||||
if os.path.isdir(dirname):
|
|
||||||
shutil.rmtree(dirname)
|
|
||||||
# six is a single file, not a package
|
|
||||||
os.unlink(os.path.join(here, 'six.py'))
|
|
||||||
|
|
||||||
def vendor():
|
|
||||||
pip.main(['install', '-t', here, '-r', 'vendor.txt'])
|
|
||||||
for dirname in glob.glob('*.egg-info'):
|
|
||||||
shutil.rmtree(dirname)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if len(sys.argv) != 2:
|
|
||||||
usage()
|
|
||||||
if sys.argv[1] == 'clean':
|
|
||||||
clean()
|
|
||||||
elif sys.argv[1] == 'vendor':
|
|
||||||
vendor()
|
|
||||||
else:
|
|
||||||
usage()
|
|
|
@ -13,4 +13,5 @@ ipaddress==1.0.17 # Only needed on 2.6 and 2.7
|
||||||
packaging==16.8
|
packaging==16.8
|
||||||
pyparsing==2.1.10
|
pyparsing==2.1.10
|
||||||
retrying==1.3.3
|
retrying==1.3.3
|
||||||
|
setuptools==28.8.0
|
||||||
webencodings==0.5
|
webencodings==0.5
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
import invoke
|
import invoke
|
||||||
|
|
||||||
from . import generate
|
from . import generate
|
||||||
|
from . import vendoring
|
||||||
|
|
||||||
ns = invoke.Collection(generate)
|
ns = invoke.Collection(generate, vendoring)
|
||||||
|
|
|
@ -4,12 +4,12 @@ import invoke
|
||||||
|
|
||||||
|
|
||||||
@invoke.task
|
@invoke.task
|
||||||
def authors():
|
def authors(ctx):
|
||||||
print("[generate.authors] Generating AUTHORS")
|
print("[generate.authors] Generating AUTHORS")
|
||||||
|
|
||||||
# Get our list of authors
|
# Get our list of authors
|
||||||
print("[generate.authors] Collecting author names")
|
print("[generate.authors] Collecting author names")
|
||||||
r = invoke.run("git log --use-mailmap --format'=%aN <%aE>'", hide=True)
|
r = ctx.run("git log --use-mailmap --format'=%aN <%aE>'", hide=True)
|
||||||
authors = []
|
authors = []
|
||||||
seen_authors = set()
|
seen_authors = set()
|
||||||
for author in r.stdout.splitlines():
|
for author in r.stdout.splitlines():
|
||||||
|
|
|
@ -0,0 +1,129 @@
|
||||||
|
""""Vendoring script, python 3.5 needed"""
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
import invoke
|
||||||
|
|
||||||
|
TASK_NAME = 'update'
|
||||||
|
|
||||||
|
FILE_WHITE_LIST = (
|
||||||
|
'Makefile',
|
||||||
|
'vendor.txt',
|
||||||
|
'__init__.py',
|
||||||
|
'README.rst',
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def drop_dir(path):
|
||||||
|
shutil.rmtree(str(path))
|
||||||
|
|
||||||
|
|
||||||
|
def remove_all(paths):
|
||||||
|
for path in paths:
|
||||||
|
if path.is_dir():
|
||||||
|
drop_dir(path)
|
||||||
|
else:
|
||||||
|
path.unlink()
|
||||||
|
|
||||||
|
|
||||||
|
def log(msg):
|
||||||
|
print('[vendoring.%s] %s' % (TASK_NAME, msg))
|
||||||
|
|
||||||
|
|
||||||
|
def clean_vendor(ctx, vendor_dir):
|
||||||
|
# Old _vendor cleanup
|
||||||
|
remove_all(vendor_dir.glob('*.pyc'))
|
||||||
|
log('Cleaning %s' % vendor_dir)
|
||||||
|
for item in vendor_dir.iterdir():
|
||||||
|
if item.is_dir():
|
||||||
|
shutil.rmtree(str(item))
|
||||||
|
elif item.name not in FILE_WHITE_LIST:
|
||||||
|
item.unlink()
|
||||||
|
else:
|
||||||
|
log('Skipping %s' % item)
|
||||||
|
|
||||||
|
|
||||||
|
def rewrite_imports(package_dir, vendored_libs):
|
||||||
|
for item in package_dir.iterdir():
|
||||||
|
if item.is_dir():
|
||||||
|
rewrite_imports(item, vendored_libs)
|
||||||
|
elif item.name.endswith('.py'):
|
||||||
|
rewrite_file_imports(item, vendored_libs)
|
||||||
|
|
||||||
|
|
||||||
|
def rewrite_file_imports(item, vendored_libs):
|
||||||
|
"""Rewrite 'import xxx' and 'from xxx import' for vendored_libs"""
|
||||||
|
text = item.read_text()
|
||||||
|
# Revendor pkg_resources.extern first
|
||||||
|
text = re.sub(r'pkg_resources.extern', r'pip._vendor', text)
|
||||||
|
for lib in vendored_libs:
|
||||||
|
text = re.sub(
|
||||||
|
r'(\n\s*)import %s' % lib,
|
||||||
|
r'\1from pip._vendor import %s' % lib,
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
text = re.sub(
|
||||||
|
r'(\n\s*)from %s' % lib,
|
||||||
|
r'\1from pip._vendor.%s' % lib,
|
||||||
|
text,
|
||||||
|
)
|
||||||
|
item.write_text(text)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_patch(ctx, patch_file_path):
|
||||||
|
log('Applying patch %s' % patch_file_path.name)
|
||||||
|
ctx.run('git apply %s' % patch_file_path)
|
||||||
|
|
||||||
|
|
||||||
|
def vendor(ctx, vendor_dir):
|
||||||
|
log('Reinstalling vendored libraries')
|
||||||
|
ctx.run(
|
||||||
|
'pip install -t {0} -r {0}/vendor.txt --no-compile'.format(
|
||||||
|
str(vendor_dir),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
remove_all(vendor_dir.glob('*.dist-info'))
|
||||||
|
remove_all(vendor_dir.glob('*.egg-info'))
|
||||||
|
|
||||||
|
# Cleanup setuptools unneeded parts
|
||||||
|
(vendor_dir / 'easy_install.py').unlink()
|
||||||
|
drop_dir(vendor_dir / 'setuptools')
|
||||||
|
drop_dir(vendor_dir / 'pkg_resources' / '_vendor')
|
||||||
|
drop_dir(vendor_dir / 'pkg_resources' / 'extern')
|
||||||
|
|
||||||
|
# Detect the vendored packages/modules
|
||||||
|
vendored_libs = []
|
||||||
|
for item in vendor_dir.iterdir():
|
||||||
|
if item.is_dir():
|
||||||
|
vendored_libs.append(item.name)
|
||||||
|
elif item.name not in FILE_WHITE_LIST:
|
||||||
|
vendored_libs.append(item.name[:-3])
|
||||||
|
log("Detected vendored libraries: %s" % ", ".join(vendored_libs))
|
||||||
|
|
||||||
|
# Global import rewrites
|
||||||
|
log("Rewriting all imports related to vendored libs")
|
||||||
|
for item in vendor_dir.iterdir():
|
||||||
|
if item.is_dir():
|
||||||
|
rewrite_imports(item, vendored_libs)
|
||||||
|
elif item.name not in FILE_WHITE_LIST:
|
||||||
|
rewrite_file_imports(item, vendored_libs)
|
||||||
|
|
||||||
|
# Special cases: apply stored patches
|
||||||
|
log("Apply patches")
|
||||||
|
patch_dir = Path(__file__).parent / 'patches'
|
||||||
|
for patch in patch_dir.glob('*.patch'):
|
||||||
|
apply_patch(ctx, patch)
|
||||||
|
|
||||||
|
|
||||||
|
@invoke.task(name=TASK_NAME)
|
||||||
|
def main(ctx):
|
||||||
|
git_root = Path(
|
||||||
|
ctx.run('git rev-parse --show-toplevel', hide=True).stdout.strip()
|
||||||
|
)
|
||||||
|
vendor_dir = git_root / 'pip' / '_vendor'
|
||||||
|
log('Using vendor dir: %s' % vendor_dir)
|
||||||
|
clean_vendor(ctx, vendor_dir)
|
||||||
|
vendor(ctx, vendor_dir)
|
||||||
|
log('Revendoring complete')
|
|
@ -0,0 +1,24 @@
|
||||||
|
diff --git a/pip/_vendor/cachecontrol/compat.py b/pip/_vendor/cachecontrol/compat.py
|
||||||
|
index 54ec211..018e6ac 100644
|
||||||
|
--- a/pip/_vendor/cachecontrol/compat.py
|
||||||
|
+++ b/pip/_vendor/cachecontrol/compat.py
|
||||||
|
@@ -10,17 +10,8 @@ except ImportError:
|
||||||
|
import pickle
|
||||||
|
|
||||||
|
|
||||||
|
-# Handle the case where the requests module has been patched to not have
|
||||||
|
-# urllib3 bundled as part of its source.
|
||||||
|
-try:
|
||||||
|
- from pip._vendor.requests.packages.urllib3.response import HTTPResponse
|
||||||
|
-except ImportError:
|
||||||
|
- from urllib3.response import HTTPResponse
|
||||||
|
-
|
||||||
|
-try:
|
||||||
|
- from pip._vendor.requests.packages.urllib3.util import is_fp_closed
|
||||||
|
-except ImportError:
|
||||||
|
- from urllib3.util import is_fp_closed
|
||||||
|
+from pip._vendor.requests.packages.urllib3.response import HTTPResponse
|
||||||
|
+from pip._vendor.requests.packages.urllib3.util import is_fp_closed
|
||||||
|
|
||||||
|
# Replicate some six behaviour
|
||||||
|
try:
|
|
@ -0,0 +1,21 @@
|
||||||
|
diff --git a/pip/_vendor/distro.py b/pip/_vendor/distro.py
|
||||||
|
index 0afa527..9e7daad 100644
|
||||||
|
--- a/pip/_vendor/distro.py
|
||||||
|
+++ b/pip/_vendor/distro.py
|
||||||
|
@@ -34,7 +34,6 @@ import sys
|
||||||
|
import json
|
||||||
|
import shlex
|
||||||
|
import logging
|
||||||
|
-import argparse
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
@@ -1052,6 +1051,8 @@ _distro = LinuxDistribution()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
+ import argparse
|
||||||
|
+
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
logger.setLevel(logging.DEBUG)
|
||||||
|
logger.addHandler(logging.StreamHandler(sys.stdout))
|
|
@ -0,0 +1,48 @@
|
||||||
|
diff --git a/pip/_vendor/requests/__init__.py b/pip/_vendor/requests/__init__.py
|
||||||
|
index 9c3b769..44f6836 100644
|
||||||
|
--- a/pip/_vendor/requests/__init__.py
|
||||||
|
+++ b/pip/_vendor/requests/__init__.py
|
||||||
|
@@ -48,11 +48,13 @@ __license__ = 'Apache 2.0'
|
||||||
|
__copyright__ = 'Copyright 2016 Kenneth Reitz'
|
||||||
|
|
||||||
|
# Attempt to enable urllib3's SNI support, if possible
|
||||||
|
-try:
|
||||||
|
- from .packages.urllib3.contrib import pyopenssl
|
||||||
|
- pyopenssl.inject_into_urllib3()
|
||||||
|
-except ImportError:
|
||||||
|
- pass
|
||||||
|
+# Note: Patched by pip to prevent using the PyOpenSSL module. On Windows this
|
||||||
|
+# prevents upgrading cryptography.
|
||||||
|
+# try:
|
||||||
|
+# from .packages.urllib3.contrib import pyopenssl
|
||||||
|
+# pyopenssl.inject_into_urllib3()
|
||||||
|
+# except ImportError:
|
||||||
|
+# pass
|
||||||
|
|
||||||
|
import warnings
|
||||||
|
|
||||||
|
diff --git a/pip/_vendor/requests/compat.py b/pip/_vendor/requests/compat.py
|
||||||
|
index eb6530d..353ec29 100644
|
||||||
|
--- a/pip/_vendor/requests/compat.py
|
||||||
|
+++ b/pip/_vendor/requests/compat.py
|
||||||
|
@@ -25,12 +25,14 @@ is_py2 = (_ver[0] == 2)
|
||||||
|
#: Python 3.x?
|
||||||
|
is_py3 = (_ver[0] == 3)
|
||||||
|
|
||||||
|
-try:
|
||||||
|
- import simplejson as json
|
||||||
|
-except (ImportError, SyntaxError):
|
||||||
|
- # simplejson does not support Python 3.2, it throws a SyntaxError
|
||||||
|
- # because of u'...' Unicode literals.
|
||||||
|
- import json
|
||||||
|
+# Note: We've patched out simplejson support in pip because it prevents
|
||||||
|
+# upgrading simplejson on Windows.
|
||||||
|
+# try:
|
||||||
|
+# import simplejson as json
|
||||||
|
+# except (ImportError, SyntaxError):
|
||||||
|
+# # simplejson does not support Python 3.2, it throws a SyntaxError
|
||||||
|
+# # because of u'...' Unicode literals.
|
||||||
|
+import json
|
||||||
|
|
||||||
|
# ---------
|
||||||
|
# Specifics
|
Loading…
Reference in New Issue