diff --git a/contrib/build-installer b/contrib/build-installer deleted file mode 100755 index a44d46d65..000000000 --- a/contrib/build-installer +++ /dev/null @@ -1,164 +0,0 @@ -#!/usr/bin/env python -import base64 -import os -import sys -import zipfile - -WRAPPER_SCRIPT = b""" -#!/usr/bin/env python -# -# Hi There! -# You may be wondering what this giant blob of binary data here is, you might -# even be worried that we're up to something nefarious (good for you for being -# paranoid!). This is a base4 encoding of a zip file, this zip file contains -# an entire copy of pip. -# -# Pip is a thing that installs packages, pip itself is a package that someone -# might want to install, especially if they're looking to run this get-pip.py -# script. Pip has a lot of code to deal with the security of installing -# packages, various edge cases on various platforms, and other such sort of -# "tribal knowledge" that has been encoded in it's code base. Because of this -# we basically include an entire copy of pip inside this blob. We do this -# because the alternatives are attempt to implement a "minipip" that probably -# doesn't do things correctly and has weird edge cases, or compress pip itself -# down into a single file. -# -# If you're wondering how this is created, the secret is -# "contrib/build-installer" from the pip repository. - -ZIPFILE = b\"\"\" -{zipfile} -\"\"\" - -import base64 -import os.path -import pkgutil -import shutil -import sys -import tempfile - - -def bootstrap(tmpdir=None): - # Import pip so we can use it to install pip and maybe setuptools too - import pip - - # We always want to install pip - packages = ["pip"] - - # Check if the user has requested us not to install setuptools - if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"): - args = [x for x in sys.argv[1:] if x != "--no-setuptools"] - else: - args = sys.argv[1:] - - # We want to see if setuptools is available before attempting to - # install it - try: - import setuptools # noqa - except ImportError: - packages += ["setuptools"] - - delete_tmpdir = False - try: - # Create a temporary directory to act as a working directory if we were - # not given one. - if tmpdir is None: - tmpdir = tempfile.mkdtemp() - delete_tmpdir = True - - # We need to extract the SSL certificates from requests so that they - # can be passed to --cert - cert_path = os.path.join(tmpdir, "cacert.pem") - with open(cert_path, "wb") as cert: - cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem")) - - # Use an environment variable here so that users can still pass - # --cert via sys.argv - os.environ.setdefault("PIP_CERT", cert_path) - - # Execute the included pip and use it to install the latest pip and - # setuptools from PyPI - sys.exit(pip.main(["install", "--upgrade"] + packages + args)) - finally: - # Remove our temporary directory - if delete_tmpdir and tmpdir: - shutil.rmtree(tmpdir, ignore_errors=True) - - -def main(): - tmpdir = None - try: - # Create a temporary working directory - tmpdir = tempfile.mkdtemp() - - # Unpack the zipfile into the temporary directory - pip_zip = os.path.join(tmpdir, "pip.zip") - with open(pip_zip, "wb") as fp: - fp.write(base64.decodestring(ZIPFILE)) - - # Add the zipfile to sys.path so that we can import it - sys.path.insert(0, pip_zip) - - # Run the bootstrap - bootstrap(tmpdir=tmpdir) - finally: - # Clean up our temporary working directory - if tmpdir: - shutil.rmtree(tmpdir, ignore_errors=True) - - -if __name__ == "__main__": - main() -""".lstrip() - - -def getmodname(rootpath, path): - parts = path.split(os.sep)[len(rootpath.split(os.sep)):] - return '/'.join(parts) - - -def main(): - sys.stdout.write("Creating pip bootstrapper...") - - here = os.path.dirname(os.path.abspath(__file__)) - toplevel = os.path.dirname(here) - get_pip = os.path.join(here, "get-pip.py") - - # Get all of the files we want to add to the zip file - all_files = [] - for root, dirs, files in os.walk(os.path.join(toplevel, "pip")): - for pyfile in files: - if os.path.splitext(pyfile)[1] in {".py", ".pem", ".cfg", ".exe"}: - all_files.append( - getmodname(toplevel, os.path.join(root, pyfile)) - ) - - with zipfile.ZipFile(get_pip, "w", compression=zipfile.ZIP_DEFLATED) as z: - # Write the pip files to the zip archive - for filename in all_files: - z.write(os.path.join(toplevel, filename), filename) - - # Get the binary data that compromises our zip file - with open(get_pip, "rb") as fp: - data = fp.read() - - # Write out the wrapper script that will take the place of the zip script - # The reason we need to do this instead of just directly executing the - # zip script is that while Python will happily execute a zip script if - # passed it on the file system, it will not however allow this to work if - # passed it via stdin. This means that this wrapper script is required to - # make ``curl https://...../get-pip.py | python`` continue to work. - with open(get_pip, "wb") as fp: - fp.write(WRAPPER_SCRIPT.format(zipfile=base64.encodestring(data))) - - # Ensure the permissions on the newly created file - if hasattr(os, "chmod"): - oldmode = os.stat(get_pip).st_mode & 0o7777 - newmode = (oldmode | 0o555) & 0o7777 - os.chmod(get_pip, newmode) - - sys.stdout.write("done.\n") - - -if __name__ == "__main__": - main() diff --git a/tasks/generate.py b/tasks/generate.py index 7a68cd0c1..c73f3f4e4 100644 --- a/tasks/generate.py +++ b/tasks/generate.py @@ -1,7 +1,12 @@ +import base64 import io +import os +import zipfile import invoke +from . import paths + @invoke.task def authors(): @@ -26,3 +31,161 @@ def authors(): with io.open("AUTHORS.txt", "w", encoding="utf8") as fp: fp.write(u"\n".join(authors)) fp.write(u"\n") + + +@invoke.task +def installer(installer_path=os.path.join(paths.CONTRIB, "get-pip.py")): + print("[generate.installer] Generating installer") + + # Define our wrapper script + WRAPPER_SCRIPT = b""" +#!/usr/bin/env python +# +# Hi There! +# You may be wondering what this giant blob of binary data here is, you might +# even be worried that we're up to something nefarious (good for you for being +# paranoid!). This is a base64 encoding of a zip file, this zip file contains +# an entire copy of pip. +# +# Pip is a thing that installs packages, pip itself is a package that someone +# might want to install, especially if they're looking to run this get-pip.py +# script. Pip has a lot of code to deal with the security of installing +# packages, various edge cases on various platforms, and other such sort of +# "tribal knowledge" that has been encoded in its code base. Because of this +# we basically include an entire copy of pip inside this blob. We do this +# because the alternatives are attempt to implement a "minipip" that probably +# doesn't do things correctly and has weird edge cases, or compress pip itself +# down into a single file. +# +# If you're wondering how this is created, it is using an invoke task located +# in tasks/generate.py called "installer". It can be invoked by using +# ``invoke generate.installer``. + +ZIPFILE = b\"\"\" +{zipfile} +\"\"\" + +import base64 +import os.path +import pkgutil +import shutil +import sys +import tempfile + + +def bootstrap(tmpdir=None): + # Import pip so we can use it to install pip and maybe setuptools too + import pip + + # We always want to install pip + packages = ["pip"] + + # Check if the user has requested us not to install setuptools + if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"): + args = [x for x in sys.argv[1:] if x != "--no-setuptools"] + else: + args = sys.argv[1:] + + # We want to see if setuptools is available before attempting to + # install it + try: + import setuptools # noqa + except ImportError: + packages += ["setuptools"] + + delete_tmpdir = False + try: + # Create a temporary directory to act as a working directory if we were + # not given one. + if tmpdir is None: + tmpdir = tempfile.mkdtemp() + delete_tmpdir = True + + # We need to extract the SSL certificates from requests so that they + # can be passed to --cert + cert_path = os.path.join(tmpdir, "cacert.pem") + with open(cert_path, "wb") as cert: + cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem")) + + # Use an environment variable here so that users can still pass + # --cert via sys.argv + os.environ.setdefault("PIP_CERT", cert_path) + + # Execute the included pip and use it to install the latest pip and + # setuptools from PyPI + sys.exit(pip.main(["install", "--upgrade"] + packages + args)) + finally: + # Remove our temporary directory + if delete_tmpdir and tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +def main(): + tmpdir = None + try: + # Create a temporary working directory + tmpdir = tempfile.mkdtemp() + + # Unpack the zipfile into the temporary directory + pip_zip = os.path.join(tmpdir, "pip.zip") + with open(pip_zip, "wb") as fp: + fp.write(base64.decodestring(ZIPFILE)) + + # Add the zipfile to sys.path so that we can import it + sys.path.insert(0, pip_zip) + + # Run the bootstrap + bootstrap(tmpdir=tmpdir) + finally: + # Clean up our temporary working directory + if tmpdir: + shutil.rmtree(tmpdir, ignore_errors=True) + + +if __name__ == "__main__": + main() +""".lstrip() + + # Get all of the files we want to add to the zip file + print("[generate.installer] Collect all the files that should be zipped") + all_files = [] + for root, dirs, files in os.walk(os.path.join(paths.PROJECT_ROOT, "pip")): + for pyfile in files: + if os.path.splitext(pyfile)[1] in {".py", ".pem", ".cfg", ".exe"}: + path = os.path.join(root, pyfile) + all_files.append( + "/".join( + path.split("/")[len(paths.PROJECT_ROOT.split("/")):] + ) + ) + + # Write the pip files to the zip archive + print("[generate.installer] Generate the bundled zip of pip") + with zipfile.ZipFile( + installer_path, "w", compression=zipfile.ZIP_DEFLATED) as z: + for filename in all_files: + z.write(os.path.join(paths.PROJECT_ROOT, filename), filename) + + # Get the binary data that compromises our zip file + with open(installer_path, "rb") as fp: + data = fp.read() + + # Write out the wrapper script that will take the place of the zip script + # The reason we need to do this instead of just directly executing the + # zip script is that while Python will happily execute a zip script if + # passed it on the file system, it will not however allow this to work if + # passed it via stdin. This means that this wrapper script is required to + # make ``curl https://...../get-pip.py | python`` continue to work. + print( + "[generate.installer] Write the wrapper script with the bundled zip " + "file" + ) + with open(installer_path, "wb") as fp: + fp.write(WRAPPER_SCRIPT.format(zipfile=base64.encodestring(data))) + + # Ensure the permissions on the newly created file + oldmode = os.stat(installer_path).st_mode & 0o7777 + newmode = (oldmode | 0o555) & 0o7777 + os.chmod(installer_path, newmode) + + print("[generate.installer] Generated installer") diff --git a/tasks/paths.py b/tasks/paths.py new file mode 100644 index 000000000..abb901153 --- /dev/null +++ b/tasks/paths.py @@ -0,0 +1,5 @@ +import os.path + +PROJECT_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) + +CONTRIB = os.path.join(PROJECT_ROOT, "contrib")