The Python package installer https://pip.pypa.io/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

289 lines
9.5 KiB

  1. """Automation using nox.
  2. """
  3. # The following comment should be removed at some point in the future.
  4. # mypy: disallow-untyped-defs=False
  5. import glob
  6. import os
  7. import shutil
  8. import sys
  9. import nox
  10. sys.path.append(".")
  11. from tools.automation import release # isort:skip # noqa
  12. sys.path.pop()
  13. nox.options.reuse_existing_virtualenvs = True
  14. nox.options.sessions = ["lint"]
  15. LOCATIONS = {
  16. "common-wheels": "tests/data/common_wheels",
  17. "protected-pip": "tools/tox_pip.py",
  18. }
  19. REQUIREMENTS = {
  20. "docs": "tools/requirements/docs.txt",
  21. "tests": "tools/requirements/tests.txt",
  22. "common-wheels": "tools/requirements/tests-common_wheels.txt",
  23. }
  24. AUTHORS_FILE = "AUTHORS.txt"
  25. VERSION_FILE = "src/pip/__init__.py"
  26. def run_with_protected_pip(session, *arguments):
  27. """Do a session.run("pip", *arguments), using a "protected" pip.
  28. This invokes a wrapper script, that forwards calls to original virtualenv
  29. (stable) version, and not the code being tested. This ensures pip being
  30. used is not the code being tested.
  31. """
  32. env = {"VIRTUAL_ENV": session.virtualenv.location}
  33. command = ("python", LOCATIONS["protected-pip"]) + arguments
  34. kwargs = {"env": env, "silent": True}
  35. session.run(*command, **kwargs)
  36. def should_update_common_wheels():
  37. # If the cache hasn't been created, create it.
  38. if not os.path.exists(LOCATIONS["common-wheels"]):
  39. return True
  40. # If the requirements was updated after cache, we'll repopulate it.
  41. cache_last_populated_at = os.path.getmtime(LOCATIONS["common-wheels"])
  42. requirements_updated_at = os.path.getmtime(REQUIREMENTS["common-wheels"])
  43. need_to_repopulate = requirements_updated_at > cache_last_populated_at
  44. # Clear the stale cache.
  45. if need_to_repopulate:
  46. shutil.rmtree(LOCATIONS["common-wheels"], ignore_errors=True)
  47. return need_to_repopulate
  48. # -----------------------------------------------------------------------------
  49. # Development Commands
  50. # These are currently prototypes to evaluate whether we want to switch over
  51. # completely to nox for all our automation. Contributors should prefer using
  52. # `tox -e ...` until this note is removed.
  53. # -----------------------------------------------------------------------------
  54. @nox.session(python=["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3"])
  55. def test(session):
  56. # Get the common wheels.
  57. if should_update_common_wheels():
  58. run_with_protected_pip(
  59. session,
  60. "wheel",
  61. "-w", LOCATIONS["common-wheels"],
  62. "-r", REQUIREMENTS["common-wheels"],
  63. )
  64. else:
  65. msg = (
  66. "Re-using existing common-wheels at {}."
  67. .format(LOCATIONS["common-wheels"])
  68. )
  69. session.log(msg)
  70. # Build source distribution
  71. sdist_dir = os.path.join(session.virtualenv.location, "sdist")
  72. if os.path.exists(sdist_dir):
  73. shutil.rmtree(sdist_dir, ignore_errors=True)
  74. session.run(
  75. "python", "setup.py", "sdist",
  76. "--formats=zip", "--dist-dir", sdist_dir,
  77. silent=True,
  78. )
  79. generated_files = os.listdir(sdist_dir)
  80. assert len(generated_files) == 1
  81. generated_sdist = os.path.join(sdist_dir, generated_files[0])
  82. # Install source distribution
  83. run_with_protected_pip(session, "install", generated_sdist)
  84. # Install test dependencies
  85. run_with_protected_pip(session, "install", "-r", REQUIREMENTS["tests"])
  86. # Parallelize tests as much as possible, by default.
  87. arguments = session.posargs or ["-n", "auto"]
  88. # Run the tests
  89. # LC_CTYPE is set to get UTF-8 output inside of the subprocesses that our
  90. # tests use.
  91. session.run("pytest", *arguments, env={"LC_CTYPE": "en_US.UTF-8"})
  92. @nox.session
  93. def docs(session):
  94. session.install("-e", ".")
  95. session.install("-r", REQUIREMENTS["docs"])
  96. def get_sphinx_build_command(kind):
  97. # Having the conf.py in the docs/html is weird but needed because we
  98. # can not use a different configuration directory vs source directory
  99. # on RTD currently. So, we'll pass "-c docs/html" here.
  100. # See https://github.com/rtfd/readthedocs.org/issues/1543.
  101. return [
  102. "sphinx-build",
  103. "-W",
  104. "-c", "docs/html", # see note above
  105. "-d", "docs/build/doctrees/" + kind,
  106. "-b", kind,
  107. "docs/" + kind,
  108. "docs/build/" + kind,
  109. ]
  110. session.run(*get_sphinx_build_command("html"))
  111. session.run(*get_sphinx_build_command("man"))
  112. @nox.session
  113. def lint(session):
  114. session.install("pre-commit")
  115. if session.posargs:
  116. args = session.posargs + ["--all-files"]
  117. else:
  118. args = ["--all-files", "--show-diff-on-failure"]
  119. session.run("pre-commit", "run", *args)
  120. @nox.session
  121. def vendoring(session):
  122. session.install("vendoring")
  123. session.run("vendoring", "sync", ".", "-v")
  124. # -----------------------------------------------------------------------------
  125. # Release Commands
  126. # -----------------------------------------------------------------------------
  127. @nox.session(name="prepare-release")
  128. def prepare_release(session):
  129. version = release.get_version_from_arguments(session)
  130. if not version:
  131. session.error("Usage: nox -s prepare-release -- <version>")
  132. session.log("# Ensure nothing is staged")
  133. if release.modified_files_in_git("--staged"):
  134. session.error("There are files staged in git")
  135. session.log(f"# Updating {AUTHORS_FILE}")
  136. release.generate_authors(AUTHORS_FILE)
  137. if release.modified_files_in_git():
  138. release.commit_file(
  139. session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}",
  140. )
  141. else:
  142. session.log(f"# No changes to {AUTHORS_FILE}")
  143. session.log("# Generating NEWS")
  144. release.generate_news(session, version)
  145. session.log(f"# Bumping for release {version}")
  146. release.update_version_file(version, VERSION_FILE)
  147. release.commit_file(session, VERSION_FILE, message="Bump for release")
  148. session.log("# Tagging release")
  149. release.create_git_tag(session, version, message=f"Release {version}")
  150. session.log("# Bumping for development")
  151. next_dev_version = release.get_next_development_version(version)
  152. release.update_version_file(next_dev_version, VERSION_FILE)
  153. release.commit_file(session, VERSION_FILE, message="Bump for development")
  154. @nox.session(name="build-release")
  155. def build_release(session):
  156. version = release.get_version_from_arguments(session)
  157. if not version:
  158. session.error("Usage: nox -s build-release -- YY.N[.P]")
  159. session.log("# Ensure no files in dist/")
  160. if release.have_files_in_folder("dist"):
  161. session.error(
  162. "There are files in dist/. Remove them and try again. "
  163. "You can use `git clean -fxdi -- dist` command to do this"
  164. )
  165. session.log("# Install dependencies")
  166. session.install("setuptools", "wheel", "twine")
  167. with release.isolated_temporary_checkout(session, version) as build_dir:
  168. session.log(
  169. "# Start the build in an isolated, "
  170. f"temporary Git checkout at {build_dir!s}",
  171. )
  172. with release.workdir(session, build_dir):
  173. tmp_dists = build_dists(session)
  174. tmp_dist_paths = (build_dir / p for p in tmp_dists)
  175. session.log(f"# Copying dists from {build_dir}")
  176. os.makedirs('dist', exist_ok=True)
  177. for dist, final in zip(tmp_dist_paths, tmp_dists):
  178. session.log(f"# Copying {dist} to {final}")
  179. shutil.copy(dist, final)
  180. def build_dists(session):
  181. """Return dists with valid metadata."""
  182. session.log(
  183. "# Check if there's any Git-untracked files before building the wheel",
  184. )
  185. has_forbidden_git_untracked_files = any(
  186. # Don't report the environment this session is running in
  187. not untracked_file.startswith('.nox/build-release/')
  188. for untracked_file in release.get_git_untracked_files()
  189. )
  190. if has_forbidden_git_untracked_files:
  191. session.error(
  192. "There are untracked files in the working directory. "
  193. "Remove them and try again",
  194. )
  195. session.log("# Build distributions")
  196. session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True)
  197. produced_dists = glob.glob("dist/*")
  198. session.log(f"# Verify distributions: {', '.join(produced_dists)}")
  199. session.run("twine", "check", *produced_dists, silent=True)
  200. return produced_dists
  201. @nox.session(name="upload-release")
  202. def upload_release(session):
  203. version = release.get_version_from_arguments(session)
  204. if not version:
  205. session.error("Usage: nox -s upload-release -- YY.N[.P]")
  206. session.log("# Install dependencies")
  207. session.install("twine")
  208. distribution_files = glob.glob("dist/*")
  209. session.log(f"# Distribution files: {distribution_files}")
  210. # Sanity check: Make sure there's 2 distribution files.
  211. count = len(distribution_files)
  212. if count != 2:
  213. session.error(
  214. f"Expected 2 distribution files for upload, got {count}. "
  215. f"Remove dist/ and run 'nox -s build-release -- {version}'"
  216. )
  217. # Sanity check: Make sure the files are correctly named.
  218. distfile_names = map(os.path.basename, distribution_files)
  219. expected_distribution_files = [
  220. f"pip-{version}-py2.py3-none-any.whl",
  221. f"pip-{version}.tar.gz",
  222. ]
  223. if sorted(distfile_names) != sorted(expected_distribution_files):
  224. session.error(
  225. f"Distribution files do not seem to be for {version} release."
  226. )
  227. session.log("# Upload distributions")
  228. session.run("twine", "upload", *distribution_files)