The Python package installer https://pip.pypa.io/
25'ten fazla konu seçemezsiniz Konular bir harf veya rakamla başlamalı, kısa çizgiler ('-') içerebilir ve en fazla 35 karakter uzunluğunda olabilir.
 
 
 

331 satır
11 KiB

  1. """Automation using nox.
  2. """
  3. # The following comment should be removed at some point in the future.
  4. # mypy: disallow-untyped-defs=False
  5. import glob
  6. import os
  7. import shutil
  8. import sys
  9. from pathlib import Path
  10. import nox
  11. sys.path.append(".")
  12. from tools.automation import release # isort:skip # noqa
  13. sys.path.pop()
  14. nox.options.reuse_existing_virtualenvs = True
  15. nox.options.sessions = ["lint"]
  16. LOCATIONS = {
  17. "common-wheels": "tests/data/common_wheels",
  18. "protected-pip": "tools/tox_pip.py",
  19. }
  20. REQUIREMENTS = {
  21. "docs": "tools/requirements/docs.txt",
  22. "tests": "tools/requirements/tests.txt",
  23. "common-wheels": "tools/requirements/tests-common_wheels.txt",
  24. }
  25. AUTHORS_FILE = "AUTHORS.txt"
  26. VERSION_FILE = "src/pip/__init__.py"
  27. def run_with_protected_pip(session, *arguments):
  28. """Do a session.run("pip", *arguments), using a "protected" pip.
  29. This invokes a wrapper script, that forwards calls to original virtualenv
  30. (stable) version, and not the code being tested. This ensures pip being
  31. used is not the code being tested.
  32. """
  33. env = {"VIRTUAL_ENV": session.virtualenv.location}
  34. command = ("python", LOCATIONS["protected-pip"]) + arguments
  35. kwargs = {"env": env, "silent": True}
  36. session.run(*command, **kwargs)
  37. def should_update_common_wheels():
  38. # If the cache hasn't been created, create it.
  39. if not os.path.exists(LOCATIONS["common-wheels"]):
  40. return True
  41. # If the requirements was updated after cache, we'll repopulate it.
  42. cache_last_populated_at = os.path.getmtime(LOCATIONS["common-wheels"])
  43. requirements_updated_at = os.path.getmtime(REQUIREMENTS["common-wheels"])
  44. need_to_repopulate = requirements_updated_at > cache_last_populated_at
  45. # Clear the stale cache.
  46. if need_to_repopulate:
  47. shutil.rmtree(LOCATIONS["common-wheels"], ignore_errors=True)
  48. return need_to_repopulate
  49. # -----------------------------------------------------------------------------
  50. # Development Commands
  51. # These are currently prototypes to evaluate whether we want to switch over
  52. # completely to nox for all our automation. Contributors should prefer using
  53. # `tox -e ...` until this note is removed.
  54. # -----------------------------------------------------------------------------
  55. @nox.session(python=["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "pypy", "pypy3"])
  56. def test(session):
  57. # Get the common wheels.
  58. if should_update_common_wheels():
  59. run_with_protected_pip(
  60. session,
  61. "wheel",
  62. "-w", LOCATIONS["common-wheels"],
  63. "-r", REQUIREMENTS["common-wheels"],
  64. )
  65. else:
  66. msg = (
  67. "Re-using existing common-wheels at {}."
  68. .format(LOCATIONS["common-wheels"])
  69. )
  70. session.log(msg)
  71. # Build source distribution
  72. sdist_dir = os.path.join(session.virtualenv.location, "sdist")
  73. if os.path.exists(sdist_dir):
  74. shutil.rmtree(sdist_dir, ignore_errors=True)
  75. session.run(
  76. "python", "setup.py", "sdist",
  77. "--formats=zip", "--dist-dir", sdist_dir,
  78. silent=True,
  79. )
  80. generated_files = os.listdir(sdist_dir)
  81. assert len(generated_files) == 1
  82. generated_sdist = os.path.join(sdist_dir, generated_files[0])
  83. # Install source distribution
  84. run_with_protected_pip(session, "install", generated_sdist)
  85. # Install test dependencies
  86. run_with_protected_pip(session, "install", "-r", REQUIREMENTS["tests"])
  87. # Parallelize tests as much as possible, by default.
  88. arguments = session.posargs or ["-n", "auto"]
  89. # Run the tests
  90. # LC_CTYPE is set to get UTF-8 output inside of the subprocesses that our
  91. # tests use.
  92. session.run("pytest", *arguments, env={"LC_CTYPE": "en_US.UTF-8"})
  93. @nox.session
  94. def docs(session):
  95. session.install("-e", ".")
  96. session.install("-r", REQUIREMENTS["docs"])
  97. def get_sphinx_build_command(kind):
  98. # Having the conf.py in the docs/html is weird but needed because we
  99. # can not use a different configuration directory vs source directory
  100. # on RTD currently. So, we'll pass "-c docs/html" here.
  101. # See https://github.com/rtfd/readthedocs.org/issues/1543.
  102. return [
  103. "sphinx-build",
  104. "-W",
  105. "-c", "docs/html", # see note above
  106. "-d", "docs/build/doctrees/" + kind,
  107. "-b", kind,
  108. "docs/" + kind,
  109. "docs/build/" + kind,
  110. ]
  111. session.run(*get_sphinx_build_command("html"))
  112. session.run(*get_sphinx_build_command("man"))
  113. @nox.session
  114. def lint(session):
  115. session.install("pre-commit")
  116. if session.posargs:
  117. args = session.posargs + ["--all-files"]
  118. else:
  119. args = ["--all-files", "--show-diff-on-failure"]
  120. session.run("pre-commit", "run", *args)
  121. @nox.session
  122. def vendoring(session):
  123. session.install("vendoring>=0.3.0")
  124. if "--upgrade" not in session.posargs:
  125. session.run("vendoring", "sync", ".", "-v")
  126. return
  127. def pinned_requirements(path):
  128. for line in path.read_text().splitlines():
  129. one, two = line.split("==", 1)
  130. name = one.strip()
  131. version = two.split("#")[0].strip()
  132. yield name, version
  133. vendor_txt = Path("src/pip/_vendor/vendor.txt")
  134. for name, old_version in pinned_requirements(vendor_txt):
  135. # update requirements.txt
  136. session.run("vendoring", "update", ".", name)
  137. # get the updated version
  138. new_version = old_version
  139. for inner_name, inner_version in pinned_requirements(vendor_txt):
  140. if inner_name == name:
  141. # this is a dedicated assignment, to make flake8 happy
  142. new_version = inner_version
  143. break
  144. else:
  145. session.error(f"Could not find {name} in {vendor_txt}")
  146. # check if the version changed.
  147. if new_version == old_version:
  148. continue # no change, nothing more to do here.
  149. # synchronize the contents
  150. session.run("vendoring", "sync", ".")
  151. # Determine the correct message
  152. message = f"Upgrade {name} to {new_version}"
  153. # Write our news fragment
  154. news_file = Path("news") / (name + ".vendor.rst")
  155. news_file.write_text(message + "\n") # "\n" appeases end-of-line-fixer
  156. # Commit the changes
  157. release.commit_file(session, ".", message=message)
  158. # -----------------------------------------------------------------------------
  159. # Release Commands
  160. # -----------------------------------------------------------------------------
  161. @nox.session(name="prepare-release")
  162. def prepare_release(session):
  163. version = release.get_version_from_arguments(session)
  164. if not version:
  165. session.error("Usage: nox -s prepare-release -- <version>")
  166. session.log("# Ensure nothing is staged")
  167. if release.modified_files_in_git("--staged"):
  168. session.error("There are files staged in git")
  169. session.log(f"# Updating {AUTHORS_FILE}")
  170. release.generate_authors(AUTHORS_FILE)
  171. if release.modified_files_in_git():
  172. release.commit_file(
  173. session, AUTHORS_FILE, message=f"Update {AUTHORS_FILE}",
  174. )
  175. else:
  176. session.log(f"# No changes to {AUTHORS_FILE}")
  177. session.log("# Generating NEWS")
  178. release.generate_news(session, version)
  179. session.log(f"# Bumping for release {version}")
  180. release.update_version_file(version, VERSION_FILE)
  181. release.commit_file(session, VERSION_FILE, message="Bump for release")
  182. session.log("# Tagging release")
  183. release.create_git_tag(session, version, message=f"Release {version}")
  184. session.log("# Bumping for development")
  185. next_dev_version = release.get_next_development_version(version)
  186. release.update_version_file(next_dev_version, VERSION_FILE)
  187. release.commit_file(session, VERSION_FILE, message="Bump for development")
  188. @nox.session(name="build-release")
  189. def build_release(session):
  190. version = release.get_version_from_arguments(session)
  191. if not version:
  192. session.error("Usage: nox -s build-release -- YY.N[.P]")
  193. session.log("# Ensure no files in dist/")
  194. if release.have_files_in_folder("dist"):
  195. session.error(
  196. "There are files in dist/. Remove them and try again. "
  197. "You can use `git clean -fxdi -- dist` command to do this"
  198. )
  199. session.log("# Install dependencies")
  200. session.install("setuptools", "wheel", "twine")
  201. with release.isolated_temporary_checkout(session, version) as build_dir:
  202. session.log(
  203. "# Start the build in an isolated, "
  204. f"temporary Git checkout at {build_dir!s}",
  205. )
  206. with release.workdir(session, build_dir):
  207. tmp_dists = build_dists(session)
  208. tmp_dist_paths = (build_dir / p for p in tmp_dists)
  209. session.log(f"# Copying dists from {build_dir}")
  210. os.makedirs('dist', exist_ok=True)
  211. for dist, final in zip(tmp_dist_paths, tmp_dists):
  212. session.log(f"# Copying {dist} to {final}")
  213. shutil.copy(dist, final)
  214. def build_dists(session):
  215. """Return dists with valid metadata."""
  216. session.log(
  217. "# Check if there's any Git-untracked files before building the wheel",
  218. )
  219. has_forbidden_git_untracked_files = any(
  220. # Don't report the environment this session is running in
  221. not untracked_file.startswith('.nox/build-release/')
  222. for untracked_file in release.get_git_untracked_files()
  223. )
  224. if has_forbidden_git_untracked_files:
  225. session.error(
  226. "There are untracked files in the working directory. "
  227. "Remove them and try again",
  228. )
  229. session.log("# Build distributions")
  230. session.run("python", "setup.py", "sdist", "bdist_wheel", silent=True)
  231. produced_dists = glob.glob("dist/*")
  232. session.log(f"# Verify distributions: {', '.join(produced_dists)}")
  233. session.run("twine", "check", *produced_dists, silent=True)
  234. return produced_dists
  235. @nox.session(name="upload-release")
  236. def upload_release(session):
  237. version = release.get_version_from_arguments(session)
  238. if not version:
  239. session.error("Usage: nox -s upload-release -- YY.N[.P]")
  240. session.log("# Install dependencies")
  241. session.install("twine")
  242. distribution_files = glob.glob("dist/*")
  243. session.log(f"# Distribution files: {distribution_files}")
  244. # Sanity check: Make sure there's 2 distribution files.
  245. count = len(distribution_files)
  246. if count != 2:
  247. session.error(
  248. f"Expected 2 distribution files for upload, got {count}. "
  249. f"Remove dist/ and run 'nox -s build-release -- {version}'"
  250. )
  251. # Sanity check: Make sure the files are correctly named.
  252. distfile_names = map(os.path.basename, distribution_files)
  253. expected_distribution_files = [
  254. f"pip-{version}-py2.py3-none-any.whl",
  255. f"pip-{version}.tar.gz",
  256. ]
  257. if sorted(distfile_names) != sorted(expected_distribution_files):
  258. session.error(
  259. f"Distribution files do not seem to be for {version} release."
  260. )
  261. session.log("# Upload distributions")
  262. session.run("twine", "upload", *distribution_files)