From 61b011293b036330b56ba87a75aa4c330ab03b2d Mon Sep 17 00:00:00 2001
From: Hugo <hugovk@users.noreply.github.com>
Date: Sat, 26 May 2018 14:16:12 +0300
Subject: [PATCH] Run functional tests for install in separate jobs (#5436)

Speeds up the end-to-end running time for Travis tests.

* Adds a new jobs to test_install* separately (using -k "[not] test_install")
* More generic cache ignore
* Move slower Python 3.5 tests before 3.4
* Factor tox envs
---
 .gitignore     |  2 +-
 .travis.yml    | 32 +++++++++++++++++++++++---------
 .travis/run.sh | 10 +++++++---
 tox.ini        |  6 +++---
 4 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/.gitignore b/.gitignore
index 1d842203b..31e69bc00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,7 +22,7 @@ docs/build/
 htmlcov/
 .coverage
 .coverage.*
-.cache
+.*cache
 nosetests.xml
 coverage.xml
 *.cover
diff --git a/.travis.yml b/.travis.yml
index aac434c96..58a71c487 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,22 +12,36 @@ matrix:
     - env: TOXENV=mypy
     - env: TOXENV=packaging
     # PyPy jobs start first -- they are the slowest
-    - env: TOXENV=pypy
-      python: pypy
-    - env: TOXENV=pypy3
+    - env: TOXENV=pypy3-functional-install
       python: pypy3
+    - env: TOXENV=pypy3-others
+      python: pypy3
+    - env: TOXENV=pypy-functional-install
+      python: pypy
+    - env: TOXENV=pypy-others
+      python: pypy
     # Latest Stable CPython jobs
-    - env: TOXENV=py27
+    - env: TOXENV=py27-functional-install
       python: 2.7
-    - env: TOXENV=py36
+    - env: TOXENV=py27-others
+      python: 2.7
+    - env: TOXENV=py36-functional-install
+      python: 3.6
+    - env: TOXENV=py36-others
       python: 3.6
     # All the other Py3 versions
-    - env: TOXENV=py34
-      python: 3.4
-    - env: TOXENV=py35
+    - env: TOXENV=py35-functional-install
       python: 3.5
+    - env: TOXENV=py35-others
+      python: 3.5
+    - env: TOXENV=py34-functional-install
+      python: 3.4
+    - env: TOXENV=py34-others
+      python: 3.4
     # Nightly Python goes last
-    - env: TOXENV=py37
+    - env: TOXENV=py37-functional-install
+      python: nightly
+    - env: TOXENV=py37-others
       python: nightly
   allow_failures:
     - python: nightly
diff --git a/.travis/run.sh b/.travis/run.sh
index 0f703b9a1..f99468b1c 100755
--- a/.travis/run.sh
+++ b/.travis/run.sh
@@ -24,11 +24,15 @@ if [[ $TOXENV != docs ]]; then
     fi
 fi
 
-if [[ $TOXENV == py* ]]; then
+if [[ $TOXENV == py*-functional-install ]]; then
+    # Only run test_install*.py integration tests
+    tox -- -m integration -n 4 --duration=5 -k test_install
+elif [[ $TOXENV == py* ]]; then
     # Run unit tests
     tox -- -m unit
-    # Run integration tests
-    tox -- -m integration -n 4 --duration=5
+
+    # Run other integration tests
+    tox -- -m integration -n 4 --duration=5 -k "not test_install"
 else
     # Run once
     tox
diff --git a/tox.ini b/tox.ini
index 34d474c1f..789fad8d8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,7 +1,7 @@
 [tox]
 envlist =
     docs, packaging, lint-py2, lint-py3, mypy,
-    py27, py34, py35, py36, py37, pypy
+    py{27,34,35,36,37,py,py3}-{functional-install,others}
 
 [testenv]
 passenv = CI GIT_SSL_CAINFO
@@ -10,13 +10,13 @@ setenv =
     # that our tests use.
     LC_CTYPE = en_US.UTF-8
 deps = -r{toxinidir}/dev-requirements.txt
-commands = py.test --timeout 300 []
+commands = pytest --timeout 300 []
 install_command = python -m pip install {opts} {packages}
 usedevelop = True
 
 [testenv:coverage-py3]
 basepython = python3
-commands = py.test --timeout 300 --cov=pip --cov-report=term-missing --cov-report=xml --cov-report=html tests/unit {posargs}
+commands = pytest --timeout 300 --cov=pip --cov-report=term-missing --cov-report=xml --cov-report=html tests/unit {posargs}
 
 [testenv:docs]
 deps = -r{toxinidir}/docs-requirements.txt