Run functional tests for install in separate jobs (#5436)

Speeds up the end-to-end running time for Travis tests.

* Adds a new jobs to test_install* separately (using -k "[not] test_install")
* More generic cache ignore
* Move slower Python 3.5 tests before 3.4
* Factor tox envs

.gitignore

@@ -22,7 +22,7 @@ docs/build/
 htmlcov/
 .coverage
 .coverage.*
-.cache
+.*cache
 nosetests.xml
 coverage.xml
 *.cover

.travis.yml

@@ -12,22 +12,36 @@ matrix:
     - env: TOXENV=mypy
     - env: TOXENV=packaging
     # PyPy jobs start first -- they are the slowest
-    - env: TOXENV=pypy
-      python: pypy
-    - env: TOXENV=pypy3
+    - env: TOXENV=pypy3-functional-install
       python: pypy3
+    - env: TOXENV=pypy3-others
+      python: pypy3
+    - env: TOXENV=pypy-functional-install
+      python: pypy
+    - env: TOXENV=pypy-others
+      python: pypy
     # Latest Stable CPython jobs
-    - env: TOXENV=py27
+    - env: TOXENV=py27-functional-install
       python: 2.7
-    - env: TOXENV=py36
+    - env: TOXENV=py27-others
+      python: 2.7
+    - env: TOXENV=py36-functional-install
+      python: 3.6
+    - env: TOXENV=py36-others
       python: 3.6
     # All the other Py3 versions
-    - env: TOXENV=py34
-      python: 3.4
-    - env: TOXENV=py35
+    - env: TOXENV=py35-functional-install
       python: 3.5
+    - env: TOXENV=py35-others
+      python: 3.5
+    - env: TOXENV=py34-functional-install
+      python: 3.4
+    - env: TOXENV=py34-others
+      python: 3.4
     # Nightly Python goes last
-    - env: TOXENV=py37
+    - env: TOXENV=py37-functional-install
+      python: nightly
+    - env: TOXENV=py37-others
       python: nightly
   allow_failures:
     - python: nightly

.travis/run.sh

@@ -24,11 +24,15 @@ if [[ $TOXENV != docs ]]; then
     fi
 fi
 
-if [[ $TOXENV == py* ]]; then
+if [[ $TOXENV == py*-functional-install ]]; then
+    # Only run test_install*.py integration tests
+    tox -- -m integration -n 4 --duration=5 -k test_install
+elif [[ $TOXENV == py* ]]; then
     # Run unit tests
     tox -- -m unit
-    # Run integration tests
-    tox -- -m integration -n 4 --duration=5
+
+    # Run other integration tests
+    tox -- -m integration -n 4 --duration=5 -k "not test_install"
 else
     # Run once
     tox

tox.ini

@@ -1,7 +1,7 @@
 [tox]
 envlist =
     docs, packaging, lint-py2, lint-py3, mypy,
-    py27, py34, py35, py36, py37, pypy
+    py{27,34,35,36,37,py,py3}-{functional-install,others}
 
 [testenv]
 passenv = CI GIT_SSL_CAINFO
@@ -10,13 +10,13 @@ setenv =
     # that our tests use.
     LC_CTYPE = en_US.UTF-8
 deps = -r{toxinidir}/dev-requirements.txt
-commands = py.test --timeout 300 []
+commands = pytest --timeout 300 []
 install_command = python -m pip install {opts} {packages}
 usedevelop = True
 
 [testenv:coverage-py3]
 basepython = python3
-commands = py.test --timeout 300 --cov=pip --cov-report=term-missing --cov-report=xml --cov-report=html tests/unit {posargs}
+commands = pytest --timeout 300 --cov=pip --cov-report=term-missing --cov-report=xml --cov-report=html tests/unit {posargs}
 
 [testenv:docs]
 deps = -r{toxinidir}/docs-requirements.txt