Add support for YAML based test files (#4637)

dev-requirements.txt

@@ -5,6 +5,7 @@ pytest-catchlog
 pytest-rerunfailures
 pytest-timeout
 pytest-xdist
+pyyaml
 mock<1.1
 scripttest>=1.3
 https://github.com/pypa/virtualenv/archive/master.zip#egg=virtualenv

tests/functional/test_yaml.py

@@ -0,0 +1,143 @@
+"""Tests for the resolver
+"""
+
+import os
+import re
+
+import pytest
+
+from tests.lib import DATA_DIR, create_basic_wheel_for_package, path_to_url
+from tests.lib.yaml_helpers import generate_yaml_tests, id_func
+
+_conflict_finder_re = re.compile(
+    # Conflicting Requirements: \
+    # A 1.0.0 requires B == 2.0.0, C 1.0.0 requires B == 1.0.0.
+    """
+        (?P<package>[\w\-_]+?)
+        [ ]
+        (?P<version>\S+?)
+        [ ]requires[ ]
+        (?P<selector>.+?)
+        (?=,|\.$)
+    """,
+    re.X
+)
+
+
+def _convert_to_dict(string):
+
+    def stripping_split(my_str, splitwith, count=None):
+        if count is None:
+            return [x.strip() for x in my_str.strip().split(splitwith)]
+        else:
+            return [x.strip() for x in my_str.strip().split(splitwith, count)]
+
+    parts = stripping_split(string, ";")
+
+    retval = {}
+    retval["depends"] = []
+    retval["extras"] = {}
+
+    retval["name"], retval["version"] = stripping_split(parts[0], " ")
+
+    for part in parts[1:]:
+        verb, args_str = stripping_split(part, " ", 1)
+        assert verb in ["depends"], "Unknown verb {!r}".format(verb)
+
+        retval[verb] = stripping_split(args_str, ",")
+
+    return retval
+
+
+def handle_install_request(script, requirement):
+    assert isinstance(requirement, str), (
+        "Need install requirement to be a string only"
+    )
+    result = script.pip(
+        "install",
+        "--no-index", "--find-links", path_to_url(script.scratch_path),
+        requirement
+    )
+
+    retval = {}
+    if result.returncode == 0:
+        # Check which packages got installed
+        retval["install"] = []
+
+        for path in result.files_created:
+            if path.endswith(".dist-info"):
+                name, version = (
+                    os.path.basename(path)[:-len(".dist-info")]
+                ).rsplit("-", 1)
+
+                # TODO: information about extras.
+
+                retval["install"].append(" ".join((name, version)))
+
+        retval["install"].sort()
+
+        # TODO: Support checking uninstallations
+        # retval["uninstall"] = []
+
+    elif "conflicting" in result.stderr.lower():
+        retval["conflicting"] = []
+
+        message = result.stderr.rsplit("\n", 1)[-1]
+
+        # XXX: There might be a better way than parsing the message
+        for match in re.finditer(message, _conflict_finder_re):
+            di = match.groupdict()
+            retval["conflicting"].append(
+                {
+                    "required_by": "{} {}".format(di["name"], di["version"]),
+                    "selector": di["selector"]
+                }
+            )
+
+    return retval
+
+
+@pytest.mark.yaml
+@pytest.mark.parametrize(
+    "case", generate_yaml_tests(DATA_DIR.folder / "yaml"), ids=id_func
+)
+def test_yaml_based(script, case):
+    available = case.get("available", [])
+    requests = case.get("request", [])
+    transaction = case.get("transaction", [])
+
+    assert len(requests) == len(transaction), (
+        "Expected requests and transaction counts to be same"
+    )
+
+    # Create a custom index of all the packages that are supposed to be
+    # available
+    # XXX: This doesn't work because this isn't making an index of files.
+    for package in available:
+        if isinstance(package, str):
+            package = _convert_to_dict(package)
+
+        assert isinstance(package, dict), "Needs to be a dictionary"
+
+        create_basic_wheel_for_package(script, **package)
+
+    available_actions = {
+        "install": handle_install_request
+    }
+
+    # use scratch path for index
+    for request, expected in zip(requests, transaction):
+        # The name of the key is what action has to be taken
+        assert len(request.keys()) == 1, "Expected only one action"
+
+        # Get the only key
+        action = list(request.keys())[0]
+
+        assert action in available_actions.keys(), (
+            "Unsupported action {!r}".format(action)
+        )
+
+        # Perform the requested action
+        effect = available_actions[action](script, request[action])
+
+        assert effect == expected, "Fixture did not succeed."

tests/lib/__init__.py

@@ -6,6 +6,7 @@ import sys
 import re
 import textwrap
 import site
+import shutil
 
 import scripttest
 import virtualenv
@@ -643,3 +644,82 @@ def create_test_package_with_setup(script, **setup_kwargs):
         setup(**kwargs)
     """) % setup_kwargs)
     return pkg_path
+
+
+def create_basic_wheel_for_package(script, name, version, depends, extras):
+    files = {
+        "{name}/__init__.py": """
+            def hello():
+                return "Hello From {name}"
+        """,
+        "{dist_info}/DESCRIPTION": """
+            UNKNOWN
+        """,
+        "{dist_info}/WHEEL": """
+            Wheel-Version: 1.0
+            Generator: pip-test-suite
+            Root-Is-Purelib: true
+            Tag: py2-none-any
+            Tag: py3-none-any
+
+
+        """,
+        "{dist_info}/METADATA": """
+            Metadata-Version: 2.0
+            Name: {name}
+            Version: {version}
+            Summary: UNKNOWN
+            Home-page: UNKNOWN
+            Author: UNKNOWN
+            Author-email: UNKNOWN
+            License: UNKNOWN
+            Platform: UNKNOWN
+            {requires_dist}
+
+            UNKNOWN
+        """,
+        "{dist_info}/top_level.txt": """
+            {name}
+        """,
+        # Have an empty RECORD becuase we don't want to be checking hashes.
+        "{dist_info}/RECORD": ""
+    }
+
+    # Some useful shorthands
+    archive_name = "{name}-{version}-py2.py3-none-any.whl".format(
+        name=name, version=version
+    )
+    dist_info = "{name}-{version}.dist-info".format(
+        name=name, version=version
+    )
+
+    requires_dist = "\n".join([
+        "Requires-Dist: {}".format(pkg) for pkg in depends
+    ] + [
+        "Provides-Extra: {}".format(pkg) for pkg in extras.keys()
+    ] + [
+        "Requires-Dist: {}; extra == \"{}\"".format(pkg, extra)
+        for extra in extras for pkg in extras[extra]
+    ])
+
+    # Replace key-values with formatted values
+    for key, value in list(files.items()):
+        del files[key]
+        key = key.format(name=name, dist_info=dist_info)
+        files[key] = textwrap.dedent(value).format(
+            name=name, version=version, requires_dist=requires_dist
+        ).strip()
+
+    for fname in files:
+        path = script.temp_path / fname
+        path.folder.mkdir()
+        path.write(files[fname])
+
+    retval = script.scratch_path / archive_name
+    generated = shutil.make_archive(retval, 'zip', script.temp_path)
+    shutil.move(generated, retval)
+
+    script.temp_path.rmtree()
+    script.temp_path.mkdir()
+
+    return retval

tests/lib/path.py

@@ -285,6 +285,10 @@ class Path(_base):
     def join(self, *parts):
         return Path(self, *parts)
 
+    def read_text(self):
+        with open(self, "r") as fp:
+            return fp.read()
+
     def write(self, content):
         with open(self, "w") as fp:
             fp.write(content)

tests/lib/yaml_helpers.py

@@ -0,0 +1,43 @@
+"""
+"""
+
+import pytest
+import yaml
+
+
+def generate_yaml_tests(directory):
+    for yml_file in directory.glob("*/*.yml"):
+        data = yaml.safe_load(yml_file.read_text())
+        assert "cases" in data, "A fixture needs cases to be used in testing"
+
+        # Strip the parts of the directory to only get a name without
+        # extension and resolver directory
+        base_name = str(yml_file)[len(str(directory)) + 1:-4]
+
+        base = data.get("base", {})
+        cases = data["cases"]
+
+        for i, case_template in enumerate(cases):
+            case = base.copy()
+            case.update(case_template)
+
+            case[":name:"] = base_name
+            if len(cases) > 1:
+                case[":name:"] += "-" + str(i)
+
+            if case.pop("skip", False):
+                case = pytest.param(case, marks=pytest.mark.xfail)
+
+            yield case
+
+
+def id_func(param):
+    """Give a nice parameter name to the generated function parameters
+    """
+    if isinstance(param, dict) and ":name:" in param:
+        return param[":name:"]
+
+    retval = str(param)
+    if len(retval) > 25:
+        retval = retval[:20] + "..." + retval[-2:]
+    return retval

tests/yaml/README.md

@@ -0,0 +1,5 @@
+# Fixtures
+
+This directory contains fixtures for testing pip's resolver. The fixtures are written as yml files, with a convinient format that allows for specifying a custom index for temporary use.
+
+<!-- TODO: Add a good description of the format and how it can be used. -->

tests/yaml/install/circular.yml

@@ -0,0 +1,45 @@
+base:
+  available:
+    - A 1.0.0; depends B == 1.0.0
+    - B 1.0.0; depends C == 1.0.0
+    - C 1.0.0; depends D == 1.0.0
+    - D 1.0.0; depends A == 1.0.0
+
+cases:
+# NOTE: Do we want to check the order?
+-
+  request:
+    - install: A
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0
+      - C 1.0.0
+      - D 1.0.0
+-
+  request:
+    - install: B
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0
+      - C 1.0.0
+      - D 1.0.0
+-
+  request:
+    - install: C
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0
+      - C 1.0.0
+      - D 1.0.0
+-
+  request:
+    - install: D
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0
+      - C 1.0.0
+      - D 1.0.0

tests/yaml/install/conflicting_diamond.yml

@@ -0,0 +1,17 @@
+cases:
+-
+  available:
+    - A 1.0.0; depends B == 1.0.0, C == 1.0.0
+    - B 1.0.0; depends D == 1.0.0
+    - C 1.0.0; depends D == 2.0.0
+    - D 1.0.0
+    - D 2.0.0
+  request:
+    - install: A
+  transaction:
+    - conflicting:
+      - required_by: [A 1.0.0, B 1.0.0]
+        selector: D == 1.0.0
+      - required_by: [A 1.0.0, C 1.0.0]
+        selector: D == 2.0.0
+  skip: true

tests/yaml/install/conflicting_triangle.yml

@@ -0,0 +1,20 @@
+cases:
+-
+  available:
+    - A 1.0.0; depends C == 1.0.0
+    - B 1.0.0; depends C == 2.0.0
+    - C 1.0.0
+    - C 2.0.0
+  request:
+    - install: A
+    - install: B
+  transaction:
+    - install:
+      - A 1.0.0
+      - C 1.0.0
+    - conflicting:
+      - required_by: [A 1.0.0]
+        selector: C == 1.0.0
+      - required_by: [B 1.0.0]
+        selector: C == 2.0.0
+  skip: true

tests/yaml/install/extras.yml

@@ -0,0 +1,42 @@
+base:
+  available:
+    - A 1.0.0; depends B == 1.0.0, C == 1.0.0, D == 1.0.0
+    - B 1.0.0; depends D[extra_1] == 1.0.0
+    - C 1.0.0; depends D[extra_2] == 1.0.0
+    - name: D
+      version: 1.0.0
+      depends: []
+      extras:
+        extra_1: [E == 1.0.0]
+        extra_2: [F == 1.0.0]
+    - E 1.0.0
+    - F 1.0.0
+cases:
+-
+  request:
+    - install: B
+  transaction:
+    - install:
+      - B 1.0.0
+      - D 1.0.0
+      - E 1.0.0
+-
+  request:
+    - install: C
+  transaction:
+    - install:
+      - C 1.0.0
+      - D 1.0.0
+      - F 1.0.0
+-
+  request:
+    - install: A
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0
+      - C 1.0.0
+      - D 1.0.0
+      - E 1.0.0
+      - F 1.0.0
+  skip: true

tests/yaml/install/non_pinned.yml

@@ -0,0 +1,24 @@
+base:
+  available:
+    - A 1.0.0; depends B < 2.0.0
+    - A 2.0.0; depends B < 3.0.0
+    - B 1.0.0
+    - B 2.0.0
+    - B 2.1.0
+    - B 3.0.0
+
+cases:
+-
+  request:
+    - install: A >= 2.0.0
+  transaction:
+    - install:
+      - A 2.0.0
+      - B 2.1.0
+-
+  request:
+    - install: A < 2.0.0
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0

tests/yaml/install/pinned.yml

@@ -0,0 +1,29 @@
+base:
+  available:
+    - A 1.0.0
+    - A 2.0.0
+    - B 1.0.0; depends A == 1.0.0
+    - B 2.0.0; depends A == 2.0.0
+
+cases:
+-
+  request:
+    - install: B
+  transaction:
+    - install:
+      - A 2.0.0
+      - B 2.0.0
+-
+  request:
+    - install: B == 2.0.0
+  transaction:
+    - install:
+      - A 2.0.0
+      - B 2.0.0
+-
+  request:
+    - install: B == 1.0.0
+  transaction:
+    - install:
+      - A 1.0.0
+      - B 1.0.0