diff --git a/pyproject.toml b/pyproject.toml index 47f701c..ea85f55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -67,6 +67,7 @@ filterwarnings = [ [dependency-groups] dev = [ + "hypothesis>=6.152.1", "mkdocs>=1.6.1", "mkdocs-material>=9.0", "mkdocstrings[python]>=0.25", diff --git a/tests/test_evaluators_property.py b/tests/test_evaluators_property.py new file mode 100644 index 0000000..ce2c2d0 --- /dev/null +++ b/tests/test_evaluators_property.py @@ -0,0 +1,261 @@ +"""Property-based tests for evalwire evaluators using Hypothesis. + +These tests verify invariants that must hold for *all* inputs, catching +edge cases that hand-crafted examples miss. +""" + +from __future__ import annotations + +import ast +import json +import math +import re + +from hypothesis import assume, given, settings +from hypothesis import strategies as st + +from evalwire.evaluators.contains import make_contains_evaluator +from evalwire.evaluators.exact_match import make_exact_match_evaluator +from evalwire.evaluators.json_match import make_json_match_evaluator +from evalwire.evaluators.membership import make_membership_evaluator +from evalwire.evaluators.numeric_tolerance import make_numeric_tolerance_evaluator +from evalwire.evaluators.regex import make_regex_evaluator +from evalwire.evaluators.top_k import make_top_k_evaluator + + +def _expected_dict(value: str | list[str]) -> dict: + """Wrap a value into the ``{"expected_output": ...}`` format.""" + return {"expected_output": value} + + +def _survives_literal_eval(s: str) -> bool: + """Return True if ``_parse_expected`` will keep *s* as a ``str`` element. + + ``_parse_expected`` runs ``ast.literal_eval`` on string values. Strings + that evaluate to non-string Python literals (e.g. ``"0"`` -> ``int(0)``) + are silently converted, causing type mismatches in downstream evaluators. + """ + try: + return isinstance(ast.literal_eval(s), str) + except (ValueError, SyntaxError): + return True + + +# Finite floats only (no NaN, no inf) -- mirrors real-world data. +finite_floats = st.floats(allow_nan=False, allow_infinity=False) + + +class TestNumericToleranceProperties: + @given(value=finite_floats) + def test_exact_match_always_passes(self, value: float): + """A value compared to itself should always pass with default tolerance.""" + evaluator = make_numeric_tolerance_evaluator() + result = evaluator(str(value), _expected_dict(str(value))) + assert result is True + + @given(value=finite_floats, atol=st.floats(min_value=0, max_value=1e10)) + def test_result_is_bool(self, value: float, atol: float): + """Return type is always bool, never crashes.""" + assume(not math.isnan(atol)) + evaluator = make_numeric_tolerance_evaluator(atol=atol) + result = evaluator(str(value), _expected_dict(str(value))) + assert isinstance(result, bool) + + @given( + value=finite_floats, + delta=st.floats(min_value=0, max_value=1e-8), + ) + def test_within_default_tolerance(self, value: float, delta: float): + """Values within default atol (1e-6) of each other should pass.""" + assume(abs(delta) <= 1e-6) + evaluator = make_numeric_tolerance_evaluator() + result = evaluator(str(value + delta), _expected_dict(str(value))) + assert result is True + + @given(output=st.text()) + def test_non_numeric_string_returns_false(self, output: str): + """Non-numeric output should return False, not crash.""" + assume(not _is_numeric(output)) + evaluator = make_numeric_tolerance_evaluator() + result = evaluator(output, _expected_dict("42.0")) + assert result is False + + @given( + a=finite_floats, + b=finite_floats, + atol=st.floats(min_value=0, max_value=1e10), + rtol=st.floats(min_value=0, max_value=1.0), + ) + def test_tolerance_formula_matches_definition( + self, a: float, b: float, atol: float, rtol: float + ): + """Result matches the formula: |a - b| <= atol + rtol * |b|.""" + assume(not math.isnan(atol) and not math.isnan(rtol)) + evaluator = make_numeric_tolerance_evaluator(atol=atol, rtol=rtol) + result = evaluator(str(a), _expected_dict(str(b))) + expected = abs(a - b) <= atol + rtol * abs(b) + assert result == expected + + +def _is_numeric(s: str) -> bool: + try: + float(s) + return True + except (ValueError, TypeError): + return False + + +class TestTopKProperties: + @given( + output=st.lists(st.text(min_size=1), min_size=1, max_size=50), + expected=st.lists(st.text(min_size=1), min_size=1, max_size=10), + k=st.integers(min_value=1, max_value=100), + ) + def test_score_in_unit_interval( + self, output: list[str], expected: list[str], k: int + ): + """Score must always be in [0.0, 1.0].""" + evaluator = make_top_k_evaluator(K=k) + score = evaluator(output, _expected_dict(expected)) + assert 0.0 <= score <= 1.0 + + @given(k=st.integers(min_value=1, max_value=100)) + def test_perfect_score_when_all_at_top(self, k: int): + """If all expected items are at position 0, score should be 1.0.""" + items = ["item"] + evaluator = make_top_k_evaluator(K=k) + score = evaluator(items, _expected_dict(items)) + assert score == 1.0 + + @given( + expected=st.lists(st.text(min_size=1), min_size=1, max_size=5), + k=st.integers(min_value=1, max_value=50), + ) + def test_score_zero_when_nothing_matches(self, expected: list[str], k: int): + """If output contains none of the expected items, score should be 0.0.""" + output = ["__no_match__" + str(i) for i in range(k)] + assume(not any(item in output for item in expected)) + evaluator = make_top_k_evaluator(K=k) + score = evaluator(output, _expected_dict(expected)) + assert score == 0.0 + + @given(output=st.lists(st.text(), max_size=20)) + def test_none_output_returns_zero(self, output: list[str]): + """None output should return 0.0.""" + evaluator = make_top_k_evaluator() + score = evaluator(None, _expected_dict(["anything"])) # ty: ignore[invalid-argument-type] + assert score == 0.0 + + +class TestJsonMatchProperties: + @given(data=st.dictionaries(st.text(min_size=1), st.text(), min_size=1)) + def test_score_in_unit_interval(self, data: dict): + """Score must always be in [0.0, 1.0].""" + json_str = json.dumps(data) + evaluator = make_json_match_evaluator() + score = evaluator(json_str, _expected_dict(json_str)) + assert 0.0 <= score <= 1.0 + + @given(data=st.dictionaries(st.text(min_size=1), st.text(), min_size=1)) + def test_identical_json_scores_one(self, data: dict): + """Identical JSON objects should score 1.0.""" + # Include a boolean so the JSON contains ``true`` which is not a valid + # Python literal, preventing ``ast.literal_eval`` from converting the + # expected string into a dict inside ``_parse_expected``. + data = {**data, "__sentinel__": True} + json_str = json.dumps(data) + evaluator = make_json_match_evaluator() + score = evaluator(json_str, _expected_dict(json_str)) + assert score == 1.0 + + @given(output=st.text()) + def test_invalid_json_returns_zero(self, output: str): + """Invalid JSON output should return 0.0, not crash.""" + assume(not _is_valid_json_object(output)) + evaluator = make_json_match_evaluator() + score = evaluator(output, _expected_dict('{"key": "val"}')) + assert score == 0.0 + + +def _is_valid_json_object(s: str) -> bool: + try: + obj = json.loads(s) + return isinstance(obj, dict) + except (json.JSONDecodeError, TypeError): + return False + + +class TestRegexProperties: + @given(output=st.text()) + @settings(max_examples=200) + def test_never_crashes_on_arbitrary_output(self, output: str): + """Evaluator should not crash on any output string.""" + evaluator = make_regex_evaluator() + result = evaluator(output, _expected_dict(r"\d+")) + assert isinstance(result, bool) + + @given(literal=st.text(min_size=1, max_size=20)) + def test_literal_pattern_matches_itself(self, literal: str): + """A regex-escaped literal should always match itself.""" + pattern = re.escape(literal) + assume(_survives_literal_eval(pattern)) + evaluator = make_regex_evaluator() + result = evaluator(literal, _expected_dict(pattern)) + assert result is True + + @given(output=st.text()) + def test_none_output_returns_false(self, output: str): + """None output always returns False.""" + evaluator = make_regex_evaluator() + result = evaluator(None, _expected_dict(r".*")) # ty: ignore[invalid-argument-type] + assert result is False + + +class TestExactMatchProperties: + @given(value=st.text()) + def test_identity(self, value: str): + """A string always exactly matches itself.""" + assume(_survives_literal_eval(value)) + evaluator = make_exact_match_evaluator() + assert evaluator(value, _expected_dict(value)) is True + + @given(a=st.text(min_size=1), b=st.text(min_size=1)) + def test_different_strings_do_not_match(self, a: str, b: str): + """Different strings should not match.""" + assume(a != b) + evaluator = make_exact_match_evaluator() + assert evaluator(a, _expected_dict(b)) is False + + +class TestContainsProperties: + @given(haystack=st.text(min_size=1), needle=st.text(min_size=1)) + def test_substring_detected(self, haystack: str, needle: str): + """If needle is a substring of haystack, evaluator returns True.""" + assume(_survives_literal_eval(needle)) + full = haystack + needle + haystack + evaluator = make_contains_evaluator() + assert evaluator(full, _expected_dict(needle)) is True + + @given(output=st.text()) + def test_none_output_returns_false(self, output: str): + """None output always returns False.""" + evaluator = make_contains_evaluator() + assert evaluator(None, _expected_dict("x")) is False # ty: ignore[invalid-argument-type] + + +class TestMembershipProperties: + @given(items=st.lists(st.text(min_size=1), min_size=1, max_size=10)) + def test_member_is_found(self, items: list[str]): + """The first item should always be found in the expected set.""" + evaluator = make_membership_evaluator() + assert evaluator(items[0], _expected_dict(items)) is True + + @given( + items=st.lists(st.text(min_size=1), min_size=1, max_size=10), + output=st.text(min_size=1), + ) + def test_non_member_not_found(self, items: list[str], output: str): + """A string not in the expected set should not be found.""" + assume(output not in items) + evaluator = make_membership_evaluator() + assert evaluator(output, _expected_dict(items)) is False diff --git a/uv.lock b/uv.lock index f2b4027..b37c2f9 100644 --- a/uv.lock +++ b/uv.lock @@ -760,6 +760,7 @@ demo = [ { name = "python-dotenv" }, ] dev = [ + { name = "hypothesis" }, { name = "mkdocs" }, { name = "mkdocs-material" }, { name = "mkdocstrings", extra = ["python"] }, @@ -800,6 +801,7 @@ demo = [ { name = "python-dotenv", specifier = ">=1.0,<2" }, ] dev = [ + { name = "hypothesis", specifier = ">=6.152.1" }, { name = "mkdocs", specifier = ">=1.6.1" }, { name = "mkdocs-material", specifier = ">=9.0" }, { name = "mkdocstrings", extras = ["python"], specifier = ">=0.25" }, @@ -1061,6 +1063,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "hypothesis" +version = "6.152.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/b1/c32bcddb9aab9e3abc700f1f56faf14e7655c64a16ca47701a57362276ea/hypothesis-6.152.1.tar.gz", hash = "sha256:4f4ed934eee295dd84ee97592477d23e8dc03e9f12ae0ee30a4e7c9ef3fca3b0", size = 465029, upload-time = "2026-04-14T22:29:24.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/83/860fb3075e00b0fc19a22a2301bc3c96f00437558c3911bdd0a3573a4a53/hypothesis-6.152.1-py3-none-any.whl", hash = "sha256:40a3619d9e0cb97b018857c7986f75cf5de2e5ec0fa8a0b172d00747758f749e", size = 530752, upload-time = "2026-04-14T22:29:20.893Z" }, +] + [[package]] name = "identify" version = "2.6.18" @@ -3414,6 +3429,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.48"