diff --git a/pyproject.toml b/pyproject.toml index 8254991..7cf616f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,19 @@ readme = "README.md" license = "BSD-3-Clause" authors = [{ name = "evalwire contributors" }] requires-python = ">=3.10" +keywords = ["evaluation", "langgraph", "phoenix", "llm", "testing", "arize"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development :: Testing", + "Typing :: Typed", +] dependencies = [ "arize-phoenix>=13.0,<14", "pandas>=2.0", diff --git a/src/evalwire/__init__.py b/src/evalwire/__init__.py index ce570ac..552d26f 100644 --- a/src/evalwire/__init__.py +++ b/src/evalwire/__init__.py @@ -1,5 +1,8 @@ """evalwire — systematic evaluation of LangGraph nodes with Arize Phoenix.""" +import logging +from importlib.metadata import version + from evalwire.evaluators import ( make_contains_evaluator, make_exact_match_evaluator, @@ -15,6 +18,10 @@ from evalwire.runner import ExperimentRunner from evalwire.uploader import DatasetUploader +__version__ = version("evalwire") + +logging.getLogger("evalwire").addHandler(logging.NullHandler()) + __all__ = [ "DatasetUploader", "ExperimentRunner", diff --git a/src/evalwire/evaluators/membership.py b/src/evalwire/evaluators/membership.py index f0d35ba..3c424f1 100644 --- a/src/evalwire/evaluators/membership.py +++ b/src/evalwire/evaluators/membership.py @@ -22,6 +22,8 @@ def make_membership_evaluator() -> Callable[[str, dict], bool]: """ def is_in(output: str, expected: dict) -> bool: + if output is None: + return False expected_items = _parse_expected(expected) return output in expected_items diff --git a/src/evalwire/evaluators/schema.py b/src/evalwire/evaluators/schema.py index 6c0eb45..2d58e72 100644 --- a/src/evalwire/evaluators/schema.py +++ b/src/evalwire/evaluators/schema.py @@ -11,8 +11,9 @@ def make_schema_evaluator(schema: dict) -> Callable[[str, dict], bool]: Schema dict using ``jsonschema``. Useful for asserting that LLM outputs conform to a declared schema regardless of the specific values produced. - The JSON schema is bound at factory-creation time so the same validator - can be reused across many evaluation rows without re-compiling. + The JSON schema and validator are both bound at factory-creation time so + the same validator can be reused across many evaluation rows without + re-compiling or re-importing. Parameters ---------- @@ -37,27 +38,24 @@ def make_schema_evaluator(schema: dict) -> Callable[[str, dict], bool]: pip install 'jsonschema>=4.0' """ + try: + import jsonschema + except ImportError as exc: + raise ImportError( + "jsonschema is required to use make_schema_evaluator. " + "Install it with: pip install 'jsonschema>=4.0'" + ) from exc + + validator = jsonschema.Draft7Validator(schema) def schema_valid(output: str, expected: dict) -> bool: # noqa: ARG001 if output is None: return False - try: - import jsonschema - except ImportError as exc: - raise ImportError( - "jsonschema is required to use make_schema_evaluator. " - "Install it with: pip install 'jsonschema>=4.0'" - ) from exc - try: instance = json.loads(output) except (json.JSONDecodeError, TypeError): return False - try: - jsonschema.validate(instance=instance, schema=schema) - except jsonschema.ValidationError: - return False - return True + return validator.is_valid(instance) schema_valid.__name__ = "schema_valid" return schema_valid diff --git a/tests/test_evaluators.py b/tests/test_evaluators.py index 664baca..b712b92 100644 --- a/tests/test_evaluators.py +++ b/tests/test_evaluators.py @@ -496,11 +496,11 @@ def test_expected_dict_is_ignored(self): def test_jsonschema_absent_raises_import_error( self, monkeypatch: pytest.MonkeyPatch ): - """If jsonschema is not importable, a helpful ImportError must be raised.""" + """If jsonschema is not importable, a helpful ImportError must be raised + at factory-creation time (not at call time).""" monkeypatch.setitem(sys.modules, "jsonschema", None) # type: ignore[arg-type] - schema_valid = make_schema_evaluator(_SIMPLE_SCHEMA) with pytest.raises(ImportError, match="jsonschema"): - schema_valid(json.dumps({"name": "Frank", "age": 1}), {}) + make_schema_evaluator(_SIMPLE_SCHEMA) # ---------------------------------------------------------------------------