fix: enhance package typing, logging, and schema evaluation (#28)

Author: zurfjereluhmie

pyproject.toml

@@ -10,6 +10,19 @@ readme          = "README.md"
 license         = "BSD-3-Clause"
 authors         = [{ name = "evalwire contributors" }]
 requires-python = ">=3.10"
+keywords        = ["evaluation", "langgraph", "phoenix", "llm", "testing", "arize"]
+classifiers     = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development :: Testing",
+    "Typing :: Typed",
+]
 dependencies = [
     "arize-phoenix>=13.0,<14",
     "pandas>=2.0",

src/evalwire/__init__.py

@@ -1,5 +1,8 @@
 """evalwire — systematic evaluation of LangGraph nodes with Arize Phoenix."""
 
+import logging
+from importlib.metadata import version
+
 from evalwire.evaluators import (
     make_contains_evaluator,
     make_exact_match_evaluator,
@@ -15,6 +18,10 @@
 from evalwire.runner import ExperimentRunner
 from evalwire.uploader import DatasetUploader
 
+__version__ = version("evalwire")
+
+logging.getLogger("evalwire").addHandler(logging.NullHandler())
+
 __all__ = [
     "DatasetUploader",
     "ExperimentRunner",

src/evalwire/evaluators/membership.py

@@ -22,6 +22,8 @@ def make_membership_evaluator() -> Callable[[str, dict], bool]:
     """
 
     def is_in(output: str, expected: dict) -> bool:
+        if output is None:
+            return False
         expected_items = _parse_expected(expected)
         return output in expected_items
 

src/evalwire/evaluators/schema.py

@@ -11,8 +11,9 @@ def make_schema_evaluator(schema: dict) -> Callable[[str, dict], bool]:
     Schema dict using ``jsonschema``.  Useful for asserting that LLM outputs
     conform to a declared schema regardless of the specific values produced.
 
-    The JSON schema is bound at factory-creation time so the same validator
-    can be reused across many evaluation rows without re-compiling.
+    The JSON schema and validator are both bound at factory-creation time so
+    the same validator can be reused across many evaluation rows without
+    re-compiling or re-importing.
 
     Parameters
     ----------
@@ -37,27 +38,24 @@ def make_schema_evaluator(schema: dict) -> Callable[[str, dict], bool]:
 
             pip install 'jsonschema>=4.0'
     """
+    try:
+        import jsonschema
+    except ImportError as exc:
+        raise ImportError(
+            "jsonschema is required to use make_schema_evaluator. "
+            "Install it with: pip install 'jsonschema>=4.0'"
+        ) from exc
+
+    validator = jsonschema.Draft7Validator(schema)
 
     def schema_valid(output: str, expected: dict) -> bool:  # noqa: ARG001
         if output is None:
             return False
-        try:
-            import jsonschema
-        except ImportError as exc:
-            raise ImportError(
-                "jsonschema is required to use make_schema_evaluator. "
-                "Install it with: pip install 'jsonschema>=4.0'"
-            ) from exc
-
         try:
             instance = json.loads(output)
         except (json.JSONDecodeError, TypeError):
             return False
-        try:
-            jsonschema.validate(instance=instance, schema=schema)
-        except jsonschema.ValidationError:
-            return False
-        return True
+        return validator.is_valid(instance)
 
     schema_valid.__name__ = "schema_valid"
     return schema_valid

tests/test_evaluators.py

@@ -496,11 +496,11 @@ def test_expected_dict_is_ignored(self):
     def test_jsonschema_absent_raises_import_error(
         self, monkeypatch: pytest.MonkeyPatch
     ):
-        """If jsonschema is not importable, a helpful ImportError must be raised."""
+        """If jsonschema is not importable, a helpful ImportError must be raised
+        at factory-creation time (not at call time)."""
         monkeypatch.setitem(sys.modules, "jsonschema", None)  # type: ignore[arg-type]
-        schema_valid = make_schema_evaluator(_SIMPLE_SCHEMA)
         with pytest.raises(ImportError, match="jsonschema"):
-            schema_valid(json.dumps({"name": "Frank", "age": 1}), {})
+            make_schema_evaluator(_SIMPLE_SCHEMA)
 
 
 # ---------------------------------------------------------------------------