fix: wrap async tasks for sync Phoenix client and handle bare-string expected_output

- runner.py: detect async task callables with inspect.iscoroutinefunction and
  wrap them in asyncio.run() so Phoenix's sync client can invoke them without
  receiving an unawaited coroutine (RuntimeError: Task is async...)
- evaluators.py: guard ast.literal_eval in both top_k and is_in with
  try/except (ValueError, SyntaxError), falling back to treating a plain
  string as a single-item list — fixes crash on bare identifiers like
  'elasticsearch' stored directly in the expected_output CSV column
- tests: add covering tests for both fixes

Author: zurfjereluhmie

src/evalwire/evaluators.py

@@ -34,7 +34,10 @@ def top_k(output: list[str], expected: dict) -> float:
 
         raw = expected.get("expected_output", [])
         if isinstance(raw, str):
-            raw = ast.literal_eval(raw)
+            try:
+                raw = ast.literal_eval(raw)
+            except (ValueError, SyntaxError):
+                raw = [raw]
         expected_items: list[str] = list(raw)
 
         if not expected_items:
@@ -74,7 +77,10 @@ def make_membership_evaluator() -> Callable[[str, dict], bool]:
     def is_in(output: str, expected: dict) -> bool:
         raw = expected.get("expected_output", [])
         if isinstance(raw, str):
-            raw = ast.literal_eval(raw)
+            try:
+                raw = ast.literal_eval(raw)
+            except (ValueError, SyntaxError):
+                raw = [raw]
         expected_items: list[str] = list(raw)
         return output in expected_items
 

src/evalwire/runner.py

@@ -2,8 +2,10 @@
 
 from __future__ import annotations
 
+import asyncio
 import importlib
 import importlib.util
+import inspect
 import logging
 import sys
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -87,6 +89,14 @@ def _run_one(exp_name: str, task: Any, evaluators: list[Any]) -> Any:
             if dataset is None:
                 return None
 
+            # Phoenix's sync client cannot await coroutine functions directly.
+            # Wrap async tasks in a sync bridge so they work transparently.
+            if inspect.iscoroutinefunction(task):
+                _async_task = task
+
+                def task(example: Any, _fn: Any = _async_task) -> Any:  # noqa: E731
+                    return asyncio.run(_fn(example))
+
             timestamp = datetime.now(tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S")
             experiment_name = f"{experiment_name_prefix}_{exp_name}_{timestamp}"
             exp_metadata = {"dataset": exp_name, "run_by": "evalwire"}

tests/test_evaluators.py

@@ -82,6 +82,19 @@ def test_missing_expected_output_key_returns_0(self):
         score = top_k(["a"], {})
         assert score == pytest.approx(0.0)
 
+    def test_bare_string_expected_output_is_treated_as_single_item(self):
+        """A plain identifier string (e.g. from a CSV column) must not crash
+        ast.literal_eval and should be treated as a single-item expected list."""
+        top_k = make_top_k_evaluator(K=10)
+        # "some_url" is a bare identifier — not a Python literal
+        score = top_k(["some_url"], {"expected_output": "some_url"})
+        assert score == pytest.approx(1.0)
+
+    def test_bare_string_expected_output_no_match(self):
+        top_k = make_top_k_evaluator(K=10)
+        score = top_k(["other_url"], {"expected_output": "some_url"})
+        assert score == pytest.approx(0.0)
+
 
 class TestMakeMembershipEvaluator:
     def test_returns_callable_named_is_in(self):
@@ -120,3 +133,14 @@ def test_single_item_list_match(self):
     def test_case_sensitive(self):
         is_in = make_membership_evaluator()
         assert is_in("ES_SEARCH", {"expected_output": ["es_search"]}) is False
+
+    def test_bare_string_expected_output_is_treated_as_single_item(self):
+        """A plain identifier string (e.g. from a CSV column) must not crash
+        ast.literal_eval and should be treated as a single-item expected list."""
+        is_in = make_membership_evaluator()
+        # "elasticsearch" is a bare identifier — not a Python literal
+        assert is_in("elasticsearch", {"expected_output": "elasticsearch"}) is True
+
+    def test_bare_string_expected_output_no_match(self):
+        is_in = make_membership_evaluator()
+        assert is_in("cms", {"expected_output": "elasticsearch"}) is False

tests/test_runner.py

@@ -227,3 +227,39 @@ def test_experiment_name_includes_dataset_name(
         runner.run(names=["es_search"])
         call_kwargs = mock_phoenix_client.experiments.run_experiment.call_args.kwargs
         assert "es_search" in call_kwargs["experiment_name"]
+
+    def test_async_task_is_wrapped_and_callable_by_sync_phoenix(
+        self, tmp_path: Path, mock_phoenix_client: MagicMock
+    ):
+        """An async task must be transparently wrapped so Phoenix's sync runner
+        can call it without getting back an unawaited coroutine."""
+        base = tmp_path / "experiments"
+        base.mkdir()
+        exp = base / "async_exp"
+        exp.mkdir()
+        (exp / "task.py").write_text("async def task(example): return 'async_result'\n")
+
+        # Capture the task callable that is passed to run_experiment.
+        captured: dict = {}
+
+        def _capture_task(**kwargs: object) -> MagicMock:
+            captured["task"] = kwargs["task"]
+            return MagicMock()
+
+        mock_phoenix_client.experiments.run_experiment.side_effect = _capture_task
+
+        runner = _make_runner(base, mock_phoenix_client)
+        runner.run()
+
+        assert "task" in captured, "run_experiment was not called"
+        wrapped = captured["task"]
+        # The wrapped task must be a plain callable (not a coroutine function)
+        # so Phoenix's sync runner can call it directly.
+        import inspect
+
+        assert not inspect.iscoroutinefunction(wrapped), (
+            "Async task should have been wrapped into a sync callable"
+        )
+        # Calling it must return the real result, not a coroutine.
+        result = wrapped(example=object())
+        assert result == "async_result"