diff --git a/src/evalwire/__init__.py b/src/evalwire/__init__.py index 552d26f..26a591d 100644 --- a/src/evalwire/__init__.py +++ b/src/evalwire/__init__.py @@ -15,6 +15,7 @@ make_top_k_evaluator, ) from evalwire.observability import setup_observability +from evalwire.results import ResultCollector from evalwire.runner import ExperimentRunner from evalwire.uploader import DatasetUploader @@ -25,6 +26,7 @@ __all__ = [ "DatasetUploader", "ExperimentRunner", + "ResultCollector", "make_contains_evaluator", "make_exact_match_evaluator", "make_json_match_evaluator", diff --git a/src/evalwire/cli.py b/src/evalwire/cli.py index 52be927..6a55d1f 100644 --- a/src/evalwire/cli.py +++ b/src/evalwire/cli.py @@ -1,4 +1,4 @@ -"""evalwire CLI — ``evalwire upload`` and ``evalwire run`` commands.""" +"""evalwire CLI — ``evalwire upload``, ``evalwire run``, ``evalwire export``, ``evalwire compare``, and ``evalwire report`` commands.""" import sys from typing import Literal, cast @@ -204,3 +204,101 @@ def run_cmd( except Exception as exc: click.echo(f"Error: {exc}", err=True) sys.exit(2) + + +@main.command("export") +@click.option( + "--experiment", + "experiment_id", + default=None, + help="Phoenix experiment ID.", +) +@click.option( + "--format", + "fmt", + type=click.Choice(["csv", "json"]), + default="csv", + show_default=True, + help="Output format.", +) +@click.option( + "--output", + "output_path", + default=None, + help="Destination file path.", +) +def export_cmd( + experiment_id: str | None, + fmt: str, + output_path: str | None, +) -> None: + """Export experiment results to a CSV or JSON file.""" + if not experiment_id: + raise click.UsageError("No experiment ID provided. Use --experiment.") + try: + from pathlib import Path as _Path + + from evalwire.results import ResultCollector + + client = _make_client() + rc = ResultCollector(client) + resolved_output = ( + _Path(output_path) if output_path else _Path(f"{experiment_id}.{fmt}") + ) + rc.export(experiment_id, format=fmt, path=resolved_output) # ty: ignore[invalid-argument-type] + click.echo(f"Exported results to {resolved_output}") + except click.UsageError: + raise + except Exception as exc: + click.echo(f"Error: {exc}", err=True) + sys.exit(2) + + +@main.command("compare") +@click.argument("experiment_id_a") +@click.argument("experiment_id_b") +def compare_cmd(experiment_id_a: str, experiment_id_b: str) -> None: + """Compare two experiment runs by their mean evaluator scores.""" + try: + from evalwire.results import ResultCollector + + client = _make_client() + rc = ResultCollector(client) + comparison = rc.compare(experiment_id_a, experiment_id_b) + + click.echo(f"Comparing {experiment_id_a!r} vs {experiment_id_b!r}") + click.echo("") + for name, info in sorted(comparison.items()): + delta = info["delta"] + sign = "+" if delta >= 0 else "" + click.echo( + f" {name}: {info['score_a']:.4f} → {info['score_b']:.4f} ({sign}{delta:.4f})" + ) + except Exception as exc: + click.echo(f"Error: {exc}", err=True) + sys.exit(2) + + +@main.command("report") +@click.option( + "--experiment", + "experiment_id", + default=None, + help="Phoenix experiment ID.", +) +def report_cmd(experiment_id: str | None) -> None: + """Generate a markdown summary report for an experiment.""" + if not experiment_id: + raise click.UsageError("No experiment ID provided. Use --experiment.") + try: + from evalwire.results import ResultCollector + + client = _make_client() + rc = ResultCollector(client) + report = rc.report(experiment_id) + click.echo(report) + except click.UsageError: + raise + except Exception as exc: + click.echo(f"Error: {exc}", err=True) + sys.exit(2) diff --git a/src/evalwire/results.py b/src/evalwire/results.py new file mode 100644 index 0000000..dd8563a --- /dev/null +++ b/src/evalwire/results.py @@ -0,0 +1,187 @@ +"""Result collection, export, comparison, and reporting for evalwire experiments.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +if TYPE_CHECKING: + from phoenix.client import Client + +_SUPPORTED_FORMATS = {"csv", "json"} + + +def _rows_from_ran_experiment(ran_experiment: dict[str, Any]) -> list[dict[str, Any]]: + """Convert a RanExperiment into a flat list of row dicts (one per task run).""" + task_runs: list[Any] = ran_experiment["task_runs"] + evaluation_runs: list[Any] = ran_experiment["evaluation_runs"] + + eval_by_run_id: dict[str, dict[str, float | None]] = {} + for ev in evaluation_runs: + run_id = ev.experiment_run_id + score: float | None = None + if ev.result is not None: + score = ev.result.get("score") + eval_by_run_id.setdefault(run_id, {})[ev.name] = score + + rows = [] + for run in task_runs: + row: dict[str, Any] = { + "run_id": run.id, + "output": run.output, + "error": run.error, + } + scores = eval_by_run_id.get(run.id, {}) + row.update(scores) + rows.append(row) + return rows + + +def _mean_scores(ran_experiment: dict[str, Any]) -> dict[str, float]: + """Return mean score per evaluator for a RanExperiment.""" + evaluation_runs: list[Any] = ran_experiment["evaluation_runs"] + totals: dict[str, list[float]] = {} + for ev in evaluation_runs: + if ev.result is not None: + score = ev.result.get("score") + if score is not None: + totals.setdefault(ev.name, []).append(float(score)) + return {name: sum(vals) / len(vals) for name, vals in totals.items()} + + +class ResultCollector: + """Fetch, export, compare, and report on evalwire experiment results. + + Parameters + ---------- + client: + An initialised ``phoenix.client.Client`` instance. + """ + + def __init__(self, client: Client) -> None: + self._client = client + + def get(self, experiment_id: str) -> Any: + """Fetch a completed experiment by its ID. + + Parameters + ---------- + experiment_id: + The Phoenix experiment ID. + + Returns + ------- + dict + A ``RanExperiment`` dict with ``task_runs`` and ``evaluation_runs``. + + Raises + ------ + ValueError + If the experiment is not found. + """ + return self._client.experiments.get_experiment(experiment_id=experiment_id) + + def export( + self, + experiment_id: str, + format: Literal["csv", "json"], + path: Path | str, + ) -> None: + """Export experiment results to a file. + + Parameters + ---------- + experiment_id: + The Phoenix experiment ID. + format: + Output format: ``"csv"`` or ``"json"``. + path: + Destination file path. + + Raises + ------ + ValueError + If *format* is not supported. + """ + if format not in _SUPPORTED_FORMATS: + raise ValueError( + f"Unsupported format {format!r}. Choose from: {sorted(_SUPPORTED_FORMATS)}" + ) + ran = self.get(experiment_id) + rows = _rows_from_ran_experiment(ran) + path = Path(path) + + if format == "csv": + fieldnames = list(rows[0].keys()) if rows else ["run_id", "output", "error"] + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + writer.writerows(rows) + else: + path.write_text(json.dumps(rows, indent=2, default=str)) + + def compare( + self, + experiment_id_a: str, + experiment_id_b: str, + ) -> dict[str, dict[str, float]]: + """Compare two experiments by their mean evaluator scores. + + Parameters + ---------- + experiment_id_a: + ID of the baseline experiment. + experiment_id_b: + ID of the comparison experiment. + + Returns + ------- + dict + Mapping of evaluator name → ``{"score_a": …, "score_b": …, "delta": …}``. + """ + ran_a = self.get(experiment_id_a) + ran_b = self.get(experiment_id_b) + scores_a = _mean_scores(ran_a) + scores_b = _mean_scores(ran_b) + all_names = set(scores_a) | set(scores_b) + result: dict[str, dict[str, float]] = {} + for name in all_names: + a = scores_a.get(name, 0.0) + b = scores_b.get(name, 0.0) + result[name] = {"score_a": a, "score_b": b, "delta": b - a} + return result + + def report(self, experiment_id: str) -> str: + """Generate a markdown summary report for an experiment. + + Parameters + ---------- + experiment_id: + The Phoenix experiment ID. + + Returns + ------- + str + A markdown-formatted summary string. + """ + ran = self.get(experiment_id) + scores = _mean_scores(ran) + task_runs: list[Any] = ran["task_runs"] + + lines = [ + f"# Experiment Report: {experiment_id}", + "", + f"**Total runs:** {len(task_runs)}", + "", + "## Evaluator Scores", + "", + ] + if scores: + for name, score in sorted(scores.items()): + lines.append(f"- **{name}**: {score:.4f}") + else: + lines.append("_No evaluator scores recorded._") + + return "\n".join(lines) diff --git a/tests/test_cli.py b/tests/test_cli.py index 783eee1..074469d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -207,3 +207,138 @@ def test_help_text_available(self): assert result_upload.exit_code == 0 result_run = _runner().invoke(main, ["run", "--help"]) assert result_run.exit_code == 0 + + +def _make_ran_experiment(experiment_id="exp-1", scores=None): + """Return a mock RanExperiment-like dict.""" + task_run = MagicMock() + task_run.id = "run-1" + task_run.output = "answer" + task_run.error = None + + eval_runs = [] + for name, score in (scores or {}).items(): + ev = MagicMock() + ev.experiment_run_id = "run-1" + ev.name = name + ev.result = MagicMock() + ev.result.get = lambda k, default=None, _s=score: {"score": _s}.get(k, default) + ev.error = None + eval_runs.append(ev) + + return { + "experiment_id": experiment_id, + "task_runs": [task_run], + "evaluation_runs": eval_runs, + "dataset_id": "ds-1", + "dataset_version_id": "dv-1", + "experiment_metadata": {}, + "project_name": None, + } + + +class TestExportCommand: + def test_export_csv_exits_zero(self, tmp_path: Path): + ran = _make_ran_experiment(scores={"accuracy": 0.9}) + client = _mock_client() + client.experiments.get_experiment.return_value = ran + out = tmp_path / "out.csv" + with patch("evalwire.cli._make_client", return_value=client): + result = _runner().invoke( + main, + [ + "export", + "--experiment", + "exp-1", + "--format", + "csv", + "--output", + str(out), + ], + ) + assert result.exit_code == 0 + assert out.exists() + + def test_export_json_exits_zero(self, tmp_path: Path): + ran = _make_ran_experiment(scores={"accuracy": 0.9}) + client = _mock_client() + client.experiments.get_experiment.return_value = ran + out = tmp_path / "out.json" + with patch("evalwire.cli._make_client", return_value=client): + result = _runner().invoke( + main, + [ + "export", + "--experiment", + "exp-1", + "--format", + "json", + "--output", + str(out), + ], + ) + assert result.exit_code == 0 + assert out.exists() + + def test_export_missing_experiment_flag_exits_nonzero(self): + result = _runner().invoke(main, ["export"]) + assert result.exit_code != 0 + + def test_export_not_found_experiment_exits_nonzero(self, tmp_path: Path): + client = _mock_client() + client.experiments.get_experiment.side_effect = ValueError("not found") + out = tmp_path / "out.csv" + with patch("evalwire.cli._make_client", return_value=client): + result = _runner().invoke( + main, + ["export", "--experiment", "missing", "--output", str(out)], + ) + assert result.exit_code != 0 + + +class TestCompareCommand: + def test_compare_exits_zero(self): + ran_a = _make_ran_experiment("exp-a", scores={"accuracy": 0.8}) + ran_b = _make_ran_experiment("exp-b", scores={"accuracy": 0.9}) + client = _mock_client() + client.experiments.get_experiment.side_effect = lambda experiment_id: ( + ran_a if experiment_id == "exp-a" else ran_b + ) + with patch("evalwire.cli._make_client", return_value=client): + result = _runner().invoke(main, ["compare", "exp-a", "exp-b"]) + assert result.exit_code == 0 + assert "accuracy" in result.output + + def test_compare_shows_delta(self): + ran_a = _make_ran_experiment("exp-a", scores={"accuracy": 0.8}) + ran_b = _make_ran_experiment("exp-b", scores={"accuracy": 0.9}) + client = _mock_client() + client.experiments.get_experiment.side_effect = lambda experiment_id: ( + ran_a if experiment_id == "exp-a" else ran_b + ) + with patch("evalwire.cli._make_client", return_value=client): + result = _runner().invoke(main, ["compare", "exp-a", "exp-b"]) + assert ( + "0.10" in result.output + or "+0.1" in result.output + or "delta" in result.output.lower() + ) + + def test_compare_missing_args_exits_nonzero(self): + result = _runner().invoke(main, ["compare"]) + assert result.exit_code != 0 + + +class TestReportCommand: + def test_report_exits_zero(self): + ran = _make_ran_experiment(scores={"accuracy": 0.75}) + client = _mock_client() + client.experiments.get_experiment.return_value = ran + with patch("evalwire.cli._make_client", return_value=client): + result = _runner().invoke(main, ["report", "--experiment", "exp-1"]) + assert result.exit_code == 0 + assert "accuracy" in result.output + + def test_report_missing_experiment_flag_exits_nonzero(self): + result = _runner().invoke(main, ["report"]) + assert result.exit_code != 0 diff --git a/tests/test_results.py b/tests/test_results.py new file mode 100644 index 0000000..18baa1d --- /dev/null +++ b/tests/test_results.py @@ -0,0 +1,222 @@ +"""Tests for evalwire.results — ResultCollector.""" + +import csv +import json +from pathlib import Path +from unittest.mock import MagicMock + +import pytest + +from evalwire.results import ResultCollector + + +def _make_eval_run(name: str, score: float | None, run_id: str = "run-1"): + ev = MagicMock() + ev.experiment_run_id = run_id + ev.name = name + ev.result = MagicMock() + ev.result.get = lambda k, default=None: {"score": score}.get(k, default) + ev.error = None + return ev + + +def _make_task_run(run_id: str = "run-1", output: str = "answer"): + run = MagicMock() + run.id = run_id + run.output = output + run.error = None + return run + + +def _make_experiment( + experiment_id: str = "exp-1", + task_runs=None, + evaluation_runs=None, +): + exp = MagicMock() + exp.__getitem__ = lambda self, k: { + "experiment_id": experiment_id, + "task_runs": task_runs or [], + "evaluation_runs": evaluation_runs or [], + }[k] + exp.get = lambda k, default=None: { + "experiment_id": experiment_id, + "task_runs": task_runs or [], + "evaluation_runs": evaluation_runs or [], + }.get(k, default) + return exp + + +def _make_ran_experiment( + experiment_id: str = "exp-1", + task_runs=None, + evaluation_runs=None, +): + exp = { + "experiment_id": experiment_id, + "task_runs": task_runs or [], + "evaluation_runs": evaluation_runs or [], + "dataset_id": "ds-1", + "dataset_version_id": "dv-1", + "experiment_metadata": {}, + "project_name": None, + } + return exp + + +def _mock_client(ran_experiment=None): + client = MagicMock() + client.experiments.get_experiment.return_value = ( + ran_experiment or _make_ran_experiment() + ) + return client + + +class TestResultCollectorGet: + def test_get_calls_get_experiment_with_id(self): + client = _mock_client() + rc = ResultCollector(client) + rc.get("exp-1") + client.experiments.get_experiment.assert_called_once_with(experiment_id="exp-1") + + def test_get_returns_ran_experiment(self): + ran = _make_ran_experiment("exp-42") + client = _mock_client(ran) + rc = ResultCollector(client) + result = rc.get("exp-42") + assert result["experiment_id"] == "exp-42" + + def test_get_raises_when_not_found(self): + client = _mock_client() + client.experiments.get_experiment.side_effect = ValueError("not found") + rc = ResultCollector(client) + with pytest.raises(ValueError, match="not found"): + rc.get("missing-id") + + +class TestResultCollectorExport: + def _build_rc_with_data(self): + task_run = _make_task_run("run-1", "42") + eval_run = _make_eval_run("accuracy", 0.9, "run-1") + ran = _make_ran_experiment( + "exp-1", + task_runs=[task_run], + evaluation_runs=[eval_run], + ) + client = _mock_client(ran) + return ResultCollector(client) + + def test_export_csv_creates_file(self, tmp_path: Path): + rc = self._build_rc_with_data() + out = tmp_path / "results.csv" + rc.export("exp-1", format="csv", path=out) + assert out.exists() + + def test_export_csv_has_header_row(self, tmp_path: Path): + rc = self._build_rc_with_data() + out = tmp_path / "results.csv" + rc.export("exp-1", format="csv", path=out) + with open(out) as f: + reader = csv.DictReader(f) + headers = reader.fieldnames or [] + assert "run_id" in headers + assert "output" in headers + + def test_export_csv_has_data_rows(self, tmp_path: Path): + rc = self._build_rc_with_data() + out = tmp_path / "results.csv" + rc.export("exp-1", format="csv", path=out) + with open(out) as f: + rows = list(csv.DictReader(f)) + assert len(rows) == 1 + assert rows[0]["run_id"] == "run-1" + + def test_export_json_creates_valid_json(self, tmp_path: Path): + rc = self._build_rc_with_data() + out = tmp_path / "results.json" + rc.export("exp-1", format="json", path=out) + assert out.exists() + data = json.loads(out.read_text()) + assert isinstance(data, list) + assert len(data) == 1 + + def test_export_json_row_has_expected_keys(self, tmp_path: Path): + rc = self._build_rc_with_data() + out = tmp_path / "results.json" + rc.export("exp-1", format="json", path=out) + data = json.loads(out.read_text()) + assert "run_id" in data[0] + assert "output" in data[0] + + def test_export_unsupported_format_raises(self, tmp_path: Path): + rc = self._build_rc_with_data() + with pytest.raises(ValueError, match="format"): + rc.export("exp-1", format="parquet", path=tmp_path / "x") + + +class TestResultCollectorCompare: + def _build_rc(self, score_a: float, score_b: float): + task_run_a = _make_task_run("run-a", "out-a") + eval_run_a = _make_eval_run("accuracy", score_a, "run-a") + ran_a = _make_ran_experiment("exp-a", [task_run_a], [eval_run_a]) + + task_run_b = _make_task_run("run-b", "out-b") + eval_run_b = _make_eval_run("accuracy", score_b, "run-b") + ran_b = _make_ran_experiment("exp-b", [task_run_b], [eval_run_b]) + + client = MagicMock() + client.experiments.get_experiment.side_effect = lambda experiment_id: ( + ran_a if experiment_id == "exp-a" else ran_b + ) + return ResultCollector(client) + + def test_compare_returns_dict_with_evaluator_names(self): + rc = self._build_rc(0.8, 0.9) + result = rc.compare("exp-a", "exp-b") + assert "accuracy" in result + + def test_compare_delta_is_b_minus_a(self): + rc = self._build_rc(0.8, 0.9) + result = rc.compare("exp-a", "exp-b") + assert result["accuracy"]["delta"] == pytest.approx(0.1) + + def test_compare_includes_scores_for_both(self): + rc = self._build_rc(0.6, 0.8) + result = rc.compare("exp-a", "exp-b") + assert result["accuracy"]["score_a"] == pytest.approx(0.6) + assert result["accuracy"]["score_b"] == pytest.approx(0.8) + + +class TestResultCollectorReport: + def test_report_returns_string(self): + task_run = _make_task_run("run-1", "42") + eval_run = _make_eval_run("accuracy", 0.75, "run-1") + ran = _make_ran_experiment("exp-1", [task_run], [eval_run]) + client = _mock_client(ran) + rc = ResultCollector(client) + report = rc.report("exp-1") + assert isinstance(report, str) + + def test_report_contains_experiment_id(self): + ran = _make_ran_experiment("exp-99") + client = _mock_client(ran) + rc = ResultCollector(client) + report = rc.report("exp-99") + assert "exp-99" in report + + def test_report_contains_evaluator_scores(self): + task_run = _make_task_run("run-1", "42") + eval_run = _make_eval_run("accuracy", 0.75, "run-1") + ran = _make_ran_experiment("exp-1", [task_run], [eval_run]) + client = _mock_client(ran) + rc = ResultCollector(client) + report = rc.report("exp-1") + assert "accuracy" in report + assert "0.75" in report + + def test_report_is_markdown(self): + ran = _make_ran_experiment("exp-1") + client = _mock_client(ran) + rc = ResultCollector(client) + report = rc.report("exp-1") + assert "#" in report