From ce6fc0507b314c4e6448c215e4ad2bac5c62ab58 Mon Sep 17 00:00:00 2001 From: Shardul D Date: Fri, 26 Jun 2026 01:35:36 +0530 Subject: [PATCH] Fix CorpusLevelF1Score(None) to report positive-class F1, not max-per-class The num_classes==2 branch returned np.max(f1_score(..., average=None)). With average=None, f1_score returns the per-class F1 array, so np.max reports the BEST class's F1 instead of the positive-class binary F1 -- inflating the score. loglikelihood_f1 (glue:mrpc, glue:qqp) uses CorpusLevelF1Score(None), so those scores were overstated. Return the positive class (fscore[1]); scalar averages (micro/macro/weighted) pass through unchanged. --- src/lighteval/metrics/metrics_corpus.py | 6 +++++- tests/test_unit_base_metrics.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/lighteval/metrics/metrics_corpus.py b/src/lighteval/metrics/metrics_corpus.py index 92c2c574a..b8b4bff07 100644 --- a/src/lighteval/metrics/metrics_corpus.py +++ b/src/lighteval/metrics/metrics_corpus.py @@ -100,7 +100,11 @@ def compute_corpus(self, items: list[LogprobCorpusMetricInput]): # Single f1 if self.num_classes == 2: fscore = sklearn.metrics.f1_score(golds, preds, average=self.average) - return np.max(fscore) + # average=None returns a per-class F1 array: report the positive class, not the + # best class (np.max inflated the score). Scalar averages pass through unchanged. + if self.average is None: + return float(fscore[1]) + return float(fscore) # Multi f1 f1s = [] diff --git a/tests/test_unit_base_metrics.py b/tests/test_unit_base_metrics.py index 575ebf595..3dd1fabfe 100644 --- a/tests/test_unit_base_metrics.py +++ b/tests/test_unit_base_metrics.py @@ -191,6 +191,19 @@ def test_prefix_quasi_exact_match(self): res = em.compute_one_item("", "") assert res == 0 + def test_corpus_level_f1_binary_positive_class(self): + from types import SimpleNamespace + + from lighteval.metrics.metrics_corpus import CorpusLevelF1Score + + # The binary (num_classes=2) path must report the positive-class F1, not the best + # per-class F1. Regression: it returned np.max over both classes, inflating the score. + golds = [0, 0, 0, 0, 0, 0, 1, 1, 1, 1] + preds = [0, 0, 0, 0, 0, 1, 0, 0, 0, 1] + items = [SimpleNamespace(golds=g, preds=p) for g, p in zip(golds, preds)] + score = CorpusLevelF1Score(None).compute_corpus(items) + assert score == pytest.approx(1 / 3) # positive-class F1; max-per-class would be ~0.714 + def test_prob(self): doc = Doc(query="Test query", choices=["A", "B", "C"], gold_index=0, task_name="test")