[Fix] Fix BBH freeform answer evaluation (#2548)

Author: ssiq

opencompass/datasets/bbh.py

@@ -47,17 +47,29 @@ def bbh_mcq_postprocess(text: str) -> str:
 @TEXT_POSTPROCESSORS.register_module('bbh-freeform')
 def bbh_freeform_postprocess(text: str) -> str:
     ans = text
-    ans_line = ans.split('answer is ')
+    ans_line = re.split(r'answer is\s*:?\s*', ans, flags=re.IGNORECASE)
     if len(ans_line) != 1:
-        ans = ans_line[1].strip()
+        ans = ans_line[-1].strip()
     ans = ans.split('\n')[0].strip()
 
     if ans.endswith('.'):
         ans = ans[:-1].strip()
 
     match = re.search(r'\*\*(.*?)\*\*', ans)
     if match:
-        return match.group(1)
+        ans = match.group(1).strip()
+
+    match = re.match(r'^(yes|no)\b', ans, flags=re.IGNORECASE)
+    if match:
+        return match.group(1).capitalize()
+
+    match = re.match(r'^(true|false)\b', ans, flags=re.IGNORECASE)
+    if match:
+        return match.group(1).capitalize()
+
+    if not re.search(r'\d\s*,\s*\d', ans):
+        ans = re.sub(r'\s*,\s*', ' ', ans)
+    ans = re.sub(r'\s+', ' ', ans).strip()
 
     return ans
 
@@ -73,12 +85,13 @@ def score(self, predictions, references):
             }
 
         predictions = [bbh_freeform_postprocess(pred) for pred in predictions]
+        references = [bbh_freeform_postprocess(ref) for ref in references]
 
         details = []
         cnt = 0
         for pred, ref in zip(predictions, references):
             detail = {'pred': pred, 'answer': ref, 'correct': False}
-            if pred == ref:
+            if pred.casefold() == ref.casefold():
                 cnt += 1
                 detail['correct'] = True
             details.append(detail)

tests/datasets/test_bbh.py

@@ -0,0 +1,53 @@
+import unittest
+
+from opencompass.datasets.bbh import BBHEvaluator, bbh_freeform_postprocess
+
+
+class TestBBHFreeform(unittest.TestCase):
+
+    def test_freeform_postprocess_extracts_short_answers(self):
+        self.assertEqual(bbh_freeform_postprocess('no'), 'No')
+        self.assertEqual(
+            bbh_freeform_postprocess(
+                'So the answer is No, because the last speaker lies.'),
+            'No',
+        )
+        self.assertEqual(bbh_freeform_postprocess('Answer is **yes**.'),
+                         'Yes')
+
+    def test_freeform_postprocess_normalizes_word_sorting_commas(self):
+        self.assertEqual(
+            bbh_freeform_postprocess('apple, banana, cherry'),
+            'apple banana cherry',
+        )
+        self.assertEqual(bbh_freeform_postprocess('1,000'), '1,000')
+
+    def test_freeform_evaluator_accepts_equivalent_formats(self):
+        result = BBHEvaluator().score(
+            predictions=[
+                'no',
+                'No, because the last speaker lies.',
+                'apple, banana, cherry',
+            ],
+            references=[
+                'No',
+                'No',
+                'apple banana cherry',
+            ],
+        )
+
+        self.assertEqual(result['score'], 100.0)
+        self.assertTrue(all(item['correct'] for item in result['details']))
+
+    def test_freeform_evaluator_rejects_wrong_word_order(self):
+        result = BBHEvaluator().score(
+            predictions=['apple, cherry, banana'],
+            references=['apple banana cherry'],
+        )
+
+        self.assertEqual(result['score'], 0.0)
+        self.assertFalse(result['details'][0]['correct'])
+
+
+if __name__ == '__main__':
+    unittest.main()