diff --git a/vlmeval/utils/matching_util.py b/vlmeval/utils/matching_util.py index d62f16c25..e454cc54e 100644 --- a/vlmeval/utils/matching_util.py +++ b/vlmeval/utils/matching_util.py @@ -9,6 +9,12 @@ logger = get_logger(__name__) +# Matches verbose chain-of-thought answers such as: +# "The correct answer is **B**. Here's why: ..." +# "The answer is C." +_VERBOSE_ANSWER_RE = re.compile(r"(?i)(?:correct\s+)?answer\s+is\s+\**([ABCD])\**") + + def can_infer_option(answer, choices): verbose = os.environ.get('VERBOSE', 0) # Choices is a dictionary @@ -47,6 +53,11 @@ def count_choice(splits, choices, prefix='', suffix=''): return ch elif count == 0 and count_choice(splits, {'Z', ''}) == 1: return 'Z' + + match = _VERBOSE_ANSWER_RE.search(answer or "") + if match and match.group(1).upper() in choices: + return match.group(1).upper() + return False