SelfElicit/eval.py at main · ZhiningLiu1998/SelfElicit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
import re
import pandas as pd
from qa_metrics.em import em_match
from qa_metrics.f1 import f1_match, f1_score_with_precision_recall


def evaluate(true_ans_list, model_ans, sel_metric="f1"):
    """
    Evaluate the model's answer against a list of ground-truth answers.

    Parameters
    ----------
    true_ans_list : list of str
        List of ground-truth answers.

    model_ans : str
        The answer generated by the model.

    sel_metric : str, optional (default="f1")
        The metric to use for selecting the best match ("f1" or "em").

    Returns
    -------
    best_ans : str
        The ground-truth answer that best matches the model's answer.

    best_eval_res : dict
        Dictionary containing evaluation metrics for the best match, including:
        - "em": Exact match score.
        - "f1m": F1 match score.
        - "f1": F1 score.
        - "pr": Precision score.
        - "re": Recall score.
    """
    best_score = -1  # Initialize the best score
    for true_ans in true_ans_list:
        # Evaluate the model's answer against the current ground-truth answer
        eval_res = evaluate_single_ans(true_ans, model_ans)
        # Update the best score and corresponding metrics if the current score is higher
        if eval_res[sel_metric] > best_score:
            best_score = eval_res[sel_metric]
            best_ans = true_ans
            best_eval_res = eval_res

    # Compute the overall exact match score
    best_eval_res["em"] = em_match(true_ans_list, model_ans)
    return best_ans, best_eval_res


def evaluate_single_ans(true_ans, model_ans):
    """
    Evaluate the model's answer against a single ground-truth answer.

    Parameters
    ----------
    true_ans : str
        The ground-truth answer.

    model_ans : str
        The answer generated by the model.

    Returns
    -------
    metrics : dict
        Dictionary containing evaluation metrics, including:
        - "em": Exact match score.
        - "f1m": F1 match score.
        - "f1": F1 score.
        - "pr": Precision score.
        - "re": Recall score.
    """
    # Compute F1, precision, and recall scores
    f1_pr = f1_score_with_precision_recall(true_ans, model_ans)
    f1, pr, re = f1_pr["f1"], f1_pr["precision"], f1_pr["recall"]
    return {
        "em": em_match(true_ans, model_ans),  # Exact match
        "f1m": f1_match(true_ans, model_ans),  # F1 match
        "f1": f1,  # F1 score
        "pr": pr,  # Precision
        "re": re,  # Recall
    }


def add_new_eval_metrics(df_res):
    """
    Add evaluation metrics (EM, F1, Precision, Recall) to the results dataframe.

    Parameters
    ----------
    df_res : pandas.DataFrame
        DataFrame containing model answers and ground-truth answers.

    Returns
    -------
    df_res : pandas.DataFrame
        Updated DataFrame with additional columns for evaluation metrics:
        - "em": Exact match.
        - "f1m": F1 match.
        - "f1": F1 score.
        - "pr": Precision.
        - "re": Recall.
    """
    df_new_eval = []  # List to store updated evaluation metrics

    for idx in range(len(df_res)):
        row = df_res.iloc[idx]  # Get the current row
        # Compute evaluation metrics for the current row
        em = em_match(row["true_ans"], row["model_ans"])
        f1m = f1_match(row["true_ans"], row["model_ans"])
        f1_pr = f1_score_with_precision_recall(row["true_ans"], row["model_ans"])
        f1, pr, re = f1_pr["f1"], f1_pr["precision"], f1_pr["recall"]
        # Append the metrics to the evaluation list
        df_new_eval.append(
            [row["model_ans"], row["true_ans"], row["is_correct"], em, f1m, f1, pr, re]
        )

    # Convert evaluation list to a new DataFrame
    df_new_eval = pd.DataFrame(
        df_new_eval,
        columns=["model_ans", "true_ans", "is_correct", "em", "f1m", "f1", "pr", "re"],
    )
    # Update the original DataFrame with new metrics
    for metric in ["em", "f1m", "f1", "pr", "re"]:
        df_res[metric] = df_new_eval[metric]

    return df_res


def remove_helpers_and_symbols(text):
    """
    Remove helper words (e.g., "a", "an", "the") and symbols from the text.

    Parameters
    ----------
    text : str
        Input text to be cleaned.

    Returns
    -------
    str
        Cleaned text with helper words and symbols removed.
    """
    # Define regex patterns for symbols and helper words
    helpers_pattern_start = r"^(a|an|the)\s+"
    helpers_pattern_end = r"\s+(a|an|the)$"
    symbols_pattern_start = r"^[\ ,\.\:\"\']+"
    symbols_pattern_end = r"[\ ,\.\:\"\']+$"

    # Iteratively remove symbols and helper words at the start of the text
    while True:
        new_text = re.sub(symbols_pattern_start, "", text)
        new_text = re.sub(helpers_pattern_start, "", new_text, flags=re.IGNORECASE)

        if new_text == text:  # Exit loop if no further changes
            break
        text = new_text

    # Iteratively remove symbols and helper words at the end of the text
    while True:
        new_text = re.sub(symbols_pattern_end, "", text)
        new_text = re.sub(helpers_pattern_end, "", new_text, flags=re.IGNORECASE)

        if new_text == text:  # Exit loop if no further changes
            break
        text = new_text

    return text.strip()  # Return the cleaned and stripped text