adaptive-rag-course/eval_results.json at main · xqs-xqs/adaptive-rag-course · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
{
  "retrieval": {
    "precision_at_k": 0.2,
    "recall_at_k": 1.0,
    "mrr": 0.6666666666666667,
    "avg_latency_ms": 3934.587001800537,
    "p95_latency_ms": 5355.758905410767,
    "per_question": [
      {
        "question": "What is the exam weighting for COMP5422?",
        "precision": 0.2,
        "recall": 1.0,
        "mrr": 1.0,
        "latency_ms": 4997.012615203857
      },
      {
        "question": "What topics does COMP5422 cover?",
        "precision": 0.2,
        "recall": 1.0,
        "mrr": 0.5,
        "latency_ms": 2812.825918197632
      },
      {
        "question": "What are the references for COMP5422?",
        "precision": 0.2,
        "recall": 1.0,
        "mrr": 0.5,
        "latency_ms": 3331.52437210083
      },
      {
        "question": "Are there any courses related to multimedia computing?",
        "precision": 0.2,
        "recall": 1.0,
        "mrr": 0.3333333333333333,
        "latency_ms": 5355.758905410767
      },
      {
        "question": "When is COMP5422's class?",
        "precision": 0.2,
        "recall": 1.0,
        "mrr": 1.0,
        "latency_ms": 3175.8131980895996
      }
    ]
  },
  "generation": {
    "avg_completeness": 5.0,
    "accuracy_rate": 1.0,
    "per_question": [
      {
        "question": "What is the exam weighting for COMP5422?",
        "completeness": 5.0,
        "accurate": true
      },
      {
        "question": "What topics does COMP5422 cover?",
        "completeness": 5.0,
        "accurate": true
      },
      {
        "question": "What are the references for COMP5422?",
        "completeness": 5.0,
        "accurate": true
      },
      {
        "question": "Are there any courses related to multimedia computing?",
        "completeness": 5.0,
        "accurate": true
      },
      {
        "question": "When is COMP5422's class?",
        "completeness": 5.0,
        "accurate": true
      }
    ]
  },
  "ablation": [
    {
      "Config": "Vector only",
      "Precision@5": "0.20",
      "Recall@5": "1.00",
      "MRR": "0.90",
      "Avg Latency": "2989 ms"
    },
    {
      "Config": "+ BM25 (RRF)",
      "Precision@5": "0.20",
      "Recall@5": "1.00",
      "MRR": "0.67",
      "Avg Latency": "2698 ms"
    },
    {
      "Config": "+ Summary Index",
      "Precision@5": "0.20",
      "Recall@5": "1.00",
      "MRR": "0.67",
      "Avg Latency": "3122 ms"
    },
    {
      "Config": "+ Multi-Query Expansion",
      "Precision@5": "0.20",
      "Recall@5": "1.00",
      "MRR": "0.67",
      "Avg Latency": "3718 ms"
    },
    {
      "Config": "Full pipeline",
      "Precision@5": "0.20",
      "Recall@5": "1.00",
      "MRR": "0.70",
      "Avg Latency": "3347 ms"
    }
  ]
}