1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
|
"""
Script to plot results with new metric using local models
"""
import os
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns
INPUT_PATH = os.getenv("INPUT_PATH")
if INPUT_PATH is None:
raise FileNotFoundError("Set INPUT_PATH or path to results directory")
OUTPUT1_PATH = os.getenv("OUTPUT1_PATH")
if OUTPUT1_PATH is None:
raise FileNotFoundError(
"Set OUTPUT1_PATH or path to save final boxplot for systems"
)
OUTPUT2_PATH = os.getenv("OUTPUT2_PATH")
if OUTPUT2_PATH is None:
raise FileNotFoundError("Set OUTPUT2_PATH or path to save final barplot for models")
def format_results(path: str) -> tuple[pd.DataFrame]:
files = os.listdir(path)
full_list = []
aggr_list = []
for f in files:
model = f.strip("_dspy_results.csv")
data = pd.read_csv(f"{path}/{f}")
aggr = pd.DataFrame(
data.aggregate("median"), columns=["satisfaction frequency (%)"]
)
aggr["model"] = model
aggr["system"] = list(aggr.index)
aggr_list.append(aggr)
full = data.transpose()
full = pd.concat([full[column] for column in full.columns])
full_df = pd.DataFrame(full, columns=["satisfaction frequency (%)"])
full_df["model"] = model
full_df["system"] = list(full.index)
full_list.append(full_df)
full_concat = pd.concat(full_list)
full_concat["model_order"] = pd.Categorical(
full_concat["model"],
categories=[
"gptoss_20b",
"qwen3_30b",
"haiku_4.5",
"qwen3_30b_opus_4.8",
"haiku_4.5_opus_4.8",
"opus_4.8",
],
ordered=False,
)
full_concat["system_order"] = pd.Categorical(
full_concat["system"],
categories=["base", "rag", "graph rag", "agent", "combi"],
ordered=False,
)
aggr_concat = pd.concat(aggr_list)
aggr_concat["model_order"] = pd.Categorical(
aggr_concat["model"],
categories=[
"gptoss_20b",
"qwen3_30b",
"haiku_4.5",
"qwen3_30b_opus_4.8",
"haiku_4.5_opus_4.8",
"opus_4.8",
],
ordered=False,
)
aggr_concat["system_order"] = pd.Categorical(
aggr_concat["system"],
categories=["base", "rag", "graph rag", "agent", "combi"],
ordered=False,
)
return full_concat.sort_values(
by=["system_order", "model_order"]
), aggr_concat.sort_values(by=["system_order", "model_order"])
def plot_results(
full_data: pd.DataFrame,
aggr_data: pd.DataFrame,
output1_path: str,
output2_path: str,
):
sns.boxplot(
data=full_data, x="system", y="satisfaction frequency (%)", color="#708090"
)
sns.swarmplot(
data=full_data,
x="system",
y="satisfaction frequency (%)",
hue="model",
palette="ocean_r",
)
plt.savefig(output1_path, dpi=1000)
plt.show()
# heatmap_data = aggr_data.pivot(
# index="model", columns="system", values="satisfaction frequency (%)"
# )
# heatmap_data = heatmap_data.reindex(
# index=[
# "opus_4.8",
# "haiku_4.5_opus_4.8",
# "qwen3_30b_opus_4.8",
# "haiku_4.5",
# "qwen3_30b",
# "gptoss_20b",
# ],
# columns=["base", "rag", "graph rag", "agent", "hybrid"],
# )
sns.barplot(
full_data,
x="system",
y="satisfaction frequency (%)",
hue="model",
palette="ocean_r",
)
plt.suptitle("Model performance with customized metric", fontsize=10)
plt.tight_layout()
plt.savefig(output2_path, dpi=1000)
plt.show()
if __name__ == "__main__":
full_df, aggr_df = format_results(INPUT_PATH)
plot_results(full_df, aggr_df, OUTPUT1_PATH, OUTPUT2_PATH)
|