diff options
Diffstat (limited to 'aisearch/scripts/plot_function.py')
| -rw-r--r-- | aisearch/scripts/plot_function.py | 137 |
1 files changed, 137 insertions, 0 deletions
diff --git a/aisearch/scripts/plot_function.py b/aisearch/scripts/plot_function.py new file mode 100644 index 0000000..1cdbc43 --- /dev/null +++ b/aisearch/scripts/plot_function.py @@ -0,0 +1,137 @@ +""" +Script to plot results with new metric using local models +""" + +import os + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +INPUT_PATH = os.getenv("INPUT_PATH") +if INPUT_PATH is None: + raise FileNotFoundError("Set INPUT_PATH or path to results directory") + +OUTPUT1_PATH = os.getenv("OUTPUT1_PATH") +if OUTPUT1_PATH is None: + raise FileNotFoundError( + "Set OUTPUT1_PATH or path to save final boxplot for systems" + ) + +OUTPUT2_PATH = os.getenv("OUTPUT2_PATH") +if OUTPUT2_PATH is None: + raise FileNotFoundError("Set OUTPUT2_PATH or path to save final barplot for models") + + +def format_results(path: str) -> tuple[pd.DataFrame]: + files = os.listdir(path) + full_list = [] + aggr_list = [] + for f in files: + model = f.strip("_dspy_results.csv") + data = pd.read_csv(f"{path}/{f}") + + aggr = pd.DataFrame( + data.aggregate("median"), columns=["satisfaction frequency (%)"] + ) + aggr["model"] = model + aggr["system"] = list(aggr.index) + aggr_list.append(aggr) + + full = data.transpose() + full = pd.concat([full[column] for column in full.columns]) + full_df = pd.DataFrame(full, columns=["satisfaction frequency (%)"]) + full_df["model"] = model + full_df["system"] = list(full.index) + full_list.append(full_df) + + full_concat = pd.concat(full_list) + full_concat["model_order"] = pd.Categorical( + full_concat["model"], + categories=[ + "gptoss_20b", + "qwen3_30b", + "haiku_4.5", + "qwen3_30b_opus_4.8", + "haiku_4.5_opus_4.8", + "opus_4.8", + ], + ordered=False, + ) + full_concat["system_order"] = pd.Categorical( + full_concat["system"], + categories=["base", "rag", "graph rag", "agent", "combi"], + ordered=False, + ) + + aggr_concat = pd.concat(aggr_list) + aggr_concat["model_order"] = pd.Categorical( + aggr_concat["model"], + categories=[ + "gptoss_20b", + "qwen3_30b", + "haiku_4.5", + "qwen3_30b_opus_4.8", + "haiku_4.5_opus_4.8", + "opus_4.8", + ], + ordered=False, + ) + aggr_concat["system_order"] = pd.Categorical( + aggr_concat["system"], + categories=["base", "rag", "graph rag", "agent", "combi"], + ordered=False, + ) + return full_concat.sort_values( + by=["system_order", "model_order"] + ), aggr_concat.sort_values(by=["system_order", "model_order"]) + + +def plot_results( + full_data: pd.DataFrame, + aggr_data: pd.DataFrame, + output1_path: str, + output2_path: str, +): + sns.boxplot( + data=full_data, x="system", y="satisfaction frequency (%)", color="#708090" + ) + sns.swarmplot( + data=full_data, + x="system", + y="satisfaction frequency (%)", + hue="model", + palette="ocean_r", + ) + plt.savefig(output1_path, dpi=1000) + plt.show() + # heatmap_data = aggr_data.pivot( + # index="model", columns="system", values="satisfaction frequency (%)" + # ) + # heatmap_data = heatmap_data.reindex( + # index=[ + # "opus_4.8", + # "haiku_4.5_opus_4.8", + # "qwen3_30b_opus_4.8", + # "haiku_4.5", + # "qwen3_30b", + # "gptoss_20b", + # ], + # columns=["base", "rag", "graph rag", "agent", "hybrid"], + # ) + sns.barplot( + full_data, + x="system", + y="satisfaction frequency (%)", + hue="model", + palette="ocean_r", + ) + plt.suptitle("Model performance with customized metric", fontsize=10) + plt.tight_layout() + plt.savefig(output2_path, dpi=1000) + plt.show() + + +if __name__ == "__main__": + full_df, aggr_df = format_results(INPUT_PATH) + plot_results(full_df, aggr_df, OUTPUT1_PATH, OUTPUT2_PATH) |
