about summary refs log tree commit diff
path: root/aisearch/scripts/barplot_metrics.py
blob: 0932f806882fbb2ab9ef8d8f0fbface2b1e7c4a5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
"""
Script to plot results with traditional metrics using local models
"""

import os

import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

INPUT1_PATH = os.getenv("INPUT1_PATH")
if INPUT1_PATH is None:
    raise FileNotFoundError("Set INPUT1_PATH or path to simple results directory")

INPUT2_PATH = os.getenv("INPUT2_PATH")
if INPUT2_PATH is None:
    raise FileNotFoundError("Set INPUT2_PATH or path to dspy results directory")

OUTPUT_PATH = os.getenv("OUTPUT_PATH")
if OUTPUT_PATH is None:
    raise FileNotFoundError("Set OUTPUT_PATH or path to save final plot")


def format_results(simple_path: str, dspy_path: str) -> pd.DataFrame:
    df_list = []
    simple_files = os.listdir(simple_path)
    for f in simple_files:
        model = f.strip("_simple_results.csv")
        data = pd.read_csv(f"{simple_path}/{f}", index_col=0)
        trans_data = data.transpose()
        trans_data = pd.concat(
            [trans_data[column] * 100 for column in trans_data.columns], axis=1
        )
        trans_data["model"] = model
        trans_data["system"] = [sys[:-2] for sys in list(trans_data.index)]
        df_list.append(trans_data)
    final_concat = pd.concat(df_list)
    dspy_files = os.listdir(dspy_path)
    cumul = []
    for f in dspy_files:
        data = pd.read_csv(f"{dspy_path}/{f}")
        aligned = pd.concat([data[column] for column in data.columns])
        cumul.extend(aligned)
    final_concat["satisfaction frequency (%)"] = cumul
    return final_concat


def plot(data: pd.DataFrame, columns: list[str], output_path: str, title: str):
    fig, axes = plt.subplots(len(columns), 1, figsize=(10, 10))
    axes = axes.flatten()
    for ind, column in enumerate(columns):
        sns.barplot(
            data=data,
            ax=axes[ind],
            x="system",
            y=column,
            hue="model",
            palette="ocean_r",
        )
    fig.suptitle(title, fontsize=15)
    plt.tight_layout()
    plt.savefig(output_path, dpi=1000)
    plt.show()


def plot_results(data: pd.DataFrame, output_path: str):
    metrics = list(data.columns)[:3]
    metrics.append(data.columns[-1])
    plot(
        data,
        metrics,
        f"{output_path}/barplot_all_metrics.png",
        "Performance comparison of standard and new metrics",
    )
    extras = list(data.columns)[3:5]
    plot(
        data,
        extras,
        f"{output_path}/barplot_extra_metrics.png",
        "Insights from additional metrics",
    )


if __name__ == "__main__":
    df = format_results(INPUT1_PATH, INPUT2_PATH)
    plot_results(df, OUTPUT_PATH)