1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
"""Script for performance evaluation of GN AI systems using mere averaging"""
import os
import time
from typing import Any
import dspy
import numpy as np
import pandas as pd
import torch
from gnais.config import Config
from gnais.evaluation.utils import (
agent_digest,
get_dataset,
graph_rag_digest,
hybrid_digest,
mark,
rag_digest,
)
def make_program(
search_func: Any,
) -> dspy.predict.react.ReAct:
search = dspy.Tool(
name="search",
desc="Answer the question asked by the user.",
args={
"query": {
"type": "string",
"desc": "question asked by the user",
},
},
func=search_func,
)
class Signature(dspy.Signature):
query: str = dspy.InputField(desc="User question")
answer: str = dspy.OutputField(
desc="Answer to user's query with details and explanations"
)
# Wrap hybrid search into a DSPy module that is compatible with DSPy evaluation
program = dspy.ReAct(
signature=Signature, tools=[search], max_iters=2
) # limit iterations
return program
def run_eval(
runner: Any, evaluation_set: list[dspy.Example], judge_llm: dspy.LM
) -> dict[str, float]:
precisions, recalls, f1s, speeds, n_tokens = [], [], [], [], []
for example in evaluation_set:
query = example.get("query")
true_response = example.get("answer")
start = time.time()
result = runner(query=query)
end = time.time()
generated_response = result.get("answer")
with dspy.context(lm=judge_llm):
precision, recall, f1 = mark(query, generated_response, true_response)
precisions.append(precision)
recalls.append(recall)
f1s.append(f1)
speeds.append(end - start)
metrics = {
"precision": np.mean(precisions).item(),
"recall": np.mean(recalls).item(),
"f1": np.mean(f1s).item(),
"speed": np.mean(speeds).item(),
}
return metrics
if __name__ == "__main__":
DATASET_PATH = os.getenv("DATASET_PATH")
if DATASET_PATH is None:
raise FileNotFoundError("DATASET_PATH must be set for evaluation")
OUTPUT_PATH = os.getenv("OUTPUT_PATH")
if OUTPUT_PATH is None:
raise FileNotFoundError("OUTPUT_PATH must be set for evaluation")
N_ITERATIONS = int(os.getenv("N_ITERATIONS"))
if N_ITERATIONS is None:
raise ValueError("N_ITERATIONS must be set for evaluation")
JUDGE_MODEL = os.getenv("JUDGE_MODEL")
if JUDGE_MODEL is None:
raise ValueError("JUDGE MODEL must be set for evaluation")
torch.manual_seed(Config.SEED)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(Config.SEED)
dspy.configure(lm=Config.DEFAULT_LLM)
judge_llm = dspy.LM(
model=JUDGE_MODEL,
api_key=Config.API_KEY,
max_tokens=1_000,
temperature=1,
cache=False,
verbose=False,
)
evaluation_set = get_dataset(DATASET_PATH)
collection = {}
# Run evaluation set with LLM only
base = dspy.ChainOfThought("query -> answer: str")
for n in range(N_ITERATIONS):
base_metrics = run_eval(base, evaluation_set, judge_llm)
collection[f"base_{n}"] = base_metrics
# Run evaluation set with GN systems
for system in [rag_digest, graph_rag_digest, agent_digest, hybrid_digest]:
system_name = " ".join(system.__name__.split("_")[:-1])
print(f"Running evaluation for {system_name}")
for n in range(N_ITERATIONS):
print(f"Iteration {n+1}")
system_metrics = run_eval(make_program(system), evaluation_set, judge_llm)
collection[f"{system_name} {n}"] = system_metrics
print(f"Evaluation completed for {system_name}")
final = pd.DataFrame(collection)
final.to_csv(OUTPUT_PATH)
|