diff options
| author | Johannes Medagbe <johanmedagbe@gmail.com> | 2026-09-21 10:03:20 +0000 |
|---|---|---|
| committer | Johannes Medagbe <johanmedagbe@gmail.com> | 2026-09-22 12:18:32 +0000 |
| commit | dce53892085c02c8571d8a6495e39c449294aa5f (patch) | |
| tree | ae64d32cc845d30ec2a4cc3514a23dc5683a09a1 /aisearch/scripts/dspy_evaluate.py | |
| download | genenetwork_ai_search-master.tar.gz | |
New: commit aisearch repo master
Diffstat (limited to 'aisearch/scripts/dspy_evaluate.py')
| -rw-r--r-- | aisearch/scripts/dspy_evaluate.py | 126 |
1 files changed, 126 insertions, 0 deletions
diff --git a/aisearch/scripts/dspy_evaluate.py b/aisearch/scripts/dspy_evaluate.py new file mode 100644 index 0000000..b7d6d6f --- /dev/null +++ b/aisearch/scripts/dspy_evaluate.py @@ -0,0 +1,126 @@ +"""Script for performance evaluation of GN AI systems using DSPy.Evaluate and custom decision function""" + +import os +from typing import Any + +import dspy +import pandas as pd +import torch +from gnais.config import Config +from gnais.evaluation.utils import ( + agent_digest, + get_dataset, + graph_rag_digest, + hybrid_digest, + mark, + rag_digest, +) + + +def evaluator( + gold: dspy.Example, + pred: str, + trace=None, + pred_name=None, + pred_trace=None, +) -> int: + query = gold["query"] + true_response = gold["answer"] + precision, recall, f1 = mark(query, pred, true_response) + return 1 if (f1 >= 0.6 or recall >= 0.7 or precision >= 0.7) else 0 + + +def run_eval( + runner: dspy.Evaluate, + evaluation_set: list[dspy.Example], + search_func: Any, +) -> dspy.evaluate.EvaluationResult: + + search = dspy.Tool( + name="search", + desc="Answer the question asked by the user.", + args={ + "query": { + "type": "string", + "desc": "question asked by the user", + }, + }, + func=search_func, + ) + + class Signature(dspy.Signature): + query: str = dspy.InputField(desc="User question") + answer: str = dspy.OutputField( + desc="Answer to user's query with details and explanations" + ) + + # Wrap hybrid search into a DSPy module that is compatible with DSPy evaluation + program = dspy.ReAct( + signature=Signature, tools=[search], max_iters=2 + ) # limit iterations + + return runner(program) + + +if __name__ == "__main__": + DATASET_PATH = os.getenv("DATASET_PATH") + if DATASET_PATH is None: + raise FileNotFoundError("DATASET_PATH must be set for evaluation") + + OUTPUT_PATH = os.getenv("OUTPUT_PATH") + if OUTPUT_PATH is None: + raise FileNotFoundError("OUTPUT_PATH must be set for evaluation") + + N_ITERATIONS = int(os.getenv("N_ITERATIONS")) + if N_ITERATIONS is None: + raise ValueError("N_ITERATIONS must be set for evaluation") + + JUDGE_MODEL = os.getenv("JUDGE_MODEL") + if JUDGE_MODEL is None: + raise ValueError("JUDGE MODEL must be set for evaluation") + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + + judge_llm = dspy.LM( + model=JUDGE_MODEL, + api_key=Config.API_KEY, + max_tokens=1_000, + temperature=1, + cache=False, + verbose=False, + ) + + evaluation_set = get_dataset(DATASET_PATH) + evaluate = dspy.Evaluate( + devset=evaluation_set, + metric=evaluator, + num_threads=1, + provide_traceback=True, + display_table=False, + display_progress=True, + lm=judge_llm, + ) + collection = {} + + # Run evaluation set with LLM only + base = dspy.ChainOfThought("query -> answer: str") + base_output = [evaluate(base).get("score") for n in range(N_ITERATIONS)] + collection["base"] = base_output + + # Run evaluation set with GN systems + for system in [rag_digest, graph_rag_digest, agent_digest, hybrid_digest]: + system_name = " ".join(system.__name__.split("_")[:-1]) + print(f"Running evaluation for {system_name}") + temp = [] + for n in range(N_ITERATIONS): + print(f"Iteration {n+1}") + system_output = run_eval(evaluate, evaluation_set, system) + temp.append(system_output.get("score")) + collection[system_name] = temp + print(f"Evaluation completed for {system_name}") + final = pd.DataFrame(collection) + final.to_csv(OUTPUT_PATH, index=False) |
