about summary refs log tree commit diff
path: root/aisearch/scripts/dspy_evaluate.py
diff options
context:
space:
mode:
Diffstat (limited to 'aisearch/scripts/dspy_evaluate.py')
-rw-r--r--aisearch/scripts/dspy_evaluate.py126
1 files changed, 126 insertions, 0 deletions
diff --git a/aisearch/scripts/dspy_evaluate.py b/aisearch/scripts/dspy_evaluate.py
new file mode 100644
index 0000000..b7d6d6f
--- /dev/null
+++ b/aisearch/scripts/dspy_evaluate.py
@@ -0,0 +1,126 @@
+"""Script for performance evaluation of GN AI systems using DSPy.Evaluate and custom decision function"""
+
+import os
+from typing import Any
+
+import dspy
+import pandas as pd
+import torch
+from gnais.config import Config
+from gnais.evaluation.utils import (
+    agent_digest,
+    get_dataset,
+    graph_rag_digest,
+    hybrid_digest,
+    mark,
+    rag_digest,
+)
+
+
+def evaluator(
+    gold: dspy.Example,
+    pred: str,
+    trace=None,
+    pred_name=None,
+    pred_trace=None,
+) -> int:
+    query = gold["query"]
+    true_response = gold["answer"]
+    precision, recall, f1 = mark(query, pred, true_response)
+    return 1 if (f1 >= 0.6 or recall >= 0.7 or precision >= 0.7) else 0
+
+
+def run_eval(
+    runner: dspy.Evaluate,
+    evaluation_set: list[dspy.Example],
+    search_func: Any,
+) -> dspy.evaluate.EvaluationResult:
+
+    search = dspy.Tool(
+        name="search",
+        desc="Answer the question asked by the user.",
+        args={
+            "query": {
+                "type": "string",
+                "desc": "question asked by the user",
+            },
+        },
+        func=search_func,
+    )
+
+    class Signature(dspy.Signature):
+        query: str = dspy.InputField(desc="User question")
+        answer: str = dspy.OutputField(
+            desc="Answer to user's query with details and explanations"
+        )
+
+    # Wrap hybrid search into a DSPy module that is compatible with DSPy evaluation
+    program = dspy.ReAct(
+        signature=Signature, tools=[search], max_iters=2
+    )  # limit iterations
+
+    return runner(program)
+
+
+if __name__ == "__main__":
+    DATASET_PATH = os.getenv("DATASET_PATH")
+    if DATASET_PATH is None:
+        raise FileNotFoundError("DATASET_PATH must be set for evaluation")
+
+    OUTPUT_PATH = os.getenv("OUTPUT_PATH")
+    if OUTPUT_PATH is None:
+        raise FileNotFoundError("OUTPUT_PATH must be set for evaluation")
+
+    N_ITERATIONS = int(os.getenv("N_ITERATIONS"))
+    if N_ITERATIONS is None:
+        raise ValueError("N_ITERATIONS must be set for evaluation")
+
+    JUDGE_MODEL = os.getenv("JUDGE_MODEL")
+    if JUDGE_MODEL is None:
+        raise ValueError("JUDGE MODEL must be set for evaluation")
+
+    torch.manual_seed(Config.SEED)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(Config.SEED)
+
+    dspy.configure(lm=Config.DEFAULT_LLM)
+
+    judge_llm = dspy.LM(
+        model=JUDGE_MODEL,
+        api_key=Config.API_KEY,
+        max_tokens=1_000,
+        temperature=1,
+        cache=False,
+        verbose=False,
+    )
+
+    evaluation_set = get_dataset(DATASET_PATH)
+    evaluate = dspy.Evaluate(
+        devset=evaluation_set,
+        metric=evaluator,
+        num_threads=1,
+        provide_traceback=True,
+        display_table=False,
+        display_progress=True,
+        lm=judge_llm,
+    )
+    collection = {}
+
+    # Run evaluation set with LLM only
+    base = dspy.ChainOfThought("query -> answer: str")
+    base_output = [evaluate(base).get("score") for n in range(N_ITERATIONS)]
+    collection["base"] = base_output
+
+    # Run evaluation set with GN systems
+    for system in [rag_digest, graph_rag_digest, agent_digest, hybrid_digest]:
+        system_name = " ".join(system.__name__.split("_")[:-1])
+        print(f"Running evaluation for {system_name}")
+        temp = []
+        for n in range(N_ITERATIONS):
+            print(f"Iteration {n+1}")
+            system_output = run_eval(evaluate, evaluation_set, system)
+            temp.append(system_output.get("score"))
+        collection[system_name] = temp
+        print(f"Evaluation completed for {system_name}")
+    final = pd.DataFrame(collection)
+    final.to_csv(OUTPUT_PATH, index=False)