about summary refs log tree commit diff
path: root/aisearch/scripts/dspy_evaluate.py
blob: b7d6d6f862b07d8a3fc9554e39f857d0e811bc3f (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
"""Script for performance evaluation of GN AI systems using DSPy.Evaluate and custom decision function"""

import os
from typing import Any

import dspy
import pandas as pd
import torch
from gnais.config import Config
from gnais.evaluation.utils import (
    agent_digest,
    get_dataset,
    graph_rag_digest,
    hybrid_digest,
    mark,
    rag_digest,
)


def evaluator(
    gold: dspy.Example,
    pred: str,
    trace=None,
    pred_name=None,
    pred_trace=None,
) -> int:
    query = gold["query"]
    true_response = gold["answer"]
    precision, recall, f1 = mark(query, pred, true_response)
    return 1 if (f1 >= 0.6 or recall >= 0.7 or precision >= 0.7) else 0


def run_eval(
    runner: dspy.Evaluate,
    evaluation_set: list[dspy.Example],
    search_func: Any,
) -> dspy.evaluate.EvaluationResult:

    search = dspy.Tool(
        name="search",
        desc="Answer the question asked by the user.",
        args={
            "query": {
                "type": "string",
                "desc": "question asked by the user",
            },
        },
        func=search_func,
    )

    class Signature(dspy.Signature):
        query: str = dspy.InputField(desc="User question")
        answer: str = dspy.OutputField(
            desc="Answer to user's query with details and explanations"
        )

    # Wrap hybrid search into a DSPy module that is compatible with DSPy evaluation
    program = dspy.ReAct(
        signature=Signature, tools=[search], max_iters=2
    )  # limit iterations

    return runner(program)


if __name__ == "__main__":
    DATASET_PATH = os.getenv("DATASET_PATH")
    if DATASET_PATH is None:
        raise FileNotFoundError("DATASET_PATH must be set for evaluation")

    OUTPUT_PATH = os.getenv("OUTPUT_PATH")
    if OUTPUT_PATH is None:
        raise FileNotFoundError("OUTPUT_PATH must be set for evaluation")

    N_ITERATIONS = int(os.getenv("N_ITERATIONS"))
    if N_ITERATIONS is None:
        raise ValueError("N_ITERATIONS must be set for evaluation")

    JUDGE_MODEL = os.getenv("JUDGE_MODEL")
    if JUDGE_MODEL is None:
        raise ValueError("JUDGE MODEL must be set for evaluation")

    torch.manual_seed(Config.SEED)
    if torch.cuda.is_available():
        torch.cuda.manual_seed_all(Config.SEED)

    dspy.configure(lm=Config.DEFAULT_LLM)

    judge_llm = dspy.LM(
        model=JUDGE_MODEL,
        api_key=Config.API_KEY,
        max_tokens=1_000,
        temperature=1,
        cache=False,
        verbose=False,
    )

    evaluation_set = get_dataset(DATASET_PATH)
    evaluate = dspy.Evaluate(
        devset=evaluation_set,
        metric=evaluator,
        num_threads=1,
        provide_traceback=True,
        display_table=False,
        display_progress=True,
        lm=judge_llm,
    )
    collection = {}

    # Run evaluation set with LLM only
    base = dspy.ChainOfThought("query -> answer: str")
    base_output = [evaluate(base).get("score") for n in range(N_ITERATIONS)]
    collection["base"] = base_output

    # Run evaluation set with GN systems
    for system in [rag_digest, graph_rag_digest, agent_digest, hybrid_digest]:
        system_name = " ".join(system.__name__.split("_")[:-1])
        print(f"Running evaluation for {system_name}")
        temp = []
        for n in range(N_ITERATIONS):
            print(f"Iteration {n+1}")
            system_output = run_eval(evaluate, evaluation_set, system)
            temp.append(system_output.get("score"))
        collection[system_name] = temp
        print(f"Evaluation completed for {system_name}")
    final = pd.DataFrame(collection)
    final.to_csv(OUTPUT_PATH, index=False)