about summary refs log tree commit diff
path: root/aisearch/src/gnais/evaluation/utils.py
diff options
context:
space:
mode:
Diffstat (limited to 'aisearch/src/gnais/evaluation/utils.py')
-rw-r--r--aisearch/src/gnais/evaluation/utils.py159
1 files changed, 159 insertions, 0 deletions
diff --git a/aisearch/src/gnais/evaluation/utils.py b/aisearch/src/gnais/evaluation/utils.py
new file mode 100644
index 0000000..8e9e335
--- /dev/null
+++ b/aisearch/src/gnais/evaluation/utils.py
@@ -0,0 +1,159 @@
+"""Common utilities for evaluation"""
+
+import asyncio
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from typing import Any
+
+import dspy
+import pandas as pd
+
+from gnais.config import Config
+from gnais.search.agent import agent_search
+from gnais.search.classification import classify_search, extract_keywords
+from gnais.search.corpus import create_ensemble_retriever, get_chroma_db, get_docs
+from gnais.search.grag import graph_rag_search
+from gnais.search.rag import rag_search
+from gnais.search.ragent import hybrid_search
+
+
+def get_dataset(
+    dataset_path: str,
+    column_names: list[str] = ["query", "answer"],
+) -> list[dspy.Example]:
+    data = pd.read_csv(
+        dataset_path,
+        usecols=column_names,
+    )
+    data_dicts = data[column_names].to_dict(orient="records")
+    evaluation_set = [
+        dspy.Example({name: x[name] for name in column_names}).with_inputs(
+            column_names[0]
+        )
+        for x in data_dicts
+    ]
+    return evaluation_set
+
+
+def mark(
+    query: str,
+    generated_response: str,
+    true_response: str,
+) -> tuple[float]:
+
+    assess = dspy.ChainOfThought("question -> answer: int")
+    context = f"""Your task is to compare elements of response in the answer generated by AI to the ones in the true answer.
+    You must compare only informations that are strictly relevant to the query.
+    User query: {query}\nAI answer: {generated_response}\nTrue answer: {true_response}"""
+
+    tp_question = f"{context}\nThe objective is to compute the number of true positives by comparing the generated answer and true answer. True positives are elements of response to the query that are mentioned in the generated answer and somewhat highlighted in the true answer as well. How many true positives do you count?"
+    tp_score = assess(question=tp_question).get("answer")
+
+    fp_question = f"{context}\nThe objective is to compute the number of false positives by comparing the generated answer and true answer. False positives are explicit and highly relevant elements of response to the query that are present in the generated answer and are not supported in any ways by the true answer. They should not be covered by a reformulation of the true answer. How many false positives do you count?"
+    fp_score = assess(question=fp_question).get("answer")
+
+    fn_question = f"{context}\nThe objective is to compute the number of false negatives by comparing the generated answer and true answer. False negatives are elements of response to the query that feature in the true answer and are not supported by the generated answer. How many false negatives do you count?"
+    fn_score = assess(question=fn_question).get("answer")
+
+    print(
+        f"Question: {query}\nTrue answer: {true_response}\nAI answer: {generated_response}"
+    )
+    print(f"TP: {tp_score}\nFP: {fp_score}\nFN: {fn_score}")
+
+    tiny_term = 1e-10
+    precision = tp_score / (tp_score + fp_score + tiny_term)
+    recall = tp_score / (tp_score + fn_score + tiny_term)
+    f1 = (2 * tp_score) / ((2 * tp_score) + fp_score + fn_score + tiny_term)
+
+    return precision, recall, f1
+
+
+_TOOL_EXECUTOR = ThreadPoolExecutor(max_workers=1, thread_name_prefix="search")
+
+
+def _run_async(async_fn, *args, **kwargs):
+    """Run async function in a separate thread with a fresh event loop"""
+
+    def _worker():
+        return asyncio.run(async_fn(*args, **kwargs))
+
+    return _TOOL_EXECUTOR.submit(_worker).result()
+
+
+def rag_digest(query: str, memory: Any = None) -> str:
+    async def _run() -> str:
+        docs = get_docs(Config.CORPUS_PATH)
+        chroma_db = get_chroma_db(embed_model="Qwen/Qwen3-Embedding-0.6B")
+        decision = classify_search(query).get("decision")
+        retriever = create_ensemble_retriever(
+            chroma_db=chroma_db,
+            docs=docs,
+            keyword_weight=0.7 if decision == "keyword" else 0.5,
+        )
+
+        parts = []
+        async for chunk in rag_search(
+            query=(
+                extract_keywords(query).get("keywords")
+                if decision == "keyword"
+                else query
+            ),
+            retriever=retriever,
+            memory=memory,
+            user_id=str(uuid.uuid4()),
+        ):
+            parts.append(str(chunk))
+        return "".join(parts)
+
+    return _run_async(_run)
+
+
+def sparql_digest(
+    query: str, handler: Any, memory: Any = None, user_id: str = "default_user"
+) -> str:
+    async def _run():
+        output = ""
+        async for chunk in handler(
+            query=query,
+            sparql_url=Config.SPARQL_ENDPOINT,
+            memory=memory,
+            user_id=user_id,
+        ):
+            if isinstance(chunk, dict) and "final" in chunk:
+                final = chunk["final"]
+                output = final
+                print(final, end="", flush=True)
+            else:
+                output += str(chunk)
+                print(chunk, end="", flush=True)
+        print()
+        return output
+
+    return _run_async(_run)
+
+
+def graph_rag_digest(query: str, memory: Any = None) -> str:
+    return sparql_digest(
+        query, graph_rag_search, memory=memory, user_id=str(uuid.uuid4())
+    )
+
+
+def agent_digest(query: str, memory: Any = None) -> str:
+    return sparql_digest(query, agent_search, memory=memory, user_id=str(uuid.uuid4()))
+
+
+def hybrid_digest(query: str, memory: Any = None) -> str:
+    async def _run():
+        final_html = None
+        async for event in hybrid_search(
+            query, memory=memory, user_id=str(uuid.uuid4())
+        ):
+            source = event["source"]
+            kind = event["kind"]
+            content = event["content"]
+            if source == "hybrid" and kind == "final":
+                final_html = content
+                break
+        return final_html
+
+    return _run_async(_run)