diff options
| author | Johannes Medagbe <johanmedagbe@gmail.com> | 2026-09-21 10:03:20 +0000 |
|---|---|---|
| committer | Johannes Medagbe <johanmedagbe@gmail.com> | 2026-09-22 12:18:32 +0000 |
| commit | dce53892085c02c8571d8a6495e39c449294aa5f (patch) | |
| tree | ae64d32cc845d30ec2a4cc3514a23dc5683a09a1 /aisearch/scripts | |
| download | genenetwork_ai_search-master.tar.gz | |
New: commit aisearch repo master
Diffstat (limited to 'aisearch/scripts')
| -rw-r--r-- | aisearch/scripts/agent_search.py | 81 | ||||
| -rw-r--r-- | aisearch/scripts/barplot_metrics.py | 86 | ||||
| -rw-r--r-- | aisearch/scripts/dspy_evaluate.py | 126 | ||||
| -rwxr-xr-x | aisearch/scripts/fetch_metadata.py | 501 | ||||
| -rw-r--r-- | aisearch/scripts/grag_search.py | 76 | ||||
| -rw-r--r-- | aisearch/scripts/new_rdf2partial_text.py | 34 | ||||
| -rw-r--r-- | aisearch/scripts/plot_function.py | 137 | ||||
| -rw-r--r-- | aisearch/scripts/rag_search.py | 111 | ||||
| -rw-r--r-- | aisearch/scripts/ragent_search.py | 170 | ||||
| -rw-r--r-- | aisearch/scripts/rdf2partial_text.py | 70 | ||||
| -rw-r--r-- | aisearch/scripts/simple_evaluate.py | 132 |
11 files changed, 1524 insertions, 0 deletions
diff --git a/aisearch/scripts/agent_search.py b/aisearch/scripts/agent_search.py new file mode 100644 index 0000000..a9e7fc8 --- /dev/null +++ b/aisearch/scripts/agent_search.py @@ -0,0 +1,81 @@ +"""This is the main module of the package using agent tool calling""" + +import argparse +import asyncio +import os +from typing import Any + +import dspy +import torch +from gnais.config import Config +from gnais.search.agent import agent_search +from mem0 import Memory +from mem0.configs.base import MemoryConfig + + +def digest(query: str, memory: Any = None, user_id: str = "default_user"): + async def _run(): + output = "" + async for chunk in agent_search( + query=query, + sparql_url=Config.SPARQL_ENDPOINT, + memory=memory, + user_id=user_id, + ): + if isinstance(chunk, dict) and "final" in chunk: + final = chunk["final"] + output = final + print(final, end="", flush=True) + else: + output += str(chunk) + print(chunk, end="", flush=True) + print() + return output + + return asyncio.run(_run()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("query", help="Search query") + parser.add_argument("--user-id", default="test-user", help="Mem0 user identity") + args = parser.parse_args() + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + os.environ[f"{Config.MEMORY_MODEL.split('/')[0].upper()}_API_KEY"] = Config.API_KEY + memory_config = MemoryConfig( + llm={ + "provider": "litellm", + "config": { + "model": Config.MEMORY_MODEL, + "temperature": 0.1, + "max_tokens": 2_000, + "api_key": Config.API_KEY, + }, + }, + embedder={ + "provider": "huggingface", + "config": { + "model": "Qwen/Qwen3-Embedding-0.6B", + "embedding_dims": 1024, + "model_kwargs": { + "trust_remote_code": True, + "device": "cuda" if torch.cuda.is_available() else "cpu", + }, + }, + }, + vector_store={ + "provider": "chroma", + "config": { + "collection_name": "mem0", + "path": os.path.join(Config.DB_PATH, "mem0_chroma"), + }, + }, + ) + memory = Memory(config=memory_config) + output = digest(args.query, memory=memory, user_id=args.user_id) + print(output) diff --git a/aisearch/scripts/barplot_metrics.py b/aisearch/scripts/barplot_metrics.py new file mode 100644 index 0000000..0932f80 --- /dev/null +++ b/aisearch/scripts/barplot_metrics.py @@ -0,0 +1,86 @@ +""" +Script to plot results with traditional metrics using local models +""" + +import os + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +INPUT1_PATH = os.getenv("INPUT1_PATH") +if INPUT1_PATH is None: + raise FileNotFoundError("Set INPUT1_PATH or path to simple results directory") + +INPUT2_PATH = os.getenv("INPUT2_PATH") +if INPUT2_PATH is None: + raise FileNotFoundError("Set INPUT2_PATH or path to dspy results directory") + +OUTPUT_PATH = os.getenv("OUTPUT_PATH") +if OUTPUT_PATH is None: + raise FileNotFoundError("Set OUTPUT_PATH or path to save final plot") + + +def format_results(simple_path: str, dspy_path: str) -> pd.DataFrame: + df_list = [] + simple_files = os.listdir(simple_path) + for f in simple_files: + model = f.strip("_simple_results.csv") + data = pd.read_csv(f"{simple_path}/{f}", index_col=0) + trans_data = data.transpose() + trans_data = pd.concat( + [trans_data[column] * 100 for column in trans_data.columns], axis=1 + ) + trans_data["model"] = model + trans_data["system"] = [sys[:-2] for sys in list(trans_data.index)] + df_list.append(trans_data) + final_concat = pd.concat(df_list) + dspy_files = os.listdir(dspy_path) + cumul = [] + for f in dspy_files: + data = pd.read_csv(f"{dspy_path}/{f}") + aligned = pd.concat([data[column] for column in data.columns]) + cumul.extend(aligned) + final_concat["satisfaction frequency (%)"] = cumul + return final_concat + + +def plot(data: pd.DataFrame, columns: list[str], output_path: str, title: str): + fig, axes = plt.subplots(len(columns), 1, figsize=(10, 10)) + axes = axes.flatten() + for ind, column in enumerate(columns): + sns.barplot( + data=data, + ax=axes[ind], + x="system", + y=column, + hue="model", + palette="ocean_r", + ) + fig.suptitle(title, fontsize=15) + plt.tight_layout() + plt.savefig(output_path, dpi=1000) + plt.show() + + +def plot_results(data: pd.DataFrame, output_path: str): + metrics = list(data.columns)[:3] + metrics.append(data.columns[-1]) + plot( + data, + metrics, + f"{output_path}/barplot_all_metrics.png", + "Performance comparison of standard and new metrics", + ) + extras = list(data.columns)[3:5] + plot( + data, + extras, + f"{output_path}/barplot_extra_metrics.png", + "Insights from additional metrics", + ) + + +if __name__ == "__main__": + df = format_results(INPUT1_PATH, INPUT2_PATH) + plot_results(df, OUTPUT_PATH) diff --git a/aisearch/scripts/dspy_evaluate.py b/aisearch/scripts/dspy_evaluate.py new file mode 100644 index 0000000..b7d6d6f --- /dev/null +++ b/aisearch/scripts/dspy_evaluate.py @@ -0,0 +1,126 @@ +"""Script for performance evaluation of GN AI systems using DSPy.Evaluate and custom decision function""" + +import os +from typing import Any + +import dspy +import pandas as pd +import torch +from gnais.config import Config +from gnais.evaluation.utils import ( + agent_digest, + get_dataset, + graph_rag_digest, + hybrid_digest, + mark, + rag_digest, +) + + +def evaluator( + gold: dspy.Example, + pred: str, + trace=None, + pred_name=None, + pred_trace=None, +) -> int: + query = gold["query"] + true_response = gold["answer"] + precision, recall, f1 = mark(query, pred, true_response) + return 1 if (f1 >= 0.6 or recall >= 0.7 or precision >= 0.7) else 0 + + +def run_eval( + runner: dspy.Evaluate, + evaluation_set: list[dspy.Example], + search_func: Any, +) -> dspy.evaluate.EvaluationResult: + + search = dspy.Tool( + name="search", + desc="Answer the question asked by the user.", + args={ + "query": { + "type": "string", + "desc": "question asked by the user", + }, + }, + func=search_func, + ) + + class Signature(dspy.Signature): + query: str = dspy.InputField(desc="User question") + answer: str = dspy.OutputField( + desc="Answer to user's query with details and explanations" + ) + + # Wrap hybrid search into a DSPy module that is compatible with DSPy evaluation + program = dspy.ReAct( + signature=Signature, tools=[search], max_iters=2 + ) # limit iterations + + return runner(program) + + +if __name__ == "__main__": + DATASET_PATH = os.getenv("DATASET_PATH") + if DATASET_PATH is None: + raise FileNotFoundError("DATASET_PATH must be set for evaluation") + + OUTPUT_PATH = os.getenv("OUTPUT_PATH") + if OUTPUT_PATH is None: + raise FileNotFoundError("OUTPUT_PATH must be set for evaluation") + + N_ITERATIONS = int(os.getenv("N_ITERATIONS")) + if N_ITERATIONS is None: + raise ValueError("N_ITERATIONS must be set for evaluation") + + JUDGE_MODEL = os.getenv("JUDGE_MODEL") + if JUDGE_MODEL is None: + raise ValueError("JUDGE MODEL must be set for evaluation") + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + + judge_llm = dspy.LM( + model=JUDGE_MODEL, + api_key=Config.API_KEY, + max_tokens=1_000, + temperature=1, + cache=False, + verbose=False, + ) + + evaluation_set = get_dataset(DATASET_PATH) + evaluate = dspy.Evaluate( + devset=evaluation_set, + metric=evaluator, + num_threads=1, + provide_traceback=True, + display_table=False, + display_progress=True, + lm=judge_llm, + ) + collection = {} + + # Run evaluation set with LLM only + base = dspy.ChainOfThought("query -> answer: str") + base_output = [evaluate(base).get("score") for n in range(N_ITERATIONS)] + collection["base"] = base_output + + # Run evaluation set with GN systems + for system in [rag_digest, graph_rag_digest, agent_digest, hybrid_digest]: + system_name = " ".join(system.__name__.split("_")[:-1]) + print(f"Running evaluation for {system_name}") + temp = [] + for n in range(N_ITERATIONS): + print(f"Iteration {n+1}") + system_output = run_eval(evaluate, evaluation_set, system) + temp.append(system_output.get("score")) + collection[system_name] = temp + print(f"Evaluation completed for {system_name}") + final = pd.DataFrame(collection) + final.to_csv(OUTPUT_PATH, index=False) diff --git a/aisearch/scripts/fetch_metadata.py b/aisearch/scripts/fetch_metadata.py new file mode 100755 index 0000000..36b9570 --- /dev/null +++ b/aisearch/scripts/fetch_metadata.py @@ -0,0 +1,501 @@ +#! /usr/bin/env python3 +"""Fetch RDF triples grouped by subject and convert to natural language sentences. + +This script fetches data from GeneNetwork's SPARQL endpoint, grouping triples by +subject and converting them to human-readable sentences. It processes different +entity types in parallel using threads and writes each type to its own file. +""" + +import argparse +import json +import os +import random +import time +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Dict, List, Optional, Tuple +from urllib.error import HTTPError + +from SPARQLWrapper import JSON, SPARQLWrapper + +SPARQL_ENDPOINT = "https://rdf.genenetwork.org/sparql" +DEFAULT_GRAPH = "http://rdf.genenetwork.org/v1" + +PREFIXES = { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "rdfs": "http://www.w3.org/2000/01/rdf-schema#", + "skos": "http://www.w3.org/2004/02/skos/core#", + "dct": "http://purl.org/dc/terms/", + "gnc": "http://rdf.genenetwork.org/v1/category/", + "gnt": "http://rdf.genenetwork.org/v1/term/", + "gn": "http://rdf.genenetwork.org/v1/id/", + "xkos": "http://rdf-vocabulary.ddialliance.org/xkos#", + "fabio": "http://purl.org/spar/fabio/", + "prism": "http://prismstandard.org/namespaces/basic/2.0/", + "owl": "http://www.w3.org/2002/07/owl#", + "xsd": "http://www.w3.org/2001/XMLSchema#", + "dcat": "http://www.w3.org/ns/dcat#", + "qb": "http://purl.org/linked-data/cube#", + "schema": "https://schema.org/", + "pubmed": "http://rdf.ncbi.nlm.nih.gov/pubmed/", + "obo": "http://purl.obolibrary.org/obo/", + "bfo": "http://purl.obolibrary.org/obo/BFO_", + "sdmx-measure": "http://purl.org/linked-data/sdmx/2009/measure#", + "foaf": "http://xmlns.com/foaf/0.1/", +} + +TEMPLATES = { + # RDF / RDFS + "http://www.w3.org/1999/02/22-rdf-syntax-ns#type": "is a {obj}", + "http://www.w3.org/2000/01/rdf-schema#label": "is labeled as '{obj}'", + "http://www.w3.org/2000/01/rdf-schema#comment": "has the note: {obj}", + # SKOS + "http://www.w3.org/2004/02/skos/core#prefLabel": "is called '{obj}'", + "http://www.w3.org/2004/02/skos/core#altLabel": "is also known as '{obj}'", + "http://www.w3.org/2004/02/skos/core#definition": "is defined as: {obj}", + "http://www.w3.org/2004/02/skos/core#member": "includes member {obj}", + "http://www.w3.org/2004/02/skos/core#notation": "has notation {obj}", + "http://www.w3.org/2004/02/skos/core#inScheme": "is in scheme {obj}", + # DCTERMS + "http://purl.org/dc/terms/title": "has title '{obj}'", + "http://purl.org/dc/terms/description": "is described as: '{obj}'", + "http://purl.org/dc/terms/identifier": "has identifier {obj}", + "http://purl.org/dc/terms/abstract": "has abstract '{obj}'", + "http://purl.org/dc/terms/creator": "was created by {obj}", + "http://purl.org/dc/terms/contributor": "has contributor {obj}", + "http://purl.org/dc/terms/created": "was created on {obj}", + "http://purl.org/dc/terms/references": "references {obj}", + "http://purl.org/dc/terms/hasVersion": "has version {obj}", + # GeneNetwork Terms + "http://rdf.genenetwork.org/v1/term/has_set_code": "provides a unique identifier code for a resource set {obj}", + "http://rdf.genenetwork.org/v1/term/genetic_type": "describes the genetic architecture of a resource set as {obj}", + "http://rdf.genenetwork.org/v1/term/has_species": "belongs to species {obj}", + "http://rdf.genenetwork.org/v1/term/has_strain": "lists all strains that belong to this resource including {obj}", + "http://rdf.genenetwork.org/v1/term/has_taxonomic_family": "links to taxonomic family {obj}", + "http://rdf.genenetwork.org/v1/term/has_uniprot_taxon_id": "has UniProt taxonomic id {obj}", + "http://rdf.genenetwork.org/v1/term/short_name": "has short name {obj}", + "http://rdf.genenetwork.org/v1/term/uses_mapping_method": "uses mapping method {obj}", + "http://rdf.genenetwork.org/v1/term/has_reference_population": "has reference population {obj}", + "http://rdf.genenetwork.org/v1/term/alias": "has alias {obj}", + "http://rdf.genenetwork.org/v1/term/has_phenotype": "measures phenotype {obj}", + "http://rdf.genenetwork.org/v1/term/abbreviation": "has abbreviation {obj}", + "http://rdf.genenetwork.org/v1/term/has_lab_code": "has lab code {obj}", + "http://rdf.genenetwork.org/v1/term/lod_score": "has LOD score (Peak -logP) {obj}", + "http://rdf.genenetwork.org/v1/term/mean": "has mean value {obj}", + "http://rdf.genenetwork.org/v1/term/submitter": "was submitted by {obj}", + "http://rdf.genenetwork.org/v1/term/has_genotype_files": "has genotype files {obj}", + "http://rdf.genenetwork.org/v1/term/has_marker_count": "has {obj} DNA markers/SNPs", + "http://rdf.genenetwork.org/v1/term/symbol": "has gene symbol {obj}", + "http://rdf.genenetwork.org/v1/term/chr": "is on chromosome {obj}", + "http://rdf.genenetwork.org/v1/term/chromosome": "is on chromosome {obj}", + "http://rdf.genenetwork.org/v1/term/mb": "is at {obj} megabases", + "http://rdf.genenetwork.org/v1/term/sequence": "has sequence {obj}", + "http://rdf.genenetwork.org/v1/term/has_target_id": "has target id {obj}", + "http://rdf.genenetwork.org/v1/term/targets_region": "targets region {obj}", + "http://rdf.genenetwork.org/v1/term/has_specificity": "has specificity {obj}", + "http://rdf.genenetwork.org/v1/term/has_blat_score": "has BLAT score {obj}", + "http://rdf.genenetwork.org/v1/term/has_blat_mb_start": "has BLAT start position {obj}", + "http://rdf.genenetwork.org/v1/term/has_blat_mb_end": "has BLAT end position {obj}", + "http://rdf.genenetwork.org/v1/term/has_blat_seq": "has BLAT sequence {obj}", + "http://rdf.genenetwork.org/v1/term/has_target_seq": "has target sequence {obj}", + "http://rdf.genenetwork.org/v1/term/has_homologene_id": "has HomoloGene ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_uniprot_id": "has UniProt ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_pub_chem_id": "has PubChem ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_kegg_id": "has KEGG ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_omim_id": "has OMIM ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_chebi_id": "has ChEBI ID {obj}", + "http://rdf.genenetwork.org/v1/term/initial": "has initial {obj}", + "http://rdf.genenetwork.org/v1/term/reason": "was modified because {obj}", + "http://rdf.genenetwork.org/v1/term/belongs_to_category": "belongs to category {obj}", + "http://rdf.genenetwork.org/v1/term/has_gene_id": "has gene ID {obj}", + "http://rdf.genenetwork.org/v1/term/gene_symbol": "has gene symbol {obj}", + "http://rdf.genenetwork.org/v1/term/transcript": "has transcript {obj}", + "http://rdf.genenetwork.org/v1/term/strand": "is on strand {obj}", + "http://rdf.genenetwork.org/v1/term/tx_start": "has transcription start {obj}", + "http://rdf.genenetwork.org/v1/term/tx_end": "has transcription end {obj}", + "http://rdf.genenetwork.org/v1/term/has_align_id": "has alignment ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_protein_id": "has protein ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_rgd_id": "has RGD ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_geo_series_id": "has GEO series ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_go_tree_value": "has GO tree value {obj}", + "http://rdf.genenetwork.org/v1/term/has_case_info": "has case info: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_citation": "has citation: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_contributors": "has contributors: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_data_processing_info": "has data processing info: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_experiment_design": "has experiment design: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_experiment_design_info": "has experiment design info: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_experiment_type": "has experiment type: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_molecular_trait": "has molecular trait {obj}", + "http://rdf.genenetwork.org/v1/term/has_phenotype_data": "has phenotype data {obj}", + "http://rdf.genenetwork.org/v1/term/has_phenotype_trait": "has phenotype trait {obj}", + "http://rdf.genenetwork.org/v1/term/has_platform_info": "has platform info: {obj}", + "http://rdf.genenetwork.org/v1/term/has_probeset_data": "has probeset data {obj}", + "http://rdf.genenetwork.org/v1/term/has_samples": "has samples {obj}", + "http://rdf.genenetwork.org/v1/term/has_specifics": "has specifics: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_summary": "has summary: '{obj}'", + "http://rdf.genenetwork.org/v1/term/has_tissue_info": "has tissue info: '{obj}'", + "http://rdf.genenetwork.org/v1/term/uses_genechip": "uses genechip platform {obj}", + "http://rdf.genenetwork.org/v1/term/uses_normalization_method": "uses normalization method {obj}", + "http://rdf.genenetwork.org/v1/term/has_probeset": "has probeset {obj}", + "http://rdf.genenetwork.org/v1/term/has_genotype_data": "has genotype data {obj}", + "http://rdf.genenetwork.org/v1/term/has_sequence": "has sequence {obj}", + "http://rdf.genenetwork.org/v1/term/source": "has source {obj}", + "http://rdf.genenetwork.org/v1/term/has_kg_id": "has KG ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_unigen_id": "has UniGene ID {obj}", + "http://rdf.genenetwork.org/v1/term/additive": "has additive effect {obj}", + "http://rdf.genenetwork.org/v1/term/se": "has standard error {obj}", + "http://rdf.genenetwork.org/v1/term/pvalue": "has p-value {obj}", + "http://rdf.genenetwork.org/v1/term/h2": "has heritability (h2) {obj}", + "http://rdf.genenetwork.org/v1/term/assigned_species": "has assigned species {obj}", + "http://rdf.genenetwork.org/v1/term/has_family_order_id": "has family order ID {obj}", + "http://rdf.genenetwork.org/v1/term/has_population_order_id": "has population order ID {obj}", + "http://rdf.genenetwork.org/v1/term/locus": "has locus {obj}", + "http://rdf.genenetwork.org/v1/term/mb_mm8": "has position (mm8) {obj} megabases", + # FOAF + "http://xmlns.com/foaf/0.1/homepage": "has homepage {obj}", + "http://xmlns.com/foaf/0.1/name": "has name '{obj}'", + "http://xmlns.com/foaf/0.1/familyName": "has family name '{obj}'", + "http://xmlns.com/foaf/0.1/givenName": "has given name '{obj}'", + "http://xmlns.com/foaf/0.1/mbox": "has email {obj}", + # XKOS + "http://rdf-vocabulary.ddialliance.org/xkos#depth": "is at depth level {obj}", + # FABIO + "http://purl.org/spar/fabio/hasPubMedId": "has PubMed ID {obj}", + "http://purl.org/spar/fabio/Journal": "was published in journal '{obj}'", + "http://purl.org/spar/fabio/page": "is on pages {obj}", + "http://purl.org/spar/fabio/hasPublicationYear": "was published in year {obj}", + # PRISM + "http://prismstandard.org/namespaces/basic/2.0/volume": "is in volume {obj}", + "http://prismstandard.org/namespaces/basic/2.0/publicationDate": "was published on {obj}", + # DCAT + "http://www.w3.org/ns/dcat#distribution": "has distribution {obj}", + "http://www.w3.org/ns/dcat#isPartOf": "is part of {obj}", + # OWL + "http://www.w3.org/2002/07/owl#equivalentClass": "is equivalent to {obj}", +} + + +TYPE_QUERIES = [ + "a dcat:Dataset", + "a gnc:set", + "a gnc:species", + "a gnc:reference_population", + "a skos:Concept", + "a gnc:molecular_trait", + "a gnc:phenotype", + "a gnc:strain", + "a fabio:ResearchPaper", + "a gnc:gene", + "a gnc:gene_symbol", + "a gnc:dna_marker", + "a gnc:ncbi_wiki_entry", +] + + +def type_pattern_to_filename(type_pattern: str) -> str: + """Convert a type pattern to a safe filename.""" + return f"{type_pattern.split(':')[-1].lower()}.json" + + +def uri_to_prefixed(uri: str) -> str: + """Convert full URI to prefixed form.""" + for prefix, ns in PREFIXES.items(): + if uri.startswith(ns): + return f"{prefix}:{uri[len(ns) :]}" + return uri + + +def triple_to_sentence(subj: str, pred: str, obj: str) -> str: + """Convert triple to natural language sentence.""" + obj_prefixed = uri_to_prefixed(obj) + if pred in TEMPLATES: + return f"{subj} {TEMPLATES[pred].format(obj=obj_prefixed)}." + pred_prefixed = uri_to_prefixed(pred) + return f"{subj} {pred_prefixed} {obj_prefixed}." + + +def fetch_with_retry( + sparql: SPARQLWrapper, max_retries: int = 3, base_delay: float = 2.0 +) -> Optional[Dict]: + """Execute SPARQL query with retry logic and exponential backoff.""" + for attempt in range(max_retries): + try: + return sparql.queryAndConvert() + except HTTPError as e: + if e.code == 504 and attempt < max_retries - 1: + # Exponential backoff with jitter + delay = base_delay * (2**attempt) + random.uniform(0, 1) + print( + f" Timeout (attempt {attempt + 1}/{max_retries}), retrying in {delay:.1f}s..." + ) + time.sleep(delay) + else: + raise + return None + + +def fetch_count_for_type(type_pattern: str) -> int: + """Get the total count of triples for a type pattern. + + Args: + type_pattern: The type pattern to count + + Returns: + Total count of triples + """ + sparql = SPARQLWrapper(SPARQL_ENDPOINT) + sparql.setReturnFormat(JSON) + sparql.setTimeout(60) + + query = f""" + PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> + PREFIX dcat: <http://www.w3.org/ns/dcat#> + PREFIX gnc: <http://rdf.genenetwork.org/v1/category/> + PREFIX skos: <http://www.w3.org/2004/02/skos/core#> + PREFIX fabio: <http://purl.org/spar/fabio/> + PREFIX foaf: <http://xmlns.com/foaf/0.1/> + + SELECT (COUNT(*) AS ?count) + FROM <{DEFAULT_GRAPH}> + WHERE {{ + ?s ?p ?o ; + {type_pattern} . + }}""" + + sparql.setQuery(query) + + try: + result = fetch_with_retry(sparql) + if result: + count_str = result["results"]["bindings"][0]["count"]["value"] + return int(count_str) + except Exception as e: + print(f" Warning: Could not get count: {e}") + + return 0 + + +def fetch_by_type_pattern(type_pattern: str, page_size: int = 5000) -> List[Dict]: + """Fetch triples for a specific type pattern using pagination. + + First gets the count, then calculates pages needed. + + Args: + type_pattern: The type pattern to query + page_size: Number of triples per page + Returns: + + List of SPARQL binding dictionaries + """ + # First, get the count + print(f" Getting count for {type_pattern}...") + total_count = fetch_count_for_type(type_pattern) + + if total_count == 0: + print(f" No data found for {type_pattern}") + return [] + + num_pages = (total_count + page_size - 1) // page_size # Ceiling division + print(f" Total: {total_count} triples, fetching {num_pages} pages...") + + all_bindings = [] + sparql = SPARQLWrapper(SPARQL_ENDPOINT) + sparql.setReturnFormat(JSON) + sparql.setTimeout(120000) + + for page in range(num_pages): + offset = page * page_size + + query = f""" + PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> + PREFIX dcat: <http://www.w3.org/ns/dcat#> + PREFIX gnc: <http://rdf.genenetwork.org/v1/category/> + PREFIX skos: <http://www.w3.org/2004/02/skos/core#> + PREFIX fabio: <http://purl.org/spar/fabio/> + PREFIX foaf: <http://xmlns.com/foaf/0.1/> + + SELECT ?s ?p ?o + FROM <{DEFAULT_GRAPH}> + WHERE {{ + ?s ?p ?o ; + {type_pattern} . + }} + LIMIT {page_size} + OFFSET {offset}""" + + sparql.setQuery(query) + + try: + result = fetch_with_retry(sparql) + if result is None: + break + + bindings = result.get("results", {}).get("bindings", []) + all_bindings.extend(bindings) + + # Progress indicator + if (page + 1) % 5 == 0 or page == num_pages - 1: + print( + f" Fetched page {page + 1}/{num_pages} ({len(all_bindings)}/{total_count} triples) :: {type_pattern_to_filename(type_pattern)}" + ) + + # Small delay between pages + if page < num_pages - 1: + time.sleep(0.3) + + except Exception as e: + print(f" ERROR on page {page + 1}: {e}") + break + + return all_bindings + + +def group_by_subject(bindings: List[Dict]) -> Dict[str, List[Tuple[str, str]]]: + """Group triples by subject URI.""" + grouped = defaultdict(list) + for b in bindings: + subj = b["s"]["value"] + pred = b["p"]["value"] + obj = b["o"]["value"] + grouped[subj].append((pred, obj)) + return grouped + + +def build_sentences(grouped: Dict[str, List[Tuple[str, str]]]) -> List[str]: + """Build list of sentence groups, one per subject.""" + result = [] + for subj_uri, triples in grouped.items(): + subject = uri_to_prefixed(subj_uri) + sentences = [triple_to_sentence(subject, pred, obj) for pred, obj in triples] + result.append(" ".join(sentences)) + return result + + +def process_type_query( + type_pattern: str, output_dir: str, page_size: int = 5000 +) -> Tuple[str, int, bool]: + """Process a single type query and write results to a file. + + Args: + type_pattern: The type pattern to query + output_dir: Directory to write the output file + page_size: Number of triples per page + + Returns: + Tuple of (type_pattern, count, success) + """ + filename = type_pattern_to_filename(type_pattern) + output_path = os.path.join(output_dir, filename) + + # Add delay at start to stagger requests when running in parallel + time.sleep(random.uniform(0.5, 2.0)) + + try: + print(f"Fetching {type_pattern}...") + bindings = fetch_by_type_pattern(type_pattern, page_size=page_size) + + if not bindings: + with open(output_path, "w") as f: + json.dump([], f, indent=2) + return type_pattern, 0, True + + print(f" Processing {len(bindings)} triples for {type_pattern}") + + grouped = group_by_subject(bindings) + print(f" Grouped into {len(grouped)} subjects") + + sentences = build_sentences(grouped) + + with open(output_path, "w") as f: + json.dump(sentences, f, indent=2) + + print(f" Wrote {len(sentences)} sentences to {filename}") + return type_pattern, len(sentences), True + + except Exception as e: + print(f" ERROR processing {type_pattern}: {e}") + return type_pattern, 0, False + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch RDF triples as natural language, one file per type" + ) + parser.add_argument( + "--output-dir", + "-o", + default="metadata_output", + help="Output directory for JSON files (default: metadata_output)", + ) + parser.add_argument( + "--workers", + "-w", + type=int, + default=2, + help="Number of parallel workers (default: 2, use 1 for sequential)", + ) + parser.add_argument( + "--page-size", + "-p", + type=int, + default=5000, + help="Number of triples per page (default: 5000)", + ) + args = parser.parse_args() + + os.makedirs(args.output_dir, exist_ok=True) + print(f"Output directory: {args.output_dir}") + print(f"Processing {len(TYPE_QUERIES)} type queries with {args.workers} workers...") + print(f"Page size: {args.page_size}") + print() + + results = [] + + if args.workers == 1: + # Sequential processing - more polite to the server + for type_pattern in TYPE_QUERIES: + result = process_type_query(type_pattern, args.output_dir, args.page_size) + results.append(result) + # Delay between types + time.sleep(1.0) + else: + # Parallel processing with limited workers + with ThreadPoolExecutor(max_workers=args.workers) as executor: + future_to_type = { + executor.submit( + process_type_query, tq, args.output_dir, args.page_size + ): tq + for tq in TYPE_QUERIES + } + + for future in as_completed(future_to_type): + type_pattern = future_to_type[future] + try: + _, count, success = future.result() + results.append((type_pattern, count, success)) + except Exception as e: + print(f" UNEXPECTED ERROR for {type_pattern}: {e}") + results.append((type_pattern, 0, False)) + + # Summary + print() + print("=" * 60) + print("SUMMARY") + print("=" * 60) + + successful = sum(1 for _, _, s in results if s) + total_sentences = sum(c for _, c, s in results if s) + + for type_pattern, count, success in sorted(results): + status = "✓" if success else "✗" + print(f"{status} {type_pattern}: {count} sentences") + + print() + print(f"Successfully processed: {successful}/{len(TYPE_QUERIES)}") + print(f"Total sentences written: {total_sentences}") + print(f"Output files in: {args.output_dir}") + + +if __name__ == "__main__": + print("script starting", flush=True) + main() diff --git a/aisearch/scripts/grag_search.py b/aisearch/scripts/grag_search.py new file mode 100644 index 0000000..29d4c09 --- /dev/null +++ b/aisearch/scripts/grag_search.py @@ -0,0 +1,76 @@ +import argparse +import asyncio +import os +import sys +from typing import Any + +import dspy +import torch +from gnais.config import Config +from gnais.search.grag import graph_rag_search +from gnais.search.prompts import GN_FACT_EXTRACTION_PROMPT, GN_UPDATE_MEMORY_PROMPT +from mem0 import Memory +from mem0.configs.base import MemoryConfig + + +def digest(query: str, memory: Memory = None, user_id: str = "default_user"): + async def _run(): + async for chunk in graph_rag_search( + query=query, + sparql_url=Config.SPARQL_ENDPOINT, + memory=memory, + user_id=user_id, + ): + print(chunk, end="", flush=True) + print() + print("Done") + + return asyncio.run(_run()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("query", help="Search query") + parser.add_argument("--user-id", default="test-user", help="Mem0 user identity") + args = parser.parse_args() + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + + os.environ[f"{Config.MEMORY_MODEL.split('/')[0].upper()}_API_KEY"] = Config.API_KEY + memory_config = MemoryConfig( + custom_fact_extraction_prompt=GN_FACT_EXTRACTION_PROMPT, + custom_update_extraction_prompt=GN_UPDATE_MEMORY_PROMPT, + llm={ + "provider": "litellm", + "config": { + "model": Config.MEMORY_MODEL, # DEFAULT_MODEL is set to be bigger + "temperature": 0.1, + "max_tokens": 2_000, + "api_key": Config.API_KEY, + }, + }, + embedder={ + "provider": "huggingface", + "config": { + "model": "Qwen/Qwen3-Embedding-0.6B", + "embedding_dims": 1024, + "model_kwargs": { + "trust_remote_code": True, + "device": "cuda" if torch.cuda.is_available() else "cpu", + }, + }, + }, + vector_store={ + "provider": "chroma", + "config": { + "collection_name": "mem0", + "path": os.path.join(Config.DB_PATH, "mem0_chroma"), + }, + }, + ) + memory = Memory(config=memory_config) + print(digest(args.query, memory=memory, user_id=args.user_id)) diff --git a/aisearch/scripts/new_rdf2partial_text.py b/aisearch/scripts/new_rdf2partial_text.py new file mode 100644 index 0000000..29b99f8 --- /dev/null +++ b/aisearch/scripts/new_rdf2partial_text.py @@ -0,0 +1,34 @@ +import json +import os +import string + +dir = os.getenv("DIR") +if dir is None: + raise ValueError("DIR not specified") + +collection = {} + +files = [os.path.join(dir, file) for file in os.listdir(dir) if "ttl" in file] + + +def clean(text) -> str: + return text.strip(string.punctuation).strip() + + +for file in files: + with open(file) as f: + lines = f.readlines() + for ind, line in enumerate(lines): + line = line.strip() + if len(line) > 1: + contents = line.split(" ") + key = clean(contents[0]) + value = " ".join(clean(content) for content in contents[1:]) + if key not in collection: + collection[key] = [value] + else: + if value not in collection[key]: + collection[key].append(value) + +with open(f"{dir}aggr_rdf.txt", "w") as new: + new.write(json.dumps(collection)) diff --git a/aisearch/scripts/plot_function.py b/aisearch/scripts/plot_function.py new file mode 100644 index 0000000..1cdbc43 --- /dev/null +++ b/aisearch/scripts/plot_function.py @@ -0,0 +1,137 @@ +""" +Script to plot results with new metric using local models +""" + +import os + +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +INPUT_PATH = os.getenv("INPUT_PATH") +if INPUT_PATH is None: + raise FileNotFoundError("Set INPUT_PATH or path to results directory") + +OUTPUT1_PATH = os.getenv("OUTPUT1_PATH") +if OUTPUT1_PATH is None: + raise FileNotFoundError( + "Set OUTPUT1_PATH or path to save final boxplot for systems" + ) + +OUTPUT2_PATH = os.getenv("OUTPUT2_PATH") +if OUTPUT2_PATH is None: + raise FileNotFoundError("Set OUTPUT2_PATH or path to save final barplot for models") + + +def format_results(path: str) -> tuple[pd.DataFrame]: + files = os.listdir(path) + full_list = [] + aggr_list = [] + for f in files: + model = f.strip("_dspy_results.csv") + data = pd.read_csv(f"{path}/{f}") + + aggr = pd.DataFrame( + data.aggregate("median"), columns=["satisfaction frequency (%)"] + ) + aggr["model"] = model + aggr["system"] = list(aggr.index) + aggr_list.append(aggr) + + full = data.transpose() + full = pd.concat([full[column] for column in full.columns]) + full_df = pd.DataFrame(full, columns=["satisfaction frequency (%)"]) + full_df["model"] = model + full_df["system"] = list(full.index) + full_list.append(full_df) + + full_concat = pd.concat(full_list) + full_concat["model_order"] = pd.Categorical( + full_concat["model"], + categories=[ + "gptoss_20b", + "qwen3_30b", + "haiku_4.5", + "qwen3_30b_opus_4.8", + "haiku_4.5_opus_4.8", + "opus_4.8", + ], + ordered=False, + ) + full_concat["system_order"] = pd.Categorical( + full_concat["system"], + categories=["base", "rag", "graph rag", "agent", "combi"], + ordered=False, + ) + + aggr_concat = pd.concat(aggr_list) + aggr_concat["model_order"] = pd.Categorical( + aggr_concat["model"], + categories=[ + "gptoss_20b", + "qwen3_30b", + "haiku_4.5", + "qwen3_30b_opus_4.8", + "haiku_4.5_opus_4.8", + "opus_4.8", + ], + ordered=False, + ) + aggr_concat["system_order"] = pd.Categorical( + aggr_concat["system"], + categories=["base", "rag", "graph rag", "agent", "combi"], + ordered=False, + ) + return full_concat.sort_values( + by=["system_order", "model_order"] + ), aggr_concat.sort_values(by=["system_order", "model_order"]) + + +def plot_results( + full_data: pd.DataFrame, + aggr_data: pd.DataFrame, + output1_path: str, + output2_path: str, +): + sns.boxplot( + data=full_data, x="system", y="satisfaction frequency (%)", color="#708090" + ) + sns.swarmplot( + data=full_data, + x="system", + y="satisfaction frequency (%)", + hue="model", + palette="ocean_r", + ) + plt.savefig(output1_path, dpi=1000) + plt.show() + # heatmap_data = aggr_data.pivot( + # index="model", columns="system", values="satisfaction frequency (%)" + # ) + # heatmap_data = heatmap_data.reindex( + # index=[ + # "opus_4.8", + # "haiku_4.5_opus_4.8", + # "qwen3_30b_opus_4.8", + # "haiku_4.5", + # "qwen3_30b", + # "gptoss_20b", + # ], + # columns=["base", "rag", "graph rag", "agent", "hybrid"], + # ) + sns.barplot( + full_data, + x="system", + y="satisfaction frequency (%)", + hue="model", + palette="ocean_r", + ) + plt.suptitle("Model performance with customized metric", fontsize=10) + plt.tight_layout() + plt.savefig(output2_path, dpi=1000) + plt.show() + + +if __name__ == "__main__": + full_df, aggr_df = format_results(INPUT_PATH) + plot_results(full_df, aggr_df, OUTPUT1_PATH, OUTPUT2_PATH) diff --git a/aisearch/scripts/rag_search.py b/aisearch/scripts/rag_search.py new file mode 100644 index 0000000..20d9863 --- /dev/null +++ b/aisearch/scripts/rag_search.py @@ -0,0 +1,111 @@ +"""This is the main module of the package""" + +import argparse +import asyncio +import os +from functools import partial + +import dspy +import torch +from mem0 import Memory +from mem0.configs.base import MemoryConfig + +from gnais.config import Config +from gnais.search.classification import classify_search, extract_keywords +from gnais.search.corpus import ( + create_ensemble_retriever, + get_chroma_db, + get_docs, + init_chroma_db, +) +from gnais.search.prompts import GN_FACT_EXTRACTION_PROMPT, GN_UPDATE_MEMORY_PROMPT +from gnais.search.rag import rag_search + + +def _digest( + query: str, memory: Memory, user_id: str, corpus_path: str, db_path: str +) -> str: + """Run the full RAG pipeline for a single query and return the answer. + + This is a convenience wrapper around the same logic used by the CLI + script, packaged as a plain synchronous function. + """ + + async def _run() -> str: + + docs = get_docs(corpus_path) + chroma_db = get_chroma_db(embed_model="Qwen/Qwen3-Embedding-0.6B") + decision = classify_search(query).get("decision") + retriever = create_ensemble_retriever( + chroma_db=chroma_db, + docs=docs, + keyword_weight=0.7 if decision == "keyword" else 0.5, + ) + + parts = [] + async for chunk in rag_search( + query=( + extract_keywords(query).get("keywords") + if decision == "keyword" + else query + ), + retriever=retriever, + memory=memory, + user_id=user_id, + ): + parts.append(str(chunk)) + return "".join(parts) + + return asyncio.run(_run()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("query", help="Search query") + parser.add_argument("--user-id", default="test-user", help="Mem0 user identity") + args = parser.parse_args() + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + + os.environ[f"{Config.MEMORY_MODEL.split('/')[0].upper()}_API_KEY"] = Config.API_KEY + memory_config = MemoryConfig( + custom_fact_extraction_prompt=GN_FACT_EXTRACTION_PROMPT, + custom_update_extraction_prompt=GN_UPDATE_MEMORY_PROMPT, + llm={ + "provider": "litellm", + "config": { + "model": Config.MEMORY_MODEL, + "temperature": 0.1, + "max_tokens": 2_000, + "api_key": Config.API_KEY, + }, + }, + embedder={ + "provider": "huggingface", + "config": { + "model": "Qwen/Qwen3-Embedding-0.6B", + "embedding_dims": 1024, + "model_kwargs": { + "trust_remote_code": True, + "device": "cuda" if torch.cuda.is_available() else "cpu", + }, + }, + }, + vector_store={ + "provider": "chroma", + "config": { + "collection_name": "mem0", + "host": "localhost", + "port": 8001, + }, + }, + ) + memory = Memory(config=memory_config) + digest = partial( + _digest, memory=memory, corpus_path=Config.CORPUS_PATH, db_path=Config.DB_PATH + ) + print(digest(query=args.query, user_id=args.user_id)) diff --git a/aisearch/scripts/ragent_search.py b/aisearch/scripts/ragent_search.py new file mode 100644 index 0000000..6f11b8c --- /dev/null +++ b/aisearch/scripts/ragent_search.py @@ -0,0 +1,170 @@ +"""Main module of hybrid search for GeneNetwork""" + +import argparse +import asyncio +import os + +import dspy +import torch +from gnais.config import Config +from gnais.search.prompts import GN_FACT_EXTRACTION_PROMPT, GN_UPDATE_MEMORY_PROMPT +from gnais.search.ragent import hybrid_search +from mem0 import Memory +from mem0.configs.base import MemoryConfig +from rich.columns import Columns +from rich.console import Console +from rich.live import Live +from rich.panel import Panel +from rich.text import Text + + +class _HybridDisplay: + """Terminal UI with three live panels for RAG / GRAG / Agent.""" + + _COLORS = { + "rag": "cyan", + "grag": "green", + "agent": "yellow", + "synthesis": "magenta", + } + + def __init__(self): + self._texts = { + "rag": Text("", end=""), + "grag": Text("", end=""), + "agent": Text("", end=""), + "synthesis": Text("", end=""), + } + self._status = { + "rag": "⏳ waiting", + "grag": "⏳ waiting", + "agent": "⏳ waiting", + "synthesis": "⏳ waiting", + } + self._live = Live(self._render(), refresh_per_second=10) + + def _render(self): + panels = [] + for src in ("rag", "grag", "agent", "synthesis"): + color = self._COLORS[src] + title = f"[bold {color}]{src.upper()}[/] — {self._status[src]}" + panels.append( + Panel( + self._texts[src], + title=title, + border_style=color, + expand=True, + ) + ) + return Columns(panels, equal=True) + + def start(self): + self._live.start() + + def stop(self): + self._live.stop() + + def append(self, source: str, text: str): + if source in self._texts: + self._texts[source].append(text) + self._live.update(self._render()) + + def set_status(self, source: str, status: str): + if source in self._status: + self._status[source] = status + self._live.update(self._render()) + + +def digest(query: str, user_id: str, memory: Memory = None): + async def _run(): + display = _HybridDisplay() + display.start() + + final_html = None + try: + async for event in hybrid_search(query, user_id=user_id, memory=memory): + source = event["source"] + kind = event["kind"] + content = event["content"] + + if source == "hybrid" and kind == "final": + final_html = content + break + + if source == "synthesis" and kind == "chunk": + display.set_status(source, "🔄 synthesizing") + display.append(source, content) + continue + + if kind == "chunk": + display.append(source, content) + elif kind == "final": + display.set_status(source, "✅ complete") + display.append(source, content) + elif kind == "error": + display.set_status(source, f"❌ error") + display.append(source, f"\n[ERROR] {content}\n") + elif kind == "done": + display.set_status(source, "✅ done") + finally: + display.stop() + + console = Console() + console.print() + if final_html: + console.rule("[bold magenta]HYBRID SYNTHESIS[/bold magenta]") + console.print(final_html) + else: + console.print("[bold red]No final synthesis received.[/bold red]") + console.print() + return final_html + + return asyncio.run(_run()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--user-id", default="test-user", help="Mem0 user identity") + parser.add_argument("query", help="Search query") + args = parser.parse_args() + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + os.environ[f"{Config.MEMORY_MODEL.split('/')[0].upper()}_API_KEY"] = Config.API_KEY + memory_config = MemoryConfig( + custom_fact_extraction_prompt=GN_FACT_EXTRACTION_PROMPT, + custom_update_extraction_prompt=GN_UPDATE_MEMORY_PROMPT, + llm={ + "provider": "litellm", + "config": { + "model": Config.MEMORY_MODEL, + "temperature": 0.1, + "max_tokens": 2_000, + "api_key": Config.API_KEY, + }, + }, + embedder={ + "provider": "huggingface", + "config": { + "model": "Qwen/Qwen3-Embedding-0.6B", + "embedding_dims": 1024, + "model_kwargs": { + "trust_remote_code": True, + "device": "cuda" if torch.cuda.is_available() else "cpu", + }, + }, + }, + vector_store={ + "provider": "chroma", + "config": { + "collection_name": "mem0", + "path": os.path.join(Config.DB_PATH, "mem0_chroma"), + }, + }, + ) + memory = Memory(config=memory_config) + + digest(args.query, args.user_id, memory=memory) diff --git a/aisearch/scripts/rdf2partial_text.py b/aisearch/scripts/rdf2partial_text.py new file mode 100644 index 0000000..632b5ea --- /dev/null +++ b/aisearch/scripts/rdf2partial_text.py @@ -0,0 +1,70 @@ +import json +import os +import string +from copy import copy + +dir = os.getenv("DIR") +if dir is None: + raise ValueError("DIR not specified") + +collection = {} + +files = [os.path.join(dir, file) for file in os.listdir(dir) if "ttl" in file] + + +def clean(text) -> str: + return text.strip(string.punctuation).strip() + + +for file in files: + with open(file) as f: + lines = f.readlines() + mem = "" + for ind, line in enumerate(lines): + if (not line.startswith("@")) and len(line) > 1: + if line.startswith("<"): + line = line.strip() + contents = line.split(" ") + key = ( + clean(contents[0].split("/")[-1]) + if "/" in contents[0] + else clean(contents[0].split(":")[-1]) + ) + mem = copy(key) + value = " ".join( + ( + clean(content.split("/")[-1]) + if "/" in content + else clean(content.split(":")[-1]) + ) + for content in contents[1:] + ) + key = clean(key) + value = clean(value) + if key not in collection: + collection[key] = [value] + else: + if value not in collection[key]: + collection[key].append(value) + else: + line = line.strip() + key = copy(mem) + contents = line.split(" ") + value = " ".join( + ( + clean(content.split("/")[-1]) + if "/" in content + else clean(content.split(":")[-1]) + ) + for content in contents + ) + key = clean(key) + value = clean(value) + if key not in collection: + collection[key] = [value] + else: + if value not in collection[key]: + collection[key].append(value) + +with open(f"{dir}aggr_rdf.txt", "w") as new: + new.write(json.dumps(collection)) diff --git a/aisearch/scripts/simple_evaluate.py b/aisearch/scripts/simple_evaluate.py new file mode 100644 index 0000000..9fc79b1 --- /dev/null +++ b/aisearch/scripts/simple_evaluate.py @@ -0,0 +1,132 @@ +"""Script for performance evaluation of GN AI systems using mere averaging""" + +import os +import time +from typing import Any + +import dspy +import numpy as np +import pandas as pd +import torch +from gnais.config import Config +from gnais.evaluation.utils import ( + agent_digest, + get_dataset, + graph_rag_digest, + hybrid_digest, + mark, + rag_digest, +) + + +def make_program( + search_func: Any, +) -> dspy.predict.react.ReAct: + + search = dspy.Tool( + name="search", + desc="Answer the question asked by the user.", + args={ + "query": { + "type": "string", + "desc": "question asked by the user", + }, + }, + func=search_func, + ) + + class Signature(dspy.Signature): + query: str = dspy.InputField(desc="User question") + answer: str = dspy.OutputField( + desc="Answer to user's query with details and explanations" + ) + + # Wrap hybrid search into a DSPy module that is compatible with DSPy evaluation + program = dspy.ReAct( + signature=Signature, tools=[search], max_iters=2 + ) # limit iterations + + return program + + +def run_eval( + runner: Any, evaluation_set: list[dspy.Example], judge_llm: dspy.LM +) -> dict[str, float]: + precisions, recalls, f1s, speeds, n_tokens = [], [], [], [], [] + for example in evaluation_set: + query = example.get("query") + true_response = example.get("answer") + + start = time.time() + result = runner(query=query) + end = time.time() + generated_response = result.get("answer") + + with dspy.context(lm=judge_llm): + precision, recall, f1 = mark(query, generated_response, true_response) + precisions.append(precision) + recalls.append(recall) + f1s.append(f1) + speeds.append(end - start) + + metrics = { + "precision": np.mean(precisions).item(), + "recall": np.mean(recalls).item(), + "f1": np.mean(f1s).item(), + "speed": np.mean(speeds).item(), + } + return metrics + + +if __name__ == "__main__": + DATASET_PATH = os.getenv("DATASET_PATH") + if DATASET_PATH is None: + raise FileNotFoundError("DATASET_PATH must be set for evaluation") + + OUTPUT_PATH = os.getenv("OUTPUT_PATH") + if OUTPUT_PATH is None: + raise FileNotFoundError("OUTPUT_PATH must be set for evaluation") + + N_ITERATIONS = int(os.getenv("N_ITERATIONS")) + if N_ITERATIONS is None: + raise ValueError("N_ITERATIONS must be set for evaluation") + + JUDGE_MODEL = os.getenv("JUDGE_MODEL") + if JUDGE_MODEL is None: + raise ValueError("JUDGE MODEL must be set for evaluation") + + torch.manual_seed(Config.SEED) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(Config.SEED) + + dspy.configure(lm=Config.DEFAULT_LLM) + + judge_llm = dspy.LM( + model=JUDGE_MODEL, + api_key=Config.API_KEY, + max_tokens=1_000, + temperature=1, + cache=False, + verbose=False, + ) + + evaluation_set = get_dataset(DATASET_PATH) + collection = {} + + # Run evaluation set with LLM only + base = dspy.ChainOfThought("query -> answer: str") + for n in range(N_ITERATIONS): + base_metrics = run_eval(base, evaluation_set, judge_llm) + collection[f"base_{n}"] = base_metrics + + # Run evaluation set with GN systems + for system in [rag_digest, graph_rag_digest, agent_digest, hybrid_digest]: + system_name = " ".join(system.__name__.split("_")[:-1]) + print(f"Running evaluation for {system_name}") + for n in range(N_ITERATIONS): + print(f"Iteration {n+1}") + system_metrics = run_eval(make_program(system), evaluation_set, judge_llm) + collection[f"{system_name} {n}"] = system_metrics + print(f"Evaluation completed for {system_name}") + final = pd.DataFrame(collection) + final.to_csv(OUTPUT_PATH) |
