From 715241466e6426c2f3650e6cd9e0990078ad80f5 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 15 May 2024 19:00:12 +0300 Subject: Pep8 formatting for llm Api file. --- gn3/api/llm.py | 64 +++++++++++++++++++++++++++++++++++----------------------- 1 file changed, 39 insertions(+), 25 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 7d860d8..91779a5 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,24 +1,26 @@ -"""API for data used to generate menus""" - -# pylint: skip-file +"""Api endpoints for gnqa""" +from datetime import timedelta +from functools import wraps +import json +import sqlite3 +from redis import Redis -from flask import jsonify, request, Blueprint, current_app +from flask import Blueprint +from flask import current_app +from flask import jsonify +from flask import request -from functools import wraps from gn3.llms.process import get_gnqa from gn3.llms.process import get_user_queries from gn3.llms.process import fetch_query_results from gn3.auth.authorisation.oauth2.resource_server import require_oauth from gn3.auth import db -from redis import Redis -import json -import sqlite3 -from datetime import timedelta GnQNA = Blueprint("GnQNA", __name__) def handle_errors(func): + """general error handling decorator function""" @wraps(func) def decorated_function(*args, **kwargs): try: @@ -30,7 +32,7 @@ def handle_errors(func): @GnQNA.route("/gnqna", methods=["POST"]) def gnqa(): - # todo add auth + """Main gnqa endpoint""" query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 @@ -38,7 +40,8 @@ def gnqa(): try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if fahamu_token is None: - return jsonify({"query": query, "error": "Use of invalid fahamu auth token"}), 500 + return jsonify({"query": query, + "error": "Use of invalid fahamu auth token"}), 500 task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -49,19 +52,22 @@ def gnqa(): } with (Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn): - # The key will be deleted after 60 seconds - redis_conn.setex(f"LLM:random_user-{query}", timedelta(days=10), json.dumps(response)) + redis_conn.setex( + f"LLM:random_user-{query}", + timedelta(days=10), json.dumps(response)) return jsonify({ **response, "prev_queries": get_user_queries("random_user", redis_conn) }) except Exception as error: - return jsonify({"query": query, "error": f"Request failed-{str(error)}"}), 500 + return jsonify({"query": query, + "error": f"Request failed-{str(error)}"}), 500 @GnQNA.route("/rating/", methods=["POST"]) @require_oauth("profile") def rating(task_id): + """Endpoint for rating qnqa query and answer""" try: llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile") as token, @@ -81,14 +87,16 @@ def rating(task_id): task_id TEXT NOT NULL UNIQUE )""" cursor.execute(create_table) - cursor.execute("""INSERT INTO Rating(user_id,query,answer,weight,task_id) + cursor.execute("""INSERT INTO Rating(user_id,query, + answer,weight,task_id) VALUES(?,?,?,?,?) ON CONFLICT(task_id) DO UPDATE SET weight=excluded.weight """, (str(user_id), query, answer, weight, task_id)) return { - "message": "You have successfully rated this query:Thank you!!" - }, 200 + "message": + "You have successfully rated this query:Thank you!!" + }, 200 except sqlite3.Error as error: return jsonify({"error": str(error)}), 500 except Exception as error: @@ -99,9 +107,10 @@ def rating(task_id): @require_oauth("profile user") @handle_errors def fetch_user_hist(query): - - with (require_oauth.acquire("profile user") as the_token, Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): + """"Endpoint to fetch previos searches for User""" + with (require_oauth.acquire("profile user") as the_token, + Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn): return jsonify({ **fetch_query_results(query, the_token.user.id, redis_conn), "prev_queries": get_user_queries("random_user", redis_conn) @@ -111,9 +120,12 @@ def fetch_user_hist(query): @GnQNA.route("/historys/", methods=["GET"]) @handle_errors def fetch_users_hist_records(query): - """method to fetch all users hist:note this is a test functionality to be replaced by fetch_user_hist""" + """method to fetch all users hist:note this is a test functionality + to be replaced by fetch_user_hist + """ - with Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn: + with Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn: return jsonify({ **fetch_query_results(query, "random_user", redis_conn), "prev_queries": get_user_queries("random_user", redis_conn) @@ -123,6 +135,8 @@ def fetch_users_hist_records(query): @GnQNA.route("/get_hist_names", methods=["GET"]) @handle_errors def fetch_prev_hist_ids(): - - with (Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True)) as redis_conn: - return jsonify({"prev_queries": get_user_queries("random_user", redis_conn)}) + """Test method for fetching history for Anony Users""" + with (Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True)) as redis_conn: + return jsonify({"prev_queries": get_user_queries("random_user", + redis_conn)}) -- cgit v1.2.3 From d2b5a1c11f6e09ee13b72669068e326b8131f65c Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 15 May 2024 19:01:56 +0300 Subject: Remove broad error handling. --- gn3/api/llm.py | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 91779a5..08783db 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,6 +1,5 @@ """Api endpoints for gnqa""" from datetime import timedelta -from functools import wraps import json import sqlite3 from redis import Redis @@ -19,17 +18,6 @@ from gn3.auth import db GnQNA = Blueprint("GnQNA", __name__) -def handle_errors(func): - """general error handling decorator function""" - @wraps(func) - def decorated_function(*args, **kwargs): - try: - return func(*args, **kwargs) - except Exception as error: - return jsonify({"error": str(error)}), 500 - return decorated_function - - @GnQNA.route("/gnqna", methods=["POST"]) def gnqa(): """Main gnqa endpoint""" @@ -105,7 +93,6 @@ def rating(task_id): @GnQNA.route("/history/", methods=["GET"]) @require_oauth("profile user") -@handle_errors def fetch_user_hist(query): """"Endpoint to fetch previos searches for User""" with (require_oauth.acquire("profile user") as the_token, @@ -118,7 +105,6 @@ def fetch_user_hist(query): @GnQNA.route("/historys/", methods=["GET"]) -@handle_errors def fetch_users_hist_records(query): """method to fetch all users hist:note this is a test functionality to be replaced by fetch_user_hist @@ -133,7 +119,6 @@ def fetch_users_hist_records(query): @GnQNA.route("/get_hist_names", methods=["GET"]) -@handle_errors def fetch_prev_hist_ids(): """Test method for fetching history for Anony Users""" with (Redis.from_url(current_app.config["REDIS_URI"], -- cgit v1.2.3 From 82c89c302a87082c47d7d773264dae4872ba6d1c Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 15 May 2024 19:08:53 +0300 Subject: Rename GnQNA blueprint to gnqa. * register gnqa api endpoint --- gn3/api/llm.py | 20 +++++++++----------- gn3/app.py | 4 ++-- 2 files changed, 11 insertions(+), 13 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 08783db..442252f 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -12,14 +12,15 @@ from flask import request from gn3.llms.process import get_gnqa from gn3.llms.process import get_user_queries from gn3.llms.process import fetch_query_results +from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth from gn3.auth import db -GnQNA = Blueprint("GnQNA", __name__) +gnqa = Blueprint("gnqa", __name__) -@GnQNA.route("/gnqna", methods=["POST"]) -def gnqa(): +@gnqa.route("/gnqna", methods=["POST"]) +def gnqna(): """Main gnqa endpoint""" query = request.json.get("querygnqa", "") if not query: @@ -47,12 +48,12 @@ def gnqa(): **response, "prev_queries": get_user_queries("random_user", redis_conn) }) - except Exception as error: + except LLMError as error: return jsonify({"query": query, "error": f"Request failed-{str(error)}"}), 500 -@GnQNA.route("/rating/", methods=["POST"]) +@gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") def rating(task_id): """Endpoint for rating qnqa query and answer""" @@ -87,11 +88,9 @@ def rating(task_id): }, 200 except sqlite3.Error as error: return jsonify({"error": str(error)}), 500 - except Exception as error: - raise error -@GnQNA.route("/history/", methods=["GET"]) +@gnqa.route("/history/", methods=["GET"]) @require_oauth("profile user") def fetch_user_hist(query): """"Endpoint to fetch previos searches for User""" @@ -104,12 +103,11 @@ def fetch_user_hist(query): }) -@GnQNA.route("/historys/", methods=["GET"]) +@gnqa.route("/historys/", methods=["GET"]) def fetch_users_hist_records(query): """method to fetch all users hist:note this is a test functionality to be replaced by fetch_user_hist """ - with Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn: return jsonify({ @@ -118,7 +116,7 @@ def fetch_users_hist_records(query): }) -@GnQNA.route("/get_hist_names", methods=["GET"]) +@gnqa.route("/get_hist_names", methods=["GET"]) def fetch_prev_hist_ids(): """Test method for fetching history for Anony Users""" with (Redis.from_url(current_app.config["REDIS_URI"], diff --git a/gn3/app.py b/gn3/app.py index 3f1e6ee..c8f0c5a 100644 --- a/gn3/app.py +++ b/gn3/app.py @@ -25,7 +25,7 @@ from gn3.api.menu import menu from gn3.api.search import search from gn3.api.metadata import metadata from gn3.api.sampledata import sampledata -from gn3.api.llm import GnQNA +from gn3.api.llm import gnqa from gn3.auth import oauth2 from gn3.case_attributes import caseattr @@ -78,7 +78,7 @@ def create_app(config: Union[Dict, str, None] = None) -> Flask: app.register_blueprint(sampledata, url_prefix="/api/sampledata") app.register_blueprint(oauth2, url_prefix="/api/oauth2") app.register_blueprint(caseattr, url_prefix="/api/case-attribute") - app.register_blueprint(GnQNA, url_prefix="/api/llm") + app.register_blueprint(gnqa, url_prefix="/api/llm") register_error_handlers(app) return app -- cgit v1.2.3 From 0832e9579d6fe6c9b46bfa499f1c7726301ac10c Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 17 May 2024 17:07:32 +0300 Subject: Add endpoint for getting prev user searches --- gn3/api/llm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 442252f..41cc376 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -90,6 +90,18 @@ def rating(task_id): return jsonify({"error": str(error)}), 500 + +@gnqa.route("/searches/", methods=["GET"]) +@require_oauth("profile user") +def fetch_prev_searches(): + with (require_oauth.acquire("profile user") as __the_token, + Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn): + return jsonify({ + "prev_queries": get_user_queries("random_user", redis_conn) + }) + + @gnqa.route("/history/", methods=["GET"]) @require_oauth("profile user") def fetch_user_hist(query): @@ -103,6 +115,7 @@ def fetch_user_hist(query): }) + @gnqa.route("/historys/", methods=["GET"]) def fetch_users_hist_records(query): """method to fetch all users hist:note this is a test functionality -- cgit v1.2.3 From 8201936afc4a8330a3dfa25a26b3786f44e8e378 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 21 May 2024 16:35:16 +0300 Subject: Add search gnqa history functionality. --- gn3/api/llm.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 41cc376..b2c9c3e 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,5 +1,4 @@ """Api endpoints for gnqa""" -from datetime import timedelta import json import sqlite3 from redis import Redis @@ -14,6 +13,7 @@ from gn3.llms.process import get_user_queries from gn3.llms.process import fetch_query_results from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth + from gn3.auth import db gnqa = Blueprint("gnqa", __name__) @@ -39,15 +39,17 @@ def gnqna(): "answer": answer, "references": refs } - with (Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): - redis_conn.setex( - f"LLM:random_user-{query}", - timedelta(days=10), json.dumps(response)) - return jsonify({ - **response, - "prev_queries": get_user_queries("random_user", redis_conn) - }) + try: + with (Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn, + require_oauth.acquire("profile user") as token): + redis_conn.set( + f"LLM:{str(token.user.user_id)}-{str(task_id['task_id'])}", + json.dumps(response) + ) + return response + except Exception: # handle specific error + return response except LLMError as error: return jsonify({"query": query, "error": f"Request failed-{str(error)}"}), 500 @@ -90,16 +92,19 @@ def rating(task_id): return jsonify({"error": str(error)}), 500 - -@gnqa.route("/searches/", methods=["GET"]) +@gnqa.route("/searches", methods=["GET"]) @require_oauth("profile user") def fetch_prev_searches(): - with (require_oauth.acquire("profile user") as __the_token, + """ api method to fetch search query records""" + with (require_oauth.acquire("profile user") as the_token, Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn): - return jsonify({ - "prev_queries": get_user_queries("random_user", redis_conn) - }) + if request.args.get("search_term"): + return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) + query_result = {} + for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): + query_result[key] = json.loads(redis_conn.get(key)) + return jsonify(query_result) @gnqa.route("/history/", methods=["GET"]) @@ -110,12 +115,11 @@ def fetch_user_hist(query): Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn): return jsonify({ - **fetch_query_results(query, the_token.user.id, redis_conn), + **fetch_query_results(query, the_token.user.user_id, redis_conn), "prev_queries": get_user_queries("random_user", redis_conn) }) - @gnqa.route("/historys/", methods=["GET"]) def fetch_users_hist_records(query): """method to fetch all users hist:note this is a test functionality -- cgit v1.2.3 From 2a99da9f46233a28e9ea0b6a297d8a6b93f61923 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 21 May 2024 16:38:53 +0300 Subject: Remove obsolete functions. --- gn3/api/llm.py | 39 +-------------------------------------- gn3/llms/process.py | 20 -------------------- 2 files changed, 1 insertion(+), 58 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index b2c9c3e..02b37f9 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -9,8 +9,6 @@ from flask import jsonify from flask import request from gn3.llms.process import get_gnqa -from gn3.llms.process import get_user_queries -from gn3.llms.process import fetch_query_results from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth @@ -46,7 +44,7 @@ def gnqna(): redis_conn.set( f"LLM:{str(token.user.user_id)}-{str(task_id['task_id'])}", json.dumps(response) - ) + ) return response except Exception: # handle specific error return response @@ -105,38 +103,3 @@ def fetch_prev_searches(): for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): query_result[key] = json.loads(redis_conn.get(key)) return jsonify(query_result) - - -@gnqa.route("/history/", methods=["GET"]) -@require_oauth("profile user") -def fetch_user_hist(query): - """"Endpoint to fetch previos searches for User""" - with (require_oauth.acquire("profile user") as the_token, - Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): - return jsonify({ - **fetch_query_results(query, the_token.user.user_id, redis_conn), - "prev_queries": get_user_queries("random_user", redis_conn) - }) - - -@gnqa.route("/historys/", methods=["GET"]) -def fetch_users_hist_records(query): - """method to fetch all users hist:note this is a test functionality - to be replaced by fetch_user_hist - """ - with Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn: - return jsonify({ - **fetch_query_results(query, "random_user", redis_conn), - "prev_queries": get_user_queries("random_user", redis_conn) - }) - - -@gnqa.route("/get_hist_names", methods=["GET"]) -def fetch_prev_hist_ids(): - """Test method for fetching history for Anony Users""" - with (Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True)) as redis_conn: - return jsonify({"prev_queries": get_user_queries("random_user", - redis_conn)}) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index 4725bcb..eba7e4b 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -135,23 +135,3 @@ def get_gnqa(query, auth_token, data_dir=""): return task_id, answer, references else: return task_id, "We couldn't provide a response,Please try to rephrase your question to receive feedback", [] - - -def fetch_query_results(query, user_id, redis_conn): - """this method fetches prev user query searches""" - result = redis_conn.get(f"LLM:{user_id}-{query}") - if result: - return json.loads(result) - return { - "query": query, - "answer": "Sorry No answer for you", - "references": [], - "task_id": None - } - - -def get_user_queries(user_id, redis_conn): - """methos to fetch all queries for a specific user""" - results = redis_conn.keys(f"LLM:{user_id}*") - return [query for query in - [result.partition("-")[2] for result in results] if query != ""] -- cgit v1.2.3 From a304037ce012516b07c17fd0dcb9e816c33a4d58 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 22 May 2024 13:38:06 +0300 Subject: Refactor error handling for llm api's. --- gn3/api/llm.py | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 02b37f9..8e8a50f 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,17 +1,20 @@ """Api endpoints for gnqa""" import json import sqlite3 +import redis from redis import Redis +from authlib.integrations.flask_oauth2.errors import _HTTPException from flask import Blueprint from flask import current_app from flask import jsonify from flask import request + from gn3.llms.process import get_gnqa from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth - +from gn3.auth.authorisation.errors import AuthorisationError from gn3.auth import db gnqa = Blueprint("gnqa", __name__) @@ -23,12 +26,10 @@ def gnqna(): query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 - try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if fahamu_token is None: - return jsonify({"query": query, - "error": "Use of invalid fahamu auth token"}), 500 + raise LLMError("Request failed:an LLM authorisation token is required ", query=query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -46,16 +47,16 @@ def gnqna(): json.dumps(response) ) return response - except Exception: # handle specific error - return response + except _HTTPException as httpe: + raise AuthorisationError("Authentication is required.") from httpe except LLMError as error: - return jsonify({"query": query, - "error": f"Request failed-{str(error)}"}), 500 + raise LLMError(f"request failed for query {str(error.args[-1])}", + query=query) from error @gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") -def rating(task_id): +def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" try: llm_db_path = current_app.config["LLM_DB_PATH"] @@ -87,19 +88,25 @@ def rating(task_id): "You have successfully rated this query:Thank you!!" }, 200 except sqlite3.Error as error: - return jsonify({"error": str(error)}), 500 + raise sqlite3.OperationalError from error + except _HTTPException as httpe: + raise AuthorisationError("Authentication is required") from httpe -@gnqa.route("/searches", methods=["GET"]) +@gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") -def fetch_prev_searches(): +def fetch_prev_history(): """ api method to fetch search query records""" - with (require_oauth.acquire("profile user") as the_token, - Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): - if request.args.get("search_term"): - return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) - query_result = {} - for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): - query_result[key] = json.loads(redis_conn.get(key)) - return jsonify(query_result) + try: + + with (require_oauth.acquire("profile user") as the_token, + Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn): + if request.args.get("search_term"): + return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) + query_result = {} + for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): + query_result[key] = json.loads(redis_conn.get(key)) + return jsonify(query_result) + except redis.exceptions.RedisError as error: + raise error -- cgit v1.2.3 From c280bae2bd97d17189763d3ce76cfdfc35588fd2 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 22 May 2024 18:11:58 +0300 Subject: This commit does the following: * Adds a new table to store the history records. * Remove the redis dependancy. --- gn3/api/llm.py | 52 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 20 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 8e8a50f..2ed52eb 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,8 +1,6 @@ """Api endpoints for gnqa""" import json import sqlite3 -import redis -from redis import Redis from authlib.integrations.flask_oauth2.errors import _HTTPException from flask import Blueprint @@ -39,16 +37,26 @@ def gnqna(): "references": refs } try: - with (Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn, + with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, require_oauth.acquire("profile user") as token): - redis_conn.set( - f"LLM:{str(token.user.user_id)}-{str(task_id['task_id'])}", - json.dumps(response) + schema = """CREATE TABLE IF NOT EXISTS + history(user_id TEXT NOT NULL, + task_id TEXT NOT NULL, + query TEXT NOT NULL, + results TEXT, + PRIMARY KEY(task_id)) WITHOUT ROWID""" + cursor = conn.cursor() + cursor.execute(schema) + cursor.execute("""INSERT INTO history(user_id,task_id,query,results) + VALUES(?,?,?,?) + """,(str(token.user.user_id),str(task_id["task_id"]),query, + json.dumps(response)) ) - return response + return response except _HTTPException as httpe: raise AuthorisationError("Authentication is required.") from httpe + except sqlite3.Error as error: + raise error except LLMError as error: raise LLMError(f"request failed for query {str(error.args[-1])}", query=query) from error @@ -92,21 +100,25 @@ def rate_queries(task_id): except _HTTPException as httpe: raise AuthorisationError("Authentication is required") from httpe - @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): """ api method to fetch search query records""" try: - - with (require_oauth.acquire("profile user") as the_token, - Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): + llm_db_path = current_app.config["LLM_DB_PATH"] + with (require_oauth.acquire("profile user") as token, + db.connection(llm_db_path) as conn): + cursor = conn.cursor() if request.args.get("search_term"): - return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) - query_result = {} - for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): - query_result[key] = json.loads(redis_conn.get(key)) - return jsonify(query_result) - except redis.exceptions.RedisError as error: - raise error + query = """SELECT results from history Where task_id=? and user_id=?""" + cursor.execute(query, (request.args.get("search_term") + ,str(token.user.user_id),)) + return dict(cursor.fetchone()) + query = """SELECT task_id,query from history WHERE user_id=?""" + cursor.execute(query, (str(token.user.user_id),)) + return [dict(item) for item in cursor.fetchall()] + + except sqlite3.Error as error: #please handle me corrrectly + return jsonify({"error":error}), 500 + except _HTTPException as httpe: + raise AuthorisationError("Authorization is required") from httpe -- cgit v1.2.3 From 7d3b2a29f5497d88b7c1391a7f5631591889ab36 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 23 May 2024 12:10:52 +0300 Subject: Refactor error handling code. --- gn3/api/llm.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 2ed52eb..172e49b 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -8,7 +8,6 @@ from flask import current_app from flask import jsonify from flask import request - from gn3.llms.process import get_gnqa from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth @@ -27,7 +26,8 @@ def gnqna(): try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if fahamu_token is None: - raise LLMError("Request failed:an LLM authorisation token is required ", query=query) + raise LLMError( + "Request failed:an LLM authorisation token is required ", query=query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -49,14 +49,14 @@ def gnqna(): cursor.execute(schema) cursor.execute("""INSERT INTO history(user_id,task_id,query,results) VALUES(?,?,?,?) - """,(str(token.user.user_id),str(task_id["task_id"]),query, - json.dumps(response)) + """, (str(token.user.user_id), str(task_id["task_id"]), query, + json.dumps(response)) ) return response except _HTTPException as httpe: raise AuthorisationError("Authentication is required.") from httpe except sqlite3.Error as error: - raise error + raise sqlite3.OperationalError(*error.args) from error except LLMError as error: raise LLMError(f"request failed for query {str(error.args[-1])}", query=query) from error @@ -96,14 +96,15 @@ def rate_queries(task_id): "You have successfully rated this query:Thank you!!" }, 200 except sqlite3.Error as error: - raise sqlite3.OperationalError from error + raise sqlite3.OperationalError(*error.args) from error except _HTTPException as httpe: raise AuthorisationError("Authentication is required") from httpe + @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): - """ api method to fetch search query records""" + """ api method to fetch search query records""" # ro only try: llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile user") as token, @@ -111,14 +112,13 @@ def fetch_prev_history(): cursor = conn.cursor() if request.args.get("search_term"): query = """SELECT results from history Where task_id=? and user_id=?""" - cursor.execute(query, (request.args.get("search_term") - ,str(token.user.user_id),)) + cursor.execute(query, (request.args.get( + "search_term"), str(token.user.user_id),)) return dict(cursor.fetchone()) query = """SELECT task_id,query from history WHERE user_id=?""" cursor.execute(query, (str(token.user.user_id),)) - return [dict(item) for item in cursor.fetchall()] - - except sqlite3.Error as error: #please handle me corrrectly - return jsonify({"error":error}), 500 + return jsonify([dict(item) for item in cursor.fetchall()]) + except sqlite3.Error as error: + raise sqlite3.OperationalError(*error.args) from error except _HTTPException as httpe: raise AuthorisationError("Authorization is required") from httpe -- cgit v1.2.3 From d4b0ae19e55a45eed7b6bca43abb5340f58ccfbe Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 23 May 2024 12:17:40 +0300 Subject: rename gnqna route to search. --- gn3/api/llm.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 172e49b..e8ffa1a 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -17,17 +17,18 @@ from gn3.auth import db gnqa = Blueprint("gnqa", __name__) -@gnqa.route("/gnqna", methods=["POST"]) -def gnqna(): +@gnqa.route("/search", methods=["POST"]) +def search(): """Main gnqa endpoint""" query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") - if fahamu_token is None: + if not fahamu_token: raise LLMError( - "Request failed:an LLM authorisation token is required ", query=query) + "Request failed:an LLM authorisation token is required ", + query=query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -39,17 +40,18 @@ def gnqna(): try: with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, require_oauth.acquire("profile user") as token): - schema = """CREATE TABLE IF NOT EXISTS + cursor = conn.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS history(user_id TEXT NOT NULL, task_id TEXT NOT NULL, query TEXT NOT NULL, results TEXT, - PRIMARY KEY(task_id)) WITHOUT ROWID""" - cursor = conn.cursor() - cursor.execute(schema) - cursor.execute("""INSERT INTO history(user_id,task_id,query,results) + PRIMARY KEY(task_id)) WITHOUT ROWID""") + cursor.execute( + """INSERT INTO history(user_id,task_id,query,results) VALUES(?,?,?,?) - """, (str(token.user.user_id), str(task_id["task_id"]), query, + """, (str(token.user.user_id), str(task_id["task_id"]), + query, json.dumps(response)) ) return response @@ -104,7 +106,7 @@ def rate_queries(task_id): @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): - """ api method to fetch search query records""" # ro only + """ api method to fetch search query records""" try: llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile user") as token, -- cgit v1.2.3 From ef955f9b456a591f64faa428b8ef83252923bb63 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 23 May 2024 12:29:04 +0300 Subject: Remove irrelevant variable assignments. --- gn3/api/llm.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index e8ffa1a..5ad58cb 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -69,9 +69,8 @@ def search(): def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" try: - llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile") as token, - db.connection(llm_db_path) as conn): + db.connection(current_app.config["LLM_DB_PATH"]) as conn): results = request.json user_id, query, answer, weight = (token.user.user_id, @@ -108,17 +107,18 @@ def rate_queries(task_id): def fetch_prev_history(): """ api method to fetch search query records""" try: - llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile user") as token, - db.connection(llm_db_path) as conn): + db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() if request.args.get("search_term"): - query = """SELECT results from history Where task_id=? and user_id=?""" - cursor.execute(query, (request.args.get( - "search_term"), str(token.user.user_id),)) + cursor.execute( + """SELECT results from history Where task_id=? and user_id=?""", + (request.args.get("search_term"), + str(token.user.user_id),)) return dict(cursor.fetchone()) - query = """SELECT task_id,query from history WHERE user_id=?""" - cursor.execute(query, (str(token.user.user_id),)) + cursor.execute( + """SELECT task_id,query from history WHERE user_id=?""", + (str(token.user.user_id),)) return jsonify([dict(item) for item in cursor.fetchall()]) except sqlite3.Error as error: raise sqlite3.OperationalError(*error.args) from error -- cgit v1.2.3 From 6bbe9763a024558f2a4a942d71c799e4583448a2 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:06:22 +0300 Subject: Remove Try/Excepts from llm api endponts. --- gn3/api/llm.py | 152 +++++++++++++++++++++++++-------------------------------- 1 file changed, 66 insertions(+), 86 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 5ad58cb..03ce815 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -2,7 +2,6 @@ import json import sqlite3 from authlib.integrations.flask_oauth2.errors import _HTTPException - from flask import Blueprint from flask import current_app from flask import jsonify @@ -14,6 +13,7 @@ from gn3.auth.authorisation.oauth2.resource_server import require_oauth from gn3.auth.authorisation.errors import AuthorisationError from gn3.auth import db + gnqa = Blueprint("gnqa", __name__) @@ -23,104 +23,84 @@ def search(): query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 - try: - fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") - if not fahamu_token: - raise LLMError( - "Request failed:an LLM authorisation token is required ", - query=query) - task_id, answer, refs = get_gnqa( - query, fahamu_token, current_app.config.get("DATA_DIR")) - response = { - "task_id": task_id, - "query": query, - "answer": answer, - "references": refs - } - try: - with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, - require_oauth.acquire("profile user") as token): - cursor = conn.cursor() - cursor.execute("""CREATE TABLE IF NOT EXISTS - history(user_id TEXT NOT NULL, - task_id TEXT NOT NULL, - query TEXT NOT NULL, - results TEXT, - PRIMARY KEY(task_id)) WITHOUT ROWID""") - cursor.execute( - """INSERT INTO history(user_id,task_id,query,results) - VALUES(?,?,?,?) - """, (str(token.user.user_id), str(task_id["task_id"]), - query, - json.dumps(response)) - ) - return response - except _HTTPException as httpe: - raise AuthorisationError("Authentication is required.") from httpe - except sqlite3.Error as error: - raise sqlite3.OperationalError(*error.args) from error - except LLMError as error: - raise LLMError(f"request failed for query {str(error.args[-1])}", - query=query) from error + fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") + if not fahamu_token: + raise LLMError( + "Request failed:an LLM authorisation token is required ", + query=query) + task_id, answer, refs = get_gnqa( + query, fahamu_token, current_app.config.get("DATA_DIR")) + response = { + "task_id": task_id, + "query": query, + "answer": answer, + "references": refs + } + with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, + require_oauth.acquire("profile user") as token): + cursor = conn.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS + history(user_id TEXT NOT NULL, + task_id TEXT NOT NULL, + query TEXT NOT NULL, + results TEXT, + PRIMARY KEY(task_id)) WITHOUT ROWID""") + cursor.execute( + """INSERT INTO history(user_id, task_id, query, results) + VALUES(?, ?, ?, ?) + """, (str(token.user.user_id), str(task_id["task_id"]), + query, + json.dumps(response)) + ) + return response @gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" - try: - with (require_oauth.acquire("profile") as token, - db.connection(current_app.config["LLM_DB_PATH"]) as conn): + with (require_oauth.acquire("profile") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): - results = request.json - user_id, query, answer, weight = (token.user.user_id, - results.get("query"), - results.get("answer"), - results.get("weight", 0)) - cursor = conn.cursor() - create_table = """CREATE TABLE IF NOT EXISTS Rating( - user_id TEXT NOT NULL, - query TEXT NOT NULL, - answer TEXT NOT NULL, - weight INTEGER NOT NULL DEFAULT 0, - task_id TEXT NOT NULL UNIQUE - )""" - cursor.execute(create_table) - cursor.execute("""INSERT INTO Rating(user_id,query, - answer,weight,task_id) - VALUES(?,?,?,?,?) - ON CONFLICT(task_id) DO UPDATE SET - weight=excluded.weight - """, (str(user_id), query, answer, weight, task_id)) + results = request.json + user_id, query, answer, weight = (token.user.user_id, + results.get("query"), + results.get("answer"), + results.get("weight", 0)) + cursor = conn.cursor() + create_table = """CREATE TABLE IF NOT EXISTS Rating( + user_id TEXT NOT NULL, + query TEXT NOT NULL, + answer TEXT NOT NULL, + weight INTEGER NOT NULL DEFAULT 0, + task_id TEXT NOT NULL UNIQUE + )""" + cursor.execute(create_table) + cursor.execute("""INSERT INTO Rating(user_id, query, + answer, weight, task_id) + VALUES(?,?,?,?,?) + ON CONFLICT(task_id) DO UPDATE SET + weight=excluded.weight + """, (str(user_id), query, answer, weight, task_id)) return { - "message": - "You have successfully rated this query:Thank you!!" + "message": "You have successfully rated this query.Thank you!" }, 200 - except sqlite3.Error as error: - raise sqlite3.OperationalError(*error.args) from error - except _HTTPException as httpe: - raise AuthorisationError("Authentication is required") from httpe @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): """ api method to fetch search query records""" - try: - with (require_oauth.acquire("profile user") as token, - db.connection(current_app.config["LLM_DB_PATH"]) as conn): - cursor = conn.cursor() - if request.args.get("search_term"): - cursor.execute( - """SELECT results from history Where task_id=? and user_id=?""", - (request.args.get("search_term"), - str(token.user.user_id),)) - return dict(cursor.fetchone()) + with (require_oauth.acquire("profile user") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): + cursor = conn.cursor() + if request.args.get("search_term"): cursor.execute( - """SELECT task_id,query from history WHERE user_id=?""", - (str(token.user.user_id),)) - return jsonify([dict(item) for item in cursor.fetchall()]) - except sqlite3.Error as error: - raise sqlite3.OperationalError(*error.args) from error - except _HTTPException as httpe: - raise AuthorisationError("Authorization is required") from httpe + """SELECT results from history Where task_id=? and user_id=?""", + (request.args.get("search_term"), + str(token.user.user_id),)) + return dict(cursor.fetchone())["results"] + cursor.execute( + """SELECT task_id,query from history WHERE user_id=?""", + (str(token.user.user_id),)) + return jsonify([dict(item) for item in cursor.fetchall()]) -- cgit v1.2.3 From 188f84ef3895de613e998c63da7ec2338e25a55c Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:30:59 +0300 Subject: Remove kwargs from LLMErrorr Exceptions and update relevant code. --- gn3/api/llm.py | 7 +------ gn3/llms/client.py | 6 +++--- gn3/llms/errors.py | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 03ce815..c1f6304 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,7 +1,5 @@ """Api endpoints for gnqa""" import json -import sqlite3 -from authlib.integrations.flask_oauth2.errors import _HTTPException from flask import Blueprint from flask import current_app from flask import jsonify @@ -10,7 +8,6 @@ from flask import request from gn3.llms.process import get_gnqa from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth -from gn3.auth.authorisation.errors import AuthorisationError from gn3.auth import db @@ -26,8 +23,7 @@ def search(): fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if not fahamu_token: raise LLMError( - "Request failed:an LLM authorisation token is required ", - query=query) + "Request failed:an LLM authorisation token is required ", query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -61,7 +57,6 @@ def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" with (require_oauth.acquire("profile") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): - results = request.json user_id, query, answer, weight = (token.user.user_id, results.get("query"), diff --git a/gn3/llms/client.py b/gn3/llms/client.py index 5c4fa0e..d29d2a1 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -117,11 +117,11 @@ class GeneNetworkQAClient(Session): continue return response else: - raise LLMError( f"Request error with code:\ + raise LLMError(f"Request error with code:\ {response.status_code} occurred with reason:\ {response_msg.get(response.status_code,response.reason)}", - query=self.query) + self.query) #time.sleep(retry_delay) raise LLMError("Time error: We couldn't provide a response,Please try\ to rephrase your question to receive feedback", - query=self.query) + self.query) diff --git a/gn3/llms/errors.py b/gn3/llms/errors.py index c5439d6..77e0f9a 100644 --- a/gn3/llms/errors.py +++ b/gn3/llms/errors.py @@ -39,4 +39,3 @@ class LLMError(Exception): """custom exception for LLMErrorMIxins""" def __init__(self, *args, **kwargs): super().__init__(*args) - self.query = kwargs.get("query") -- cgit v1.2.3 From a3284926aaa73ce27565e4a1131d4447a3a7face Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 17:18:55 +0300 Subject: Add created_at timestamp to History table. --- gn3/api/llm.py | 1 + 1 file changed, 1 insertion(+) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index c1f6304..5924f12 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -40,6 +40,7 @@ def search(): task_id TEXT NOT NULL, query TEXT NOT NULL, results TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(task_id)) WITHOUT ROWID""") cursor.execute( """INSERT INTO history(user_id, task_id, query, results) -- cgit v1.2.3 From 22a1517e71ff6058090596498b9e829fb4e19664 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 17:34:59 +0300 Subject: Add created_at timestamp for Rating table. --- gn3/api/llm.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 5924f12..ab33c7a 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,5 +1,6 @@ """Api endpoints for gnqa""" import json +from datetime import datetime, timezone from flask import Blueprint from flask import current_app from flask import jsonify @@ -69,15 +70,17 @@ def rate_queries(task_id): query TEXT NOT NULL, answer TEXT NOT NULL, weight INTEGER NOT NULL DEFAULT 0, - task_id TEXT NOT NULL UNIQUE + task_id TEXT NOT NULL UNIQUE, + created_at TIMESTAMP )""" cursor.execute(create_table) cursor.execute("""INSERT INTO Rating(user_id, query, - answer, weight, task_id) - VALUES(?,?,?,?,?) + answer, weight, task_id, created_at) + VALUES(?, ?, ?, ?, ?, ?) ON CONFLICT(task_id) DO UPDATE SET weight=excluded.weight - """, (str(user_id), query, answer, weight, task_id)) + """, (str(user_id), query, answer, weight, task_id, + datetime.now(timezone.utc))) return { "message": "You have successfully rated this query.Thank you!" }, 200 -- cgit v1.2.3 From 105f2b36eb62b9b097e1cbf6fa815f98da77bc16 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 14:09:02 +0300 Subject: Update Docstrings for Api endpoints and functions. --- gn3/api/llm.py | 6 +++--- gn3/llms/process.py | 48 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index ab33c7a..4b8ec52 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -17,7 +17,7 @@ gnqa = Blueprint("gnqa", __name__) @gnqa.route("/search", methods=["POST"]) def search(): - """Main gnqa endpoint""" + """Api endpoint for searching queries in fahamu Api""" query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 @@ -56,7 +56,7 @@ def search(): @gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") def rate_queries(task_id): - """Endpoint for rating qnqa query and answer""" + """Api endpoint for rating GNQA query and answer""" with (require_oauth.acquire("profile") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): results = request.json @@ -89,7 +89,7 @@ def rate_queries(task_id): @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): - """ api method to fetch search query records""" + """Api endpoint to fetch GNQA previous search.""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() diff --git a/gn3/llms/process.py b/gn3/llms/process.py index ab2a80e..ade4104 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -35,7 +35,7 @@ class DocIDs(): raise FileNotFoundError(f"{file_path}-- FIle does not exist\n") def format_doc_ids(self, docs): - """method to format doc_ids for list items""" + """method to format doc_ids for list items doc_id and doc_name""" for _key, val in docs.items(): if isinstance(val, list): for doc_obj in val: @@ -43,7 +43,14 @@ class DocIDs(): self.doc_ids.update({doc_obj["id"]: doc_name}) def get_info(self, doc_id): - """ interface to make read from doc_ids""" + """ interface to make read from doc_ids + and extract info data else returns + doc_id + Args: + doc_id: str: a search key for doc_ids + Returns: + an object with doc_info if doc_id in doc_ids + """ if doc_id in self.doc_ids.keys(): return self.doc_ids[doc_id] else: @@ -51,7 +58,8 @@ class DocIDs(): def format_bibliography_info(bib_info): - """Function for formatting bibliography info""" + """Utility function for formatting bibliography info + """ if isinstance(bib_info, str): return bib_info.removesuffix('.txt') elif isinstance(bib_info, dict): @@ -66,7 +74,15 @@ def filter_response_text(val): def parse_context(context, get_info_func, format_bib_func): - """function to parse doc_ids content""" + """Function to parse doc_ids content + Args: + context: raw references from fahamu api + get_info_func: function to get doc_ids info + format_bib_func: function to foramt bibliography info + Returns: + an list with each item having (doc_id,bib_info, + combined reference text) + """ results = [] for doc_ids, summary in context.items(): combo_txt = "" @@ -81,7 +97,12 @@ def parse_context(context, get_info_func, format_bib_func): def load_file(filename, dir_path): - """function to open and load json file""" + """Utility function to read json file + Args: + filename: file name to read + dir_path: base directory for the file + Returns: json data read to a dict + """ file_path = os.path.join(dir_path, f"{filename}") if not os.path.isfile(file_path): raise FileNotFoundError(f"{filename} was not found or is a directory") @@ -90,8 +111,19 @@ def load_file(filename, dir_path): def fetch_pubmed(references, file_name, data_dir=""): - """method to fetch and populate references with pubmed""" + """ + Fetches PubMed data from a JSON file and populates the\ + references dictionary. + + Args: + references (dict): Dictionary with document IDs as keys\ + and reference data as values. + filename (str): Name of the JSON file containing PubMed data. + data_dir (str): Base directory where the data files are located. + Returns: + dict: Updated references dictionary populated with the PubMed data. + """ try: pubmed = load_file(file_name, os.path.join(data_dir, "gn-meta/lit")) for reference in references: @@ -123,4 +155,6 @@ def get_gnqa(query, auth_token, data_dir=""): answer = resp_text['data']['answer'] context = resp_text['data']['context'] return task_id, answer, fetch_pubmed(parse_context( - context, DocIDs().get_info, format_bibliography_info), "pubmed.json", data_dir) + context, DocIDs().get_info, + format_bibliography_info), + "pubmed.json", data_dir) -- cgit v1.2.3 From 7d79812db623d12474422a9613c81f35e25aef55 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 29 May 2024 14:05:00 +0300 Subject: Add delete functionality for gnqa history. --- gn3/api/llm.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 4b8ec52..228d3fa 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -86,16 +86,24 @@ def rate_queries(task_id): }, 200 -@gnqa.route("/history", methods=["GET"]) +@gnqa.route("/history", methods=["GET", "POST"]) @require_oauth("profile user") def fetch_prev_history(): """Api endpoint to fetch GNQA previous search.""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() + if request.method == "POST": + task_ids = list(request.json.values()) + query = """DELETE FROM history + WHERE task_id IN ({}) + and user_id=?""".format(",".join("?" * len(task_ids))) + cursor.execute(query, (*task_ids, str(token.user.user_id),)) + return jsonify({}) if request.args.get("search_term"): cursor.execute( - """SELECT results from history Where task_id=? and user_id=?""", + """SELECT results from history + Where task_id=? and user_id=?""", (request.args.get("search_term"), str(token.user.user_id),)) return dict(cursor.fetchone())["results"] -- cgit v1.2.3 From 231cbd69ed5292451b976091e117d819e5466b30 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 09:51:38 +0300 Subject: Update gnqa search endpoint from POST to PUT. --- gn3/api/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 228d3fa..d56a3d2 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -15,7 +15,7 @@ from gn3.auth import db gnqa = Blueprint("gnqa", __name__) -@gnqa.route("/search", methods=["POST"]) +@gnqa.route("/search", methods=["PUT"]) def search(): """Api endpoint for searching queries in fahamu Api""" query = request.json.get("querygnqa", "") -- cgit v1.2.3 From c32378420d58d6770045cf8d2025dabccb4d1492 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 10:09:24 +0300 Subject: Use correct http method `Delete` for search history. --- gn3/api/llm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index d56a3d2..a94badd 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -86,21 +86,22 @@ def rate_queries(task_id): }, 200 -@gnqa.route("/history", methods=["GET", "POST"]) +@gnqa.route("/history", methods=["GET", "DELETE"]) @require_oauth("profile user") def fetch_prev_history(): """Api endpoint to fetch GNQA previous search.""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() - if request.method == "POST": + if request.method == "DELETE": task_ids = list(request.json.values()) query = """DELETE FROM history WHERE task_id IN ({}) and user_id=?""".format(",".join("?" * len(task_ids))) cursor.execute(query, (*task_ids, str(token.user.user_id),)) return jsonify({}) - if request.args.get("search_term"): + elif (request.method == "GET" and + request.args.get("search_term")): cursor.execute( """SELECT results from history Where task_id=? and user_id=?""", -- cgit v1.2.3 From ade1b4cb03de8a1c670a3e876b8a18692dfc7694 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 10:26:05 +0300 Subject: Check for empty values when fetching search history. --- gn3/api/llm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index a94badd..5901ef5 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -107,7 +107,10 @@ def fetch_prev_history(): Where task_id=? and user_id=?""", (request.args.get("search_term"), str(token.user.user_id),)) - return dict(cursor.fetchone())["results"] + record = cursor.fetchone() + if record: + return dict(record).get("results") + return {} cursor.execute( """SELECT task_id,query from history WHERE user_id=?""", (str(token.user.user_id),)) -- cgit v1.2.3 From a45ab3e62df6e1dfcc6fff03916369fbdaf68ab8 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 11:05:39 +0300 Subject: Add default timestamp and and primary key for Rating table. --- gn3/api/llm.py | 4 ++-- sql/update/llm_db_update.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 5901ef5..952a5b9 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -71,8 +71,8 @@ def rate_queries(task_id): answer TEXT NOT NULL, weight INTEGER NOT NULL DEFAULT 0, task_id TEXT NOT NULL UNIQUE, - created_at TIMESTAMP - )""" + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(task_id))""" cursor.execute(create_table) cursor.execute("""INSERT INTO Rating(user_id, query, answer, weight, task_id, created_at) diff --git a/sql/update/llm_db_update.sql b/sql/update/llm_db_update.sql index 6608a90..a4eb848 100644 --- a/sql/update/llm_db_update.sql +++ b/sql/update/llm_db_update.sql @@ -34,4 +34,4 @@ CREATE TABLE IF NOT EXISTS history ( CREATE INDEX IF NOT EXISTS idx_tbl_history_cols_task_id_user_id ON history (task_id, user_id); -ALTER TABLE Rating ADD COLUMN created_at TIMESTAMP; +ALTER TABLE Rating ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP; -- cgit v1.2.3 From 9c6f74d0a3f16a940b333d2a56803ed2e7d7c462 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 2 Sep 2024 15:05:25 +0300 Subject: Add spacing after punctuation. --- gn3/api/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 952a5b9..836a880 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -24,7 +24,7 @@ def search(): fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if not fahamu_token: raise LLMError( - "Request failed:an LLM authorisation token is required ", query) + "Request failed: an LLM authorisation token is required ", query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { -- cgit v1.2.3 From 748ebc5baff3412df163e17ee18a8fc688329b1d Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 2 Sep 2024 15:08:59 +0300 Subject: Use default datetime for table. --- gn3/api/llm.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 836a880..a617faf 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -75,12 +75,11 @@ def rate_queries(task_id): PRIMARY KEY(task_id))""" cursor.execute(create_table) cursor.execute("""INSERT INTO Rating(user_id, query, - answer, weight, task_id, created_at) - VALUES(?, ?, ?, ?, ?, ?) + answer, weight, task_id) + VALUES(?, ?, ?, ?, ?) ON CONFLICT(task_id) DO UPDATE SET weight=excluded.weight - """, (str(user_id), query, answer, weight, task_id, - datetime.now(timezone.utc))) + """, (str(user_id), query, answer, weight, task_id)) return { "message": "You have successfully rated this query.Thank you!" }, 200 -- cgit v1.2.3 From 884f57de47dfb7e80fd8ff25760ce5353267964e Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 2 Sep 2024 15:09:56 +0300 Subject: Fix spacing after punctuation. --- gn3/api/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index a617faf..8d17bc2 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -81,7 +81,7 @@ def rate_queries(task_id): weight=excluded.weight """, (str(user_id), query, answer, weight, task_id)) return { - "message": "You have successfully rated this query.Thank you!" + "message": "You have successfully rated this query. Thank you!" }, 200 -- cgit v1.2.3 From c34d75a29b7d144030542b9de4fd0e0d614303a9 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 3 Sep 2024 11:08:31 +0300 Subject: Use Jsonb for storing results. --- gn3/api/llm.py | 2 +- sql/update/llm_db_tables.sql | 2 +- sql/update/llm_db_update.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 8d17bc2..9ee4a79 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -40,7 +40,7 @@ def search(): history(user_id TEXT NOT NULL, task_id TEXT NOT NULL, query TEXT NOT NULL, - results TEXT, + results JSONB, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(task_id)) WITHOUT ROWID""") cursor.execute( diff --git a/sql/update/llm_db_tables.sql b/sql/update/llm_db_tables.sql index a6c0479..b501832 100644 --- a/sql/update/llm_db_tables.sql +++ b/sql/update/llm_db_tables.sql @@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS history ( user_id TEXT NOT NULL, task_id TEXT NOT NULL, query TEXT NOT NULL, - results TEXT, + results JSONB, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (task_id) ) WITHOUT ROWID; diff --git a/sql/update/llm_db_update.sql b/sql/update/llm_db_update.sql index a4eb848..7f1a9f9 100644 --- a/sql/update/llm_db_update.sql +++ b/sql/update/llm_db_update.sql @@ -25,7 +25,7 @@ CREATE TABLE IF NOT EXISTS history ( user_id TEXT NOT NULL, task_id TEXT NOT NULL, query TEXT NOT NULL, - results TEXT, + results JSONB, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (task_id) ) WITHOUT ROWID; -- cgit v1.2.3 From dbe79d8a4ca31e23d18b9fdb352dc783c7e0db64 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 3 Sep 2024 11:46:21 +0300 Subject: Remove unused imports. --- gn3/api/llm.py | 1 - gn3/llms/client.py | 2 -- 2 files changed, 3 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 9ee4a79..20831e5 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,6 +1,5 @@ """Api endpoints for gnqa""" import json -from datetime import datetime, timezone from flask import Blueprint from flask import current_app from flask import jsonify diff --git a/gn3/llms/client.py b/gn3/llms/client.py index 9bcb2e3..54a7a17 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -2,8 +2,6 @@ # pylint: disable=C0301 import json import time - -import requests from requests import Session from requests.adapters import HTTPAdapter from requests.adapters import Retry -- cgit v1.2.3 From 4dc60a7de36134b416d3414bea0c89f49f9420f4 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 5 Sep 2024 13:58:27 +0300 Subject: Create new endpoints for fetching user previous records. --- gn3/api/llm.py | 72 +++++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 51 insertions(+), 21 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 20831e5..192af23 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -84,32 +84,62 @@ def rate_queries(task_id): }, 200 -@gnqa.route("/history", methods=["GET", "DELETE"]) +@gnqa.route("/search/records", methods=["GET"]) @require_oauth("profile user") -def fetch_prev_history(): - """Api endpoint to fetch GNQA previous search.""" +def get_user_search_records(): + """get all history records for a given user using their + user id + """ with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() - if request.method == "DELETE": - task_ids = list(request.json.values()) - query = """DELETE FROM history - WHERE task_id IN ({}) - and user_id=?""".format(",".join("?" * len(task_ids))) - cursor.execute(query, (*task_ids, str(token.user.user_id),)) - return jsonify({}) - elif (request.method == "GET" and - request.args.get("search_term")): - cursor.execute( - """SELECT results from history - Where task_id=? and user_id=?""", - (request.args.get("search_term"), - str(token.user.user_id),)) - record = cursor.fetchone() - if record: - return dict(record).get("results") - return {} cursor.execute( """SELECT task_id,query from history WHERE user_id=?""", (str(token.user.user_id),)) return jsonify([dict(item) for item in cursor.fetchall()]) + + +@gnqa.route("/search/record/", methods=["GET"]) +@require_oauth("profile user") +def get_user_record_by_task(task_id): + """Get user record by task id """ + with (require_oauth.acquire("profile user") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): + cursor = conn.cursor() + cursor.execute( + """SELECT results from history + Where task_id=? and user_id=?""", + (task_id, + str(token.user.user_id),)) + record = cursor.fetchone() + if record: + return dict(record).get("results") + return {} + + +@gnqa.route("/search/record/", methods=["DELETE"]) +@require_oauth("profile user") +def delete_record(task_id): + """Delete user record by task-id""" + with (require_oauth.acquire("profile user") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): + cursor = conn.cursor() + query = """DELETE FROM history + WHERE task_id=? and user_id=?""" + cursor.execute(query, (task_id, token.user.user_id,)) + return {"msg": f"Successfully Deleted the task {task_id}"} + + +@gnqa.route("/search/records", methods=["DELETE"]) +@require_oauth("profile user") +def delete_records(): + """ Delete a users records using for all given task ids""" + with (require_oauth.acquire("profile user") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): + task_ids = list(request.json.values()) + cursor = conn.cursor() + query = """DELETE FROM history + WHERE task_id IN ({}) + and user_id=?""".format(",".join("?" * len(task_ids))) + cursor.execute(query, (*task_ids, str(token.user.user_id),)) + return jsonify({}) -- cgit v1.2.3 From 57986c976c702c590cc814fa9863c4fd9be42c6f Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 5 Sep 2024 14:00:07 +0300 Subject: Apply pep8 formatting. --- gn3/api/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 192af23..cdf427e 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -80,7 +80,7 @@ def rate_queries(task_id): weight=excluded.weight """, (str(user_id), query, answer, weight, task_id)) return { - "message": "You have successfully rated this query. Thank you!" + "message": "You have successfully rated this query. Thank you!" }, 200 -- cgit v1.2.3 From 8cb85c8f8c12180702cfc3a257bf9a513ac4da3d Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 5 Sep 2024 15:27:00 +0300 Subject: Sort previos records by datetime. --- gn3/api/llm.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'gn3/api') diff --git a/gn3/api/llm.py b/gn3/api/llm.py index cdf427e..7e60271 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,5 +1,7 @@ """Api endpoints for gnqa""" import json +from datetime import datetime + from flask import Blueprint from flask import current_app from flask import jsonify @@ -94,15 +96,18 @@ def get_user_search_records(): db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() cursor.execute( - """SELECT task_id,query from history WHERE user_id=?""", + """SELECT task_id, query, created_at from history WHERE user_id=?""", (str(token.user.user_id),)) - return jsonify([dict(item) for item in cursor.fetchall()]) + results = [dict(item) for item in cursor.fetchall()] + return jsonify(sorted(results, reverse=True, + key=lambda x: datetime.strptime(x.get("created_at"), + '%Y-%m-%d %H:%M:%S'))) @gnqa.route("/search/record/", methods=["GET"]) @require_oauth("profile user") def get_user_record_by_task(task_id): - """Get user record by task id """ + """Get user previous search record by task id """ with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() @@ -120,7 +125,7 @@ def get_user_record_by_task(task_id): @gnqa.route("/search/record/", methods=["DELETE"]) @require_oauth("profile user") def delete_record(task_id): - """Delete user record by task-id""" + """Delete user previous seach record by task-id""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() -- cgit v1.2.3