From 0832e9579d6fe6c9b46bfa499f1c7726301ac10c Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 17 May 2024 17:07:32 +0300 Subject: Add endpoint for getting prev user searches --- gn3/api/llm.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 442252f..41cc376 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -90,6 +90,18 @@ def rating(task_id): return jsonify({"error": str(error)}), 500 + +@gnqa.route("/searches/", methods=["GET"]) +@require_oauth("profile user") +def fetch_prev_searches(): + with (require_oauth.acquire("profile user") as __the_token, + Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn): + return jsonify({ + "prev_queries": get_user_queries("random_user", redis_conn) + }) + + @gnqa.route("/history/", methods=["GET"]) @require_oauth("profile user") def fetch_user_hist(query): @@ -103,6 +115,7 @@ def fetch_user_hist(query): }) + @gnqa.route("/historys/", methods=["GET"]) def fetch_users_hist_records(query): """method to fetch all users hist:note this is a test functionality -- cgit v1.2.3 From 8201936afc4a8330a3dfa25a26b3786f44e8e378 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 21 May 2024 16:35:16 +0300 Subject: Add search gnqa history functionality. --- gn3/api/llm.py | 40 ++++++++++++++++++++++------------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 41cc376..b2c9c3e 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,5 +1,4 @@ """Api endpoints for gnqa""" -from datetime import timedelta import json import sqlite3 from redis import Redis @@ -14,6 +13,7 @@ from gn3.llms.process import get_user_queries from gn3.llms.process import fetch_query_results from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth + from gn3.auth import db gnqa = Blueprint("gnqa", __name__) @@ -39,15 +39,17 @@ def gnqna(): "answer": answer, "references": refs } - with (Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): - redis_conn.setex( - f"LLM:random_user-{query}", - timedelta(days=10), json.dumps(response)) - return jsonify({ - **response, - "prev_queries": get_user_queries("random_user", redis_conn) - }) + try: + with (Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn, + require_oauth.acquire("profile user") as token): + redis_conn.set( + f"LLM:{str(token.user.user_id)}-{str(task_id['task_id'])}", + json.dumps(response) + ) + return response + except Exception: # handle specific error + return response except LLMError as error: return jsonify({"query": query, "error": f"Request failed-{str(error)}"}), 500 @@ -90,16 +92,19 @@ def rating(task_id): return jsonify({"error": str(error)}), 500 - -@gnqa.route("/searches/", methods=["GET"]) +@gnqa.route("/searches", methods=["GET"]) @require_oauth("profile user") def fetch_prev_searches(): - with (require_oauth.acquire("profile user") as __the_token, + """ api method to fetch search query records""" + with (require_oauth.acquire("profile user") as the_token, Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn): - return jsonify({ - "prev_queries": get_user_queries("random_user", redis_conn) - }) + if request.args.get("search_term"): + return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) + query_result = {} + for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): + query_result[key] = json.loads(redis_conn.get(key)) + return jsonify(query_result) @gnqa.route("/history/", methods=["GET"]) @@ -110,12 +115,11 @@ def fetch_user_hist(query): Redis.from_url(current_app.config["REDIS_URI"], decode_responses=True) as redis_conn): return jsonify({ - **fetch_query_results(query, the_token.user.id, redis_conn), + **fetch_query_results(query, the_token.user.user_id, redis_conn), "prev_queries": get_user_queries("random_user", redis_conn) }) - @gnqa.route("/historys/", methods=["GET"]) def fetch_users_hist_records(query): """method to fetch all users hist:note this is a test functionality -- cgit v1.2.3 From 2a99da9f46233a28e9ea0b6a297d8a6b93f61923 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 21 May 2024 16:38:53 +0300 Subject: Remove obsolete functions. --- gn3/api/llm.py | 39 +-------------------------------------- gn3/llms/process.py | 20 -------------------- 2 files changed, 1 insertion(+), 58 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index b2c9c3e..02b37f9 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -9,8 +9,6 @@ from flask import jsonify from flask import request from gn3.llms.process import get_gnqa -from gn3.llms.process import get_user_queries -from gn3.llms.process import fetch_query_results from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth @@ -46,7 +44,7 @@ def gnqna(): redis_conn.set( f"LLM:{str(token.user.user_id)}-{str(task_id['task_id'])}", json.dumps(response) - ) + ) return response except Exception: # handle specific error return response @@ -105,38 +103,3 @@ def fetch_prev_searches(): for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): query_result[key] = json.loads(redis_conn.get(key)) return jsonify(query_result) - - -@gnqa.route("/history/", methods=["GET"]) -@require_oauth("profile user") -def fetch_user_hist(query): - """"Endpoint to fetch previos searches for User""" - with (require_oauth.acquire("profile user") as the_token, - Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): - return jsonify({ - **fetch_query_results(query, the_token.user.user_id, redis_conn), - "prev_queries": get_user_queries("random_user", redis_conn) - }) - - -@gnqa.route("/historys/", methods=["GET"]) -def fetch_users_hist_records(query): - """method to fetch all users hist:note this is a test functionality - to be replaced by fetch_user_hist - """ - with Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn: - return jsonify({ - **fetch_query_results(query, "random_user", redis_conn), - "prev_queries": get_user_queries("random_user", redis_conn) - }) - - -@gnqa.route("/get_hist_names", methods=["GET"]) -def fetch_prev_hist_ids(): - """Test method for fetching history for Anony Users""" - with (Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True)) as redis_conn: - return jsonify({"prev_queries": get_user_queries("random_user", - redis_conn)}) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index 4725bcb..eba7e4b 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -135,23 +135,3 @@ def get_gnqa(query, auth_token, data_dir=""): return task_id, answer, references else: return task_id, "We couldn't provide a response,Please try to rephrase your question to receive feedback", [] - - -def fetch_query_results(query, user_id, redis_conn): - """this method fetches prev user query searches""" - result = redis_conn.get(f"LLM:{user_id}-{query}") - if result: - return json.loads(result) - return { - "query": query, - "answer": "Sorry No answer for you", - "references": [], - "task_id": None - } - - -def get_user_queries(user_id, redis_conn): - """methos to fetch all queries for a specific user""" - results = redis_conn.keys(f"LLM:{user_id}*") - return [query for query in - [result.partition("-")[2] for result in results] if query != ""] -- cgit v1.2.3 From 12100489a73094016602926183e0ee51002fb9c6 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 22 May 2024 13:37:32 +0300 Subject: Register LLM error in app. * do refactoring for gn3:llm:errors --- gn3/errors.py | 17 ++++++++++++++++- gn3/llms/errors.py | 14 +++++++++----- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/gn3/errors.py b/gn3/errors.py index 1833bf6..ac9e070 100644 --- a/gn3/errors.py +++ b/gn3/errors.py @@ -16,7 +16,7 @@ from authlib.oauth2.rfc6749.errors import OAuth2Error from flask import Flask, jsonify, Response, current_app from gn3.auth.authorisation.errors import AuthorisationError - +from gn3.llms.errors import LLMError def add_trace(exc: Exception, jsonmsg: dict) -> dict: """Add the traceback to the error handling object.""" @@ -106,6 +106,20 @@ def handle_generic(exc: Exception) -> Response: return resp +def handle_llm_error(exc: Exception) -> Response: + """ Handle llm erros if not handled anywhere else. """ + resp = jsonify({ + "query": exc.query if exc.query else "", + "error_type": type(exc).__name__, + "error": ( + exc.args if bool(exc.args) else "Fahamu gnqa error occurred" + ), + "trace": traceback.format_exc() + }) + resp.status_code = 500 + return resp + + def register_error_handlers(app: Flask): """Register application-level error handlers.""" app.register_error_handler(NotFound, page_not_found) @@ -115,6 +129,7 @@ def register_error_handlers(app: Flask): app.register_error_handler(AuthorisationError, handle_authorisation_error) app.register_error_handler(RemoteDisconnected, internal_server_error) app.register_error_handler(URLError, url_server_error) + app.register_error_handler(LLMError, handle_llm_error) for exc in ( EndPointInternalError, EndPointNotFound, diff --git a/gn3/llms/errors.py b/gn3/llms/errors.py index af3d7b0..3512f4d 100644 --- a/gn3/llms/errors.py +++ b/gn3/llms/errors.py @@ -35,8 +35,12 @@ class UnprocessableEntity(HTTPError): msg, request=request, response=response) -class LLMError(HTTPError): - """Custom error from making Fahamu APi request """ - def __init__(self, request, response, msg): - super(HTTPError, self).__init__( - msg, request=request, response=response) +class LLMErrorMIxins(Exception): + """base class for llm errors""" + + +class LLMError(LLMErrorMIxins): + """custom exception for LLMErrorMIxins""" + def __init__(self, *args, **kwargs): + super().__init__(*args) + self.query = kwargs.get("query") -- cgit v1.2.3 From a304037ce012516b07c17fd0dcb9e816c33a4d58 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 22 May 2024 13:38:06 +0300 Subject: Refactor error handling for llm api's. --- gn3/api/llm.py | 49 ++++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 02b37f9..8e8a50f 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,17 +1,20 @@ """Api endpoints for gnqa""" import json import sqlite3 +import redis from redis import Redis +from authlib.integrations.flask_oauth2.errors import _HTTPException from flask import Blueprint from flask import current_app from flask import jsonify from flask import request + from gn3.llms.process import get_gnqa from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth - +from gn3.auth.authorisation.errors import AuthorisationError from gn3.auth import db gnqa = Blueprint("gnqa", __name__) @@ -23,12 +26,10 @@ def gnqna(): query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 - try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if fahamu_token is None: - return jsonify({"query": query, - "error": "Use of invalid fahamu auth token"}), 500 + raise LLMError("Request failed:an LLM authorisation token is required ", query=query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -46,16 +47,16 @@ def gnqna(): json.dumps(response) ) return response - except Exception: # handle specific error - return response + except _HTTPException as httpe: + raise AuthorisationError("Authentication is required.") from httpe except LLMError as error: - return jsonify({"query": query, - "error": f"Request failed-{str(error)}"}), 500 + raise LLMError(f"request failed for query {str(error.args[-1])}", + query=query) from error @gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") -def rating(task_id): +def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" try: llm_db_path = current_app.config["LLM_DB_PATH"] @@ -87,19 +88,25 @@ def rating(task_id): "You have successfully rated this query:Thank you!!" }, 200 except sqlite3.Error as error: - return jsonify({"error": str(error)}), 500 + raise sqlite3.OperationalError from error + except _HTTPException as httpe: + raise AuthorisationError("Authentication is required") from httpe -@gnqa.route("/searches", methods=["GET"]) +@gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") -def fetch_prev_searches(): +def fetch_prev_history(): """ api method to fetch search query records""" - with (require_oauth.acquire("profile user") as the_token, - Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): - if request.args.get("search_term"): - return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) - query_result = {} - for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): - query_result[key] = json.loads(redis_conn.get(key)) - return jsonify(query_result) + try: + + with (require_oauth.acquire("profile user") as the_token, + Redis.from_url(current_app.config["REDIS_URI"], + decode_responses=True) as redis_conn): + if request.args.get("search_term"): + return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) + query_result = {} + for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): + query_result[key] = json.loads(redis_conn.get(key)) + return jsonify(query_result) + except redis.exceptions.RedisError as error: + raise error -- cgit v1.2.3 From c280bae2bd97d17189763d3ce76cfdfc35588fd2 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 22 May 2024 18:11:58 +0300 Subject: This commit does the following: * Adds a new table to store the history records. * Remove the redis dependancy. --- gn3/api/llm.py | 52 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 8e8a50f..2ed52eb 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,8 +1,6 @@ """Api endpoints for gnqa""" import json import sqlite3 -import redis -from redis import Redis from authlib.integrations.flask_oauth2.errors import _HTTPException from flask import Blueprint @@ -39,16 +37,26 @@ def gnqna(): "references": refs } try: - with (Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn, + with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, require_oauth.acquire("profile user") as token): - redis_conn.set( - f"LLM:{str(token.user.user_id)}-{str(task_id['task_id'])}", - json.dumps(response) + schema = """CREATE TABLE IF NOT EXISTS + history(user_id TEXT NOT NULL, + task_id TEXT NOT NULL, + query TEXT NOT NULL, + results TEXT, + PRIMARY KEY(task_id)) WITHOUT ROWID""" + cursor = conn.cursor() + cursor.execute(schema) + cursor.execute("""INSERT INTO history(user_id,task_id,query,results) + VALUES(?,?,?,?) + """,(str(token.user.user_id),str(task_id["task_id"]),query, + json.dumps(response)) ) - return response + return response except _HTTPException as httpe: raise AuthorisationError("Authentication is required.") from httpe + except sqlite3.Error as error: + raise error except LLMError as error: raise LLMError(f"request failed for query {str(error.args[-1])}", query=query) from error @@ -92,21 +100,25 @@ def rate_queries(task_id): except _HTTPException as httpe: raise AuthorisationError("Authentication is required") from httpe - @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): """ api method to fetch search query records""" try: - - with (require_oauth.acquire("profile user") as the_token, - Redis.from_url(current_app.config["REDIS_URI"], - decode_responses=True) as redis_conn): + llm_db_path = current_app.config["LLM_DB_PATH"] + with (require_oauth.acquire("profile user") as token, + db.connection(llm_db_path) as conn): + cursor = conn.cursor() if request.args.get("search_term"): - return jsonify(json.loads(redis_conn.get(request.args.get("search_term")))) - query_result = {} - for key in redis_conn.scan_iter(f"LLM:{str(the_token.user.user_id)}*"): - query_result[key] = json.loads(redis_conn.get(key)) - return jsonify(query_result) - except redis.exceptions.RedisError as error: - raise error + query = """SELECT results from history Where task_id=? and user_id=?""" + cursor.execute(query, (request.args.get("search_term") + ,str(token.user.user_id),)) + return dict(cursor.fetchone()) + query = """SELECT task_id,query from history WHERE user_id=?""" + cursor.execute(query, (str(token.user.user_id),)) + return [dict(item) for item in cursor.fetchall()] + + except sqlite3.Error as error: #please handle me corrrectly + return jsonify({"error":error}), 500 + except _HTTPException as httpe: + raise AuthorisationError("Authorization is required") from httpe -- cgit v1.2.3 From 7d3b2a29f5497d88b7c1391a7f5631591889ab36 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 23 May 2024 12:10:52 +0300 Subject: Refactor error handling code. --- gn3/api/llm.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 2ed52eb..172e49b 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -8,7 +8,6 @@ from flask import current_app from flask import jsonify from flask import request - from gn3.llms.process import get_gnqa from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth @@ -27,7 +26,8 @@ def gnqna(): try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if fahamu_token is None: - raise LLMError("Request failed:an LLM authorisation token is required ", query=query) + raise LLMError( + "Request failed:an LLM authorisation token is required ", query=query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -49,14 +49,14 @@ def gnqna(): cursor.execute(schema) cursor.execute("""INSERT INTO history(user_id,task_id,query,results) VALUES(?,?,?,?) - """,(str(token.user.user_id),str(task_id["task_id"]),query, - json.dumps(response)) + """, (str(token.user.user_id), str(task_id["task_id"]), query, + json.dumps(response)) ) return response except _HTTPException as httpe: raise AuthorisationError("Authentication is required.") from httpe except sqlite3.Error as error: - raise error + raise sqlite3.OperationalError(*error.args) from error except LLMError as error: raise LLMError(f"request failed for query {str(error.args[-1])}", query=query) from error @@ -96,14 +96,15 @@ def rate_queries(task_id): "You have successfully rated this query:Thank you!!" }, 200 except sqlite3.Error as error: - raise sqlite3.OperationalError from error + raise sqlite3.OperationalError(*error.args) from error except _HTTPException as httpe: raise AuthorisationError("Authentication is required") from httpe + @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): - """ api method to fetch search query records""" + """ api method to fetch search query records""" # ro only try: llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile user") as token, @@ -111,14 +112,13 @@ def fetch_prev_history(): cursor = conn.cursor() if request.args.get("search_term"): query = """SELECT results from history Where task_id=? and user_id=?""" - cursor.execute(query, (request.args.get("search_term") - ,str(token.user.user_id),)) + cursor.execute(query, (request.args.get( + "search_term"), str(token.user.user_id),)) return dict(cursor.fetchone()) query = """SELECT task_id,query from history WHERE user_id=?""" cursor.execute(query, (str(token.user.user_id),)) - return [dict(item) for item in cursor.fetchall()] - - except sqlite3.Error as error: #please handle me corrrectly - return jsonify({"error":error}), 500 + return jsonify([dict(item) for item in cursor.fetchall()]) + except sqlite3.Error as error: + raise sqlite3.OperationalError(*error.args) from error except _HTTPException as httpe: raise AuthorisationError("Authorization is required") from httpe -- cgit v1.2.3 From d4b0ae19e55a45eed7b6bca43abb5340f58ccfbe Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 23 May 2024 12:17:40 +0300 Subject: rename gnqna route to search. --- gn3/api/llm.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 172e49b..e8ffa1a 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -17,17 +17,18 @@ from gn3.auth import db gnqa = Blueprint("gnqa", __name__) -@gnqa.route("/gnqna", methods=["POST"]) -def gnqna(): +@gnqa.route("/search", methods=["POST"]) +def search(): """Main gnqa endpoint""" query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 try: fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") - if fahamu_token is None: + if not fahamu_token: raise LLMError( - "Request failed:an LLM authorisation token is required ", query=query) + "Request failed:an LLM authorisation token is required ", + query=query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -39,17 +40,18 @@ def gnqna(): try: with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, require_oauth.acquire("profile user") as token): - schema = """CREATE TABLE IF NOT EXISTS + cursor = conn.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS history(user_id TEXT NOT NULL, task_id TEXT NOT NULL, query TEXT NOT NULL, results TEXT, - PRIMARY KEY(task_id)) WITHOUT ROWID""" - cursor = conn.cursor() - cursor.execute(schema) - cursor.execute("""INSERT INTO history(user_id,task_id,query,results) + PRIMARY KEY(task_id)) WITHOUT ROWID""") + cursor.execute( + """INSERT INTO history(user_id,task_id,query,results) VALUES(?,?,?,?) - """, (str(token.user.user_id), str(task_id["task_id"]), query, + """, (str(token.user.user_id), str(task_id["task_id"]), + query, json.dumps(response)) ) return response @@ -104,7 +106,7 @@ def rate_queries(task_id): @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): - """ api method to fetch search query records""" # ro only + """ api method to fetch search query records""" try: llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile user") as token, -- cgit v1.2.3 From ef955f9b456a591f64faa428b8ef83252923bb63 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 23 May 2024 12:29:04 +0300 Subject: Remove irrelevant variable assignments. --- gn3/api/llm.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index e8ffa1a..5ad58cb 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -69,9 +69,8 @@ def search(): def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" try: - llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile") as token, - db.connection(llm_db_path) as conn): + db.connection(current_app.config["LLM_DB_PATH"]) as conn): results = request.json user_id, query, answer, weight = (token.user.user_id, @@ -108,17 +107,18 @@ def rate_queries(task_id): def fetch_prev_history(): """ api method to fetch search query records""" try: - llm_db_path = current_app.config["LLM_DB_PATH"] with (require_oauth.acquire("profile user") as token, - db.connection(llm_db_path) as conn): + db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() if request.args.get("search_term"): - query = """SELECT results from history Where task_id=? and user_id=?""" - cursor.execute(query, (request.args.get( - "search_term"), str(token.user.user_id),)) + cursor.execute( + """SELECT results from history Where task_id=? and user_id=?""", + (request.args.get("search_term"), + str(token.user.user_id),)) return dict(cursor.fetchone()) - query = """SELECT task_id,query from history WHERE user_id=?""" - cursor.execute(query, (str(token.user.user_id),)) + cursor.execute( + """SELECT task_id,query from history WHERE user_id=?""", + (str(token.user.user_id),)) return jsonify([dict(item) for item in cursor.fetchall()]) except sqlite3.Error as error: raise sqlite3.OperationalError(*error.args) from error -- cgit v1.2.3 From d9233e0a4811203e59161b635c33a7c1b753b3d8 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 14:20:02 +0300 Subject: Remove redundant llm base class exception. --- gn3/llms/errors.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gn3/llms/errors.py b/gn3/llms/errors.py index 3512f4d..c5439d6 100644 --- a/gn3/llms/errors.py +++ b/gn3/llms/errors.py @@ -35,11 +35,7 @@ class UnprocessableEntity(HTTPError): msg, request=request, response=response) -class LLMErrorMIxins(Exception): - """base class for llm errors""" - - -class LLMError(LLMErrorMIxins): +class LLMError(Exception): """custom exception for LLMErrorMIxins""" def __init__(self, *args, **kwargs): super().__init__(*args) -- cgit v1.2.3 From 2518b60722e5248db363405dee68d9edb0e79601 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 15:14:31 +0300 Subject: Update custom_request to raise LLMError for Exceptions. --- gn3/llms/client.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/gn3/llms/client.py b/gn3/llms/client.py index d57bca2..99df36a 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -105,22 +105,25 @@ class GeneNetworkQAClient(Session): """ make custom request to fahamu api ask and get response""" max_retries = 50 retry_delay = 3 + response_msg = { + 404: "Api endpoint Does not exist", + 500: "Use of Invalid Token/or the Fahamu Api is currently down", + 400: "You sent a bad Fahamu request", + 401: "You do not have authorization to perform the request", + } for _i in range(max_retries): - try: - response = super().request(method, url, *args, **kwargs) - response.raise_for_status() - if response.ok: - if method.lower() == "get" and response.json().get("data") is None: - time.sleep(retry_delay) - continue - return response - else: + response = super().request(method, url, *args, **kwargs) + if response.ok: + if method.lower() == "get" and response.json().get("data") is None: + # note this is a dirty trick to check if fahamu has returned the results + # the issue is that the api only returns 500 or 200 satus code + # TODO: fix this on their end time.sleep(retry_delay) - except requests.exceptions.HTTPError as error: - if error.response.status_code == 500: - raise LLMError(error.request, error.response, f"Response Error with:status_code:{error.response.status_code},Reason for error: Use of Invalid Fahamu Token") from error - raise LLMError(error.request, error.response, - f"HTTP error occurred with error status:{error.response.status_code}") from error - except requests.exceptions.RequestException as error: - raise error - raise TimeoutError + continue + return response + else: + raise LLMError( f"Request error with code:\ + {response.status_code} occurred with reason:\ + {response_msg.get(response.status_code,response.reason)}") + #time.sleep(retry_delay) + raise LLMError("Time error occurred when querying the fahamu Api,Please a try the search again") -- cgit v1.2.3 From e75d484bf786176edcac4edfff65e3035fd493eb Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 15:22:08 +0300 Subject: Remove try/block for get_answer/ask methods:Exception already raised --- gn3/llms/client.py | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/gn3/llms/client.py b/gn3/llms/client.py index 99df36a..07e9506 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -86,20 +86,13 @@ class GeneNetworkQAClient(Session): def ask(self, ex_url, *args, **kwargs): """fahamu ask api interface""" res = self.custom_request('POST', f"{self.base_url}{ex_url}", *args, **kwargs) - if res.status_code != 200: - return f"Error: Status code -{res.status_code}- Reason::{res.reason}", 0 return res, json.loads(res.text) def get_answer(self, taskid, *args, **kwargs): """Fahamu get answer interface""" - try: - query = f"{self.answer_url}?task_id={taskid['task_id']}" - res = self.custom_request('GET', query, *args, **kwargs) - if res.status_code != 200: - return f"Error: Status code -{res.status_code}- Reason::{res.reason}", 0 - return res, 1 - except TimeoutError: - return "Timeout error occured:try to rephrase your query", 0 + query = f"{self.answer_url}?task_id={taskid['task_id']}" + res = self.custom_request('GET', query, *args, **kwargs) + return res, 1 def custom_request(self, method, url, *args, **kwargs): """ make custom request to fahamu api ask and get response""" @@ -126,4 +119,4 @@ class GeneNetworkQAClient(Session): {response.status_code} occurred with reason:\ {response_msg.get(response.status_code,response.reason)}") #time.sleep(retry_delay) - raise LLMError("Time error occurred when querying the fahamu Api,Please a try the search again") + raise LLMError("Time error: Please try to rephrase of query to get an answer") -- cgit v1.2.3 From 9806f3cf708f8eb4e1248eb5059deee53a3887c6 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 15:33:51 +0300 Subject: Initialize new class attribute self.query for to pass to LLMError. --- gn3/llms/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gn3/llms/client.py b/gn3/llms/client.py index 07e9506..fa1e36e 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -55,6 +55,7 @@ class GeneNetworkQAClient(Session): self.base_url = "https://genenetwork.fahamuai.com/api/tasks" self.answer_url = f"{self.base_url}/answers" self.feedback_url = f"{self.base_url}/feedback" + self.query = "" adapter = TimeoutHTTPAdapter( timeout=timeout, @@ -83,8 +84,9 @@ class GeneNetworkQAClient(Session): """ handler for non 200 response from fahamu api""" return f"Error: Status code -{response.status_code}- Reason::{response.reason}" - def ask(self, ex_url, *args, **kwargs): + def ask(self, ex_url, query, *args, **kwargs): """fahamu ask api interface""" + self.query = query res = self.custom_request('POST', f"{self.base_url}{ex_url}", *args, **kwargs) return res, json.loads(res.text) @@ -117,6 +119,8 @@ class GeneNetworkQAClient(Session): else: raise LLMError( f"Request error with code:\ {response.status_code} occurred with reason:\ - {response_msg.get(response.status_code,response.reason)}") + {response_msg.get(response.status_code,response.reason)}", + query=self.query) #time.sleep(retry_delay) - raise LLMError("Time error: Please try to rephrase of query to get an answer") + raise LLMError("Time error: Please try to rephrase of query to get an answer", + query=self.query) -- cgit v1.2.3 From 13bb57cbd191ffe6e40e830ca08b9191b2dc5700 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 15:34:53 +0300 Subject: Pass query as an argument to api_client ask method. --- gn3/llms/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index eba7e4b..d53a7fd 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -118,7 +118,7 @@ def get_gnqa(query, auth_token, data_dir=""): """ api_client = GeneNetworkQAClient(api_key=auth_token) - res, task_id = api_client.ask('?ask=' + quote(query), auth_token) + res, task_id = api_client.ask('?ask=' + quote(query), query=query) if task_id == 0: raise RuntimeError(f"Error connecting to Fahamu Api: {str(res)}") res, status = api_client.get_answer(task_id) -- cgit v1.2.3 From ee1345535817a6b3e6943b268057f53840d78b8f Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 15:48:45 +0300 Subject: Check for null and empty data results and update timeout message --- gn3/llms/client.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gn3/llms/client.py b/gn3/llms/client.py index fa1e36e..5c4fa0e 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -109,7 +109,7 @@ class GeneNetworkQAClient(Session): for _i in range(max_retries): response = super().request(method, url, *args, **kwargs) if response.ok: - if method.lower() == "get" and response.json().get("data") is None: + if method.lower() == "get" and not (response.json().get("data")): # note this is a dirty trick to check if fahamu has returned the results # the issue is that the api only returns 500 or 200 satus code # TODO: fix this on their end @@ -122,5 +122,6 @@ class GeneNetworkQAClient(Session): {response_msg.get(response.status_code,response.reason)}", query=self.query) #time.sleep(retry_delay) - raise LLMError("Time error: Please try to rephrase of query to get an answer", + raise LLMError("Time error: We couldn't provide a response,Please try\ + to rephrase your question to receive feedback", query=self.query) -- cgit v1.2.3 From 651f307a4b8e60aaea0c8a7649a5b02aafce7a98 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 15:59:30 +0300 Subject: Removed status check on get_gnqa function. --- gn3/llms/process.py | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index d53a7fd..ab2a80e 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -116,22 +116,11 @@ def get_gnqa(query, auth_token, data_dir=""): answer references: contains doc_name,reference,pub_med_info """ - api_client = GeneNetworkQAClient(api_key=auth_token) res, task_id = api_client.ask('?ask=' + quote(query), query=query) - if task_id == 0: - raise RuntimeError(f"Error connecting to Fahamu Api: {str(res)}") - res, status = api_client.get_answer(task_id) - if status == 1: - resp_text = filter_response_text(res.text) - if resp_text.get("data") is None: - return task_id, "Please try to rephrase your question to receive feedback", [] - answer = resp_text['data']['answer'] - context = resp_text['data']['context'] - references = parse_context( - context, DocIDs().get_info, format_bibliography_info) - references = fetch_pubmed(references, "pubmed.json", data_dir) - - return task_id, answer, references - else: - return task_id, "We couldn't provide a response,Please try to rephrase your question to receive feedback", [] + res, _status = api_client.get_answer(task_id) + resp_text = filter_response_text(res.text) + answer = resp_text['data']['answer'] + context = resp_text['data']['context'] + return task_id, answer, fetch_pubmed(parse_context( + context, DocIDs().get_info, format_bibliography_info), "pubmed.json", data_dir) -- cgit v1.2.3 From 6bbe9763a024558f2a4a942d71c799e4583448a2 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:06:22 +0300 Subject: Remove Try/Excepts from llm api endponts. --- gn3/api/llm.py | 152 +++++++++++++++++++++++++-------------------------------- 1 file changed, 66 insertions(+), 86 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 5ad58cb..03ce815 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -2,7 +2,6 @@ import json import sqlite3 from authlib.integrations.flask_oauth2.errors import _HTTPException - from flask import Blueprint from flask import current_app from flask import jsonify @@ -14,6 +13,7 @@ from gn3.auth.authorisation.oauth2.resource_server import require_oauth from gn3.auth.authorisation.errors import AuthorisationError from gn3.auth import db + gnqa = Blueprint("gnqa", __name__) @@ -23,104 +23,84 @@ def search(): query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 - try: - fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") - if not fahamu_token: - raise LLMError( - "Request failed:an LLM authorisation token is required ", - query=query) - task_id, answer, refs = get_gnqa( - query, fahamu_token, current_app.config.get("DATA_DIR")) - response = { - "task_id": task_id, - "query": query, - "answer": answer, - "references": refs - } - try: - with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, - require_oauth.acquire("profile user") as token): - cursor = conn.cursor() - cursor.execute("""CREATE TABLE IF NOT EXISTS - history(user_id TEXT NOT NULL, - task_id TEXT NOT NULL, - query TEXT NOT NULL, - results TEXT, - PRIMARY KEY(task_id)) WITHOUT ROWID""") - cursor.execute( - """INSERT INTO history(user_id,task_id,query,results) - VALUES(?,?,?,?) - """, (str(token.user.user_id), str(task_id["task_id"]), - query, - json.dumps(response)) - ) - return response - except _HTTPException as httpe: - raise AuthorisationError("Authentication is required.") from httpe - except sqlite3.Error as error: - raise sqlite3.OperationalError(*error.args) from error - except LLMError as error: - raise LLMError(f"request failed for query {str(error.args[-1])}", - query=query) from error + fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") + if not fahamu_token: + raise LLMError( + "Request failed:an LLM authorisation token is required ", + query=query) + task_id, answer, refs = get_gnqa( + query, fahamu_token, current_app.config.get("DATA_DIR")) + response = { + "task_id": task_id, + "query": query, + "answer": answer, + "references": refs + } + with (db.connection(current_app.config["LLM_DB_PATH"]) as conn, + require_oauth.acquire("profile user") as token): + cursor = conn.cursor() + cursor.execute("""CREATE TABLE IF NOT EXISTS + history(user_id TEXT NOT NULL, + task_id TEXT NOT NULL, + query TEXT NOT NULL, + results TEXT, + PRIMARY KEY(task_id)) WITHOUT ROWID""") + cursor.execute( + """INSERT INTO history(user_id, task_id, query, results) + VALUES(?, ?, ?, ?) + """, (str(token.user.user_id), str(task_id["task_id"]), + query, + json.dumps(response)) + ) + return response @gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" - try: - with (require_oauth.acquire("profile") as token, - db.connection(current_app.config["LLM_DB_PATH"]) as conn): + with (require_oauth.acquire("profile") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): - results = request.json - user_id, query, answer, weight = (token.user.user_id, - results.get("query"), - results.get("answer"), - results.get("weight", 0)) - cursor = conn.cursor() - create_table = """CREATE TABLE IF NOT EXISTS Rating( - user_id TEXT NOT NULL, - query TEXT NOT NULL, - answer TEXT NOT NULL, - weight INTEGER NOT NULL DEFAULT 0, - task_id TEXT NOT NULL UNIQUE - )""" - cursor.execute(create_table) - cursor.execute("""INSERT INTO Rating(user_id,query, - answer,weight,task_id) - VALUES(?,?,?,?,?) - ON CONFLICT(task_id) DO UPDATE SET - weight=excluded.weight - """, (str(user_id), query, answer, weight, task_id)) + results = request.json + user_id, query, answer, weight = (token.user.user_id, + results.get("query"), + results.get("answer"), + results.get("weight", 0)) + cursor = conn.cursor() + create_table = """CREATE TABLE IF NOT EXISTS Rating( + user_id TEXT NOT NULL, + query TEXT NOT NULL, + answer TEXT NOT NULL, + weight INTEGER NOT NULL DEFAULT 0, + task_id TEXT NOT NULL UNIQUE + )""" + cursor.execute(create_table) + cursor.execute("""INSERT INTO Rating(user_id, query, + answer, weight, task_id) + VALUES(?,?,?,?,?) + ON CONFLICT(task_id) DO UPDATE SET + weight=excluded.weight + """, (str(user_id), query, answer, weight, task_id)) return { - "message": - "You have successfully rated this query:Thank you!!" + "message": "You have successfully rated this query.Thank you!" }, 200 - except sqlite3.Error as error: - raise sqlite3.OperationalError(*error.args) from error - except _HTTPException as httpe: - raise AuthorisationError("Authentication is required") from httpe @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): """ api method to fetch search query records""" - try: - with (require_oauth.acquire("profile user") as token, - db.connection(current_app.config["LLM_DB_PATH"]) as conn): - cursor = conn.cursor() - if request.args.get("search_term"): - cursor.execute( - """SELECT results from history Where task_id=? and user_id=?""", - (request.args.get("search_term"), - str(token.user.user_id),)) - return dict(cursor.fetchone()) + with (require_oauth.acquire("profile user") as token, + db.connection(current_app.config["LLM_DB_PATH"]) as conn): + cursor = conn.cursor() + if request.args.get("search_term"): cursor.execute( - """SELECT task_id,query from history WHERE user_id=?""", - (str(token.user.user_id),)) - return jsonify([dict(item) for item in cursor.fetchall()]) - except sqlite3.Error as error: - raise sqlite3.OperationalError(*error.args) from error - except _HTTPException as httpe: - raise AuthorisationError("Authorization is required") from httpe + """SELECT results from history Where task_id=? and user_id=?""", + (request.args.get("search_term"), + str(token.user.user_id),)) + return dict(cursor.fetchone())["results"] + cursor.execute( + """SELECT task_id,query from history WHERE user_id=?""", + (str(token.user.user_id),)) + return jsonify([dict(item) for item in cursor.fetchall()]) -- cgit v1.2.3 From 67c71507c84d474ac13681f16d994e7967321ddb Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:21:50 +0300 Subject: Return first argument as error message. --- gn3/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gn3/errors.py b/gn3/errors.py index ac9e070..ad08ae5 100644 --- a/gn3/errors.py +++ b/gn3/errors.py @@ -112,7 +112,7 @@ def handle_llm_error(exc: Exception) -> Response: "query": exc.query if exc.query else "", "error_type": type(exc).__name__, "error": ( - exc.args if bool(exc.args) else "Fahamu gnqa error occurred" + exc.args[0] if bool(exc.args) else "Fahamu gnqa error occurred" ), "trace": traceback.format_exc() }) -- cgit v1.2.3 From 8512d9a606fbfff864345d82c210e281a6d943bf Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:27:51 +0300 Subject: Initiliaze second args to LLMError as query parameter. --- gn3/errors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gn3/errors.py b/gn3/errors.py index ad08ae5..8331028 100644 --- a/gn3/errors.py +++ b/gn3/errors.py @@ -109,7 +109,7 @@ def handle_generic(exc: Exception) -> Response: def handle_llm_error(exc: Exception) -> Response: """ Handle llm erros if not handled anywhere else. """ resp = jsonify({ - "query": exc.query if exc.query else "", + "query": exc.args[1], "error_type": type(exc).__name__, "error": ( exc.args[0] if bool(exc.args) else "Fahamu gnqa error occurred" -- cgit v1.2.3 From 188f84ef3895de613e998c63da7ec2338e25a55c Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:30:59 +0300 Subject: Remove kwargs from LLMErrorr Exceptions and update relevant code. --- gn3/api/llm.py | 7 +------ gn3/llms/client.py | 6 +++--- gn3/llms/errors.py | 1 - 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 03ce815..c1f6304 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,7 +1,5 @@ """Api endpoints for gnqa""" import json -import sqlite3 -from authlib.integrations.flask_oauth2.errors import _HTTPException from flask import Blueprint from flask import current_app from flask import jsonify @@ -10,7 +8,6 @@ from flask import request from gn3.llms.process import get_gnqa from gn3.llms.errors import LLMError from gn3.auth.authorisation.oauth2.resource_server import require_oauth -from gn3.auth.authorisation.errors import AuthorisationError from gn3.auth import db @@ -26,8 +23,7 @@ def search(): fahamu_token = current_app.config.get("FAHAMU_AUTH_TOKEN") if not fahamu_token: raise LLMError( - "Request failed:an LLM authorisation token is required ", - query=query) + "Request failed:an LLM authorisation token is required ", query) task_id, answer, refs = get_gnqa( query, fahamu_token, current_app.config.get("DATA_DIR")) response = { @@ -61,7 +57,6 @@ def rate_queries(task_id): """Endpoint for rating qnqa query and answer""" with (require_oauth.acquire("profile") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): - results = request.json user_id, query, answer, weight = (token.user.user_id, results.get("query"), diff --git a/gn3/llms/client.py b/gn3/llms/client.py index 5c4fa0e..d29d2a1 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -117,11 +117,11 @@ class GeneNetworkQAClient(Session): continue return response else: - raise LLMError( f"Request error with code:\ + raise LLMError(f"Request error with code:\ {response.status_code} occurred with reason:\ {response_msg.get(response.status_code,response.reason)}", - query=self.query) + self.query) #time.sleep(retry_delay) raise LLMError("Time error: We couldn't provide a response,Please try\ to rephrase your question to receive feedback", - query=self.query) + self.query) diff --git a/gn3/llms/errors.py b/gn3/llms/errors.py index c5439d6..77e0f9a 100644 --- a/gn3/llms/errors.py +++ b/gn3/llms/errors.py @@ -39,4 +39,3 @@ class LLMError(Exception): """custom exception for LLMErrorMIxins""" def __init__(self, *args, **kwargs): super().__init__(*args) - self.query = kwargs.get("query") -- cgit v1.2.3 From 3ea881f3bc28e8087be08f1d507991ac9b2a4230 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 16:36:57 +0300 Subject: Pylint fixes. --- gn3/llms/client.py | 4 ++-- gn3/llms/errors.py | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/gn3/llms/client.py b/gn3/llms/client.py index d29d2a1..ad6c400 100644 --- a/gn3/llms/client.py +++ b/gn3/llms/client.py @@ -109,7 +109,7 @@ class GeneNetworkQAClient(Session): for _i in range(max_retries): response = super().request(method, url, *args, **kwargs) if response.ok: - if method.lower() == "get" and not (response.json().get("data")): + if method.lower() == "get" and not response.json().get("data"): # note this is a dirty trick to check if fahamu has returned the results # the issue is that the api only returns 500 or 200 satus code # TODO: fix this on their end @@ -122,6 +122,6 @@ class GeneNetworkQAClient(Session): {response_msg.get(response.status_code,response.reason)}", self.query) #time.sleep(retry_delay) - raise LLMError("Time error: We couldn't provide a response,Please try\ + raise LLMError("Timeout error: We couldn't provide a response,Please try\ to rephrase your question to receive feedback", self.query) diff --git a/gn3/llms/errors.py b/gn3/llms/errors.py index 77e0f9a..a3a47a3 100644 --- a/gn3/llms/errors.py +++ b/gn3/llms/errors.py @@ -37,5 +37,3 @@ class UnprocessableEntity(HTTPError): class LLMError(Exception): """custom exception for LLMErrorMIxins""" - def __init__(self, *args, **kwargs): - super().__init__(*args) -- cgit v1.2.3 From a3284926aaa73ce27565e4a1131d4447a3a7face Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 17:18:55 +0300 Subject: Add created_at timestamp to History table. --- gn3/api/llm.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index c1f6304..5924f12 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -40,6 +40,7 @@ def search(): task_id TEXT NOT NULL, query TEXT NOT NULL, results TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY(task_id)) WITHOUT ROWID""") cursor.execute( """INSERT INTO history(user_id, task_id, query, results) -- cgit v1.2.3 From 22a1517e71ff6058090596498b9e829fb4e19664 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 17:34:59 +0300 Subject: Add created_at timestamp for Rating table. --- gn3/api/llm.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 5924f12..ab33c7a 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -1,5 +1,6 @@ """Api endpoints for gnqa""" import json +from datetime import datetime, timezone from flask import Blueprint from flask import current_app from flask import jsonify @@ -69,15 +70,17 @@ def rate_queries(task_id): query TEXT NOT NULL, answer TEXT NOT NULL, weight INTEGER NOT NULL DEFAULT 0, - task_id TEXT NOT NULL UNIQUE + task_id TEXT NOT NULL UNIQUE, + created_at TIMESTAMP )""" cursor.execute(create_table) cursor.execute("""INSERT INTO Rating(user_id, query, - answer, weight, task_id) - VALUES(?,?,?,?,?) + answer, weight, task_id, created_at) + VALUES(?, ?, ?, ?, ?, ?) ON CONFLICT(task_id) DO UPDATE SET weight=excluded.weight - """, (str(user_id), query, answer, weight, task_id)) + """, (str(user_id), query, answer, weight, task_id, + datetime.now(timezone.utc))) return { "message": "You have successfully rated this query.Thank you!" }, 200 -- cgit v1.2.3 From f6dfb1a3d06b75ef74cfdc295bd7f30390d2a4d8 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 18:38:18 +0300 Subject: sql: update: llm_db_update.sql: New file. --- sql/update/llm_db_update.sql | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 sql/update/llm_db_update.sql diff --git a/sql/update/llm_db_update.sql b/sql/update/llm_db_update.sql new file mode 100644 index 0000000..71f7491 --- /dev/null +++ b/sql/update/llm_db_update.sql @@ -0,0 +1,37 @@ +-- llm_db_update.sql --- + +-- Copyright (C) 2022 Alexander kabui + +-- Author: Alexander Kabui + +-- This program is free software; you can redistribute it and/or +-- modify it under the terms of the GNU General Public License +-- as published by the Free Software Foundation; either version 3 +-- of the License, or (at your option) any later version. + +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. + +-- You should have received a copy of the GNU General Public License +-- along with this program. If not, see . + +-- Sql file to create the history table, adding indexing for the history table +-- and adding timestamp column the Rating table + + +CREATE TABLE IF NOT EXISTS history ( + user_id TEXT NOT NULL, + task_id TEXT NOT NULL, + query TEXT NOT NULL, + results TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (task_id) +) WITHOUT ROWID; + + +CREATE INDEX IF NOT EXISTS idx_tbl_history_cols_task_id_user_id +ON history (task_id, user_id); + +ALTER TABLE Rating ADD COLUMN created_at TIMESTAMP; -- cgit v1.2.3 From 666461bcf6afc811e4c21dd23dbef2711a07049a Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 24 May 2024 18:55:49 +0300 Subject: Update copyright year and email. --- sql/update/llm_db_update.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/update/llm_db_update.sql b/sql/update/llm_db_update.sql index 71f7491..6608a90 100644 --- a/sql/update/llm_db_update.sql +++ b/sql/update/llm_db_update.sql @@ -1,6 +1,6 @@ -- llm_db_update.sql --- --- Copyright (C) 2022 Alexander kabui +-- Copyright (C) 2024 Alexander kabui -- Author: Alexander Kabui -- cgit v1.2.3 From 105f2b36eb62b9b097e1cbf6fa815f98da77bc16 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 14:09:02 +0300 Subject: Update Docstrings for Api endpoints and functions. --- gn3/api/llm.py | 6 +++--- gn3/llms/process.py | 48 +++++++++++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index ab33c7a..4b8ec52 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -17,7 +17,7 @@ gnqa = Blueprint("gnqa", __name__) @gnqa.route("/search", methods=["POST"]) def search(): - """Main gnqa endpoint""" + """Api endpoint for searching queries in fahamu Api""" query = request.json.get("querygnqa", "") if not query: return jsonify({"error": "querygnqa is missing in the request"}), 400 @@ -56,7 +56,7 @@ def search(): @gnqa.route("/rating/", methods=["POST"]) @require_oauth("profile") def rate_queries(task_id): - """Endpoint for rating qnqa query and answer""" + """Api endpoint for rating GNQA query and answer""" with (require_oauth.acquire("profile") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): results = request.json @@ -89,7 +89,7 @@ def rate_queries(task_id): @gnqa.route("/history", methods=["GET"]) @require_oauth("profile user") def fetch_prev_history(): - """ api method to fetch search query records""" + """Api endpoint to fetch GNQA previous search.""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() diff --git a/gn3/llms/process.py b/gn3/llms/process.py index ab2a80e..ade4104 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -35,7 +35,7 @@ class DocIDs(): raise FileNotFoundError(f"{file_path}-- FIle does not exist\n") def format_doc_ids(self, docs): - """method to format doc_ids for list items""" + """method to format doc_ids for list items doc_id and doc_name""" for _key, val in docs.items(): if isinstance(val, list): for doc_obj in val: @@ -43,7 +43,14 @@ class DocIDs(): self.doc_ids.update({doc_obj["id"]: doc_name}) def get_info(self, doc_id): - """ interface to make read from doc_ids""" + """ interface to make read from doc_ids + and extract info data else returns + doc_id + Args: + doc_id: str: a search key for doc_ids + Returns: + an object with doc_info if doc_id in doc_ids + """ if doc_id in self.doc_ids.keys(): return self.doc_ids[doc_id] else: @@ -51,7 +58,8 @@ class DocIDs(): def format_bibliography_info(bib_info): - """Function for formatting bibliography info""" + """Utility function for formatting bibliography info + """ if isinstance(bib_info, str): return bib_info.removesuffix('.txt') elif isinstance(bib_info, dict): @@ -66,7 +74,15 @@ def filter_response_text(val): def parse_context(context, get_info_func, format_bib_func): - """function to parse doc_ids content""" + """Function to parse doc_ids content + Args: + context: raw references from fahamu api + get_info_func: function to get doc_ids info + format_bib_func: function to foramt bibliography info + Returns: + an list with each item having (doc_id,bib_info, + combined reference text) + """ results = [] for doc_ids, summary in context.items(): combo_txt = "" @@ -81,7 +97,12 @@ def parse_context(context, get_info_func, format_bib_func): def load_file(filename, dir_path): - """function to open and load json file""" + """Utility function to read json file + Args: + filename: file name to read + dir_path: base directory for the file + Returns: json data read to a dict + """ file_path = os.path.join(dir_path, f"{filename}") if not os.path.isfile(file_path): raise FileNotFoundError(f"{filename} was not found or is a directory") @@ -90,8 +111,19 @@ def load_file(filename, dir_path): def fetch_pubmed(references, file_name, data_dir=""): - """method to fetch and populate references with pubmed""" + """ + Fetches PubMed data from a JSON file and populates the\ + references dictionary. + + Args: + references (dict): Dictionary with document IDs as keys\ + and reference data as values. + filename (str): Name of the JSON file containing PubMed data. + data_dir (str): Base directory where the data files are located. + Returns: + dict: Updated references dictionary populated with the PubMed data. + """ try: pubmed = load_file(file_name, os.path.join(data_dir, "gn-meta/lit")) for reference in references: @@ -123,4 +155,6 @@ def get_gnqa(query, auth_token, data_dir=""): answer = resp_text['data']['answer'] context = resp_text['data']['context'] return task_id, answer, fetch_pubmed(parse_context( - context, DocIDs().get_info, format_bibliography_info), "pubmed.json", data_dir) + context, DocIDs().get_info, + format_bibliography_info), + "pubmed.json", data_dir) -- cgit v1.2.3 From d0801cea229d00d5d4ce19fa1cb36242e56070d1 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 14:18:48 +0300 Subject: Delete filter response text method and update relevant code. --- gn3/llms/process.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index ade4104..2ce6b2b 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -67,12 +67,6 @@ def format_bibliography_info(bib_info): return bib_info -def filter_response_text(val): - """helper function for filtering non-printable chars""" - return json.loads(''.join([str(char) - for char in val if char in string.printable])) - - def parse_context(context, get_info_func, format_bib_func): """Function to parse doc_ids content Args: @@ -151,7 +145,8 @@ def get_gnqa(query, auth_token, data_dir=""): api_client = GeneNetworkQAClient(api_key=auth_token) res, task_id = api_client.ask('?ask=' + quote(query), query=query) res, _status = api_client.get_answer(task_id) - resp_text = filter_response_text(res.text) + resp_text = json.loads(''.join([str(char) + for char in res.text if char in string.printable])) answer = resp_text['data']['answer'] context = resp_text['data']['context'] return task_id, answer, fetch_pubmed(parse_context( -- cgit v1.2.3 From 8aeb4e00af2651cf7ec55f2ace23c600f537ff77 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 14:33:44 +0300 Subject: Delete llm obsolete unittests --- tests/unit/test_llm.py | 65 -------------------------------------------------- 1 file changed, 65 deletions(-) diff --git a/tests/unit/test_llm.py b/tests/unit/test_llm.py index 7b8a970..6e0f2af 100644 --- a/tests/unit/test_llm.py +++ b/tests/unit/test_llm.py @@ -35,68 +35,3 @@ def test_parse_context(): ] assert parsed_result == expected_result - - -@dataclass(frozen=True) -class MockResponse: - """mock a response object""" - text: str - - def __getattr__(self, name: str): - return self.__dict__[f"_{name}"] - - -class MockGeneNetworkQAClient: - """mock the GeneNetworkQAClient class""" - - def __init__(self, session, api_key): - pass - - def ask(self, query, auth_token): - """mock method for ask query""" - # Simulate the ask method - return MockResponse("Mock response"), "F400995EAFE104EA72A5927CE10C73B7" - - def get_answer(self, task_id): - """mock get_answer method""" - return MockResponse("Mock answer"), 1 - - -def mock_filter_response_text(text): - """ method to simulate the filterResponseText method""" - return {"data": {"answer": "Mock answer for what is a gene", "context": {}}} - - -def mock_parse_context(context, get_info_func, format_bib_func): - """method to simulate the parse context method""" - return [] - - -@pytest.mark.unit_test -def test_get_gnqa(monkeypatch): - """test for process.get_gnqa functoin""" - monkeypatch.setattr( - "gn3.llms.process.GeneNetworkQAClient", - MockGeneNetworkQAClient - ) - - monkeypatch.setattr( - 'gn3.llms.process.filter_response_text', - mock_filter_response_text - ) - monkeypatch.setattr( - 'gn3.llms.process.parse_context', - mock_parse_context - ) - - query = "What is a gene" - auth_token = "test_token" - result = get_gnqa(query, auth_token) - - expected_result = ( - "F400995EAFE104EA72A5927CE10C73B7", - 'Mock answer for what is a gene', - [] - ) - - assert result == expected_result -- cgit v1.2.3 From 58fbc6527537cb229ded87eea57949c3cf02621f Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 14:39:38 +0300 Subject: Remove duplicate code for loading files. --- gn3/llms/process.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index 2ce6b2b..40e53c5 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -21,19 +21,10 @@ class DocIDs(): * doc_ids.json: opens doc)ids for gn references * sugar_doc_ids: open doci_ids for diabetes references """ - self.doc_ids = self.load_file("doc_ids.json") - self.sugar_doc_ids = self.load_file("all_files.json") + self.doc_ids = load_file("doc_ids.json", BASEDIR) + self.sugar_doc_ids = load_file("all_files.json", BASEDIR) self.format_doc_ids(self.sugar_doc_ids) - def load_file(self, file_name): - """Method to load and read doc_id files""" - file_path = os.path.join(BASEDIR, file_name) - if os.path.isfile(file_path): - with open(file_path, "rb") as file_handler: - return json.load(file_handler) - else: - raise FileNotFoundError(f"{file_path}-- FIle does not exist\n") - def format_doc_ids(self, docs): """method to format doc_ids for list items doc_id and doc_name""" for _key, val in docs.items(): -- cgit v1.2.3 From 59a27f884b2821ab9142f5285cd713ec374ea820 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 14:47:19 +0300 Subject: Pylint fixes. --- tests/unit/test_llm.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/unit/test_llm.py b/tests/unit/test_llm.py index 6e0f2af..c32e888 100644 --- a/tests/unit/test_llm.py +++ b/tests/unit/test_llm.py @@ -1,8 +1,5 @@ -# pylint: disable=unused-argument -"""Test cases for procedures defined in llms module""" -from dataclasses import dataclass +"""Test cases for procedures defined in llms """ import pytest -from gn3.llms.process import get_gnqa from gn3.llms.process import parse_context -- cgit v1.2.3 From d3f87b9a02bfec223d23c16eb1374d53065fea92 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Mon, 27 May 2024 17:37:13 +0300 Subject: Add regular expressions for parsing links in texts. --- gn3/llms/process.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/gn3/llms/process.py b/gn3/llms/process.py index 40e53c5..55c27a0 100644 --- a/gn3/llms/process.py +++ b/gn3/llms/process.py @@ -1,6 +1,7 @@ """this module contains code for processing response from fahamu client.py""" # pylint: disable=C0301 import os +import re import string import json import logging @@ -76,8 +77,13 @@ def parse_context(context, get_info_func, format_bib_func): doc_info = get_info_func(doc_ids) bib_info = doc_ids if doc_ids == doc_info else format_bib_func( doc_info) + pattern = r'(https?://|www\.)[\w.-]+(\.[a-zA-Z]{2,})([/\w.-]*)*' + combo_text = re.sub(pattern, + lambda x: f" {x[0]} ", + combo_txt) results.append( - {"doc_id": doc_ids, "bibInfo": bib_info, "comboTxt": combo_txt}) + {"doc_id": doc_ids, "bibInfo": bib_info, + "comboTxt": combo_text}) return results @@ -137,8 +143,10 @@ def get_gnqa(query, auth_token, data_dir=""): res, task_id = api_client.ask('?ask=' + quote(query), query=query) res, _status = api_client.get_answer(task_id) resp_text = json.loads(''.join([str(char) - for char in res.text if char in string.printable])) - answer = resp_text['data']['answer'] + for char in res.text if char in string.printable])) + answer = re.sub(r'(https?://|www\.)[\w.-]+(\.[a-zA-Z]{2,})([/\w.-]*)*', + lambda x: f" {x[0]} ", + resp_text["data"]["answer"]) context = resp_text['data']['context'] return task_id, answer, fetch_pubmed(parse_context( context, DocIDs().get_info, -- cgit v1.2.3 From 7d79812db623d12474422a9613c81f35e25aef55 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 29 May 2024 14:05:00 +0300 Subject: Add delete functionality for gnqa history. --- gn3/api/llm.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 4b8ec52..228d3fa 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -86,16 +86,24 @@ def rate_queries(task_id): }, 200 -@gnqa.route("/history", methods=["GET"]) +@gnqa.route("/history", methods=["GET", "POST"]) @require_oauth("profile user") def fetch_prev_history(): """Api endpoint to fetch GNQA previous search.""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() + if request.method == "POST": + task_ids = list(request.json.values()) + query = """DELETE FROM history + WHERE task_id IN ({}) + and user_id=?""".format(",".join("?" * len(task_ids))) + cursor.execute(query, (*task_ids, str(token.user.user_id),)) + return jsonify({}) if request.args.get("search_term"): cursor.execute( - """SELECT results from history Where task_id=? and user_id=?""", + """SELECT results from history + Where task_id=? and user_id=?""", (request.args.get("search_term"), str(token.user.user_id),)) return dict(cursor.fetchone())["results"] -- cgit v1.2.3 From 231cbd69ed5292451b976091e117d819e5466b30 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 09:51:38 +0300 Subject: Update gnqa search endpoint from POST to PUT. --- gn3/api/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 228d3fa..d56a3d2 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -15,7 +15,7 @@ from gn3.auth import db gnqa = Blueprint("gnqa", __name__) -@gnqa.route("/search", methods=["POST"]) +@gnqa.route("/search", methods=["PUT"]) def search(): """Api endpoint for searching queries in fahamu Api""" query = request.json.get("querygnqa", "") -- cgit v1.2.3 From c32378420d58d6770045cf8d2025dabccb4d1492 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 10:09:24 +0300 Subject: Use correct http method `Delete` for search history. --- gn3/api/llm.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index d56a3d2..a94badd 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -86,21 +86,22 @@ def rate_queries(task_id): }, 200 -@gnqa.route("/history", methods=["GET", "POST"]) +@gnqa.route("/history", methods=["GET", "DELETE"]) @require_oauth("profile user") def fetch_prev_history(): """Api endpoint to fetch GNQA previous search.""" with (require_oauth.acquire("profile user") as token, db.connection(current_app.config["LLM_DB_PATH"]) as conn): cursor = conn.cursor() - if request.method == "POST": + if request.method == "DELETE": task_ids = list(request.json.values()) query = """DELETE FROM history WHERE task_id IN ({}) and user_id=?""".format(",".join("?" * len(task_ids))) cursor.execute(query, (*task_ids, str(token.user.user_id),)) return jsonify({}) - if request.args.get("search_term"): + elif (request.method == "GET" and + request.args.get("search_term")): cursor.execute( """SELECT results from history Where task_id=? and user_id=?""", -- cgit v1.2.3 From ade1b4cb03de8a1c670a3e876b8a18692dfc7694 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 10:26:05 +0300 Subject: Check for empty values when fetching search history. --- gn3/api/llm.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index a94badd..5901ef5 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -107,7 +107,10 @@ def fetch_prev_history(): Where task_id=? and user_id=?""", (request.args.get("search_term"), str(token.user.user_id),)) - return dict(cursor.fetchone())["results"] + record = cursor.fetchone() + if record: + return dict(record).get("results") + return {} cursor.execute( """SELECT task_id,query from history WHERE user_id=?""", (str(token.user.user_id),)) -- cgit v1.2.3 From 7b17ab9d98f6a12f2a5b9b0eec8ff4c2c3ef2a5e Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 10:28:34 +0300 Subject: Add logging for gn llm errors. --- gn3/errors.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gn3/errors.py b/gn3/errors.py index 8331028..c53604f 100644 --- a/gn3/errors.py +++ b/gn3/errors.py @@ -108,6 +108,7 @@ def handle_generic(exc: Exception) -> Response: def handle_llm_error(exc: Exception) -> Response: """ Handle llm erros if not handled anywhere else. """ + current_app.logger.error(exc) resp = jsonify({ "query": exc.args[1], "error_type": type(exc).__name__, -- cgit v1.2.3 From a45ab3e62df6e1dfcc6fff03916369fbdaf68ab8 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 11:05:39 +0300 Subject: Add default timestamp and and primary key for Rating table. --- gn3/api/llm.py | 4 ++-- sql/update/llm_db_update.sql | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gn3/api/llm.py b/gn3/api/llm.py index 5901ef5..952a5b9 100644 --- a/gn3/api/llm.py +++ b/gn3/api/llm.py @@ -71,8 +71,8 @@ def rate_queries(task_id): answer TEXT NOT NULL, weight INTEGER NOT NULL DEFAULT 0, task_id TEXT NOT NULL UNIQUE, - created_at TIMESTAMP - )""" + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(task_id))""" cursor.execute(create_table) cursor.execute("""INSERT INTO Rating(user_id, query, answer, weight, task_id, created_at) diff --git a/sql/update/llm_db_update.sql b/sql/update/llm_db_update.sql index 6608a90..a4eb848 100644 --- a/sql/update/llm_db_update.sql +++ b/sql/update/llm_db_update.sql @@ -34,4 +34,4 @@ CREATE TABLE IF NOT EXISTS history ( CREATE INDEX IF NOT EXISTS idx_tbl_history_cols_task_id_user_id ON history (task_id, user_id); -ALTER TABLE Rating ADD COLUMN created_at TIMESTAMP; +ALTER TABLE Rating ADD COLUMN created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP; -- cgit v1.2.3 From 57b4a4fd5bcb8a2b7f9af856d8f1212c0fbbe0da Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Thu, 29 Aug 2024 11:06:07 +0300 Subject: Add sql file for creating llm db tables. --- sql/update/llm_db_tables.sql | 47 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 sql/update/llm_db_tables.sql diff --git a/sql/update/llm_db_tables.sql b/sql/update/llm_db_tables.sql new file mode 100644 index 0000000..a6c0479 --- /dev/null +++ b/sql/update/llm_db_tables.sql @@ -0,0 +1,47 @@ +-- llm_db_update.sql --- + +-- Copyright (C) 2024 Alexander kabui + +-- Author: Alexander Kabui + +-- This program is free software; you can redistribute it and/or +-- modify it under the terms of the GNU General Public License +-- as published by the Free Software Foundation; either version 3 +-- of the License, or (at your option) any later version. + +-- This program is distributed in the hope that it will be useful, +-- but WITHOUT ANY WARRANTY; without even the implied warranty of +-- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-- GNU General Public License for more details. + +-- You should have received a copy of the GNU General Public License +-- along with this program. If not, see . + +-- Sql file to create the tables for history rating and adding indexing for the history table +-- this targets setting up a new db +-- and adding timestamp column the Rating table + + +CREATE TABLE IF NOT EXISTS history ( + user_id TEXT NOT NULL, + task_id TEXT NOT NULL, + query TEXT NOT NULL, + results TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (task_id) +) WITHOUT ROWID; + + +CREATE INDEX IF NOT EXISTS idx_tbl_history_cols_task_id_user_id +ON history (task_id, user_id); + + + +CREATE TABLE IF NOT EXISTS Rating( + user_id TEXT NOT NULL, + query TEXT NOT NULL, + answer TEXT NOT NULL, + weight INTEGER NOT NULL DEFAULT 0, + task_id TEXT NOT NULL UNIQUE, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (task_id)); -- cgit v1.2.3