From 656014370380951577900f31e66e9a92c995b0bd Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:16:08 +0300 Subject: feat: implement ednpoint for computing qtl using rqtl2. Capture stdout results to a file. --- gn3/api/rqtl2.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 gn3/api/rqtl2.py diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py new file mode 100644 index 0000000..e746918 --- /dev/null +++ b/gn3/api/rqtl2.py @@ -0,0 +1,38 @@ +""" File contains endpoints for rqlt2""" + +import subprocess +import uuid +import os +from flask import current_app +from flask import jsonify +from flask import Blueprint + +rqtl2 = Blueprint("rqtl2", __name__) + +@rqtl2.route("/compute", methods=["GET"]) +def compute(): + """Endpoint for computing QTL analysis using R/QTL2""" + wkdir = current_app.config.get("TMPDIR") + output_file = os.path.join(wkdir, "output.txt") + rscript_cmd = ( + f"Rscript ./scripts/rqtl2_wrapper.R " + f"-i /home/kabui/r_playground/meta_grav.json " + f"-d /home/kabui/r_playground " + f"-o /home/kabui/r_playground/rqtl_output.json " + f"--nperm 100 --threshold 1 --cores 0" + ) + process = subprocess.Popen( + rscript_cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + # TODO rethink where we write this file + with open(output_file, "a+") as file_handler: + for line in iter(process.stdout.readline, b""): + file_handler.write(line.decode("utf-8")) + process.stdout.close() + process.wait() + if process.returncode == 0: + return jsonify({"msg": "success", "results": "file_here"}) + else: + return jsonify({"msg": "fail", "error": "Process failed"}) -- cgit 1.4.1 From a47a49a96684108630061f6ada0285fd05c28d4a Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:17:19 +0300 Subject: feat: register rqtl2 blueprint to app. --- gn3/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gn3/app.py b/gn3/app.py index eb9e5d3..0bec32a 100644 --- a/gn3/app.py +++ b/gn3/app.py @@ -27,6 +27,7 @@ from gn3.api.search import search from gn3.api.metadata import metadata from gn3.api.sampledata import sampledata from gn3.api.llm import gnqa +from gn3.api.rqtl2 import rqtl2 from gn3.case_attributes import caseattr @@ -107,6 +108,7 @@ def create_app(config: Union[Dict, str, None] = None) -> Flask: app.register_blueprint(sampledata, url_prefix="/api/sampledata") app.register_blueprint(caseattr, url_prefix="/api/case-attribute") app.register_blueprint(gnqa, url_prefix="/api/llm") + app.register_blueprint(rqtl2, url_prefix="/api/rqtl2") register_error_handlers(app) return app -- cgit 1.4.1 From 9773b216d1f92111e2ba9f4f77e78b2ba64ce5f1 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:33:16 +0300 Subject: feat: Add new endpoint to read stdout given a file identifier. --- gn3/api/rqtl2.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index e746918..03cf340 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -36,3 +36,15 @@ def compute(): return jsonify({"msg": "success", "results": "file_here"}) else: return jsonify({"msg": "fail", "error": "Process failed"}) + + +@rqtl2.route("/stream/", methods=["GET"]) +def stream(indetifier): + """ This endpoints streams stdout from a file expects + the indetifier to be the file """ + output_file = os.path.join(current_app.config.get("TMPDIR"), + f"{indetifier}.txt") + # raise error if file does not exist + with open(output_file) as file_handler: + # rethink how we do the read should this be stream / yield ???? + return jsonify({"data": file_handler.readlines()}) -- cgit 1.4.1 From a408abf9539de083c0d58b002c647ab67c2c622d Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:37:49 +0300 Subject: feat: return pointer position from where the file was last read. --- gn3/api/rqtl2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index 03cf340..cc857da 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -6,6 +6,7 @@ import os from flask import current_app from flask import jsonify from flask import Blueprint +from flask import request rqtl2 = Blueprint("rqtl2", __name__) @@ -42,9 +43,11 @@ def compute(): def stream(indetifier): """ This endpoints streams stdout from a file expects the indetifier to be the file """ + # add seek position to this output_file = os.path.join(current_app.config.get("TMPDIR"), f"{indetifier}.txt") # raise error if file does not exist with open(output_file) as file_handler: - # rethink how we do the read should this be stream / yield ???? - return jsonify({"data": file_handler.readlines()}) + # rethink how we do the read should this be stream / yield/peak ???? + return jsonify({"data": file_handler.readlines(), + "pointer": file_handler.tell()}) -- cgit 1.4.1 From 56cc3f94e9b66ea3bd677eb1e46584bad5bfaa39 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 26 Nov 2024 15:10:43 +0300 Subject: feat: Implement reading from file functionality Implement read from the last position for a file. --- gn3/api/rqtl2.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index cc857da..8ac5ce3 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -15,12 +15,13 @@ def compute(): """Endpoint for computing QTL analysis using R/QTL2""" wkdir = current_app.config.get("TMPDIR") output_file = os.path.join(wkdir, "output.txt") + # this should be computed locally not via files rscript_cmd = ( f"Rscript ./scripts/rqtl2_wrapper.R " f"-i /home/kabui/r_playground/meta_grav.json " f"-d /home/kabui/r_playground " f"-o /home/kabui/r_playground/rqtl_output.json " - f"--nperm 100 --threshold 1 --cores 0" + f"--nperm 100 --threshold 1 --cores 0" ) process = subprocess.Popen( rscript_cmd, shell=True, @@ -28,9 +29,11 @@ def compute(): stderr=subprocess.STDOUT ) # TODO rethink where we write this file - with open(output_file, "a+") as file_handler: - for line in iter(process.stdout.readline, b""): + for line in iter(process.stdout.readline, b""): + # dont modify + with open(output_file, "a+") as file_handler: file_handler.write(line.decode("utf-8")) + process.stdout.close() process.wait() if process.returncode == 0: @@ -46,8 +49,9 @@ def stream(indetifier): # add seek position to this output_file = os.path.join(current_app.config.get("TMPDIR"), f"{indetifier}.txt") - # raise error if file does not exist + seek_position = int(request.args.get("peak", 0)) with open(output_file) as file_handler: # rethink how we do the read should this be stream / yield/peak ???? + file_handler.seek(seek_position) return jsonify({"data": file_handler.readlines(), "pointer": file_handler.tell()}) -- cgit 1.4.1 From f59b24d618fafd8030e9191267b99a86ab4298e2 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 27 Nov 2024 00:59:14 +0300 Subject: feat: return run id for compute and streaming api's. --- gn3/api/rqtl2.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index 8ac5ce3..ffa8755 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -13,8 +13,10 @@ rqtl2 = Blueprint("rqtl2", __name__) @rqtl2.route("/compute", methods=["GET"]) def compute(): """Endpoint for computing QTL analysis using R/QTL2""" - wkdir = current_app.config.get("TMPDIR") - output_file = os.path.join(wkdir, "output.txt") + # get the run id to act as file identifier default to output + run_id = request.args.get("id", "output") + output_file = os.path.join(current_app.config.get("TMPDIR"), + f"{run_id}.txt") # this should be computed locally not via files rscript_cmd = ( f"Rscript ./scripts/rqtl2_wrapper.R " @@ -28,30 +30,32 @@ def compute(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - # TODO rethink where we write this file for line in iter(process.stdout.readline, b""): - # dont modify + # these allow endpoint stream to read the file since + # no read and write file same tiem with open(output_file, "a+") as file_handler: file_handler.write(line.decode("utf-8")) - process.stdout.close() process.wait() if process.returncode == 0: - return jsonify({"msg": "success", "results": "file_here"}) - else: - return jsonify({"msg": "fail", "error": "Process failed"}) + return jsonify({"msg": "success", + "results": "file_here", + "run_id": run_id}) + return jsonify({"msg": "fail", + "error": "Process failed", + "run_id": run_id}) -@rqtl2.route("/stream/", methods=["GET"]) -def stream(indetifier): +@rqtl2.route("/stream/", methods=["GET"]) +def stream(identifier="output"): """ This endpoints streams stdout from a file expects the indetifier to be the file """ - # add seek position to this output_file = os.path.join(current_app.config.get("TMPDIR"), - f"{indetifier}.txt") + f"{identifier}.txt") seek_position = int(request.args.get("peak", 0)) with open(output_file) as file_handler: - # rethink how we do the read should this be stream / yield/peak ???? + # read to the last position default to 0 file_handler.seek(seek_position) return jsonify({"data": file_handler.readlines(), + "run_id": identifier, "pointer": file_handler.tell()}) -- cgit 1.4.1 From 09b6934461cc14d9c957baa2a51b031ea4c5c7bc Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 27 Nov 2024 19:45:46 +0300 Subject: feat: define general endpoint for streaming stdout using unique identifier. --- gn3/api/general.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/gn3/api/general.py b/gn3/api/general.py index b984361..fcaf21b 100644 --- a/gn3/api/general.py +++ b/gn3/api/general.py @@ -1,5 +1,6 @@ """General API endpoints. Put endpoints that can't be grouped together nicely here.""" +import os from flask import Blueprint from flask import current_app from flask import jsonify @@ -68,3 +69,31 @@ def run_r_qtl(geno_filestr, pheno_filestr): cmd = (f"Rscript {rqtl_wrapper} " f"{geno_filestr} {pheno_filestr}") return jsonify(run_cmd(cmd)), 201 + + +@general.route("/stream ", methods=["GET"]) +def stream(): + """ + This endpoint streams the stdout content from a file. + It expects an identifier to be passed as a query parameter. + Example: `/stream?id=` + The `id` will be used to locate the corresponding file. + You can also pass an optional `peak` parameter + to specify the file position to start reading from. + Query Parameters: + - `id` (required): The identifier used to locate the file. + - `peak` (optional): The position in the file to start reading from. + Returns: + - dict with data(stdout), run_id unique id for file, + pointer last read position for file + """ + run_id = request.args.get("id", "") + output_file = os.path.join(current_app.config.get("TMPDIR"), + f"{run_id}.txt") + seek_position = int(request.args.get("peak", 0)) + with open(output_file) as file_handler: + # read to the last position default to 0 + file_handler.seek(seek_position) + return jsonify({"data": file_handler.readlines(), + "run_id": run_id, + "pointer": file_handler.tell()}) -- cgit 1.4.1