From 656014370380951577900f31e66e9a92c995b0bd Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:16:08 +0300 Subject: feat: implement ednpoint for computing qtl using rqtl2. Capture stdout results to a file. --- gn3/api/rqtl2.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 gn3/api/rqtl2.py (limited to 'gn3/api/rqtl2.py') diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py new file mode 100644 index 0000000..e746918 --- /dev/null +++ b/gn3/api/rqtl2.py @@ -0,0 +1,38 @@ +""" File contains endpoints for rqlt2""" + +import subprocess +import uuid +import os +from flask import current_app +from flask import jsonify +from flask import Blueprint + +rqtl2 = Blueprint("rqtl2", __name__) + +@rqtl2.route("/compute", methods=["GET"]) +def compute(): + """Endpoint for computing QTL analysis using R/QTL2""" + wkdir = current_app.config.get("TMPDIR") + output_file = os.path.join(wkdir, "output.txt") + rscript_cmd = ( + f"Rscript ./scripts/rqtl2_wrapper.R " + f"-i /home/kabui/r_playground/meta_grav.json " + f"-d /home/kabui/r_playground " + f"-o /home/kabui/r_playground/rqtl_output.json " + f"--nperm 100 --threshold 1 --cores 0" + ) + process = subprocess.Popen( + rscript_cmd, shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + # TODO rethink where we write this file + with open(output_file, "a+") as file_handler: + for line in iter(process.stdout.readline, b""): + file_handler.write(line.decode("utf-8")) + process.stdout.close() + process.wait() + if process.returncode == 0: + return jsonify({"msg": "success", "results": "file_here"}) + else: + return jsonify({"msg": "fail", "error": "Process failed"}) -- cgit 1.4.1 From 9773b216d1f92111e2ba9f4f77e78b2ba64ce5f1 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:33:16 +0300 Subject: feat: Add new endpoint to read stdout given a file identifier. --- gn3/api/rqtl2.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'gn3/api/rqtl2.py') diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index e746918..03cf340 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -36,3 +36,15 @@ def compute(): return jsonify({"msg": "success", "results": "file_here"}) else: return jsonify({"msg": "fail", "error": "Process failed"}) + + +@rqtl2.route("/stream/", methods=["GET"]) +def stream(indetifier): + """ This endpoints streams stdout from a file expects + the indetifier to be the file """ + output_file = os.path.join(current_app.config.get("TMPDIR"), + f"{indetifier}.txt") + # raise error if file does not exist + with open(output_file) as file_handler: + # rethink how we do the read should this be stream / yield ???? + return jsonify({"data": file_handler.readlines()}) -- cgit 1.4.1 From a408abf9539de083c0d58b002c647ab67c2c622d Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Fri, 22 Nov 2024 12:37:49 +0300 Subject: feat: return pointer position from where the file was last read. --- gn3/api/rqtl2.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'gn3/api/rqtl2.py') diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index 03cf340..cc857da 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -6,6 +6,7 @@ import os from flask import current_app from flask import jsonify from flask import Blueprint +from flask import request rqtl2 = Blueprint("rqtl2", __name__) @@ -42,9 +43,11 @@ def compute(): def stream(indetifier): """ This endpoints streams stdout from a file expects the indetifier to be the file """ + # add seek position to this output_file = os.path.join(current_app.config.get("TMPDIR"), f"{indetifier}.txt") # raise error if file does not exist with open(output_file) as file_handler: - # rethink how we do the read should this be stream / yield ???? - return jsonify({"data": file_handler.readlines()}) + # rethink how we do the read should this be stream / yield/peak ???? + return jsonify({"data": file_handler.readlines(), + "pointer": file_handler.tell()}) -- cgit 1.4.1 From 56cc3f94e9b66ea3bd677eb1e46584bad5bfaa39 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Tue, 26 Nov 2024 15:10:43 +0300 Subject: feat: Implement reading from file functionality Implement read from the last position for a file. --- gn3/api/rqtl2.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'gn3/api/rqtl2.py') diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index cc857da..8ac5ce3 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -15,12 +15,13 @@ def compute(): """Endpoint for computing QTL analysis using R/QTL2""" wkdir = current_app.config.get("TMPDIR") output_file = os.path.join(wkdir, "output.txt") + # this should be computed locally not via files rscript_cmd = ( f"Rscript ./scripts/rqtl2_wrapper.R " f"-i /home/kabui/r_playground/meta_grav.json " f"-d /home/kabui/r_playground " f"-o /home/kabui/r_playground/rqtl_output.json " - f"--nperm 100 --threshold 1 --cores 0" + f"--nperm 100 --threshold 1 --cores 0" ) process = subprocess.Popen( rscript_cmd, shell=True, @@ -28,9 +29,11 @@ def compute(): stderr=subprocess.STDOUT ) # TODO rethink where we write this file - with open(output_file, "a+") as file_handler: - for line in iter(process.stdout.readline, b""): + for line in iter(process.stdout.readline, b""): + # dont modify + with open(output_file, "a+") as file_handler: file_handler.write(line.decode("utf-8")) + process.stdout.close() process.wait() if process.returncode == 0: @@ -46,8 +49,9 @@ def stream(indetifier): # add seek position to this output_file = os.path.join(current_app.config.get("TMPDIR"), f"{indetifier}.txt") - # raise error if file does not exist + seek_position = int(request.args.get("peak", 0)) with open(output_file) as file_handler: # rethink how we do the read should this be stream / yield/peak ???? + file_handler.seek(seek_position) return jsonify({"data": file_handler.readlines(), "pointer": file_handler.tell()}) -- cgit 1.4.1 From f59b24d618fafd8030e9191267b99a86ab4298e2 Mon Sep 17 00:00:00 2001 From: Alexander_Kabui Date: Wed, 27 Nov 2024 00:59:14 +0300 Subject: feat: return run id for compute and streaming api's. --- gn3/api/rqtl2.py | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) (limited to 'gn3/api/rqtl2.py') diff --git a/gn3/api/rqtl2.py b/gn3/api/rqtl2.py index 8ac5ce3..ffa8755 100644 --- a/gn3/api/rqtl2.py +++ b/gn3/api/rqtl2.py @@ -13,8 +13,10 @@ rqtl2 = Blueprint("rqtl2", __name__) @rqtl2.route("/compute", methods=["GET"]) def compute(): """Endpoint for computing QTL analysis using R/QTL2""" - wkdir = current_app.config.get("TMPDIR") - output_file = os.path.join(wkdir, "output.txt") + # get the run id to act as file identifier default to output + run_id = request.args.get("id", "output") + output_file = os.path.join(current_app.config.get("TMPDIR"), + f"{run_id}.txt") # this should be computed locally not via files rscript_cmd = ( f"Rscript ./scripts/rqtl2_wrapper.R " @@ -28,30 +30,32 @@ def compute(): stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - # TODO rethink where we write this file for line in iter(process.stdout.readline, b""): - # dont modify + # these allow endpoint stream to read the file since + # no read and write file same tiem with open(output_file, "a+") as file_handler: file_handler.write(line.decode("utf-8")) - process.stdout.close() process.wait() if process.returncode == 0: - return jsonify({"msg": "success", "results": "file_here"}) - else: - return jsonify({"msg": "fail", "error": "Process failed"}) + return jsonify({"msg": "success", + "results": "file_here", + "run_id": run_id}) + return jsonify({"msg": "fail", + "error": "Process failed", + "run_id": run_id}) -@rqtl2.route("/stream/", methods=["GET"]) -def stream(indetifier): +@rqtl2.route("/stream/", methods=["GET"]) +def stream(identifier="output"): """ This endpoints streams stdout from a file expects the indetifier to be the file """ - # add seek position to this output_file = os.path.join(current_app.config.get("TMPDIR"), - f"{indetifier}.txt") + f"{identifier}.txt") seek_position = int(request.args.get("peak", 0)) with open(output_file) as file_handler: - # rethink how we do the read should this be stream / yield/peak ???? + # read to the last position default to 0 file_handler.seek(seek_position) return jsonify({"data": file_handler.readlines(), + "run_id": identifier, "pointer": file_handler.tell()}) -- cgit 1.4.1