From aff6704cdcaea388f14b71cc013b1a224afb87fc Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 14:30:39 +0300 Subject: add function to capture output in real time --- gn3/computations/wgcna.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index fd508fa..0fb56e8 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -3,8 +3,9 @@ import os import json import uuid -from gn3.settings import TMPDIR +import subprocess +from gn3.settings import TMPDIR from gn3.commands import run_cmd @@ -20,6 +21,16 @@ def dump_wgcna_data(request_data: dict): return temp_file_path +def stream_cmd_output(socket, cmd: str): + """function to stream in realtime""" + # xtodo syncing and closing /edge cases + + results = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) + for line in iter(results.stdout.readline, b""): + line = line.decode("utf-8").rstrip() + + def compose_wgcna_cmd(rscript_path: str, temp_file_path: str): """function to componse wgcna cmd""" # (todo):issue relative paths to abs paths -- cgit v1.2.3 From f73262a56745de17bdb582474e27726cb19bd95e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 15:09:10 +0300 Subject: add function to process images --- gn3/computations/wgcna.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 0fb56e8..26040d3 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -4,6 +4,7 @@ import os import json import uuid import subprocess +import base64 from gn3.settings import TMPDIR from gn3.commands import run_cmd @@ -15,6 +16,8 @@ def dump_wgcna_data(request_data: dict): temp_file_path = os.path.join(TMPDIR, filename) + request_data["TMPDIR"] = TMPDIR + with open(temp_file_path, "w") as output_file: json.dump(request_data, output_file) @@ -31,6 +34,16 @@ def stream_cmd_output(socket, cmd: str): line = line.decode("utf-8").rstrip() +def process_image(image_loc: str) ->bytes: + """encode the image""" + + try: + with open(image_loc, "rb") as image_file: + return base64.b64encode(image_file.read()) + except FileNotFoundError as e: + return b"" + + def compose_wgcna_cmd(rscript_path: str, temp_file_path: str): """function to componse wgcna cmd""" # (todo):issue relative paths to abs paths @@ -47,12 +60,18 @@ def call_wgcna_script(rscript_path: str, request_data: dict): run_cmd_results = run_cmd(cmd) + print(run_cmd_results["output"]) + with open(generated_file, "r") as outputfile: + output_file_data = json.load(outputfile) + output_file_data["image_1"] = process_image( + output_file_data["output"]["imageLoc"]) + if run_cmd_results["code"] != 0: return run_cmd_results return { - "data": json.load(outputfile), + "data": output_file_data, **run_cmd_results } except FileNotFoundError: -- cgit v1.2.3 From 838276a6c4effc6985a888a32d148ef8942b526a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 15:13:06 +0300 Subject: rename key to image_data --- gn3/computations/wgcna.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 26040d3..421e09c 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -65,7 +65,7 @@ def call_wgcna_script(rscript_path: str, request_data: dict): with open(generated_file, "r") as outputfile: output_file_data = json.load(outputfile) - output_file_data["image_1"] = process_image( + output_file_data["image_data"] = process_image( output_file_data["output"]["imageLoc"]) if run_cmd_results["code"] != 0: -- cgit v1.2.3 From 453570cbf22a92de72fa0401895c99a923831942 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 15:14:25 +0300 Subject: script modification : add debug statements --- scripts/wgcna_analysis.R | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/wgcna_analysis.R b/scripts/wgcna_analysis.R index 17b3537..a5cbbe3 100644 --- a/scripts/wgcna_analysis.R +++ b/scripts/wgcna_analysis.R @@ -6,11 +6,15 @@ library(rjson) options(stringsAsFactors = FALSE); -imgDir = Sys.getenv("GENERATED_IMAGE_DIR") +cat("Running the wgcna analysis script\n") + # load expression data **assumes from json files row(traits)(columns info+samples) # pass the file_path as arg # pass the file path to read json data + + + args = commandArgs(trailingOnly=TRUE) if (length(args)==0) { @@ -21,6 +25,7 @@ if (length(args)==0) { } inputData <- fromJSON(file = json_file_path) +imgDir = inputData$TMPDIR trait_sample_data <- do.call(rbind, inputData$trait_sample_data) @@ -83,6 +88,11 @@ network <- blockwiseModules(dataExpr, +cat("Generated network \n") + +network + + genImageRandStr <- function(prefix){ randStr <- paste(prefix,stri_rand_strings(1, 9, pattern = "[A-Za-z0-9]"),sep="_") @@ -95,6 +105,10 @@ mergedColors <- labels2colors(network$colors) imageLoc <- file.path(imgDir,genImageRandStr("WGCNAoutput")) png(imageLoc,width=1000,height=600,type='cairo-png') + +cat("Generating the CLuster dendrogram\n") + + plotDendroAndColors(network$dendrograms[[1]],mergedColors[network$blockGenes[[1]]], "Module colors", dendroLabels = FALSE, hang = 0.03, -- cgit v1.2.3 From 84f5c8a4384f29dfcb187ac5eed43551860b182b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 16:57:13 +0300 Subject: fix issues serializing byte string --- gn3/computations/wgcna.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 421e09c..4dd0985 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -33,8 +33,10 @@ def stream_cmd_output(socket, cmd: str): for line in iter(results.stdout.readline, b""): line = line.decode("utf-8").rstrip() + # xtodo emit the data -def process_image(image_loc: str) ->bytes: + +def process_image(image_loc: str) -> bytes: """encode the image""" try: @@ -65,8 +67,9 @@ def call_wgcna_script(rscript_path: str, request_data: dict): with open(generated_file, "r") as outputfile: output_file_data = json.load(outputfile) - output_file_data["image_data"] = process_image( - output_file_data["output"]["imageLoc"]) + # json format only supports unicode string// to get image data reconvert + output_file_data["output"]["image_data"] = process_image( + output_file_data["output"]["imageLoc"]).decode("ascii") if run_cmd_results["code"] != 0: return run_cmd_results -- cgit v1.2.3 From c6bce638cfac9df0a4f3f380971fde65b1947ad7 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 17:14:23 +0300 Subject: add socket obj and emit process --- gn3/computations/wgcna.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 4dd0985..94e2a40 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -28,12 +28,18 @@ def stream_cmd_output(socket, cmd: str): """function to stream in realtime""" # xtodo syncing and closing /edge cases + socket.emit("output", {"data", f"calling you script {cmd}"}) + results = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) for line in iter(results.stdout.readline, b""): line = line.decode("utf-8").rstrip() - # xtodo emit the data + socket.emit("output", + {"data": line}) + # close above make sure the process is closed + + socket.emit("output", {"data": "parsing the output results"}) def process_image(image_loc: str) -> bytes: @@ -62,8 +68,6 @@ def call_wgcna_script(rscript_path: str, request_data: dict): run_cmd_results = run_cmd(cmd) - print(run_cmd_results["output"]) - with open(generated_file, "r") as outputfile: output_file_data = json.load(outputfile) -- cgit v1.2.3 From fd8650ca108cf61cd16b264e09568df9ad7bf428 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 5 Oct 2021 17:34:59 +0300 Subject: pep8 formatting,pylint fixes --- gn3/computations/wgcna.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 94e2a40..d1d6d39 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -37,7 +37,7 @@ def stream_cmd_output(socket, cmd: str): socket.emit("output", {"data": line}) - # close above make sure the process is closed + # wait for process to complete code socket.emit("output", {"data": "parsing the output results"}) @@ -48,7 +48,7 @@ def process_image(image_loc: str) -> bytes: try: with open(image_loc, "rb") as image_file: return base64.b64encode(image_file.read()) - except FileNotFoundError as e: + except FileNotFoundError: return b"" -- cgit v1.2.3 From dda47e74d38b76e59f10ac184a0394066f086481 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 13 Oct 2021 07:21:05 +0300 Subject: minor modification for emitting data:add namespaces --- gn3/computations/wgcna.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index d1d6d39..f7463c5 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -6,6 +6,7 @@ import uuid import subprocess import base64 + from gn3.settings import TMPDIR from gn3.commands import run_cmd @@ -24,22 +25,23 @@ def dump_wgcna_data(request_data: dict): return temp_file_path -def stream_cmd_output(socket, cmd: str): +def stream_cmd_output(request_data, cmd: str): """function to stream in realtime""" # xtodo syncing and closing /edge cases - socket.emit("output", {"data", f"calling you script {cmd}"}) - + socketio.emit("output", {"data": f"calling you script {cmd}"}, + namespace="/", room=request_data["socket_id"]) results = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) for line in iter(results.stdout.readline, b""): + line = line.decode("utf-8").rstrip() - socket.emit("output", - {"data": line}) - # wait for process to complete code + socketio.emit("output", + {"data": line}, namespace="/", room=request_data["socket_id"]) - socket.emit("output", {"data": "parsing the output results"}) + socketio.emit( + "output", {"data": "parsing the output results"}, namespace="/", room=request_data["socket_id"]) def process_image(image_loc: str) -> bytes: @@ -64,9 +66,11 @@ def call_wgcna_script(rscript_path: str, request_data: dict): generated_file = dump_wgcna_data(request_data) cmd = compose_wgcna_cmd(rscript_path, generated_file) + stream_cmd_output(request_data, cmd) + try: - run_cmd_results = run_cmd(cmd) + # run_cmd_results = run_cmd(cmd) with open(generated_file, "r") as outputfile: @@ -74,12 +78,14 @@ def call_wgcna_script(rscript_path: str, request_data: dict): # json format only supports unicode string// to get image data reconvert output_file_data["output"]["image_data"] = process_image( output_file_data["output"]["imageLoc"]).decode("ascii") + output_file_data["output"]["image_data2"] = process_image( + output_file_data["output"]["heatMap"]).decode("ascii") - if run_cmd_results["code"] != 0: - return run_cmd_results + # if run_cmd_results["code"] != 0: + # return run_cmd_results return { "data": output_file_data, - **run_cmd_results + "output": "" } except FileNotFoundError: # relook at handling errors gn3 -- cgit v1.2.3 From c6f411416ba70f2013f1eb69e13a84ab70adfca4 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 13 Oct 2021 07:22:02 +0300 Subject: add flask-socketio input to gn3 --- guix.scm | 1 + 1 file changed, 1 insertion(+) diff --git a/guix.scm b/guix.scm index 9b8f399..d8b1596 100644 --- a/guix.scm +++ b/guix.scm @@ -100,6 +100,7 @@ ("python-redis" ,python-redis) ("python-requests" ,python-requests) ("python-scipy" ,python-scipy) + ("python-flask-socketio" ,python-flask-socketio) ("python-sqlalchemy-stubs" ,python-sqlalchemy-stubs) ("r-optparse" ,r-optparse) -- cgit v1.2.3 From d1be5270b99959f99802a7704562eeaaeb504122 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 13 Oct 2021 07:29:24 +0300 Subject: minor fixes for sript enable annotations --- scripts/wgcna_analysis.R | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/wgcna_analysis.R b/scripts/wgcna_analysis.R index a5cbbe3..b0d25a9 100644 --- a/scripts/wgcna_analysis.R +++ b/scripts/wgcna_analysis.R @@ -13,8 +13,6 @@ cat("Running the wgcna analysis script\n") # pass the file path to read json data - - args = commandArgs(trailingOnly=TRUE) if (length(args)==0) { @@ -100,6 +98,7 @@ genImageRandStr <- function(prefix){ return(paste(randStr,".png",sep="")) } + mergedColors <- labels2colors(network$colors) imageLoc <- file.path(imgDir,genImageRandStr("WGCNAoutput")) @@ -111,7 +110,7 @@ cat("Generating the CLuster dendrogram\n") plotDendroAndColors(network$dendrograms[[1]],mergedColors[network$blockGenes[[1]]], "Module colors", -dendroLabels = FALSE, hang = 0.03, +dendroLabels = NULL, hang = 0.03, addGuide = TRUE, guideHang = 0.05) -- cgit v1.2.3 From 4c2cf2aceb044d00fe5e41ac40a07fe614737ef2 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 13 Oct 2021 08:11:25 +0300 Subject: fix unittests --- gn3/computations/wgcna.py | 15 +++++++-------- tests/unit/computations/test_wgcna.py | 14 ++++++++++---- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index f7463c5..90db455 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -66,26 +66,25 @@ def call_wgcna_script(rscript_path: str, request_data: dict): generated_file = dump_wgcna_data(request_data) cmd = compose_wgcna_cmd(rscript_path, generated_file) - stream_cmd_output(request_data, cmd) + # stream_cmd_output(request_data, cmd) disable streaming of data try: - # run_cmd_results = run_cmd(cmd) + run_cmd_results = run_cmd(cmd) with open(generated_file, "r") as outputfile: + if run_cmd_results["code"] != 0: + return run_cmd_results + output_file_data = json.load(outputfile) - # json format only supports unicode string// to get image data reconvert output_file_data["output"]["image_data"] = process_image( output_file_data["output"]["imageLoc"]).decode("ascii") - output_file_data["output"]["image_data2"] = process_image( - output_file_data["output"]["heatMap"]).decode("ascii") + # json format only supports unicode string// to get image data reconvert - # if run_cmd_results["code"] != 0: - # return run_cmd_results return { "data": output_file_data, - "output": "" + **run_cmd_results } except FileNotFoundError: # relook at handling errors gn3 diff --git a/tests/unit/computations/test_wgcna.py b/tests/unit/computations/test_wgcna.py index ec81d94..5f23a86 100644 --- a/tests/unit/computations/test_wgcna.py +++ b/tests/unit/computations/test_wgcna.py @@ -10,13 +10,16 @@ from gn3.computations.wgcna import call_wgcna_script class TestWgcna(TestCase): """test class for wgcna""" + @mock.patch("gn3.computations.wgcna.process_image") @mock.patch("gn3.computations.wgcna.run_cmd") @mock.patch("gn3.computations.wgcna.compose_wgcna_cmd") @mock.patch("gn3.computations.wgcna.dump_wgcna_data") def test_call_wgcna_script(self, mock_dumping_data, mock_compose_wgcna, - mock_run_cmd): + mock_run_cmd, + mock_img, + ): """test for calling wgcna script""" # pylint: disable = line-too-long @@ -50,7 +53,7 @@ class TestWgcna(TestCase): "output": "Flagging genes and samples with too many missing values...\n ..step 1\nAllowing parallel execution with up to 3 working processes.\npickSoftThreshold: will use block size 7.\n pickSoftThreshold: calculating connectivity for given powers...\n ..working on genes 1 through 7 of 7\n Flagging genes and samples with too many missing values...\n ..step 1\n ..Working on block 1 .\n TOM calculation: adjacency..\n ..will not use multithreading.\nclustering..\n ....detecting modules..\n ....calculating module eigengenes..\n ....checking kME in modules..\n ..merging modules that are too close..\n mergeCloseModules: Merging modules whose distance is less than 0.15\n mergeCloseModules: less than two proper modules.\n ..color levels are turquoise\n ..there is nothing to merge.\n Calculating new MEs...\n" } - json_output = "{\"inputdata\":{\"trait_sample_data \":{},\"minModuleSize\":30,\"TOMtype\":\"unsigned\"},\"outputdata\":{\"eigengenes\":[],\"colors\":[]}}" + json_output = "{\"inputdata\":{\"trait_sample_data \":{},\"minModuleSize\":30,\"TOMtype\":\"unsigned\"},\"output\":{\"eigengenes\":[],\"imageLoc\":[],\"colors\":[]}}" expected_output = { @@ -61,9 +64,11 @@ class TestWgcna(TestCase): "TOMtype": "unsigned" }, - "outputdata": { + "output": { "eigengenes": [], - "colors": [] + "imageLoc": [], + "colors": [], + "image_data": "AFDSFNBSDGJJHH" } }, @@ -74,6 +79,7 @@ class TestWgcna(TestCase): with mock.patch("builtins.open", mock.mock_open(read_data=json_output)): mock_run_cmd.return_value = mock_run_cmd_results + mock_img.return_value = b"AFDSFNBSDGJJHH" results = call_wgcna_script( "Rscript/GUIX_PATH/scripts/r_file.R", request_data) -- cgit v1.2.3 From 7b8efdf2040b0f623d1736e5608338ddcbe4a9da Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 13 Oct 2021 08:15:49 +0300 Subject: pylint and pep8 formatting --- gn3/computations/wgcna.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 90db455..76c71d5 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -25,7 +25,7 @@ def dump_wgcna_data(request_data: dict): return temp_file_path -def stream_cmd_output(request_data, cmd: str): +def stream_cmd_output(socketio, request_data, cmd: str): """function to stream in realtime""" # xtodo syncing and closing /edge cases @@ -41,7 +41,9 @@ def stream_cmd_output(request_data, cmd: str): {"data": line}, namespace="/", room=request_data["socket_id"]) socketio.emit( - "output", {"data": "parsing the output results"}, namespace="/", room=request_data["socket_id"]) + "output", {"data": + "parsing the output results"}, namespace="/", + room=request_data["socket_id"]) def process_image(image_loc: str) -> bytes: -- cgit v1.2.3 From a016f291176a974e654c9387937057cb8f6aa250 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 20 Oct 2021 13:17:35 +0300 Subject: mypy fix for none stdout --- gn3/computations/wgcna.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 76c71d5..0e49527 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -33,17 +33,20 @@ def stream_cmd_output(socketio, request_data, cmd: str): namespace="/", room=request_data["socket_id"]) results = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True) - for line in iter(results.stdout.readline, b""): - line = line.decode("utf-8").rstrip() + if results.stdout is not None: - socketio.emit("output", - {"data": line}, namespace="/", room=request_data["socket_id"]) + for line in iter(results.stdout.readline, b""): - socketio.emit( - "output", {"data": - "parsing the output results"}, namespace="/", - room=request_data["socket_id"]) + line = line.decode("utf-8").rstrip() + + socketio.emit("output", + {"data": line}, namespace="/", room=request_data["socket_id"]) + + socketio.emit( + "output", {"data": + "parsing the output results"}, namespace="/", + room=request_data["socket_id"]) def process_image(image_loc: str) -> bytes: -- cgit v1.2.3 From 5440bfcd6940db08c4479a39ba66dbc802b2c426 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 20 Oct 2021 13:26:02 +0300 Subject: mypy:Incompatible types in assignment fix --- gn3/computations/wgcna.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gn3/computations/wgcna.py b/gn3/computations/wgcna.py index 0e49527..ab12fe7 100644 --- a/gn3/computations/wgcna.py +++ b/gn3/computations/wgcna.py @@ -37,11 +37,9 @@ def stream_cmd_output(socketio, request_data, cmd: str): if results.stdout is not None: for line in iter(results.stdout.readline, b""): - - line = line.decode("utf-8").rstrip() - socketio.emit("output", - {"data": line}, namespace="/", room=request_data["socket_id"]) + {"data": line.decode("utf-8").rstrip()}, + namespace="/", room=request_data["socket_id"]) socketio.emit( "output", {"data": -- cgit v1.2.3