From 869f84ee791ce2761ef240a07c641fc5a829b0c3 Mon Sep 17 00:00:00 2001 From: jgart Date: Mon, 17 Jan 2022 05:24:38 -0500 Subject: update live jupyter notebooks --- wqflask/wqflask/jupyter_notebooks.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index 7d76828e..ba0b0e0d 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -1,17 +1,31 @@ from flask import Blueprint, render_template -jupyter_notebooks = Blueprint('jupyter_notebooks', __name__) +jupyter_notebooks = Blueprint("jupyter_notebooks", __name__) + @jupyter_notebooks.route("/launcher", methods=("GET",)) def launcher(): links = ( - { - "main_url": "http://notebook.genenetwork.org/34301/notebooks/genenetwork-api-using-r.ipynb", - "notebook_name": "R notebook showing how to query the GeneNetwork API.", - "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook"}, - { - "main_url": "http://notebook.genenetwork.org/57675/notebooks/genenetwork.ipynb", - "notebook_name": "Querying the GeneNetwork API declaratively with python.", - "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example"}) + { + "main_url": "http://notebook.genenetwork.org/59373/notebooks/BXD%20Analysis.ipynb", + "notebook_name": "This shows how to model BXD mouse weight data using an AR(1) process", + "src_link_url": "https://github.com/BonfaceKilz/tsaf-analysis-of-bxd-mouse-colonies", + }, + { + "main_url": "http://notebook.genenetwork.org/50939/notebooks/2020-05-08/solberg-rat-analysis.ipynb", + "notebook_name": "Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions - Webinar Series", + "src_link_url": "https://github.com/senresearch/quant-genetics-webinars", + }, + { + "main_url": "http://notebook.genenetwork.org/34447/notebooks/genenetwork.ipynb", + "notebook_name": "Querying the GeneNetwork API declaratively with python.", + "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example", + }, + { + "main_url": "http://notebook.genenetwork.org/53923/notebooks/genenetwork-api-using-r.ipynb", + "notebook_name": "R notebook showing how to query the GeneNetwork API.", + "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook", + }, + ) return render_template("jupyter_notebooks.html", links=links) -- cgit v1.2.3 From 885b5fb52fd123832fa829484de949d4afaa38d6 Mon Sep 17 00:00:00 2001 From: jgart Date: Mon, 17 Jan 2022 06:17:50 -0500 Subject: update saunak's live jupyter notebook --- wqflask/wqflask/jupyter_notebooks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index ba0b0e0d..fc3a0af3 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -12,7 +12,7 @@ def launcher(): "src_link_url": "https://github.com/BonfaceKilz/tsaf-analysis-of-bxd-mouse-colonies", }, { - "main_url": "http://notebook.genenetwork.org/50939/notebooks/2020-05-08/solberg-rat-analysis.ipynb", + "main_url": "http://notebook.genenetwork.org/53457/notebooks/2020-05-08/solberg-rat-analysis.ipynb", "notebook_name": "Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions - Webinar Series", "src_link_url": "https://github.com/senresearch/quant-genetics-webinars", }, -- cgit v1.2.3 From b8a3ac8a4d60e74f893a6714b8440ad6758a7fbd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 25 Jan 2022 14:51:32 +0300 Subject: user_login: Remove noisy debug statement --- wqflask/wqflask/user_login.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py index ff77982f..45a12f77 100644 --- a/wqflask/wqflask/user_login.py +++ b/wqflask/wqflask/user_login.py @@ -177,8 +177,6 @@ def verify_email(): @app.route("/n/login", methods=('GET', 'POST')) def login(): params = request.form if request.form else request.args - logger.debug("in login params are:", params) - if not params: # ZS: If coming to page for first time from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID external_login = {} -- cgit v1.2.3 From ea146928a397d8a813ccd169e1e532fd623f8998 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 Jan 2022 11:13:43 +0300 Subject: wqflask: Ignore diffs with |ε| < 0.001 * wqflask/wqflask/metadata_edits.py (update_phenotype): Filter out values with |ε| < 0.001 when generating the diff file. --- wqflask/wqflask/metadata_edits.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/metadata_edits.py b/wqflask/wqflask/metadata_edits.py index dc738f88..c1121774 100644 --- a/wqflask/wqflask/metadata_edits.py +++ b/wqflask/wqflask/metadata_edits.py @@ -257,9 +257,29 @@ def update_phenotype(dataset_id: str, name: str): r = run_cmd(cmd=("csvdiff " f"'{uploaded_file_name}' '{new_file_name}' " "--format json")) + json_data = json.loads(r.get("output")) + + # Only consider values where |ε| < 0.001; otherwise, use the + # old value in "Original". + _modifications = [] + for m in json_data.get("Modifications"): + _original = m.get("Original").split(",") + _current = m.get("Current").split(",") + for i, (x, y) in enumerate(zip(_original, _current)): + if (x.replace('.', '').isdigit() + and y.replace('.', '').isdigit() + and abs(float(x) - float(y)) < 0.001): + _current[i] = x + if not (__o:=",".join(_original)) == (__c:=",".join(_current)): + _modifications.append( + { + "Original": __o, + "Current": __c, + }) + json_data['Modifications'] = _modifications # Edge case where the csv file has not been edited! - if not any(json.loads(r.get("output")).values()): + if not any(json_data.values()): flash(f"You have not modified the csv file you downloaded!", "warning") return redirect(f"/datasets/{dataset_id}/traits/{name}" @@ -267,7 +287,7 @@ def update_phenotype(dataset_id: str, name: str): diff_output = (f"{TMPDIR}/sample-data/diffs/" f"{_file_name}.json") with open(diff_output, "w") as f: - dict_ = json.loads(r.get("output")) + dict_ = json_data dict_.update({ "trait_name": str(name), "phenotype_id": str(phenotype_id), -- cgit v1.2.3 From f6490120975eebe879a9d22f6bd741d887ec6ca5 Mon Sep 17 00:00:00 2001 From: jgart Date: Wed, 26 Jan 2022 20:58:58 -0500 Subject: update live jupyter notebooks Removed bonface's notebook until seaborn gets fixed. --- wqflask/wqflask/jupyter_notebooks.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/jupyter_notebooks.py b/wqflask/wqflask/jupyter_notebooks.py index fc3a0af3..10386d33 100644 --- a/wqflask/wqflask/jupyter_notebooks.py +++ b/wqflask/wqflask/jupyter_notebooks.py @@ -7,22 +7,17 @@ jupyter_notebooks = Blueprint("jupyter_notebooks", __name__) def launcher(): links = ( { - "main_url": "http://notebook.genenetwork.org/59373/notebooks/BXD%20Analysis.ipynb", - "notebook_name": "This shows how to model BXD mouse weight data using an AR(1) process", - "src_link_url": "https://github.com/BonfaceKilz/tsaf-analysis-of-bxd-mouse-colonies", - }, - { - "main_url": "http://notebook.genenetwork.org/53457/notebooks/2020-05-08/solberg-rat-analysis.ipynb", + "main_url": "http://notebook.genenetwork.org/40771/notebooks/2020-05-08/solberg-rat-analysis.ipynb", "notebook_name": "Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions - Webinar Series", "src_link_url": "https://github.com/senresearch/quant-genetics-webinars", }, { - "main_url": "http://notebook.genenetwork.org/34447/notebooks/genenetwork.ipynb", + "main_url": "http://notebook.genenetwork.org/45837/notebooks/genenetwork.ipynb", "notebook_name": "Querying the GeneNetwork API declaratively with python.", "src_link_url": "https://github.com/jgarte/genenetwork-jupyter-notebook-example", }, { - "main_url": "http://notebook.genenetwork.org/53923/notebooks/genenetwork-api-using-r.ipynb", + "main_url": "http://notebook.genenetwork.org/38079/notebooks/genenetwork-api-using-r.ipynb", "notebook_name": "R notebook showing how to query the GeneNetwork API.", "src_link_url": "https://github.com/jgarte/genenetwork-api-r-jupyter-notebook", }, -- cgit v1.2.3 From 978da92d6fdf6c1f3f9f4c56b963339725921248 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jan 2022 20:42:44 +0000 Subject: This fixes an issue where you couldn't select more than 150-200 rows when submitting search result traits to a collection; it previously just used JQuery and needed to be changed to use DataTables API to select from rows --- wqflask/wqflask/static/new/javascript/search_results.js | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 48b9b7be..9e507123 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -132,9 +132,13 @@ $(function() { add_to_collection = function() { var traits; - traits = $("#trait_table input:checked").map(function() { - return $(this).val(); - }).get(); + table_api = $('#trait_table').DataTable(); + check_nodes = table_api.column(0).nodes().to$(); + traits = Array.from(check_nodes.map(function() { + if ($(this)[0].childNodes[0].checked){ + return $(this)[0].childNodes[0].value + } + })) var traits_hash = md5(traits.toString()); @@ -337,4 +341,4 @@ $(function() { } apply_default(); -}); \ No newline at end of file +}); -- cgit v1.2.3 From 2e4442e0d552200a79769680bdcd1dab9ae98078 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jan 2022 20:49:08 +0000 Subject: Increase Y of search result table to 1000px so more rows are visible without scrolling --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index f73cba17..5e111882 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -447,7 +447,7 @@ "destroy": true, "autoWidth": false, "bSortClasses": false, - "scrollY": "500px", + "scrollY": "1000px", "scrollCollapse": true, {% if trait_list|length > 5 %} "scroller": true, -- cgit v1.2.3 From 0761eb67c601efe7d285e556fb1a07d52398eb6b Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jan 2022 21:15:01 +0000 Subject: Fix issue where Geno query was fetching from the chr_num column instead of the Chr column (and the chr_num column isn't always populated for certain Genotype datasets) --- wqflask/wqflask/do_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 99272ee3..ee5a4d91 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -343,7 +343,7 @@ class GenotypeSearch(DoSearch): GenoFreeze.createtime as thistable, Geno.Name as Geno_Name, Geno.Source2 as Geno_Source2, - Geno.chr_num as Geno_chr_num, + Geno.Chr as Geno_Chr, Geno.Mb as Geno_Mb FROM GenoXRef, GenoFreeze, Geno """ -- cgit v1.2.3 From 03c0557192242d28a35e0f9f1c20510cf457d985 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jan 2022 22:26:45 +0000 Subject: Replaced the info icon in the Peak LOD and Effect Size columns with a superscript question mark, since the icon was appearing strangely in some browsers (namely Rob's) but in a while I couldn't reproducd --- wqflask/wqflask/templates/search_result_page.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 5e111882..065d9049 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -245,7 +245,7 @@ 'orderSequence': [ "desc", "asc"] }, { - 'title': "
Peak  
LOD  
", + 'title': "
Peak  ?
LOD  
", 'type': "natural-minus-na", 'data': "lod_score", 'width': "60px", @@ -260,7 +260,7 @@ 'data': "lrs_location" }, { - 'title': "
Effect  
Size  
", + 'title': "
Effect  ?
Size  
", 'type': "natural-minus-na", 'data': "additive", 'width': "65px", @@ -339,7 +339,7 @@ 'orderSequence': [ "desc", "asc"] }, { - 'title': "
Peak  
LOD  
", + 'title': "
Peak  ?
LOD  
", 'type': "natural-minus-na", 'data': "lod_score", 'targets': 7, @@ -354,7 +354,7 @@ 'data': "lrs_location" }, { - 'title': "
Effect  
Size  
", + 'title': "
Effect  ?
Size  
", 'type': "natural-minus-na", 'width': "60px", 'data': "additive", -- cgit v1.2.3 From da49ee44334723358957dab3d1393a98ea9d7f51 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jan 2022 22:28:24 +0000 Subject: Increased the width of the location columns a bit, since they were being pushed onto a second line in some browsers (specifically Robs, but couldn't reproduce in mine, so I just changed it to something that should work for him) --- wqflask/wqflask/templates/search_result_page.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 065d9049..63e06237 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -232,7 +232,7 @@ { 'title': "
Location
", 'type': "natural-minus-na", - 'width': "125px", + 'width': "130px", 'targets': 5, 'data': "location" }, @@ -255,7 +255,7 @@ { 'title': "
Peak Location
", 'type': "natural-minus-na", - 'width': "125px", + 'width': "130px", 'targets': 8, 'data': "lrs_location" }, @@ -349,7 +349,7 @@ { 'title': "
Peak Location
", 'type': "natural-minus-na", - 'width': "120px", + 'width': "125px", 'targets': 8, 'data': "lrs_location" }, @@ -374,7 +374,7 @@ { 'title': "
Location
", 'type': "natural-minus-na", - 'width': "120px", + 'width': "125px", 'targets': 2, 'data': "location" }{% endif %} -- cgit v1.2.3 From 3e3335ec46a61056ffd8130969bca1d6857d182f Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 Jan 2022 22:30:53 +0000 Subject: Made the search result table wider for genotype results, since previously the Location text was being pushed to a second line (this was actually visible for me as well, not just Rob) --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 63e06237..eb4fe6a1 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -126,7 +126,7 @@ {% endif %} {% endif %} -
+
-- cgit v1.2.3 From 77f15b22adf297a8d46c3d590183b8f52b2baf51 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 20:05:42 +0000 Subject: Make glossary link question marks italicized and red --- wqflask/wqflask/templates/search_result_page.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index eb4fe6a1..c86aae39 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -245,7 +245,7 @@ 'orderSequence': [ "desc", "asc"] }, { - 'title': "
Peak  ?
LOD  
", + 'title': "
Peak ?
LOD  
", 'type': "natural-minus-na", 'data': "lod_score", 'width': "60px", @@ -260,7 +260,7 @@ 'data': "lrs_location" }, { - 'title': "
Effect  ?
Size  
", + 'title': "
Effect ?
Size  
", 'type': "natural-minus-na", 'data': "additive", 'width': "65px", @@ -339,7 +339,7 @@ 'orderSequence': [ "desc", "asc"] }, { - 'title': "
Peak  ?
LOD  
", + 'title': "
Peak ?
LOD  
", 'type': "natural-minus-na", 'data': "lod_score", 'targets': 7, @@ -354,7 +354,7 @@ 'data': "lrs_location" }, { - 'title': "
Effect  ?
Size  
", + 'title': "
Effect ?
Size  
", 'type': "natural-minus-na", 'width': "60px", 'data': "additive", -- cgit v1.2.3 From 7a8bfd0d99801e07c18dd5f9606e069b7643df89 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 20:09:38 +0000 Subject: Reordered search result options as described in Rob's 1/19/2022 e-mail --- wqflask/wqflask/templates/search_result_page.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index c86aae39..054c1a98 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -94,12 +94,12 @@ {% endif %} - - - - + + + + -- cgit v1.2.3 From 8f6248c71d04bf111d1cc38a398ea331f48d4fae Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 20:10:17 +0000 Subject: Moved the Show/Hide Columns text before the column buttons to save vertical space --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 054c1a98..a7d72807 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -105,7 +105,7 @@ {% if dataset.type != 'Geno' %}
- Show/Hide Columns: + Show/Hide Columns: 
{% if dataset.type == 'ProbeSet' %} -- cgit v1.2.3 From 07412521f896a609ae0011bc0a5ac0adf252b637 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 20:47:51 +0000 Subject: Replaced the Select Top feature with one that functions more like the Block By Index feature from the trait page; you can now use text indicating index ranges or specific indices, and the filter triggers on either hitting Enter or clicking outside of the text area --- .../static/new/javascript/search_results.js | 74 ++++++++++++---------- wqflask/wqflask/templates/search_result_page.html | 2 +- 2 files changed, 43 insertions(+), 33 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 9e507123..355fd18e 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -93,41 +93,51 @@ $(function() { $('#trait_table').DataTable().search($(this).val()).draw(); }); - $('#select_top').keyup(function(){ - num_rows = $(this).val() - - if (num_rows = parseInt(num_rows)){ - table_api = $('#trait_table').DataTable(); - - check_cells = table_api.column(0).nodes().to$(); - for (let i = 0; i < num_rows; i++) { - check_cells[i].childNodes[0].checked = true; - } - - check_rows = table_api.rows().nodes(); - for (let i=0; i < num_rows; i++) { - if (check_rows[i].classList.contains("selected")){ - continue - } else { - check_rows[i].classList.add("selected") - } - } - for (let i = num_rows; i < check_rows.length; i++){ - check_cells[i].childNodes[0].checked = false; - if (check_rows[i].classList.contains("selected")){ - check_rows[i].classList.remove("selected") - } - } - } - else { - for (let i = 0; i < check_rows.length; i++){ - check_cells[i].childNodes[0].checked = false; - if (check_rows[i].classList.contains("selected")){ - check_rows[i].classList.remove("selected") + parse_index_string = function(idx_string) { + index_list = []; + _ref = idx_string.split(","); + for (_i = 0, _len = _ref.length; _i < _len; _i++) { + index_set = _ref[_i]; + if (index_set.indexOf('-') !== -1) { + try { + start_index = parseInt(index_set.split("-")[0]); + end_index = parseInt(index_set.split("-")[1]); + for (index = _j = start_index; start_index <= end_index ? _j <= end_index : _j >= end_index; index = start_index <= end_index ? ++_j : --_j) { + index_list.push(index); } + } catch (_error) { + error = _error; + alert("Syntax error"); } + } else { + index = parseInt(index_set); + index_list.push(index); + } + } + return new Set(index_list) + } + + filter_by_index = function() { + index_string = $('#select_top').val() + index_set = parse_index_string(index_string) + + table_api = $('#trait_table').DataTable(); + check_nodes = table_api.column(0).nodes().to$(); + check_nodes.each(function(index) { + if (index_set.has(index + 1)){ + $(this)[0].childNodes[0].checked = true } - change_buttons(); + }) + } + + $('#select_top').keyup(function(event){ + if (event.keyCode === 13) { + filter_by_index() + } + }); + + $('#select_top').blur(function() { + filter_by_index() }); add_to_collection = function() { diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index a7d72807..fb335fd3 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -97,7 +97,7 @@ - + -- cgit v1.2.3 From 43706c381ed211ff92338be689a7d32c010dcf70 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 21:01:54 +0000 Subject: Changed search result index column to be static (so it doesn't change when the user sorts) --- wqflask/wqflask/templates/search_result_page.html | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index fb335fd3..56ac6e82 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -195,6 +195,8 @@ 'title': "Index", 'type': "natural", 'width': "35px", + "searchable": false, + "orderable": false, 'targets': 1, 'data': "index" } @@ -491,6 +493,12 @@ } } + trait_table.on( 'order.dt search.dt', function () { + trait_table.column(1, {search:'applied', order:'applied'}).nodes().each( function (cell, i) { + cell.innerHTML = i+1; + } ); + } ).draw(); + window.addEventListener('resize', function(){ trait_table.columns.adjust(); }); -- cgit v1.2.3 From 90103e1847256c8123cdb309b0acfa5e116751e4 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 21:19:48 +0000 Subject: Changed the glossary question mark to appear at the very end of the colum text, in the hopes that this will show up normally for Rob --- wqflask/wqflask/templates/search_result_page.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 56ac6e82..62c0164b 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -247,7 +247,7 @@ 'orderSequence': [ "desc", "asc"] }, { - 'title': "
Peak ?
LOD  
", + 'title': "
Peak
LOD ?
", 'type': "natural-minus-na", 'data': "lod_score", 'width': "60px", @@ -262,7 +262,7 @@ 'data': "lrs_location" }, { - 'title': "
Effect ?
Size  
", + 'title': "
Effect
Size ?
", 'type': "natural-minus-na", 'data': "additive", 'width': "65px", @@ -341,7 +341,7 @@ 'orderSequence': [ "desc", "asc"] }, { - 'title': "
Peak ?
LOD  
", + 'title': "
Peak
LOD ?
", 'type': "natural-minus-na", 'data': "lod_score", 'targets': 7, @@ -356,7 +356,7 @@ 'data': "lrs_location" }, { - 'title': "
Effect ?
Size  
", + 'title': "
Effect
Size ?
", 'type': "natural-minus-na", 'width': "60px", 'data': "additive", -- cgit v1.2.3 From 65ebd0ae48d77120569fb830b593c16f52a15cca Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 19 Jan 2022 21:20:19 +0000 Subject: Moved the Show/Hide Columns text onto the same line as the buttons, to save a little vertical space --- wqflask/wqflask/templates/search_result_page.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 62c0164b..419f9512 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -105,8 +105,7 @@ {% if dataset.type != 'Geno' %}
- Show/Hide Columns:  -
+ Show/Hide Columns:   {% if dataset.type == 'ProbeSet' %} -- cgit v1.2.3 From fb8e8c5f56b122e4561d448ce2873183da73f561 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 20 Jan 2022 19:40:47 +0000 Subject: Added some JS disabling form submission on hitting Enter, so that users can hit Enter in the Select Rows text area --- wqflask/wqflask/static/new/javascript/search_results.js | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 355fd18e..8ab41f25 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -130,6 +130,13 @@ $(function() { }) } + $(window).keydown(function(event){ + if((event.keyCode == 13)) { + event.preventDefault(); + return false; + } + }); + $('#select_top').keyup(function(event){ if (event.keyCode === 13) { filter_by_index() -- cgit v1.2.3 From 95d06e4b9028e25527239911d99bf584666073a8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 20 Jan 2022 19:43:42 +0000 Subject: Changed the placeholder text for the table search to Search For --- wqflask/wqflask/templates/search_result_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 419f9512..a5b2242c 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -94,7 +94,7 @@ {% endif %} - + -- cgit v1.2.3 From 58cda1ea8740792beb5d0f3f1de15c51b2f47f82 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 26 Jan 2022 18:41:31 +0000 Subject: Removed unnecessary variable from for-loop in parse_index_string --- wqflask/wqflask/static/new/javascript/search_results.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 8ab41f25..17b41812 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -96,7 +96,7 @@ $(function() { parse_index_string = function(idx_string) { index_list = []; _ref = idx_string.split(","); - for (_i = 0, _len = _ref.length; _i < _len; _i++) { + for (_i = 0; _i < _ref.length; _i++) { index_set = _ref[_i]; if (index_set.indexOf('-') !== -1) { try { -- cgit v1.2.3 From e374cac6eec2d05345d6f7a05061ee38a63db9f1 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 26 Jan 2022 19:21:24 +0000 Subject: Changed functions in search_results.js to be camelCased, except for change_buttons because it's referenced in many other files --- .../static/new/javascript/search_results.js | 68 +++++++++++----------- 1 file changed, 34 insertions(+), 34 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 17b41812..7124151e 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -25,10 +25,10 @@ change_buttons = function(check_node = 0) { }; $(function() { - var add, checked_traits, deselect_all, invert, remove, removed_traits, select_all; + var add, checked_traits, deselectAll, invert, selectAll; checked_traits = null; - select_all = function() { + selectAll = function() { table_api = $('#trait_table').DataTable(); check_cells = table_api.column(0).nodes().to$(); @@ -44,7 +44,7 @@ $(function() { change_buttons(); }; - deselect_all = function() { + deselectAll = function() { table_api = $('#trait_table').DataTable(); check_cells = table_api.column(0).nodes().to$(); @@ -93,7 +93,7 @@ $(function() { $('#trait_table').DataTable().search($(this).val()).draw(); }); - parse_index_string = function(idx_string) { + parseIndexString = function(idx_string) { index_list = []; _ref = idx_string.split(","); for (_i = 0; _i < _ref.length; _i++) { @@ -117,9 +117,9 @@ $(function() { return new Set(index_list) } - filter_by_index = function() { + filterByIndex = function() { index_string = $('#select_top').val() - index_set = parse_index_string(index_string) + index_set = parseIndexString(index_string) table_api = $('#trait_table').DataTable(); check_nodes = table_api.column(0).nodes().to$(); @@ -139,15 +139,15 @@ $(function() { $('#select_top').keyup(function(event){ if (event.keyCode === 13) { - filter_by_index() + filterByIndex() } }); $('#select_top').blur(function() { - filter_by_index() + filterByIndex() }); - add_to_collection = function() { + addToCollection = function() { var traits; table_api = $('#trait_table').DataTable(); check_nodes = table_api.column(0).nodes().to$(); @@ -174,19 +174,19 @@ $(function() { }; - removed_traits = function() { + removedTraits = function() { return checked_traits.closest("tr").fadeOut(); }; - submit_bnw = function() { - trait_data = submit_traits_to_export_or_bnw("trait_table", "submit_bnw") + submitBnw = function() { + trait_data = submitTraitsToExportOrBnw("trait_table", "submit_bnw") } - export_traits = function() { - trait_data = submit_traits_to_export_or_bnw("trait_table", "export_csv") + exportTraits = function() { + trait_data = submitTraitsToExportOrBnw("trait_table", "export_csv") }; - submit_traits_to_export_or_bnw = function(table_name, destination) { + submitTraitsToExportOrBnw = function(table_name, destination) { trait_table = $('#'+table_name); table_dict = {}; @@ -227,7 +227,7 @@ $(function() { $('#export_form').submit(); }; - get_traits_from_table = function(){ + getTraitsFromTable = function(){ traits = $("#trait_table input:checked").map(function() { return $(this).val(); }).get(); @@ -243,42 +243,42 @@ $(function() { } $("#corr_matrix").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) $("input[name=tool_used]").val("Correlation Matrix") $("input[name=form_url]").val($(this).data("url")) return submit_special("/loading") }); $("#network_graph").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) $("input[name=tool_used]").val("Network Graph") $("input[name=form_url]").val($(this).data("url")) return submit_special("/loading") }); $("#wgcna_setup").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) $("input[name=tool_used]").val("WGCNA Setup") $("input[name=form_url]").val($(this).data("url")) return submit_special("/loading") }); $("#ctl_setup").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) $("input[name=tool_used]").val("CTL Setup") $("input[name=form_url]").val($(this).data("url")) return submit_special("/loading") }); $("#heatmap").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) $("input[name=tool_used]").val("Heatmap") $("input[name=form_url]").val($(this).data("url")) return submit_special("/loading") }); $("#comp_bar_chart").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) $("input[name=tool_used]").val("Comparison Bar Chart") $("input[name=form_url]").val($(this).data("url")) @@ -286,32 +286,32 @@ $(function() { }); $("#send_to_webgestalt, #send_to_bnw, #send_to_geneweaver").on("click", function() { - traits = get_traits_from_table() + traits = getTraitsFromTable() $("#trait_list").val(traits) url = $(this).data("url") return submit_special(url) }); - $("#select_all").click(select_all); - $("#deselect_all").click(deselect_all); + $("#select_all").click(selectAll); + $("#deselect_all").click(deselectAll); $("#invert").click(invert); - $("#add").click(add_to_collection); - $("#submit_bnw").click(submit_bnw); - $("#export_traits").click(export_traits); + $("#add").click(addToCollection); + $("#submit_bnw").click(submitBnw); + $("#export_traits").click(exportTraits); let naturalAsc = $.fn.dataTableExt.oSort["natural-ci-asc"] let naturalDesc = $.fn.dataTableExt.oSort["natural-ci-desc"] let na_equivalent_vals = ["N/A", "--", ""]; //ZS: Since there are multiple values that should be treated the same as N/A - function extract_inner_text(the_string){ + function extractInnerText(the_string){ var span = document.createElement('span'); span.innerHTML = the_string; return span.textContent || span.innerText; } - function sort_NAs(a, b, sort_function){ + function sortNAs(a, b, sort_function){ if ( na_equivalent_vals.includes(a) && na_equivalent_vals.includes(b)) { return 0; } @@ -326,10 +326,10 @@ $(function() { $.extend( $.fn.dataTableExt.oSort, { "natural-minus-na-asc": function (a, b) { - return sort_NAs(extract_inner_text(a), extract_inner_text(b), naturalAsc) + return sortNAs(extractInnerText(a), extractInnerText(b), naturalAsc) }, "natural-minus-na-desc": function (a, b) { - return sort_NAs(extract_inner_text(a), extract_inner_text(b), naturalDesc) + return sortNAs(extractInnerText(a), extractInnerText(b), naturalDesc) } }); @@ -347,7 +347,7 @@ $(function() { } ); } - apply_default = function() { + applyDefault = function() { let default_collection_id = $.cookie('default_collection'); if (default_collection_id) { let the_option = $('[name=existing_collection] option').filter(function() { @@ -356,6 +356,6 @@ $(function() { the_option.prop('selected', true); } } - apply_default(); + applyDefault(); }); -- cgit v1.2.3 From 6ab213921932d904c64c8b37a327a971a3c46cfc Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 26 Jan 2022 19:36:38 +0000 Subject: Replaced a confusing for loop with something that is (hopefully) easier to read and explicitly inverts start_index and end_index when the former is greater than the latter --- wqflask/wqflask/static/new/javascript/search_results.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 7124151e..a665e3d2 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -102,7 +102,13 @@ $(function() { try { start_index = parseInt(index_set.split("-")[0]); end_index = parseInt(index_set.split("-")[1]); - for (index = _j = start_index; start_index <= end_index ? _j <= end_index : _j >= end_index; index = start_index <= end_index ? ++_j : --_j) { + + // If start index is higher than end index (for example is the string "10-5" exists) swap values so it'll be interpreted as "5-10" instead + if (start_index > end_index) { + [start_index, end_index] = [end_index, start_index] + } + + for (index = start_index; index <= end_index; index++) { index_list.push(index); } } catch (_error) { -- cgit v1.2.3 From 480d71d9e12878449f7526cacb644f686a9169af Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 26 Jan 2022 19:42:30 +0000 Subject: Added a comment explaining what the parseIndexString function does --- wqflask/wqflask/static/new/javascript/search_results.js | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index a665e3d2..b7ad359f 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -93,6 +93,13 @@ $(function() { $('#trait_table').DataTable().search($(this).val()).draw(); }); + /** + * parseIndexString takes a string consisting of integers, + * hyphens, and/or commas to indicate range(s) of indices + * to select a rows and returns the corresponding set of indices + * For example - "1, 5-10, 15" would return a set of 8 rows + * @return {Set} The list of indices as a Set + */ parseIndexString = function(idx_string) { index_list = []; _ref = idx_string.split(","); -- cgit v1.2.3 From e5f179330b024671e066edb0672a5a8b628766ed Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 26 Jan 2022 20:54:03 +0000 Subject: Replaced the try/catch with something checking for the correct pattern with regex, for validating the Select Rows input --- wqflask/wqflask/static/new/javascript/search_results.js | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index b7ad359f..a3e21081 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -102,11 +102,17 @@ $(function() { */ parseIndexString = function(idx_string) { index_list = []; + _ref = idx_string.split(","); for (_i = 0; _i < _ref.length; _i++) { index_set = _ref[_i]; + if (!/^ *([0-9]+$) *| *([0-9]+ *- *[0-9]+$) *|(^$)$/.test(index_set)) { + $('#select_samples_invalid').show(); + break + } else { + $('#select_samples_invalid').hide(); + } if (index_set.indexOf('-') !== -1) { - try { start_index = parseInt(index_set.split("-")[0]); end_index = parseInt(index_set.split("-")[1]); @@ -118,10 +124,6 @@ $(function() { for (index = start_index; index <= end_index; index++) { index_list.push(index); } - } catch (_error) { - error = _error; - alert("Syntax error"); - } } else { index = parseInt(index_set); index_list.push(index); -- cgit v1.2.3 From b25d68790fdf8d66644297dc104408140baf77e5 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 26 Jan 2022 20:55:37 +0000 Subject: Added the div for showing the error when the Select Rows text has wrong syntax --- wqflask/wqflask/templates/search_result_page.html | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index a5b2242c..26cdc437 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -102,6 +102,10 @@
+ {% if dataset.type != 'Geno' %}
-- cgit v1.2.3 From 3dc47b73cd597291c8410cad05d1143819303e1d Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 Jan 2022 19:21:41 +0000 Subject: Removed unused variables code --- wqflask/wqflask/static/new/javascript/search_results.js | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index a3e21081..4c634399 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -25,9 +25,8 @@ change_buttons = function(check_node = 0) { }; $(function() { - var add, checked_traits, deselectAll, invert, selectAll; + var selectAll, deselectAll, invert; - checked_traits = null; selectAll = function() { table_api = $('#trait_table').DataTable(); @@ -189,10 +188,6 @@ $(function() { }; - removedTraits = function() { - return checked_traits.closest("tr").fadeOut(); - }; - submitBnw = function() { trait_data = submitTraitsToExportOrBnw("trait_table", "submit_bnw") } -- cgit v1.2.3 From 63cfd61c030312158aba242ff239e79e2025f2e0 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 Jan 2022 19:24:04 +0000 Subject: Changed var to let --- wqflask/wqflask/static/new/javascript/search_results.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index 4c634399..c5f9fe00 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -25,7 +25,7 @@ change_buttons = function(check_node = 0) { }; $(function() { - var selectAll, deselectAll, invert; + let selectAll, deselectAll, invert; selectAll = function() { table_api = $('#trait_table').DataTable(); -- cgit v1.2.3 From 04ef9ee2f18e812b4a24d478bdbf78303923623e Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 Jan 2022 19:40:54 +0000 Subject: Highlight rows selected by filterByIndex This adds the "selected" class to rows selected by filterByIndex, which adds highlighting to them I also made the variables in this part of the code camel cased --- .../wqflask/static/new/javascript/search_results.js | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index c5f9fe00..ff2452af 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -132,16 +132,23 @@ $(function() { } filterByIndex = function() { - index_string = $('#select_top').val() - index_set = parseIndexString(index_string) + indexString = $('#select_top').val() + indexSet = parseIndexString(indexString) - table_api = $('#trait_table').DataTable(); - check_nodes = table_api.column(0).nodes().to$(); - check_nodes.each(function(index) { - if (index_set.has(index + 1)){ + tableApi = $('#trait_table').DataTable(); + checkNodes = tableApi.column(0).nodes().to$(); + checkNodes.each(function(index) { + if (indexSet.has(index + 1)){ $(this)[0].childNodes[0].checked = true } }) + + checkRows = tableApi.rows().nodes().to$(); + checkRows.each(function(index) { + if (indexSet.has(index + 1)){ + $(this)[0].classList.add("selected"); + } + }) } $(window).keydown(function(event){ -- cgit v1.2.3 From a01033438a85e3097e4ade30e9a208fd5109d4e1 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 6 Dec 2021 17:58:28 +0300 Subject: Add "Partial Correlations" UI elements on collections page Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Add UI elements to trigger the partial correlations feature * Connect partial correlation view to the application * Point to correct application element for gunicorn --- bin/genenetwork2 | 2 +- wqflask/wqflask/__init__.py | 1 + wqflask/wqflask/partial_correlations_views.py | 263 +++++++++++++++++++++ wqflask/wqflask/templates/collections/view.html | 17 ++ .../wqflask/templates/partial_correlations.html | 199 ++++++++++++++++ 5 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 wqflask/wqflask/partial_correlations_views.py create mode 100644 wqflask/wqflask/templates/partial_correlations.html (limited to 'wqflask') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 1d645c23..0a2c18cf 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -192,7 +192,7 @@ if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi - cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" + cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload run_gunicorn:app" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 05e040ed..ab8b9e66 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -98,3 +98,4 @@ from wqflask import user_login from wqflask import user_session import wqflask.views +import wqflask.partial_correlations_views diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py new file mode 100644 index 00000000..bee0a033 --- /dev/null +++ b/wqflask/wqflask/partial_correlations_views.py @@ -0,0 +1,263 @@ +from typing import Union, Tuple + +import MySQLdb +from gn3.db.traits import retrieve_trait_info +from flask import flash, request, current_app, render_template +from gn3.computations.partial_correlations import partial_correlations_entry + +from wqflask import app + +def parse_trait(trait_str: str) -> Union[dict, None]: + keys = ("name", "dataset", "symbol", "description", "data_hmac") + parts = tuple(part.strip() for part in trait_str.split(":::")) + if len(parts) == len(keys): + return dict(zip(keys, parts)) + return None + +def process_step_select_primary( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + if primary_trait is None: + flash("You must select a primary trait", "alert-danger") + return ( + "select-primary", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + return ( + "select-controls", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list + if trait["data_hmac"] != primary_trait["data_hmac"]), + corr_method) + +def process_step_select_controls( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + if len(control_traits) == 0 or len(control_traits) > 3: + flash( + ("You must select a minimum of one control trait, up to a maximum " + "of three control traits."), + "alert-danger") + return ( + "select-controls", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + hmacs =(primary_trait["data_hmac"],) + tuple( + trait["data_hmac"] for trait in control_traits) + return ( + "select-targets", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list if trait["data_hmac"] not in hmacs), + corr_method) + +def process_step_select_targets( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + if len(target_traits) == 0: + flash( + "You must select at least one target trait.", "alert-danger") + return ( + "select-targets", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + hmacs =(primary_trait["data_hmac"],) + tuple( + trait["data_hmac"] for trait in (control_traits + target_traits)) + return ( + "select-corr-method", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list if trait["data_hmac"] not in hmacs), + corr_method) + +def process_step_select_corr_method( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + methods = ( + "genetic correlation, pearson's r", + "genetic correlation, spearman's rho", + "sgo literature correlation", + "tissue correlation, pearson's r", + "tissue correlation, spearman's rho") + if corr_method.lower() not in methods: + flash( + "Selected method is unknown.", "alert-danger") + return ( + "select-corr-method", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + hmacs =(primary_trait["data_hmac"],) + tuple( + trait["data_hmac"] for trait in (control_traits + target_traits)) + return ( + "run-correlation", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list if trait["data_hmac"] not in hmacs), + corr_method) + +def process_step( + step: str, primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], traits_list: Tuple[dict, ...], + corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + processor_functions = { + # "select-traits": lambda arg: arg, + "select-primary": process_step_select_primary, + "select-controls": process_step_select_controls, + "select-targets": process_step_select_targets, + "select-corr-method": process_step_select_corr_method + } + return processor_functions[(step or "select-primary")]( + primary_trait, control_traits, target_traits, traits_list, corr_method) + +def sequence_of_traits(trait_strs) -> Tuple[dict, ...]: + return tuple(filter( + lambda trt: trt is not None, + (parse_trait(tstr.strip()) for tstr in trait_strs))) + +def publish_target_dabases(conn, group, threshold): + query = ( + "SELECT PublishFreeze.FullName,PublishFreeze.Name " + "FROM PublishFreeze, InbredSet " + "WHERE PublishFreeze.InbredSetId = InbredSet.Id " + "AND InbredSet.Name = %(group)s " + "AND PublishFreeze.public > %(threshold)s") + with conn.cursor() as cursor: + cursor.execute(query, {"group": group, "threshold": threshold}) + res = cursor.fetchall() + if res: + return tuple( + dict(zip(("description", "value"), row)) for row in res) + + return tuple() + +def geno_target_databases(conn, group, threshold): + query = ( + "SELECT GenoFreeze.FullName,GenoFreeze.Name " + "FROM GenoFreeze, InbredSet " + "WHERE GenoFreeze.InbredSetId = InbredSet.Id " + "AND InbredSet.Name = %(group)s " + "AND GenoFreeze.public > %(threshold)s") + with conn.cursor() as cursor: + cursor.execute(query, {"group": group, "threshold": threshold}) + res = cursor.fetchall() + if res: + return tuple( + dict(zip(("description", "value"), row)) for row in res) + + return tuple() + +def probeset_target_databases(conn, group, threshold): + query1 = "SELECT Id, Name FROM Tissue order by Name" + query2 = ( + "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, ProbeSetFreeze.Name " + "FROM ProbeSetFreeze, ProbeFreeze, InbredSet " + "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND ProbeFreeze.TissueId IN %(tissue_ids)s " + "AND ProbeSetFreeze.public > %(threshold)s " + "AND ProbeFreeze.InbredSetId = InbredSet.Id " + "AND InbredSet.Name like %(group)s " + "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId") + with conn.cursor() as cursor: + cursor.execute(query1) + tissue_res = cursor.fetchall() + if tissue_res: + tissue_ids = tuple(row[0] for row in tissue_res) + cursor.execute( + query2,{ + "tissue_ids": tissue_ids, "group": f"{group}%%", + "threshold": threshold + }) + db_res = cursor.fetchall() + if db_res: + databases = tuple( + dict(zip(("tissue_id", "description", "value"), row)) + for row in db_res) + return tuple( + {tissue_name: tuple( + { + "value": item["value"], + "description": item["description"] + } for item in databases + if item["tissue_id"] == tissue_id)} + for tissue_id, tissue_name in tissue_res) + + return tuple() + +def target_databases(conn, step, trait, threshold): + """ + Retrieves the names of possible target databases from the database. + """ + if step != "select-corr-method": + return None + + trait_info = retrieve_trait_info( + threshold, f"{trait['dataset']}::{trait['name']}", conn) + group = trait_info["group"] + return ( + publish_target_dabases(conn, group, threshold) + + geno_target_databases(conn, group, threshold) + + probeset_target_databases(conn, group, threshold)) + +def pcorrelations(conn, values): + if values["step"] != "run-correlation": + return None + + def trait_fullname(trait): + return f"{trait['dataset']}::{trait['name']}" + + return partial_correlations_entry( + conn, trait_fullname(values["primary_trait"]), + tuple(trait_fullname(trait) for trait in values["control_traits"]), + values["method"], values["criteria"], values["target_db"]) + +@app.route("/partial_correlations", methods=("POST",)) +def partial_correlations(): + form = request.form + traits_list = tuple(filter( + lambda trt: trt is not None, + (parse_trait(tstr) for tstr in form.get("traits_list", "").split("|||")))) + + args_dict = dict(zip( + ("step", "primary_trait", "control_traits", "target_traits", + "traits_list", "method"), + process_step( + form.get("step", None), + parse_trait(form.get("primary_trait", "")), + sequence_of_traits( + form.getlist("control_traits[]") or + form.get("control_traits", "").split("|||")), + sequence_of_traits( + form.getlist("target_traits[]") or + form.get("target_traits", "").split("|||")), + sequence_of_traits(form.get("traits_list", "").split("|||")), + form.get("method")))) + + conn = MySQLdb.Connect( + db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + target_dbs = target_databases( + conn, args_dict["step"], args_dict["primary_trait"], 0) + + if args_dict["step"] == "run-correlation": + args_dict = { + **args_dict, "target_db": form.get("target_db"), + "criteria": int(form.get("criteria", 500))} + + corr_results = pcorrelations(conn, args_dict) + + return render_template( + "partial_correlations.html", **args_dict, target_dbs=target_dbs, + corr_results=corr_results) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 0ded66a6..f4270b67 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -34,6 +34,23 @@
+
+ + + + +

diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html new file mode 100644 index 00000000..b61f7fc4 --- /dev/null +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -0,0 +1,199 @@ +{%extends "base.html"%} + +{%block content%} +
+ + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} +
    + {%for category, message in messages:%} +
  • {{message}}
    • + {%endfor%} +
+ {%endif%} + {%endwith%} + + + + + {%if primary_trait:%} + +

+ Primary Trait: {{primary_trait["name"]}} -  + {{primary_trait["symbol"]}} - {{primary_trait["description"]}} +

+ {%endif%} + + {%if control_traits:%} + +

+ Control Traits: +

    + {%for trait in control_traits:%} +
  • + {{trait["name"]}} - {{trait["symbol"]}} -  + {{trait["description"]}}
    • + {%endfor%} +
+

+ {%endif%} + + {%if target_traits:%} + +

+ Target Traits: +

    + {%for trait in target_traits:%} +
  • + {{trait["name"]}} - {{trait["symbol"]}} -  + {{trait["description"]}}
    • + {%endfor%} +
+

+ {%endif%} + + + + {%if step == "select-primary":%} +

Please select the primary trait (X)

+ {%for trait in traits_list:%} +
+ + +
+ {%endfor%} + + {%endif%} + + + + {%if step == "select-controls":%} + +

Select a maximum of three (3) control traits (Z)

+ {%for trait in traits_list:%} +
+ + +
+ {%endfor%} + + {%endif%} + + + + {%if step == "select-targets":%} +

Select at least one target trait (Y)

+ {%for trait in traits_list:%} +
+ + +
+ {%endfor%} + + {%endif%} + + + + {%if step == "select-corr-method":%} +
+ + +
+ +
+ + +
+ +
+ + +
+ + + {%endif%} + + {%if step == "run-correlation":%} + + + + + {{corr_results}} + {%endif%} + +
+{%endblock%} -- cgit v1.2.3 From db0fc679dcef0360c4b99d28d9126026b893af70 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 10 Dec 2021 09:06:31 +0300 Subject: Implement better styling Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Separate the form elements that allow selection from the elements that display previous selections. This allows styling of the "active" area of the form in a different way from the "display" area of the form, allowing the user to see information on their previous choices, even while making further selections for the partial correlations. --- .../static/new/css/partial_correlations.css | 30 +++++ .../wqflask/templates/partial_correlations.html | 140 ++++++++++++--------- 2 files changed, 108 insertions(+), 62 deletions(-) create mode 100644 wqflask/wqflask/static/new/css/partial_correlations.css (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css new file mode 100644 index 00000000..ebc00302 --- /dev/null +++ b/wqflask/wqflask/static/new/css/partial_correlations.css @@ -0,0 +1,30 @@ +#partial-correlations-form { + width: 100%; + display: grid; + grid-column-gap: 1em; +} + +#main-form { + grid-column-start: 1; + grid-column-end: 2; +} + +#form-display-area { + grid-column-start: 2; + grid-column-end: 3; +} + +.label-element-combo { + width: 100%; + display: grid; +} + +.with-trait > label { + width: 80%; + grid-column-start: 2; +} + +.with-trait > input { + width: 20%; + grid-column-start: 2; +} diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index b61f7fc4..ddf6fb4e 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -1,10 +1,17 @@ {%extends "base.html"%} +{%block title%}Partial Correlations:{%endblock%} + +{%block css%} + +{%endblock%} + {%block content%} -
-
+ + +
{%with messages = get_flashed_messages(with_categories=true)%} {%if messages:%}
    @@ -15,60 +22,10 @@ {%endif%} {%endwith%} - - - - {%if primary_trait:%} - -

    - Primary Trait: {{primary_trait["name"]}} -  - {{primary_trait["symbol"]}} - {{primary_trait["description"]}} -

    - {%endif%} - - {%if control_traits:%} - -

    - Control Traits: -

      - {%for trait in control_traits:%} -
    • - {{trait["name"]}} - {{trait["symbol"]}} -  - {{trait["description"]}}
      • - {%endfor%} -
    -

    - {%endif%} - - {%if target_traits:%} - -

    - Target Traits: -

      - {%for trait in target_traits:%} -
    • - {{trait["name"]}} - {{trait["symbol"]}} -  - {{trait["description"]}}
      • - {%endfor%} -
    -

    - {%endif%} - - - {%if step == "select-primary":%}

    Please select the primary trait (X)

    {%for trait in traits_list:%} -
    +
    Select a maximum of three (3) control traits (Z)

    {%for trait in traits_list:%} -
    +
    Select at least one target trait (Y)

    {%for trait in traits_list:%} -
    +
    +
    -
    +
    -
    +
    + + + {%if primary_trait:%} + + +
    +
    Primary Trait (X)
    +
    + {{primary_trait["name"]}} -  + {{primary_trait["symbol"]}} - {{primary_trait["description"]}} +
    +
    + {%endif%} + + {%if target_traits:%} + +
    +
    Target Traits (Y)
    +
    +
      + {%for trait in target_traits:%} +
    • + {{trait["name"]}} - {{trait["symbol"]}} -  + {{trait["description"]}}
      • + {%endfor%} +
    +
    +
    + {%endif%} + + {%if control_traits:%} + +
    +
    Control Traits (Z)
    +
    +
      + {%for trait in control_traits:%} +
    • + {{trait["name"]}} - {{trait["symbol"]}} -  + {{trait["description"]}}
      • + {%endfor%} +
    +
    +
    + {%endif%} +
    + + {%endblock%} -- cgit v1.2.3 From 2822237883b9a91fb35f33fb9094d2dcb9e44855 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 13 Dec 2021 07:24:57 +0300 Subject: Skip the target traits selection step Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Skip the target traits selection step, and only allow running the partial correlations against one of the available databases. --- wqflask/wqflask/partial_correlations_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index bee0a033..9a443897 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -51,7 +51,7 @@ def process_step_select_controls( hmacs =(primary_trait["data_hmac"],) + tuple( trait["data_hmac"] for trait in control_traits) return ( - "select-targets", primary_trait, control_traits, target_traits, + "select-corr-method", primary_trait, control_traits, target_traits, tuple( trait for trait in traits_list if trait["data_hmac"] not in hmacs), corr_method) -- cgit v1.2.3 From 6c7231c0dec704ad2722e12190914ce5cf8822fd Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 17 Dec 2021 11:01:32 +0300 Subject: Use list rather than tuple Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Using a tuple, rather than a list for the HTTP methods in the route definition causes an error. This fixes that by using a list. --- wqflask/wqflask/partial_correlations_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index 9a443897..0b7ffd37 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -221,7 +221,7 @@ def pcorrelations(conn, values): tuple(trait_fullname(trait) for trait in values["control_traits"]), values["method"], values["criteria"], values["target_db"]) -@app.route("/partial_correlations", methods=("POST",)) +@app.route("/partial_correlations", methods=["POST"]) def partial_correlations(): form = request.form traits_list = tuple(filter( -- cgit v1.2.3 From 5027abd962dfc599a9885d12a556fd12c37cadfa Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 17 Dec 2021 11:03:23 +0300 Subject: Add elements to display results. Provide PoC JS Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Add some html elements to be used to display the results of running the partial correlations * Provide some initial proof-of-concept javascript code to animate the various elements and to use for displaying both successful and error results. --- wqflask/wqflask/partial_correlations_views.py | 3 +- .../static/new/javascript/partial_correlations.js | 117 +++++++++++++++++++++ .../wqflask/templates/partial_correlations.html | 23 +++- 3 files changed, 141 insertions(+), 2 deletions(-) create mode 100644 wqflask/wqflask/static/new/javascript/partial_correlations.js (limited to 'wqflask') diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index 0b7ffd37..82287312 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -6,6 +6,7 @@ from flask import flash, request, current_app, render_template from gn3.computations.partial_correlations import partial_correlations_entry from wqflask import app +from utility.tools import GN_SERVER_URL def parse_trait(trait_str: str) -> Union[dict, None]: keys = ("name", "dataset", "symbol", "description", "data_hmac") @@ -260,4 +261,4 @@ def partial_correlations(): return render_template( "partial_correlations.html", **args_dict, target_dbs=target_dbs, - corr_results=corr_results) + corr_results=corr_results, part_corr_url=f"{GN_SERVER_URL}correlation/partial") diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js new file mode 100644 index 00000000..71fac21d --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -0,0 +1,117 @@ +/** + * This is, hopefully, a short-term stop-gap measure to get the system working + * and to get some feedback, even as better optimisation is done in the + * background to get better response/performance for the partial correlation + * computations + */ + +function key_value(keys, values) { + if(!(keys.length == values.length)) { + Error("The 'keys' and 'values' objects MUST be the same length"); + return null; + } + return values.reduce(function(accumulator, item, index) { + accumulator[keys[index]] = item; + return accumulator; + }, {}); +} + +function trait(trait_str) { + return key_value( + ["name", "dataset", "symbol", "description", "data_hmac"], + trait_str.split(":::")); +} + +function primary_trait() { + trait_string = document.querySelector( + "#partial-correlations-form input[name=primary_trait]").value; + return trait(trait_string); +} + +function control_traits() { + return document.querySelector( + "#partial-correlations-form input[name=control_traits]").value.split( + "|||").map(trait).filter(trait => !(trait === null)); +} + +function correlation_method() { + return document.querySelector( + "#partial-correlations-form select[name=method]").value; +} + +function criteria() { + return document.querySelector( + "#partial-correlations-form select[name=criteria]").value; +} + +function target_db() { + return document.querySelector( + "#partial-correlations-form select[name=target_db]").value; +} + +function partial_corr_request_data() { + return { + "primary_trait": primary_trait(), + "control_traits": control_traits(), + "method": correlation_method(), + "criteria": criteria(), + "target_db": target_db() + } +} + +function display_partial_corr_results(data, status, xhr) { + progress_indicator = document.getElementById( + "partial-correlations-progress-indicator").style.display = "none"; + parent = document.getElementById("part-corr-success"); + child = document.createElement("p"); + child.textContent = data; + parent.appendChild(child); +} + +function display_partial_corr_error(xhr, status, error) { + document.getElementById( + "partial-correlations-progress-indicator").style.display = "none"; + error_element = document.getElementById("part-corr-error"); + panel = document.createElement("div"); + panel.setAttribute("class", "panel panel-danger"); + error_element.appendChild(panel); + + panel_header = document.createElement("div"); + panel_header.setAttribute("class", "panel-heading"); + panel_header.textContent = "Error: " + xhr.status; + panel.appendChild(panel_header); + + panel_body = document.createElement("div"); + panel_body.setAttribute("class", "panel-body"); + panel_body.textContent = xhr.statusText; + panel.appendChild(panel_body); + console.log(xhr) +} + +function send_data_and_process_results( + remote_url, request_data, success_fn, error_fn, indicator_id) { + document.getElementById(indicator_id).style.display = "block"; + $.ajax({ + type: "POST", + url: remote_url, + contentType: "application/json", + data: JSON.stringify(request_data), + dataType: "JSON", + success: success_fn, + error: error_fn + }); +} + +$("#partial-correlations-form").submit(function(e) { + e.preventDefault(); +}); + +$("#run-partial-corr-button").click(function(evt) { + send_data_and_process_results( + document.getElementById( + "run-partial-corr-button").getAttribute("data-url"), + partial_corr_request_data(), + display_partial_corr_results, + display_partial_corr_error, + "partial-correlations-progress-indicator"); +}) diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index ddf6fb4e..08d6fc2a 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -139,7 +139,10 @@
    - {%endif%} @@ -212,4 +215,22 @@
    + +
    + + + +
    +
    +
    +
    +
    +{%endblock%} + +{%block js%} +{%if step == "select-corr-method":%} + +{%endif%} {%endblock%} -- cgit v1.2.3 From 37c668eb1c3d9ad06f8caaf78ac77b69515b6956 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Sat, 18 Dec 2021 08:11:37 +0300 Subject: Add templates for the results tables Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Add templates for the tables that will display the results. --- .../static/new/javascript/partial_correlations.js | 1 + .../wqflask/templates/partial_correlations.html | 121 +++++++++++++++++++++ 2 files changed, 122 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index 71fac21d..b1b12ee7 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -66,6 +66,7 @@ function display_partial_corr_results(data, status, xhr) { child = document.createElement("p"); child.textContent = data; parent.appendChild(child); + console.log(data); } function display_partial_corr_error(xhr, status, error) { diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index 08d6fc2a..215ba32e 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -222,6 +222,127 @@
    +

Loading...
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
-- cgit v1.2.3 From aea641e0a6fbd0ea68241efffb7c280222491238 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 20 Dec 2021 07:58:39 +0300 Subject: Display data for "Publish" datasets Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Display the correlations results for "Publish" datasets. --- .../static/new/css/partial_correlations.css | 10 ++++ .../static/new/javascript/partial_correlations.js | 65 ++++++++++++++++++++-- 2 files changed, 71 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css index ebc00302..41dd04df 100644 --- a/wqflask/wqflask/static/new/css/partial_correlations.css +++ b/wqflask/wqflask/static/new/css/partial_correlations.css @@ -7,6 +7,7 @@ #main-form { grid-column-start: 1; grid-column-end: 2; + text-align: left; } #form-display-area { @@ -28,3 +29,12 @@ width: 20%; grid-column-start: 2; } + +td, th { + border: 1px solid; + text-align: left; +} + +tr:nth-of-type(2n) { + background: #DDDDDD; +} diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index b1b12ee7..e61dc250 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -59,14 +59,71 @@ function partial_corr_request_data() { } } +function rho_or_r(method) { + if (method === "spearman") { + return "rho"; + } + return "r"; +} + +function display_publish_results(primary, controls, correlations, method) { + table = document.getElementById("part-corr-results-publish"); + table.setAttribute("style", "display: block;"); + table_body = document.querySelector("#part-corr-results-publish tbody"); + template_row = document.querySelector( + "#part-corr-results-publish tr.template-publish-results-row"); + correlations.forEach(function(item, index, arr) { + new_row = template_row.cloneNode(true); + new_row.setAttribute("class", "results-row"); + new_row.querySelector( + 'td[data-column-heading="Record"]').innerHTML = item["trait_name"]; + new_row.querySelector( + 'td[data-column-heading="Phenotype"]').innerHTML = ( + item["post_publication_description"]); + new_row.querySelector( + 'td[data-column-heading="Authors"]').innerHTML = item["authors"]; + new_row.querySelector( + 'td[data-column-heading="Year"]').innerHTML = item["year"]; + new_row.querySelector( + 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; + new_row.querySelector( + `td[data-column-heading="Partial ${rho_or_r(method)}"]` + ).innerHTML = item["partial_corr"]; + new_row.querySelector( + `td[data-column-heading="p(partial ${rho_or_r(method)})"]` + ).innerHTML = item["partial_corr_p_value"]; + new_row.querySelector( + `td[data-column-heading="${rho_or_r(method)}"]` + ).innerHTML = item["corr"]; + new_row.querySelector( + `td[data-column-heading="p(${rho_or_r(method)})"]` + ).innerHTML = item["corr_p_value"]; + new_row.querySelector( + `td[data-column-heading="delta ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["delta"]); + table_body.appendChild(new_row); + }); + table_body.removeChild(template_row); +} + +function display_geno_results(primary, controls, correlations) {} +function display_probeset_results(primary, controls, correlations) {} + function display_partial_corr_results(data, status, xhr) { progress_indicator = document.getElementById( "partial-correlations-progress-indicator").style.display = "none"; - parent = document.getElementById("part-corr-success"); - child = document.createElement("p"); - child.textContent = data; - parent.appendChild(child); console.log(data); + display_functions = { + "Publish": display_publish_results, + "Geno": display_geno_results, + "ProbeSet": display_probeset_results + } + + display_functions[data["results"]["dataset_type"]]( + data["results"]["primary_traits"], + data["results"]["control_traits"], + data["results"]["correlations"], + data["results"]["method"]); } function display_partial_corr_error(xhr, status, error) { -- cgit v1.2.3 From 53985f5ac92d31f38748d101c9bb1574b11e5f73 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 20 Dec 2021 11:49:22 +0300 Subject: Format numbers for display Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Format the numbers for display, to reduce the number of columns needed for display. --- .../static/new/javascript/partial_correlations.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index e61dc250..fdba247f 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -66,6 +66,16 @@ function rho_or_r(method) { return "r"; } +function format_number(num) { + if(num === null) { + return NaN; + } + if(Math.abs(num) <= 1.04e-4) { + return num.toExponential(2); + } + return num.toFixed(5); +} + function display_publish_results(primary, controls, correlations, method) { table = document.getElementById("part-corr-results-publish"); table.setAttribute("style", "display: block;"); @@ -88,16 +98,16 @@ function display_publish_results(primary, controls, correlations, method) { 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; new_row.querySelector( `td[data-column-heading="Partial ${rho_or_r(method)}"]` - ).innerHTML = item["partial_corr"]; + ).innerHTML = format_number(item["partial_corr"]); new_row.querySelector( `td[data-column-heading="p(partial ${rho_or_r(method)})"]` - ).innerHTML = item["partial_corr_p_value"]; + ).innerHTML = format_number(item["partial_corr_p_value"]); new_row.querySelector( `td[data-column-heading="${rho_or_r(method)}"]` - ).innerHTML = item["corr"]; + ).innerHTML = format_number(item["corr"]); new_row.querySelector( `td[data-column-heading="p(${rho_or_r(method)})"]` - ).innerHTML = item["corr_p_value"]; + ).innerHTML = format_number(item["corr_p_value"]); new_row.querySelector( `td[data-column-heading="delta ${rho_or_r(method)}"]` ).innerHTML = format_number(item["delta"]); -- cgit v1.2.3 From 81355831321642f09f2dc606f0670e6b7ded79c9 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 20 Dec 2021 11:50:36 +0300 Subject: Improve UI elements Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Force results area to use up the entire width of the page * Set background colour and line-height of header row * Add some padding to the cells * Reorganise layout * Set up appropriate classes for .with-trait items --- .../static/new/css/partial_correlations.css | 30 +++++++++++++++++----- .../wqflask/templates/partial_correlations.html | 12 +++++---- 2 files changed, 31 insertions(+), 11 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css index 41dd04df..a4f45ace 100644 --- a/wqflask/wqflask/static/new/css/partial_correlations.css +++ b/wqflask/wqflask/static/new/css/partial_correlations.css @@ -15,26 +15,44 @@ grid-column-end: 3; } +#part-corr-success { + grid-column-start: 1; + grid-column-end: 3; +} + .label-element-combo { width: 100%; display: grid; } -.with-trait > label { - width: 80%; - grid-column-start: 2; +.with-trait { + margin-left: 0.7em; + text-align: left; } -.with-trait > input { - width: 20%; +.with-trait .label-element { + width: 90%; grid-column-start: 2; + grid-column-end: 3 +} + +.with-trait .selector-element { + width: 1em; + grid-column-start: 1; + grid-column-end: 2 } td, th { border: 1px solid; text-align: left; + padding: 0.2em 0.5em 0.2em 0.7em; } tr:nth-of-type(2n) { - background: #DDDDDD; + background: #F9F9F9; +} + +thead tr { + background: #336699; + line-height: 1.5em; } diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index 215ba32e..800222ba 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -29,8 +29,9 @@ -
@@ -46,12 +47,13 @@

Select a maximum of three (3) control traits (Z)

{%for trait in traits_list:%} -
+
-
-- cgit v1.2.3 From ea728239a571e1991530d17d48d7b93b939b3b11 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Tue, 21 Dec 2021 13:28:02 +0300 Subject: Display results for Geno and ProbeSet tables Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Add function to display the results for the **Geno** and **ProbeSet** tables. --- .../static/new/javascript/partial_correlations.js | 92 +++++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index fdba247f..a6a07558 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -116,8 +116,96 @@ function display_publish_results(primary, controls, correlations, method) { table_body.removeChild(template_row); } -function display_geno_results(primary, controls, correlations) {} -function display_probeset_results(primary, controls, correlations) {} +function display_geno_results(primary, controls, correlations) { + table = document.getElementById("part-corr-results-geno"); + table.setAttribute("style", "display: block;"); + table_body = document.querySelector("#part-corr-results-geno tbody"); + template_row = document.querySelector( + "#part-corr-results-geno tr.template-geno-results-row"); + correlations.forEach(function(item, index, arr) { + new_row = template_row.cloneNode(true); + new_row.setAttribute("class", "results-row"); + new_row.querySelector( + 'td[data-column-heading="Chr"]').innerHTML = item["chr"]; + new_row.querySelector( + 'td[data-column-heading="Megabase"]').innerHTML = item["mb"]; + new_row.querySelector( + 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; + new_row.querySelector( + `td[data-column-heading="Partial ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["partial_corr"]); + new_row.querySelector( + `td[data-column-heading="p(partial ${rho_or_r(method)})"]` + ).innerHTML = format_number(item["partial_corr_p_value"]); + new_row.querySelector( + `td[data-column-heading="${rho_or_r(method)}"]` + ).innerHTML = format_number(item["corr"]); + new_row.querySelector( + `td[data-column-heading="p(${rho_or_r(method)})"]` + ).innerHTML = format_number(item["corr_p_value"]); + new_row.querySelector( + `td[data-column-heading="delta ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["delta"]); + table_body.appendChild(new_row); + }); + table_body.removeChild(template_row); +} + +function display_probeset_results(primary, controls, correlations) { + table = document.getElementById("part-corr-results-probeset"); + table.setAttribute("style", "display: block;"); + table_body = document.querySelector("#part-corr-results-probeset tbody"); + template_row = document.querySelector( + "#part-corr-results-probeset tr.template-probeset-results-row"); + correlations.forEach(function(item, index, arr) { + new_row = template_row.cloneNode(true); + new_row.setAttribute("class", "results-row"); + new_row.querySelector( + 'td[data-column-heading="Record"]').innerHTML = item["trait_name"]; + new_row.querySelector( + 'td[data-column-heading="Gene ID"]').innerHTML = item["geneid"]; + new_row.querySelector( + 'td[data-column-heading="Homologene ID"]').innerHTML = item["homologeneid"]; + new_row.querySelector( + 'td[data-column-heading="Symbol"]').innerHTML = item["symbol"]; + new_row.querySelector( + 'td[data-column-heading="Description"]').innerHTML = item["description"]; + new_row.querySelector( + 'td[data-column-heading="Chr"]').innerHTML = item["chr"]; + new_row.querySelector( + 'td[data-column-heading="Megabase"]').innerHTML = item["mb"]; + new_row.querySelector( + 'td[data-column-heading="Mean Expr"]').innerHTML = item["mean_expr"]; + new_row.querySelector( + 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; + new_row.querySelector( + `td[data-column-heading="Sample Partial ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["partial_corr"]); + new_row.querySelector( + `td[data-column-heading="Sample p(partial ${rho_or_r(method)})"]` + ).innerHTML = format_number(item["partial_corr_p_value"]); + new_row.querySelector( + `td[data-column-heading="Sample ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["corr"]); + new_row.querySelector( + `td[data-column-heading="Sample p(${rho_or_r(method)})"]` + ).innerHTML = format_number(item["corr_p_value"]); + new_row.querySelector( + `td[data-column-heading="delta ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["delta"]); + new_row.querySelector( + `td[data-column-heading="Lit Corr"]` + ).innerHTML = format_number(item["l_corr"]); + new_row.querySelector( + `td[data-column-heading="Tissue ${rho_or_r(method)}"]` + ).innerHTML = format_number(item["tissue_corr"]); + new_row.querySelector( + `td[data-column-heading="Tissue p(${rho_or_r(method)})"]` + ).innerHTML = format_number(item["tissue_p_value"]); + table_body.appendChild(new_row); + }); + table_body.removeChild(template_row); +} function display_partial_corr_results(data, status, xhr) { progress_indicator = document.getElementById( -- cgit v1.2.3 From ddd8e18cb236d6131e9f12a56d4f869649466a09 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 3 Jan 2022 11:03:21 +0300 Subject: Include template to get rid of repetitive template code Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * The generation of input elements that relate to the traits is very similar, therefore this commit pulls it out into a separate template that can be included where needed. --- .../wqflask/templates/partial_correlations.html | 30 ++++------------------ wqflask/wqflask/templates/with-trait-items.html | 18 +++++++++++++ 2 files changed, 23 insertions(+), 25 deletions(-) create mode 100644 wqflask/wqflask/templates/with-trait-items.html (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index 800222ba..c565d4bf 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -24,18 +24,8 @@ {%if step == "select-primary":%}

Please select the primary trait (X)

- {%for trait in traits_list:%} -
- - -
- {%endfor%} + {%include "with-trait-items.html" %} + @@ -46,20 +36,10 @@ {%if step == "select-controls":%}

Select a maximum of three (3) control traits (Z)

- {%for trait in traits_list:%} -
- - -
- {%endfor%} + {%include "with-trait-items.html" %} + {%endif%} diff --git a/wqflask/wqflask/templates/with-trait-items.html b/wqflask/wqflask/templates/with-trait-items.html new file mode 100644 index 00000000..66d6fd22 --- /dev/null +++ b/wqflask/wqflask/templates/with-trait-items.html @@ -0,0 +1,18 @@ +{%for trait in traits_list:%} +
+ + +
+{%endfor%} -- cgit v1.2.3 From cd2e1760751bbffb729c0d8df85e65f1641f93fa Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 3 Jan 2022 11:05:11 +0300 Subject: Update styling: Display traits in a tabular-like form Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Display the trait details in a tabular-life form without using a table. --- .../static/new/css/partial_correlations.css | 94 +++++++++++++++++----- 1 file changed, 75 insertions(+), 19 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css index a4f45ace..8ff5eae7 100644 --- a/wqflask/wqflask/static/new/css/partial_correlations.css +++ b/wqflask/wqflask/static/new/css/partial_correlations.css @@ -20,39 +20,95 @@ grid-column-end: 3; } -.label-element-combo { - width: 100%; - display: grid; +td, th { + border: 1px solid; + text-align: left; + padding: 0.2em 0.5em 0.2em 0.7em; +} + +tr:nth-of-type(2n) { + background: #F9F9F9; +} + +thead tr { + background: #336699; + line-height: 1.5em; } .with-trait { margin-left: 0.7em; + position: relative; + display: grid; + width: 100%; + grid-column-gap: 1em; + grid-template-columns: 1em 1fr; text-align: left; } -.with-trait .label-element { - width: 90%; - grid-column-start: 2; - grid-column-end: 3 +.with-trait:nth-of-type(2n) { + background: #E5E5FF; } .with-trait .selector-element { - width: 1em; - grid-column-start: 1; - grid-column-end: 2 + grid-column: 1 / 2; + grid-row: 1 / 1; } -td, th { - border: 1px solid; - text-align: left; - padding: 0.2em 0.5em 0.2em 0.7em; +.with-trait:first-of-type { + padding-top: 2.5em; } -tr:nth-of-type(2n) { - background: #F9F9F9; +.with-trait:first-of-type label *:before{ + position: absolute; + top: 0px; + + text-transform: capitalize; + font-weight: bolder; + content: attr(data-title); + background: #336699; /*#FFCCCC;*/ + color: #FFFFFF; + padding: 0.5em; } -thead tr { - background: #336699; - line-height: 1.5em; +.with-trait .label-element { + display: grid; + grid-column-gap: 0.5em; + grid-template-columns: 4fr 2fr 2fr 9fr 2fr 1fr 2fr; + grid-column: 2 / 2; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-dataset { + grid-column: 1; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-name { + grid-column: 2; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-symbol { + grid-column: 3; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-description { + grid-column: 4; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-location { + grid-column: 5; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-mean-expr { + grid-column: 6; + grid-row: 1 / 1; +} + +.with-trait .label-element .trait-max-lrs { + grid-column: 7; + grid-row: 1 / 1; } -- cgit v1.2.3 From bfc3d9483ec97c8ed70034f2de4a3ec414e402b4 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 3 Jan 2022 11:40:33 +0300 Subject: Add more items to trait string Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi --- wqflask/wqflask/partial_correlations_views.py | 4 +++- wqflask/wqflask/static/new/javascript/partial_correlations.js | 3 ++- wqflask/wqflask/templates/collections/view.html | 2 +- wqflask/wqflask/templates/partial_correlations.html | 8 ++++---- 4 files changed, 10 insertions(+), 7 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index 82287312..f3ec0f78 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -9,7 +9,9 @@ from wqflask import app from utility.tools import GN_SERVER_URL def parse_trait(trait_str: str) -> Union[dict, None]: - keys = ("name", "dataset", "symbol", "description", "data_hmac") + keys = ( + "name", "dataset", "symbol", "description", "location", "mean_expr", + "max_lrs", "data_hmac") parts = tuple(part.strip() for part in trait_str.split(":::")) if len(parts) == len(keys): return dict(zip(keys, parts)) diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index a6a07558..dea4c675 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -18,7 +18,8 @@ function key_value(keys, values) { function trait(trait_str) { return key_value( - ["name", "dataset", "symbol", "description", "data_hmac"], + ["name", "dataset", "symbol", "description", "location", "mean_expr", + "max_lrs", "data_hmac"], trait_str.split(":::")); } diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index f4270b67..432393a7 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -42,7 +42,7 @@ value="{{uc.uc_id}}" /> + value="{% for this_trait in trait_obs %}{{ this_trait.name }}:::{{ this_trait.dataset.name }}:::{{this_trait.symbol}}:::{{this_trait.description_display}}:::{{this_trait.location_repr}}:::{{this_trait.mean}}:::{{this_trait['LRS_location_repr']}}:::{{data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name))}}|||{% endfor %}" /> - +
+ +

Use tools like cytoscape to visualize the network

+ +Download Sif file + +Download Node file
@@ -78,7 +88,6 @@ @@ -121,8 +74,6 @@ - + @@ -86,4 +85,5 @@ $('#significance').DataTable({ }) }); -{% endblock %} \ No newline at end of file +{% endblock %} + -- cgit v1.2.3 From d082c40370e45187994e90b146ae6a5e15a2c9c9 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 31 Jan 2022 22:28:15 +0300 Subject: autopep8 formatting --- wqflask/wqflask/ctl/gn3_ctl_analysis.py | 22 ++-------------------- 1 file changed, 2 insertions(+), 20 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/ctl/gn3_ctl_analysis.py b/wqflask/wqflask/ctl/gn3_ctl_analysis.py index 4afae119..67f76a30 100644 --- a/wqflask/wqflask/ctl/gn3_ctl_analysis.py +++ b/wqflask/wqflask/ctl/gn3_ctl_analysis.py @@ -11,21 +11,13 @@ from base import data_set def process_significance_data(significant_data): - """function to process significance the data for - datatables""" - col_names = ["trait", "marker", "trait_2", "LOD", "dcor"] - dataset_rows = [[] for _ in range(len(significant_data["trait"]))] - for col in col_names: for (index, col_data) in enumerate(significant_data[col]): - if col in ["dcor", "LOD"]: - dataset_rows[index].append(round(float(col_data), 2)) else: - dataset_rows[index].append(col_data) return { @@ -35,16 +27,11 @@ def process_significance_data(significant_data): def parse_geno_data(dataset_group_name) -> dict: - """function to parse geno file data""" genofile_location = locate(dataset_group_name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofile_location) - parser.process_csv() - markers = [] - markernames = [] - for marker in parser.markers: markernames.append(marker["name"]) markers.append(marker["genotypes"]) @@ -60,8 +47,6 @@ def parse_geno_data(dataset_group_name) -> dict: def parse_phenotype_data(trait_db_list, dataset, individuals): - """function to parse and generate phenodata""" - traits = [] for trait in trait_db_list: if trait != "": @@ -90,27 +75,24 @@ def parse_form_data(form_data: dict): trait_db_list = [trait.strip() for trait in form_data['trait_list'].split(',')] - form_data["trait_db_list"] = [x for x in trait_db_list if x] + form_data["trait_db_list"] = [x for x in trait_db_list if x] form_data["nperm"] = int(form_data["nperm"]) form_data["significance"] = float(form_data["significance"]) form_data["strategy"] = form_data["strategy"].capitalize() + return form_data def run_ctl(requestform): """function to make an api call to gn3 and run ctl""" - ctl_api = f"{GN3_LOCAL_URL}/api/ctl/run_ctl" form_data = parse_form_data(requestform.to_dict()) - trait_db_list = form_data["trait_db_list"] dataset = data_set.create_dataset(trait_db_list[0].split(":")[1]) - geno_data = parse_geno_data(dataset.group.name) - pheno_data = parse_phenotype_data( trait_db_list, dataset, geno_data["individuals"]) -- cgit v1.2.3 From 6d3f48d3fa257e5fc22e874b3e0746b809a278a4 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 1 Feb 2022 00:42:41 +0300 Subject: handle exceptions --- wqflask/wqflask/ctl/gn3_ctl_analysis.py | 28 +++++++++++++++++--------- wqflask/wqflask/templates/gn3_ctl_results.html | 14 ++++++++++++- 2 files changed, 32 insertions(+), 10 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/ctl/gn3_ctl_analysis.py b/wqflask/wqflask/ctl/gn3_ctl_analysis.py index 67f76a30..7af0013f 100644 --- a/wqflask/wqflask/ctl/gn3_ctl_analysis.py +++ b/wqflask/wqflask/ctl/gn3_ctl_analysis.py @@ -96,18 +96,28 @@ def run_ctl(requestform): pheno_data = parse_phenotype_data( trait_db_list, dataset, geno_data["individuals"]) - response = requests.post(ctl_api, json={ + # todo refactor this chunk;;similar to wgcna check - "genoData": geno_data, - "phenoData": pheno_data, + try: - **form_data, + response = requests.post(ctl_api, json={ - }).json()["results"] + "genoData": geno_data, + "phenoData": pheno_data, + **form_data, - response["significance_data"] = process_significance_data( - response["significance_data"]) + }) + if response.status_code != 200: + return {"error": response.json()} + response = response.json()["results"] + response["significance_data"] = process_significance_data( + response["significance_data"]) + + return response + + except requests.exceptions.ConnectionError: + return { + "error": "A connection error to perform computation occurred" + } - # todo check for errors - return response diff --git a/wqflask/wqflask/templates/gn3_ctl_results.html b/wqflask/wqflask/templates/gn3_ctl_results.html index fedddc40..c42707f6 100644 --- a/wqflask/wqflask/templates/gn3_ctl_results.html +++ b/wqflask/wqflask/templates/gn3_ctl_results.html @@ -14,6 +14,14 @@ }
+ +{% if error %} +
+

{{error}}

+
+ +{% else %} +

CTL/QTL Plots for Individual Traits

@@ -68,7 +76,11 @@
- +{% endif %} +
+ + + -- cgit v1.2.3 From 78a544ea268c28b34c2761e6d65465882578d235 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 2 Feb 2022 09:03:20 +0300 Subject: variable naming and docstrings fix --- wqflask/wqflask/ctl/gn3_ctl_analysis.py | 39 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/ctl/gn3_ctl_analysis.py b/wqflask/wqflask/ctl/gn3_ctl_analysis.py index 7af0013f..8f790597 100644 --- a/wqflask/wqflask/ctl/gn3_ctl_analysis.py +++ b/wqflask/wqflask/ctl/gn3_ctl_analysis.py @@ -10,11 +10,11 @@ from base.trait import retrieve_sample_data from base import data_set -def process_significance_data(significant_data): +def process_significance_data(dataset): col_names = ["trait", "marker", "trait_2", "LOD", "dcor"] - dataset_rows = [[] for _ in range(len(significant_data["trait"]))] + dataset_rows = [[] for _ in range(len(dataset["trait"]))] for col in col_names: - for (index, col_data) in enumerate(significant_data[col]): + for (index, col_data) in enumerate(dataset[col]): if col in ["dcor", "LOD"]: dataset_rows[index].append(round(float(col_data), 2)) else: @@ -27,6 +27,12 @@ def process_significance_data(significant_data): def parse_geno_data(dataset_group_name) -> dict: + """ + Args: + dataset_group_name: string name + + @returns : dict with keys genotypes,markernames & individuals + """ genofile_location = locate(dataset_group_name + ".geno", "genotype") parser = genofile_parser.ConvertGenoFile(genofile_location) parser.process_csv() @@ -46,9 +52,21 @@ def parse_geno_data(dataset_group_name) -> dict: } -def parse_phenotype_data(trait_db_list, dataset, individuals): +def parse_phenotype_data(trait_list, dataset, individuals): + """ + Args: + trait_list:list contains the traits + dataset: object + individuals:a list contains the individual vals + Returns: + traits_db_List:parsed list of traits + traits: list contains trait names + individuals + + """ + traits = [] - for trait in trait_db_list: + for trait in trait_list: if trait != "": ts = trait.split(':') gt = create_trait(name=ts[0], dataset_name=ts[1]) @@ -60,18 +78,13 @@ def parse_phenotype_data(trait_db_list, dataset, individuals): traits.append("-999") return { - "trait_db_list": trait_db_list, + "trait_db_list": trait_list, "traits": traits, "individuals": individuals } def parse_form_data(form_data: dict): - """function to parse/validate form data - input: dict containing required data - output: dict with parsed data - - """ trait_db_list = [trait.strip() for trait in form_data['trait_list'].split(',')] @@ -96,8 +109,6 @@ def run_ctl(requestform): pheno_data = parse_phenotype_data( trait_db_list, dataset, geno_data["individuals"]) - # todo refactor this chunk;;similar to wgcna check - try: response = requests.post(ctl_api, json={ @@ -119,5 +130,3 @@ def run_ctl(requestform): return { "error": "A connection error to perform computation occurred" } - - -- cgit v1.2.3 From 91691799956450cac4e763f96e2152aa1cd44f3b Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 14 Feb 2022 21:31:07 +0000 Subject: Fix CisTrans search bug ', Geno' was being added to the FROM clause, which is wrong becausee Geno is included in the query via a JOIN (and caused an error making those searches not work) --- wqflask/wqflask/do_search.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index ee5a4d91..5c182260 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -609,9 +609,6 @@ class PhenotypeLrsSearch(LrsSearch, PhenotypeSearch): class CisTransLrsSearch(DoSearch): - def get_from_clause(self): - return ", Geno" - def get_where_clause(self, cis_trans): self.mb_buffer = 5 # default chromosome = None -- cgit v1.2.3 From c147a6e75a9496263fd76f2eb9711ddf77a643ff Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 14 Feb 2022 21:44:40 +0000 Subject: Add search text for cis/transLRS/LOD searches There wasn't any search text for cis/trans searches including both chromosome and exclusion zone, so I added somee --- wqflask/wqflask/templates/search_result_page.html | 2 ++ 1 file changed, 2 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index 26cdc437..70c10946 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -40,6 +40,8 @@ with {{ word.key|upper }} between {{ word.search_term[0] }} and {{ word.search_term[1] }}{% if loop.last %}.{% else %} and {% endif %} {% elif word.search_term|length == 3 %} with {{ word.key|upper }} between {{ word.search_term[0] }} and {{ word.search_term[1] }} on chromosome {{ word.search_term[2] }}{% if loop.last %}.{% else %} and {% endif %} + {% elif word.search_term|length == 4 %} + with {{ word.key|upper }} between {{ word.search_term[0] }} and {{ word.search_term[1] }} on chromosome {{ word.search_term[3] }} with an exclusion zone of {{ word.search_term[2] }} Mb {% elif word.search_term|length == 5 %} with {{ word.key|upper }} between {{ word.search_term[0] }} and {{ word.search_term[1] }} on chromosome {{ word.search_term[2] }} between {{ word.search_term[3] }} and {{ word.search_term[4] }} Mb{% if loop.last %}.{% else %} and {% endif %} {% endif %} -- cgit v1.2.3 From 17652b17455bd58bf82d130b60b3e80c57b7f80c Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Fri, 18 Feb 2022 16:01:01 +0530 Subject: wqflask: Remove python pre-2.2 checks. These checks are only relevant for python pre-2.2. * wqflask/utility/svg.py: Remove python pre-2.2 checks. --- wqflask/utility/svg.py | 6 ------ 1 file changed, 6 deletions(-) (limited to 'wqflask') diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index eddb97da..912cd04c 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -108,12 +108,6 @@ if use_dom_implementation != 0: # The implementation is used for the creating the XML document. # The prettyprint module is used for converting the xml document object to a xml file -assert sys.version_info[0] >= 2 -if sys.version_info[1] < 2: - True = 1 - False = 0 - file = open - sys.setrecursionlimit = 50 # The recursion limit is set conservative so mistakes like s=svg() s.addElement(s) # won't eat up too much processor time. -- cgit v1.2.3 From 7c9e73f196575cd6d1de7df4430bc2b4ecb28466 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 21 Feb 2022 21:18:46 +0000 Subject: Fix incorrect dataset trait data caching Trait data caching wasn't working correctly because it didn't account for the samplelist, causing caching to work incorrect in any situation where the target dataset's samplelist wasn't the same as that of the trait being correlated against. Trait data is stored as a dictionary where the keys are trait IDs and values are *lists* of sample values. This means that the caching needs to account for the exact same set of samples; otherwise you'll end up with samples being mismatched (since "the third sample with a value" for one dataset's trait might not be the same as "the third sample with a value" for another dataset's trait). To fix this, I added the samplelist to the functions that generate and fetch the hash file. This will require more cache files, though, so this should probably be reexamined later to make the code work with only a single cache file for each dataset. --- wqflask/base/data_set.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'wqflask') diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index af248659..d7e4e62f 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -756,7 +756,7 @@ class DataSet: chunk_size = 50 number_chunks = int(math.ceil(len(sample_ids) / chunk_size)) - cached_results = fetch_cached_results(self.name, self.type) + cached_results = fetch_cached_results(self.name, self.type, self.samplelist) if cached_results is None: trait_sample_data = [] for sample_ids_step in chunks.divide_into_chunks(sample_ids, number_chunks): @@ -812,9 +812,8 @@ class DataSet: trait_sample_data[chunk_counter][trait_counter][data_start_pos:]) cache_dataset_results( - self.name, self.type, self.trait_data) + self.name, self.type, self.samplelist, self.trait_data) else: - self.trait_data = cached_results @@ -1278,14 +1277,14 @@ def query_table_timestamp(dataset_type: str): return date_time_obj.strftime("%Y-%m-%d %H:%M:%S") -def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str): +def generate_hash_file(dataset_name: str, dataset_type: str, dataset_timestamp: str, samplelist: str): """given the trait_name generate a unique name for this""" - string_unicode = f"{dataset_name}{dataset_timestamp}".encode() + string_unicode = f"{dataset_name}{dataset_timestamp}{samplelist}".encode() md5hash = hashlib.md5(string_unicode) return md5hash.hexdigest() -def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: List): +def cache_dataset_results(dataset_name: str, dataset_type: str, samplelist: List, query_results: List): """function to cache dataset query results to file input dataset_name and type query_results(already processed in default dict format) """ @@ -1293,21 +1292,22 @@ def cache_dataset_results(dataset_name: str, dataset_type: str, query_results: L # store the file path on redis table_timestamp = query_table_timestamp(dataset_type) + samplelist_as_str = ",".join(samplelist) - - file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp, samplelist_as_str) file_path = os.path.join(TMPDIR, f"{file_name}.json") with open(file_path, "w") as file_handler: json.dump(query_results, file_handler) -def fetch_cached_results(dataset_name: str, dataset_type: str): +def fetch_cached_results(dataset_name: str, dataset_type: str, samplelist: List): """function to fetch the cached results""" table_timestamp = query_table_timestamp(dataset_type) + samplelist_as_str = ",".join(samplelist) - file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp) + file_name = generate_hash_file(dataset_name, dataset_type, table_timestamp, samplelist_as_str) file_path = os.path.join(TMPDIR, f"{file_name}.json") try: with open(file_path, "r") as file_handler: -- cgit v1.2.3 From abdc642cea9ab40f5e71d65abaab0362bc5df74b Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 23 Feb 2022 21:27:11 +0000 Subject: Add top margin to Partial Correlations button Later this button should just be included with the other function buttons. There's no need for it to be in a separate form, since it should be able to take the same trait list input as all the other collection page functions (with just the action url for the form changing) --- wqflask/wqflask/templates/collections/view.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 432393a7..3e3d3972 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -46,7 +46,7 @@ -- cgit v1.2.3 From 348e506ce811f4f314b7c9e52073bf03aeb70d50 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 23 Feb 2022 21:43:57 +0000 Subject: Reposition Partial Correlations and Heatmap buttons Moved Partial Correlations and Generate Heatmap buttons to be lined up below the other buttons and hide the Heatmap orientation selection (JS will be added to show it after Heatmap is run) --- wqflask/wqflask/templates/collections/view.html | 81 +++++++++++++------------ 1 file changed, 42 insertions(+), 39 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 3e3d3972..447a821a 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -33,48 +33,51 @@
- -
- - - -
+
+
+ + + +
+   +
+
+
+ Heatmap Orientation + + + + +
+
+ +
+

-
-
- Heatmap Orientation - - - - -
- -
-
-- cgit v1.2.3 From 42cb64c8d89766cd28276c03b37674a227d39aad Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 23 Feb 2022 22:04:34 +0000 Subject: Show heatmap options when heatmap is run Heatmap options are now shown after the user clicks the button I also reordered the Heatmap and Partial Correlation buttons, since it's awkward to get the Heatmap options to display well if its button is second. We'll probably completely rework this later, but for now this looks a little less ugly. --- wqflask/wqflask/templates/collections/view.html | 49 +++++++++++++------------ 1 file changed, 25 insertions(+), 24 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 447a821a..804c52e9 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -34,6 +34,30 @@
+
+ +
+ +
+  
@@ -50,30 +74,6 @@ Partial Correlations
-   -
-
-
- Heatmap Orientation - - - - -
-
- -
@@ -342,6 +342,7 @@ $("#clustered-heatmap").on("click", function() { clear_heatmap_area(); + $("#heatmap-options").show(); intv = window.setInterval(generate_progress_indicator(), 300); vert_element = document.getElementById("heatmap-orient-vertical"); vert_true = vert_element == null ? false : vert_element.checked; -- cgit v1.2.3 From 82018ffcb426812c3d986e262e0a1523fbe973aa Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 23 Feb 2022 22:15:16 +0000 Subject: Improve Heatmap options' appearance Moved the Heatmap display area above the table selection options (since it didn't make sense to have it below those) Added a button for clearing the Heatmap area and hiding the Heatmap options Added some margins to the Heatmap options --- wqflask/wqflask/templates/collections/view.html | 42 ++++++++++++++++--------- 1 file changed, 28 insertions(+), 14 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 804c52e9..f896471a 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -43,18 +43,27 @@
  @@ -77,6 +86,7 @@
+

@@ -94,8 +104,6 @@
-
-
Show/Hide Columns:
@@ -369,6 +377,12 @@ } }); }); + + $("#clear-heatmap").on("click", function() { + clear_heatmap_area(); + $("#heatmap-options").hide(); + }); + }); -- cgit v1.2.3 From 510829d9ed7ff47d52989d79e235d86205bc5e2e Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 23 Feb 2022 22:19:02 +0000 Subject: Fix Heatmap GN3 URL --- wqflask/wqflask/collect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 76ef5ca4..3475ae5d 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -222,7 +222,7 @@ def view_collection(): collection_info = dict( trait_obs=trait_obs, uc=uc, - heatmap_data_url=f"{GN_SERVER_URL}heatmaps/clustered") + heatmap_data_url=f"{GN_SERVER_URL}api/heatmaps/clustered") if "json" in params: return json.dumps(json_version) -- cgit v1.2.3 From 86ead9b3d823e46350b5566b197463b5fdc46102 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 24 Feb 2022 19:27:28 +0300 Subject: init replace rpy2 for pca --- .../wqflask/correlation_matrix/show_corr_matrix.py | 28 ++++++++++------------ 1 file changed, 13 insertions(+), 15 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index e7b16e77..9462f973 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -23,8 +23,6 @@ import math import random import string -import rpy2.robjects as ro -from rpy2.robjects.packages import importr import numpy as np import scipy @@ -38,6 +36,12 @@ from utility import helper_functions from utility import corr_result_helpers from utility.redis_tools import get_redis_conn + + +from gn3.computations.principal_component_analysis import compute_pca + +from gn3.computations.principal_component_analysis import process_factor_loadings_tdata + Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -174,7 +178,7 @@ class CorrelationMatrix: self.pca_trait_ids = [] pca = self.calculate_pca( list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors) - self.loadings_array = self.process_loadings() + self.loadings_array = process_factor_loadings_tdata(self.loadings,len(self.trait_list)) else: self.pca_works = "False" except: @@ -188,18 +192,12 @@ class CorrelationMatrix: sample_data=self.sample_data,) def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors): - base = importr('base') - stats = importr('stats') - - corr_results_to_list = ro.FloatVector( - [item for sublist in self.pca_corr_results for item in sublist]) - - m = ro.r.matrix(corr_results_to_list, nrow=len(cols)) - eigen = base.eigen(m) - pca = stats.princomp(m, cor="TRUE") - self.loadings = pca.rx('loadings') - self.scores = pca.rx('scores') - self.scale = pca.rx('scale') + + + pca = compute_pca(self.pca_corr_results) + + self.loadings = pca["components"] + self.scores = pca["scores"] trait_array = zScore(self.trait_data_array) trait_array_vectors = np.dot(corr_eigen_vectors, trait_array) -- cgit v1.2.3 From 0ee723d14957c01162a67f4f6b99a25d43908b5b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 24 Feb 2022 22:19:00 +0300 Subject: remove redundant functions and code --- .../wqflask/correlation_matrix/show_corr_matrix.py | 70 ++-------------------- 1 file changed, 6 insertions(+), 64 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 9462f973..bcd73436 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -41,6 +41,7 @@ from utility.redis_tools import get_redis_conn from gn3.computations.principal_component_analysis import compute_pca from gn3.computations.principal_component_analysis import process_factor_loadings_tdata +from gn3.computations.principal_component_analysis import generate_pca_traits_vals Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -169,15 +170,12 @@ class CorrelationMatrix: self.pca_works = "False" try: - corr_result_eigen = np.linalg.eig(np.array(self.pca_corr_results)) - corr_eigen_value, corr_eigen_vectors = sortEigenVectors( - corr_result_eigen) if self.do_PCA == True: self.pca_works = "True" self.pca_trait_ids = [] pca = self.calculate_pca( - list(range(len(self.traits))), corr_eigen_value, corr_eigen_vectors) + list(range(len(self.traits)))) self.loadings_array = process_factor_loadings_tdata(self.loadings,len(self.trait_list)) else: self.pca_works = "False" @@ -191,7 +189,7 @@ class CorrelationMatrix: samples=self.all_sample_list, sample_data=self.sample_data,) - def calculate_pca(self, cols, corr_eigen_value, corr_eigen_vectors): + def calculate_pca(self, cols): pca = compute_pca(self.pca_corr_results) @@ -199,8 +197,9 @@ class CorrelationMatrix: self.loadings = pca["components"] self.scores = pca["scores"] - trait_array = zScore(self.trait_data_array) - trait_array_vectors = np.dot(corr_eigen_vectors, trait_array) + trait_array_vectors = generate_pca_traits_vals(self.trait_data_array,self.pca_corr_results) + + pca_traits = [] for i, vector in enumerate(trait_array_vectors): @@ -231,21 +230,6 @@ class CorrelationMatrix: return pca - def process_loadings(self): - loadings_array = [] - loadings_row = [] - for i in range(len(self.trait_list)): - loadings_row = [] - if len(self.trait_list) > 2: - the_range = 3 - else: - the_range = 2 - for j in range(the_range): - position = i + len(self.trait_list) * j - loadings_row.append(self.loadings[0][position]) - loadings_array.append(loadings_row) - return loadings_array - def export_corr_matrix(corr_results): corr_matrix_filename = "corr_matrix_" + \ @@ -285,45 +269,3 @@ def export_corr_matrix(corr_results): return corr_matrix_filename, matrix_export_path - -def zScore(trait_data_array): - NN = len(trait_data_array[0]) - if NN < 10: - return trait_data_array - else: - i = 0 - for data in trait_data_array: - N = len(data) - S = reduce(lambda x, y: x + y, data, 0.) - SS = reduce(lambda x, y: x + y * y, data, 0.) - mean = S / N - var = SS - S * S / N - stdev = math.sqrt(var / (N - 1)) - if stdev == 0: - stdev = 1e-100 - data2 = [(x - mean) / stdev for x in data] - trait_data_array[i] = data2 - i += 1 - return trait_data_array - - -def sortEigenVectors(vector): - try: - eigenValues = vector[0].tolist() - eigenVectors = vector[1].T.tolist() - combines = [] - i = 0 - for item in eigenValues: - combines.append([eigenValues[i], eigenVectors[i]]) - i += 1 - sorted(combines, key=cmp_to_key(webqtlUtil.cmpEigenValue)) - A = [] - B = [] - for item in combines: - A.append(item[0]) - B.append(item[1]) - sum = reduce(lambda x, y: x + y, A, 0.0) - A = [x * 100.0 / sum for x in A] - return [A, B] - except: - return [] -- cgit v1.2.3 From 494e91a2dd04cdb6b8cd20fc1709d847ac2c012a Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 24 Feb 2022 21:32:42 +0000 Subject: Change parser.py to also detect hyphen in search terms --- wqflask/wqflask/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/parser.py b/wqflask/wqflask/parser.py index bd1c4407..7a808ac9 100644 --- a/wqflask/wqflask/parser.py +++ b/wqflask/wqflask/parser.py @@ -33,7 +33,7 @@ def parse(pstring): pstring = re.split(r"""(?:(\w+\s*=\s*[\('"\[][^)'"]*[\)\]'"]) | # LRS=(1 2 3), cisLRS=[4 5 6], etc (\w+\s*[=:\>\<][\w\*]+) | # wiki=bar, GO:foobar, etc (".*?") | ('.*?') | # terms in quotes, i.e. "brain weight" - ([\w\*\?]+)) # shh, brain, etc """, pstring, + ([\w\*\?\-]+)) # shh, brain, etc """, pstring, flags=re.VERBOSE) pstring = [item.strip() for item in pstring if item and item.strip()] -- cgit v1.2.3 From bbeba72a82e0a5698b4fb413430203e8aaee1830 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 24 Feb 2022 21:33:38 +0000 Subject: Fix alias query --- wqflask/wqflask/search_results.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index cf2905c9..6062f354 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -360,7 +360,8 @@ def get_aliases(symbol_list, species): filtered_aliases = [] response = requests.get( - GN2_BASE_URL + "/gn3/gene/aliases2/" + symbols_string) + GN2_BASE_URL + "gn3/gene/aliases/" + symbols_string) + if response: alias_lists = json.loads(response.content) seen = set() -- cgit v1.2.3 From 6496f75af521d6801c25de682d2852e14d597fed Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 24 Feb 2022 21:53:56 +0000 Subject: Fix ProbeSet query for aliases ProbeSet queries previously weren't dealing with aliases correctly, because it was doing a MATCH/AGAINST against the ProbeSet.alias field, but that field usually contains a list of gene symbols separated by semi-colons (so it wouldn't detect the alias unless there was only a single alias. To fix this, I added some LIKE conditions, searching for the possible variations. This is a little awkward, because I needed to make sure to avoid a situation where, for example, an alias like 'LPD-1' matches a search for 'PD-1'. I don't think the way it currently works is efficient, but I don't know of any good alternative without changing the way we store aliases in the database. --- wqflask/wqflask/do_search.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 5c182260..f8cc482f 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -137,15 +137,18 @@ class MrnaAssaySearch(DoSearch): search_string = escape(self.search_term[0]) if self.search_term[0] != "*": - match_clause = """((MATCH (ProbeSet.Name, + match_clause = f"""((MATCH (ProbeSet.Name, ProbeSet.description, ProbeSet.symbol, - alias, GenbankId, UniGeneId, Probe_Target_Description) - AGAINST ('%s' IN BOOLEAN MODE))) AND - """ % (search_string) + AGAINST ('{search_string}' IN BOOLEAN MODE)) OR ( + alias LIKE '%%; {search_string};%%' OR + alias LIKE '{search_string};%%' OR + alias LIKE '%%; {search_string}' OR + alias LIKE '{search_string}' + )) AND """ else: match_clause = "" -- cgit v1.2.3 From d5d0e0f3c271e056057c0311083ab3684ccc4386 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 25 Feb 2022 17:33:19 +0300 Subject: integrating generating temp dataset for pca --- .../wqflask/correlation_matrix/show_corr_matrix.py | 63 +++++++++++----------- 1 file changed, 32 insertions(+), 31 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index bcd73436..d5ec738b 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -19,7 +19,6 @@ # This module is used by GeneNetwork project (www.genenetwork.org) import datetime -import math import random import string @@ -29,9 +28,6 @@ import scipy from base import data_set from base.webqtlConfig import GENERATED_TEXT_DIR -from functools import reduce -from functools import cmp_to_key -from utility import webqtlUtil from utility import helper_functions from utility import corr_result_helpers from utility.redis_tools import get_redis_conn @@ -42,6 +38,8 @@ from gn3.computations.principal_component_analysis import compute_pca from gn3.computations.principal_component_analysis import process_factor_loadings_tdata from gn3.computations.principal_component_analysis import generate_pca_traits_vals +from gn3.computations.principal_component_analysis import generate_pca_temp_dataset +from gn3.computations.principal_component_analysis import cache_pca_dataset Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -171,11 +169,11 @@ class CorrelationMatrix: self.pca_works = "False" try: + if self.do_PCA == True: self.pca_works = "True" self.pca_trait_ids = [] - pca = self.calculate_pca( - list(range(len(self.traits)))) + pca = self.calculate_pca() self.loadings_array = process_factor_loadings_tdata(self.loadings,len(self.trait_list)) else: self.pca_works = "False" @@ -189,7 +187,7 @@ class CorrelationMatrix: samples=self.all_sample_list, sample_data=self.sample_data,) - def calculate_pca(self, cols): + def calculate_pca(self): pca = compute_pca(self.pca_corr_results) @@ -197,36 +195,37 @@ class CorrelationMatrix: self.loadings = pca["components"] self.scores = pca["scores"] - trait_array_vectors = generate_pca_traits_vals(self.trait_data_array,self.pca_corr_results) - - - - pca_traits = [] - for i, vector in enumerate(trait_array_vectors): - # ZS: Check if below check is necessary - # if corr_eigen_value[i-1] > 100.0/len(self.trait_list): - pca_traits.append((vector * -1.0).tolist()) this_group_name = self.trait_list[0][1].group.name temp_dataset = data_set.create_dataset( dataset_name="Temp", dataset_type="Temp", group_name=this_group_name) temp_dataset.group.get_samplelist() - for i, pca_trait in enumerate(pca_traits): - trait_id = "PCA" + str(i + 1) + "_" + temp_dataset.group.species + "_" + \ - this_group_name + "_" + datetime.datetime.now().strftime("%m%d%H%M%S") - this_vals_string = "" - position = 0 - for sample in temp_dataset.group.all_samples_ordered(): - if sample in self.shared_samples_list: - this_vals_string += str(pca_trait[position]) - this_vals_string += " " - position += 1 - else: - this_vals_string += "x " - this_vals_string = this_vals_string[:-1] - Redis.set(trait_id, this_vals_string, ex=THIRTY_DAYS) - self.pca_trait_ids.append(trait_id) + + species = temp_dataset.group.species + + group =this_group_name + + trait_data_array = self.trait_data_array + + pca_corr = self.pca_corr_results + + sample_list = temp_dataset.group.all_samples_ordered() + + + shared = self.shared_samples_list + + dt_time = datetime.datetime.now().strftime("%m%d%H%M%S") + + + + results = generate_pca_temp_dataset(species = species, group= group,traits_data = self.trait_data_array,corr_array = self.pca_corr_results,dataset_samples = sample_list, shared_samples=shared,create_time=dt_time) + + + + cache_pca_dataset(Redis,THIRTY_DAYS,results) + + self.pca_trait_ids = list(results.keys()) return pca @@ -269,3 +268,5 @@ def export_corr_matrix(corr_results): return corr_matrix_filename, matrix_export_path + + -- cgit v1.2.3 From b1629deb7822b165dbb42de4662e9ab968170e95 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 22 Feb 2022 22:02:09 +0000 Subject: Add JS initializing DataTables --- .../static/new/javascript/partial_correlations.js | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index b3a89c5e..030b63f2 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -249,6 +249,8 @@ function display_partial_corr_results(data, status, xhr) { data["results"]["control_traits"], data["results"]["correlations"], data["results"]["method"]); + + initializePcorrTable(data["results"]["dataset_type"]); } function display_partial_corr_error(xhr, status, error) { @@ -285,6 +287,40 @@ function send_data_and_process_results( }); } +function initializePcorrTable(dataType){ + tableId = "part-corr-results-" + dataType.toLowerCase(); + if (dataType == "Publish") { + orderCol = 7; + } else if (dataType == "ProbeSet") { + orderCol = 11; + } else { + orderCol = 6; + } + + $('#' + tableId).dataTable( { + 'drawCallback': function( settings ) { + $('#' + tableId + ' tr').off().on("click", function(event) { + if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { + var obj =$(this).find('input'); + obj.prop('checked', !obj.is(':checked')); + } + if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ + $(this).removeClass("selected") + } else if (event.target.tagName.toLowerCase() !== 'a') { + $(this).addClass("selected") + } + }); + }, + "order": [[orderCol, "asc" ]], + "sDom": "itir", + "iDisplayLength": -1, + "autoWidth": false, + "bDeferRender": true, + "bSortClasses": false, + "paging": false + } ); +} + $("#partial-correlations-form").submit(function(e) { e.preventDefault(); }); -- cgit v1.2.3 From 140707eab56f8878d0b149e6fe92819fe5054d43 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 22 Feb 2022 22:02:52 +0000 Subject: Include DataTables imports and necessary CSS --- wqflask/wqflask/templates/partial_correlations.html | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index 4be4890e..ad1c674a 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -4,6 +4,9 @@ {%block css%} + + + {%endblock%} {%block content%} @@ -204,7 +207,7 @@
- + @@ -337,5 +340,7 @@ {%if step == "select-corr-method":%} + {%endif%} {%endblock%} -- cgit v1.2.3 From d7d88b138fa4b418ba8590410999fdff8c4eed2b Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 22 Feb 2022 22:09:41 +0000 Subject: Add checkbox and index columns --- wqflask/wqflask/static/new/javascript/partial_correlations.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index 030b63f2..3bf2acc4 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -290,11 +290,11 @@ function send_data_and_process_results( function initializePcorrTable(dataType){ tableId = "part-corr-results-" + dataType.toLowerCase(); if (dataType == "Publish") { - orderCol = 7; + orderCol = 8; } else if (dataType == "ProbeSet") { - orderCol = 11; + orderCol = 12; } else { - orderCol = 6; + orderCol = 7; } $('#' + tableId).dataTable( { -- cgit v1.2.3 From 73a0ad695f4c7ad9d4a313b9bb9231dcaab5fcb7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 22 Feb 2022 22:11:10 +0000 Subject: Add container div --- wqflask/wqflask/templates/partial_correlations.html | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html index ad1c674a..0cf09afc 100644 --- a/wqflask/wqflask/templates/partial_correlations.html +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -10,6 +10,7 @@ {%endblock%} {%block content%} +
@@ -210,7 +211,8 @@
- + + @@ -226,7 +228,8 @@ - + + @@ -249,7 +252,8 @@ - + + @@ -264,7 +268,8 @@ - + + @@ -286,7 +291,8 @@ - + + @@ -309,6 +315,8 @@ + + @@ -334,6 +342,7 @@
+ {%endblock%} {%block js%} -- cgit v1.2.3 From f1e0a91802e7236412e7de211ec249cd8725c64a Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 22 Feb 2022 22:11:59 +0000 Subject: Remove CSS from partial_correlations.css This CSS is overwritten by CSS from trait_list.css and show_trait.css --- wqflask/wqflask/static/new/css/partial_correlations.css | 5 ----- 1 file changed, 5 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/static/new/css/partial_correlations.css b/wqflask/wqflask/static/new/css/partial_correlations.css index 8ff5eae7..84a0877f 100644 --- a/wqflask/wqflask/static/new/css/partial_correlations.css +++ b/wqflask/wqflask/static/new/css/partial_correlations.css @@ -30,11 +30,6 @@ tr:nth-of-type(2n) { background: #F9F9F9; } -thead tr { - background: #336699; - line-height: 1.5em; -} - .with-trait { margin-left: 0.7em; position: relative; -- cgit v1.2.3 From 78d8ed64a072851ddc58281553dfc9806c25b332 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 27 Feb 2022 10:21:46 +0300 Subject: code refactoring --- .../wqflask/correlation_matrix/show_corr_matrix.py | 50 ++++++---------------- 1 file changed, 14 insertions(+), 36 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index d5ec738b..499a4e13 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -33,7 +33,6 @@ from utility import corr_result_helpers from utility.redis_tools import get_redis_conn - from gn3.computations.principal_component_analysis import compute_pca from gn3.computations.principal_component_analysis import process_factor_loadings_tdata @@ -44,6 +43,7 @@ from gn3.computations.principal_component_analysis import cache_pca_dataset Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 + class CorrelationMatrix: def __init__(self, start_vars): @@ -54,7 +54,6 @@ class CorrelationMatrix: self.all_sample_list = [] self.traits = [] - self.insufficient_shared_samples = False self.do_PCA = True # ZS: Getting initial group name before verifying all traits are in the same group in the following loop this_group = self.trait_list[0][1].group.name @@ -169,12 +168,12 @@ class CorrelationMatrix: self.pca_works = "False" try: - if self.do_PCA == True: self.pca_works = "True" self.pca_trait_ids = [] pca = self.calculate_pca() - self.loadings_array = process_factor_loadings_tdata(self.loadings,len(self.trait_list)) + self.loadings_array = process_factor_loadings_tdata( + self.loadings, len(self.trait_list)) else: self.pca_works = "False" except: @@ -189,43 +188,25 @@ class CorrelationMatrix: def calculate_pca(self): - pca = compute_pca(self.pca_corr_results) self.loadings = pca["components"] self.scores = pca["scores"] - this_group_name = self.trait_list[0][1].group.name temp_dataset = data_set.create_dataset( dataset_name="Temp", dataset_type="Temp", group_name=this_group_name) temp_dataset.group.get_samplelist() + pca_dataset = generate_pca_temp_dataset(species=temp_dataset.group.species, group=this_group_name, + traits_data=self.trait_data_array, corr_array=self.pca_corr_results, + dataset_samples=temp_dataset.group.all_samples_ordered(), + shared_samples=self.shared_samples_list, + create_time=datetime.datetime.now().strftime("%m%d%H%M%S")) - species = temp_dataset.group.species - - group =this_group_name - - trait_data_array = self.trait_data_array - - pca_corr = self.pca_corr_results - - sample_list = temp_dataset.group.all_samples_ordered() - - - shared = self.shared_samples_list + cache_pca_dataset(Redis, THIRTY_DAYS, pca_dataset) - dt_time = datetime.datetime.now().strftime("%m%d%H%M%S") - - - - results = generate_pca_temp_dataset(species = species, group= group,traits_data = self.trait_data_array,corr_array = self.pca_corr_results,dataset_samples = sample_list, shared_samples=shared,create_time=dt_time) - - - - cache_pca_dataset(Redis,THIRTY_DAYS,results) - - self.pca_trait_ids = list(results.keys()) + self.pca_trait_ids = list(pca_dataset.keys()) return pca @@ -242,11 +223,11 @@ def export_corr_matrix(corr_results): output_file.write("\n") output_file.write("Correlation ") for i, item in enumerate(corr_results[0]): - output_file.write("Trait" + str(i + 1) + ": " + \ + output_file.write("Trait" + str(i + 1) + ": " + str(item[0].dataset.name) + "::" + str(item[0].name) + "\t") output_file.write("\n") for i, row in enumerate(corr_results): - output_file.write("Trait" + str(i + 1) + ": " + \ + output_file.write("Trait" + str(i + 1) + ": " + str(row[0][0].dataset.name) + "::" + str(row[0][0].name) + "\t") for item in row: output_file.write(str(item[1]) + "\t") @@ -256,17 +237,14 @@ def export_corr_matrix(corr_results): output_file.write("\n") output_file.write("N ") for i, item in enumerate(corr_results[0]): - output_file.write("Trait" + str(i) + ": " + \ + output_file.write("Trait" + str(i) + ": " + str(item[0].dataset.name) + "::" + str(item[0].name) + "\t") output_file.write("\n") for i, row in enumerate(corr_results): - output_file.write("Trait" + str(i) + ": " + \ + output_file.write("Trait" + str(i) + ": " + str(row[0][0].dataset.name) + "::" + str(row[0][0].name) + "\t") for item in row: output_file.write(str(item[2]) + "\t") output_file.write("\n") return corr_matrix_filename, matrix_export_path - - - -- cgit v1.2.3 From f61bf4beb6622c1553ea441cd63192d4ec778670 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 28 Feb 2022 20:22:49 +0000 Subject: Add distinct to fix issue where GeneRIF search returns duplicates --- wqflask/wqflask/do_search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index f8cc482f..9a06ea80 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -82,7 +82,7 @@ class MrnaAssaySearch(DoSearch): DoSearch.search_types['ProbeSet'] = "MrnaAssaySearch" base_query = """ - SELECT + SELECT DISTINCT ProbeSetFreeze.`Name`, ProbeSetFreeze.`FullName`, ProbeSet.`Name`, -- cgit v1.2.3 From c71218d9221789f850f5d93329f2d466527d3c52 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 28 Feb 2022 21:16:20 +0000 Subject: Fix fulltext search for certain searches Hyphens in fulltext searches were causing problems, but a recent commit I made to fix the issue apparently had some side effects for other types of searches, in addition to make such searches someewhat slower Apparently the issue wasn't just the hyphens, but also the text to either side of the hyphen being lower than the minimum word length (which is either 2 or 3 for us, can't remember). To try and address this, I did a regular expression check for a pattern with text of <3 legnth to either side of a hyphen, and when that's the case I add quotes from the search term plus an asterisk, which seeems to be necessary to get it to not treat the hyphen as a delimiter and to correctly detect the search term --- wqflask/wqflask/do_search.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 9a06ea80..2cc6aa61 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -1,6 +1,7 @@ -import string -import requests import json +import re +import requests +import string from flask import Flask, g @@ -137,18 +138,17 @@ class MrnaAssaySearch(DoSearch): search_string = escape(self.search_term[0]) if self.search_term[0] != "*": + if re.search("\w{1,2}\-\w+|\w+\-\w{1,2}", self.search_term[0]): + search_string = f'"{search_string}*"' + match_clause = f"""((MATCH (ProbeSet.Name, ProbeSet.description, ProbeSet.symbol, + alias, GenbankId, UniGeneId, Probe_Target_Description) - AGAINST ('{search_string}' IN BOOLEAN MODE)) OR ( - alias LIKE '%%; {search_string};%%' OR - alias LIKE '{search_string};%%' OR - alias LIKE '%%; {search_string}' OR - alias LIKE '{search_string}' - )) AND """ + AGAINST ('{search_string}' IN BOOLEAN MODE))) AND """ else: match_clause = "" -- cgit v1.2.3 From f733cf955ed5137c4a1a67860b26bbaf23650c83 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 7 Mar 2022 11:05:24 +0300 Subject: Fix/caching (#679) * unlink file for JSONDecodeError * fix for avoiding caching empty dicts * fix for checking null dicts--- wqflask/wqflask/correlation/pre_computes.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation/pre_computes.py b/wqflask/wqflask/correlation/pre_computes.py index 975a53b8..cb2f4470 100644 --- a/wqflask/wqflask/correlation/pre_computes.py +++ b/wqflask/wqflask/correlation/pre_computes.py @@ -6,29 +6,40 @@ from pathlib import Path from base.data_set import query_table_timestamp from base.webqtlConfig import TMPDIR +from json.decoder import JSONDecodeError + def fetch_all_cached_metadata(dataset_name): """in a gvein dataset fetch all the traits metadata""" file_name = generate_filename(dataset_name, suffix="metadata") - file_path = os.path.join(TMPDIR, file_name) + file_path = Path(TMPDIR, file_name) try: with open(file_path, "r+") as file_handler: dataset_metadata = json.load(file_handler) + return (file_path, dataset_metadata) except FileNotFoundError: - Path(file_path).touch(exist_ok=True) - return (file_path, {}) + pass + + except JSONDecodeError: + file_path.unlink() + + file_path.touch(exist_ok=True) + + return (file_path, {}) def cache_new_traits_metadata(dataset_metadata: dict, new_traits_metadata, file_path: str): """function to cache the new traits metadata""" - if bool(new_traits_metadata): - dataset_metadata.update(new_traits_metadata) - + if (dataset_metadata == {} and new_traits_metadata == {}): + return + + dataset_metadata.update(new_traits_metadata) + with open(file_path, "w+") as file_handler: json.dump(dataset_metadata, file_handler) -- cgit v1.2.3 From aed325dcf84629bc26809eae6d537f81dcc40cf7 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 8 Mar 2022 17:33:27 +0300 Subject: make fixes;variable names and kwargs --- .../wqflask/correlation_matrix/show_corr_matrix.py | 29 +++++++++++----------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 499a4e13..9b4cb2eb 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -33,12 +33,11 @@ from utility import corr_result_helpers from utility.redis_tools import get_redis_conn -from gn3.computations.principal_component_analysis import compute_pca +from gn3.computations.pca import compute_pca -from gn3.computations.principal_component_analysis import process_factor_loadings_tdata -from gn3.computations.principal_component_analysis import generate_pca_traits_vals -from gn3.computations.principal_component_analysis import generate_pca_temp_dataset -from gn3.computations.principal_component_analysis import cache_pca_dataset +from gn3.computations.pca import process_factor_loadings_tdata +from gn3.computations.pca import generate_pca_temp_traits +from gn3.computations.pca import cache_pca_dataset Redis = get_redis_conn() THIRTY_DAYS = 60 * 60 * 24 * 30 @@ -168,12 +167,12 @@ class CorrelationMatrix: self.pca_works = "False" try: - if self.do_PCA == True: + if self.do_PCA: self.pca_works = "True" self.pca_trait_ids = [] pca = self.calculate_pca() self.loadings_array = process_factor_loadings_tdata( - self.loadings, len(self.trait_list)) + factor_loadings=self.loadings, traits_num=len(self.trait_list)) else: self.pca_works = "False" except: @@ -198,15 +197,17 @@ class CorrelationMatrix: dataset_name="Temp", dataset_type="Temp", group_name=this_group_name) temp_dataset.group.get_samplelist() - pca_dataset = generate_pca_temp_dataset(species=temp_dataset.group.species, group=this_group_name, - traits_data=self.trait_data_array, corr_array=self.pca_corr_results, - dataset_samples=temp_dataset.group.all_samples_ordered(), - shared_samples=self.shared_samples_list, - create_time=datetime.datetime.now().strftime("%m%d%H%M%S")) + pca_temp_traits = generate_pca_temp_traits(species=temp_dataset.group.species, group=this_group_name, + traits_data=self.trait_data_array, corr_array=self.pca_corr_results, + dataset_samples=temp_dataset.group.all_samples_ordered(), + shared_samples=self.shared_samples_list, + create_time=datetime.datetime.now().strftime("%m%d%H%M%S")) - cache_pca_dataset(Redis, THIRTY_DAYS, pca_dataset) + + cache_pca_dataset(redis_conn=get_redis_conn( + ), exp_days=60 * 60 * 24 * 30, pca_trait_dict=pca_temp_traits) - self.pca_trait_ids = list(pca_dataset.keys()) + self.pca_trait_ids = list(pca_temp_traits.keys()) return pca -- cgit v1.2.3 From 6359dc2bf8973991072634e6a2b8d6a8a038166a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 8 Mar 2022 17:39:22 +0300 Subject: remove global variables;pep8 formatting --- .../wqflask/correlation_matrix/show_corr_matrix.py | 24 +++++++++------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py index 9b4cb2eb..88d62045 100644 --- a/wqflask/wqflask/correlation_matrix/show_corr_matrix.py +++ b/wqflask/wqflask/correlation_matrix/show_corr_matrix.py @@ -21,27 +21,23 @@ import datetime import random import string - - import numpy as np import scipy -from base import data_set +from base.data_set import create_dataset from base.webqtlConfig import GENERATED_TEXT_DIR -from utility import helper_functions -from utility import corr_result_helpers + + +from utility.helper_functions import get_trait_db_obs +from utility.corr_result_helpers import normalize_values from utility.redis_tools import get_redis_conn from gn3.computations.pca import compute_pca - from gn3.computations.pca import process_factor_loadings_tdata from gn3.computations.pca import generate_pca_temp_traits from gn3.computations.pca import cache_pca_dataset -Redis = get_redis_conn() -THIRTY_DAYS = 60 * 60 * 24 * 30 - class CorrelationMatrix: @@ -49,7 +45,7 @@ class CorrelationMatrix: trait_db_list = [trait.strip() for trait in start_vars['trait_list'].split(',')] - helper_functions.get_trait_db_obs(self, trait_db_list) + get_trait_db_obs(self, trait_db_list) self.all_sample_list = [] self.traits = [] @@ -117,7 +113,7 @@ class CorrelationMatrix: if sample in self.shared_samples_list: self.shared_samples_list.remove(sample) - this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values( + this_trait_vals, target_vals, num_overlap = normalize_values( this_trait_vals, target_vals) if num_overlap < self.lowest_overlap: @@ -193,8 +189,9 @@ class CorrelationMatrix: self.scores = pca["scores"] this_group_name = self.trait_list[0][1].group.name - temp_dataset = data_set.create_dataset( - dataset_name="Temp", dataset_type="Temp", group_name=this_group_name) + temp_dataset = create_dataset( + dataset_name="Temp", dataset_type="Temp", + group_name=this_group_name) temp_dataset.group.get_samplelist() pca_temp_traits = generate_pca_temp_traits(species=temp_dataset.group.species, group=this_group_name, @@ -203,7 +200,6 @@ class CorrelationMatrix: shared_samples=self.shared_samples_list, create_time=datetime.datetime.now().strftime("%m%d%H%M%S")) - cache_pca_dataset(redis_conn=get_redis_conn( ), exp_days=60 * 60 * 24 * 30, pca_trait_dict=pca_temp_traits) -- cgit v1.2.3 From 80b02d37f60d172be01bf8cd62bd84b406b1e0dd Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Sat, 26 Feb 2022 08:11:09 +0300 Subject: Use one-step process for the partial correlations computations This commit gets rid of the multi-step partial correlations process replacing it with a single-step process. Summary of changes: * wqflask/wqflask/collect.py: Add function to format the trait details in a format that is usable for the partial correlations system. * wqflask/wqflask/database.py: Provide function to create a connection to the database * wqflask/wqflask/partial_correlations_views.py: Rework the code to enable the one-step process for the partial correlations computations * wqflask/wqflask/static/new/javascript/partial_correlations.js: Get rid of code that supported the multi-step process * wqflask/wqflask/templates/collections/view.html: Remove inconsistent UI elements. Attach traits info in a form usable for the partial correlations * wqflask/wqflask/templates/partial_correlations.html: delete html template * wqflask/wqflask/templates/partial_correlations/pcorrs_error.html: provide a html template to display errors in the partial correlations computation process * wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html: UI template to provide user with feedback as the computations continue in the background * wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html: UI template to present the results of successful computation * wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html: UI template to trigger the partial correlations computations * wqflask/wqflask/templates/tool_buttons.html: Add the partial correlations button to the template to ensure a consistent look and feel --- wqflask/wqflask/collect.py | 31 +- wqflask/wqflask/database.py | 12 + wqflask/wqflask/partial_correlations_views.py | 415 +++++++++++---------- .../static/new/javascript/partial_correlations.js | 358 ++---------------- wqflask/wqflask/templates/collections/view.html | 27 +- .../wqflask/templates/partial_correlations.html | 355 ------------------ .../partial_correlations/pcorrs_error.html | 54 +++ .../partial_correlations/pcorrs_poll_results.html | 15 + .../pcorrs_results_presentation.html | 249 +++++++++++++ .../pcorrs_select_operations.html | 146 ++++++++ wqflask/wqflask/templates/tool_buttons.html | 7 + 11 files changed, 762 insertions(+), 907 deletions(-) delete mode 100644 wqflask/wqflask/templates/partial_correlations.html create mode 100644 wqflask/wqflask/templates/partial_correlations/pcorrs_error.html create mode 100644 wqflask/wqflask/templates/partial_correlations/pcorrs_poll_results.html create mode 100644 wqflask/wqflask/templates/partial_correlations/pcorrs_results_presentation.html create mode 100644 wqflask/wqflask/templates/partial_correlations/pcorrs_select_operations.html (limited to 'wqflask') diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 3475ae5d..815bb7c1 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -189,6 +189,30 @@ def delete_collection(): return redirect(url_for('list_collections')) +def trait_info_str(trait): + """Provide a string representation for given trait""" + def __trait_desc(trt): + if trait.dataset.type == "Geno": + return f"Marker: {trt.name}" + return trt.description_display or "N/A" + + def __symbol(trt): + return (trt.symbol or trt.abbreviation or "N/A")[:20] + + def __lrs(trt): + return ( + f"{float(trait.LRS_score_repr):0.3f}" if float(trait.LRS_score_repr) > 0 + else f"{trait.LRS_score_repr}") + + def __location(trt): + if hasattr(trt, "location_repr"): + return trt.location_repr + return None + + return "{}|||{}|||{}|||{}|||{}|||{:0.3f}|||{}|||{}".format( + trait.name, trait.dataset.name, __trait_desc(trait), __symbol(trait), + __location(trait), trait.mean, __lrs(trait), trait.LRS_location_repr) + @app.route("/collections/view") def view_collection(): params = request.args @@ -227,9 +251,10 @@ def view_collection(): if "json" in params: return json.dumps(json_version) else: - return render_template("collections/view.html", - **collection_info - ) + return render_template( + "collections/view.html", + trait_info_str=trait_info_str, + **collection_info) @app.route("/collections/change_name", methods=('POST',)) diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 11f8d287..e485bcf1 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -1,10 +1,13 @@ # Module to initialize sqlalchemy with flask +import MySQLdb from sqlalchemy import create_engine from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.ext.declarative import declarative_base from utility.tools import SQL_URI +from flask import current_app + engine = create_engine(SQL_URI, encoding="latin1") @@ -16,3 +19,12 @@ Base.query = db_session.query_property() # Initialise the db Base.metadata.create_all(bind=engine) + + +def database_connection(): + """Returns a database connection""" + return MySQLdb.Connect( + db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py index 6bc5efee..659b49e9 100644 --- a/wqflask/wqflask/partial_correlations_views.py +++ b/wqflask/wqflask/partial_correlations_views.py @@ -1,142 +1,31 @@ +import json +import math +import requests +from functools import reduce from typing import Union, Tuple -import MySQLdb -from gn3.db.traits import retrieve_trait_info -from flask import flash, request, current_app, render_template -from gn3.computations.partial_correlations import partial_correlations_entry +from flask import ( + flash, + request, + url_for, + redirect, + current_app, + render_template) from wqflask import app from utility.tools import GN_SERVER_URL +from wqflask.database import database_connection +from gn3.db.partial_correlations import traits_info -def parse_trait(trait_str: str) -> Union[dict, None]: - keys = ( - "name", "dataset", "symbol", "description", "location", "mean_expr", - "max_lrs", "data_hmac") - parts = tuple(part.strip() for part in trait_str.split(":::")) - if len(parts) == len(keys): - return dict(zip(keys, parts)) - return None - -def process_step_select_primary( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - if primary_trait is None: - flash("You must select a primary trait", "alert-danger") - return ( - "select-primary", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - return ( - "select-controls", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list - if trait["data_hmac"] != primary_trait["data_hmac"]), - corr_method) - -def process_step_select_controls( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - if len(control_traits) == 0 or len(control_traits) > 3: - flash( - ("You must select a minimum of one control trait, up to a maximum " - "of three control traits."), - "alert-danger") - return ( - "select-controls", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - hmacs =(primary_trait["data_hmac"],) + tuple( - trait["data_hmac"] for trait in control_traits) - return ( - "select-corr-method", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list if trait["data_hmac"] not in hmacs), - corr_method) - -def process_step_select_targets( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - if len(target_traits) == 0: - flash( - "You must select at least one target trait.", "alert-danger") - return ( - "select-targets", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - hmacs =(primary_trait["data_hmac"],) + tuple( - trait["data_hmac"] for trait in (control_traits + target_traits)) - return ( - "select-corr-method", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list if trait["data_hmac"] not in hmacs), - corr_method) - -def process_step_select_corr_method( - primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], - traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - methods = ( - "genetic correlation, pearson's r", - "genetic correlation, spearman's rho", - "sgo literature correlation", - "tissue correlation, pearson's r", - "tissue correlation, spearman's rho") - if corr_method.lower() not in methods: - flash( - "Selected method is unknown.", "alert-danger") - return ( - "select-corr-method", primary_trait, control_traits, target_traits, - traits_list, corr_method) - - hmacs =(primary_trait["data_hmac"],) + tuple( - trait["data_hmac"] for trait in (control_traits + target_traits)) - return ( - "run-correlation", primary_trait, control_traits, target_traits, - tuple( - trait for trait in traits_list if trait["data_hmac"] not in hmacs), - corr_method) - -def process_step( - step: str, primary_trait: dict, control_traits: Tuple[dict, ...], - target_traits: Tuple[dict, ...], traits_list: Tuple[dict, ...], - corr_method: str) -> Tuple[ - str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], - str]: - processor_functions = { - # "select-traits": lambda arg: arg, - "select-primary": process_step_select_primary, - "select-controls": process_step_select_controls, - "select-targets": process_step_select_targets, - "select-corr-method": process_step_select_corr_method - } - return processor_functions[(step or "select-primary")]( - primary_trait, control_traits, target_traits, traits_list, corr_method) - -def sequence_of_traits(trait_strs) -> Tuple[dict, ...]: - return tuple(filter( - lambda trt: trt is not None, - (parse_trait(tstr.strip()) for tstr in trait_strs))) - -def publish_target_dabases(conn, group, threshold): +def publish_target_databases(conn, groups, threshold): query = ( "SELECT PublishFreeze.FullName,PublishFreeze.Name " "FROM PublishFreeze, InbredSet " "WHERE PublishFreeze.InbredSetId = InbredSet.Id " - "AND InbredSet.Name = %(group)s " - "AND PublishFreeze.public > %(threshold)s") + f"AND InbredSet.Name IN ({', '.join(['%s'] * len(groups))}) " + "AND PublishFreeze.public > %s") with conn.cursor() as cursor: - cursor.execute(query, {"group": group, "threshold": threshold}) + cursor.execute(query, tuple(groups) + (threshold,)) res = cursor.fetchall() if res: return tuple( @@ -144,15 +33,15 @@ def publish_target_dabases(conn, group, threshold): return tuple() -def geno_target_databases(conn, group, threshold): +def geno_target_databases(conn, groups, threshold): query = ( "SELECT GenoFreeze.FullName,GenoFreeze.Name " "FROM GenoFreeze, InbredSet " "WHERE GenoFreeze.InbredSetId = InbredSet.Id " - "AND InbredSet.Name = %(group)s " - "AND GenoFreeze.public > %(threshold)s") + f"AND InbredSet.Name IN ({', '.join(['%s'] * len(groups))}) " + "AND GenoFreeze.public > %s") with conn.cursor() as cursor: - cursor.execute(query, {"group": group, "threshold": threshold}) + cursor.execute(query, tuple(groups) + (threshold,)) res = cursor.fetchall() if res: return tuple( @@ -160,27 +49,26 @@ def geno_target_databases(conn, group, threshold): return tuple() -def probeset_target_databases(conn, group, threshold): +def probeset_target_databases(conn, groups, threshold): query1 = "SELECT Id, Name FROM Tissue order by Name" - query2 = ( - "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, ProbeSetFreeze.Name " - "FROM ProbeSetFreeze, ProbeFreeze, InbredSet " - "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " - "AND ProbeFreeze.TissueId IN %(tissue_ids)s " - "AND ProbeSetFreeze.public > %(threshold)s " - "AND ProbeFreeze.InbredSetId = InbredSet.Id " - "AND InbredSet.Name like %(group)s " - "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId") with conn.cursor() as cursor: cursor.execute(query1) tissue_res = cursor.fetchall() if tissue_res: tissue_ids = tuple(row[0] for row in tissue_res) - cursor.execute( - query2,{ - "tissue_ids": tissue_ids, "group": f"{group}%%", - "threshold": threshold - }) + groups_clauses = ["InbredSet.Name like %s"] * len(groups) + query2 = ( + "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, " + "ProbeSetFreeze.Name " + "FROM ProbeSetFreeze, ProbeFreeze, InbredSet " + "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND ProbeFreeze.TissueId IN " + f"({', '.join(['%s'] * len(tissue_ids))}) " + "AND ProbeSetFreeze.public > %s " + "AND ProbeFreeze.InbredSetId = InbredSet.Id " + f"AND ({' OR '.join(groups_clauses)}) " + "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId") + cursor.execute(query2, tissue_ids + (threshold,) + tuple(groups)) db_res = cursor.fetchall() if db_res: databases = tuple( @@ -197,70 +85,201 @@ def probeset_target_databases(conn, group, threshold): return tuple() -def target_databases(conn, step, trait, threshold): +def target_databases(conn, traits, threshold): """ Retrieves the names of possible target databases from the database. """ - if step != "select-corr-method": - return None - - trait_info = retrieve_trait_info( - threshold, f"{trait['dataset']}::{trait['name']}", conn) - group = trait_info["group"] + trait_info = traits_info( + conn, threshold, + tuple(f"{trait['dataset']}::{trait['trait_name']}" for trait in traits)) + groups = tuple(set(row["db"]["group"] for row in trait_info)) return ( - publish_target_dabases(conn, group, threshold) + - geno_target_databases(conn, group, threshold) + - probeset_target_databases(conn, group, threshold)) + publish_target_databases(conn, groups, threshold) + + geno_target_databases(conn, groups, threshold) + + probeset_target_databases(conn, groups, threshold)) + +def primary_error(args): + if len(args["primary_trait"]) == 0 or len(args["primary_trait"]) > 1: + return { + **args, + "errors": (args.get("errors", tuple()) + + ("You must provide one, and only one primary trait",))} + return args + +def controls_error(args): + if len(args["control_traits"]) == 0 or len(args["control_traits"]) > 3: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + (("You must provide at least one control trait, and a maximum " + "of three control traits"),))} + return args + +def target_db_error(args): + if not args["target_db"]: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + ("The target database must be provided",))} + return args + +def method_error(args): + methods = ( + "genetic correlation, pearson's r", + "genetic correlation, spearman's rho", + "sgo literature correlation", + "tissue correlation, pearson's r", + "tissue correlation, spearman's rho") + if not args["method"] or args["method"].lower() not in methods: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + ("Invalid correlation method provided",))} + return args -def pcorrelations(conn, values): - if values["step"] != "run-correlation": - return None +def criteria_error(args): + try: + int(args.get("criteria", "invalid")) + return args + except ValueError: + return { + **args, + "errors": ( + args.get("errors", tuple()) + + ("Invalid return number provided",))} - def trait_fullname(trait): - return f"{trait['dataset']}::{trait['name']}" +def errors(args): + return criteria_error(method_error(target_db_error(controls_error( + primary_error(args))))) - return partial_correlations_entry( - conn, trait_fullname(values["primary_trait"]), - tuple(trait_fullname(trait) for trait in values["control_traits"]), - values["method"], values["criteria"], values["target_db"]) +def __classify_args(acc, item): + if item[1].startswith("primary_"): + return { + **acc, + "primary_trait": (acc.get("primary_trait", tuple()) + (item,))} + if item[1].startswith("controls_"): + return {**acc, "control_traits": (acc.get("control_traits", tuple()) + (item,))} + if item[0] == "target_db": + return {**acc, "target_db": item[1]} + if item[0] == "method": + return {**acc, "method": item[1]} + if item[0] == "criteria": + return {**acc, "criteria": item[1]} + return acc + +def __build_args(raw_form, traits): + args = reduce(__classify_args, raw_form.items(), {}) + return { + **args, + "primary_trait": [ + item for item in traits if item["trait_name"] in + (name[1][8:] for name in args["primary_trait"])], + "control_traits": [ + item for item in traits if item["trait_name"] in + (name[1][9:] for name in args["control_traits"])] + } + +def parse_trait(trait_str): + return dict(zip( + ("trait_name", "dataset", "description", "symbol", "location", "mean", + "lrs", "lrs_location"), + trait_str.strip().split("|||"))) + +def response_error_message(response): + error_messages = { + 404: ("We could not connect to the API server at this time. " + "Try again later."), + 500: ("The API server experienced a problem. We will be working on a " + "fix. Please try again later.") + } + return error_messages.get( + response.status_code, + "General API server error!!") + +def render_error(error_message): + return render_template( + "partial_correlations/pcorrs_error.html", + message = error_message) + +def handle_200_response(response): + if response["status"] == "success": + return redirect( + url_for( + "poll_partial_correlation_results", + command_id=response["results"]), + code=303) + return render_error(response["results"]) + +def handle_response(response): + if response.status_code != 200: + return render_template( + "partial_correlations/pcorrs_error.html", + message = response_error_message(response)) + return handle_200_response(response.json()) @app.route("/partial_correlations", methods=["POST"]) def partial_correlations(): form = request.form - traits_list = tuple(filter( - lambda trt: trt is not None, - (parse_trait(tstr) for tstr in form.get("traits_list", "").split("|||")))) + traits = tuple( + parse_trait(trait) for trait in + form.get("trait_list").split(";;;")) - args_dict = dict(zip( - ("step", "primary_trait", "control_traits", "target_traits", - "traits_list", "method"), - process_step( - form.get("step", None), - parse_trait(form.get("primary_trait", "")), - sequence_of_traits( - form.getlist("control_traits[]") or - form.get("control_traits", "").split("|||")), - sequence_of_traits( - form.getlist("target_traits[]") or - form.get("target_traits", "").split("|||")), - sequence_of_traits(form.get("traits_list", "").split("|||")), - form.get("method")))) + if form.get("submit") == "Run Partial Correlations": + args = errors(__build_args(form, traits)) + if len(args.get("errors", [])) == 0: + post_data = { + **args, + "primary_trait": args["primary_trait"][0] + } + return handle_response(requests.post( + url=f"{GN_SERVER_URL}api/correlation/partial", + json=json.dumps(post_data))) - conn = MySQLdb.Connect( - db=current_app.config.get("DB_NAME"), - user=current_app.config.get("DB_USER"), - passwd=current_app.config.get("DB_PASS"), - host=current_app.config.get("DB_HOST")) - target_dbs = target_databases( - conn, args_dict["step"], args_dict["primary_trait"], 0) + for error in args["errors"]: + flash(error, "alert-danger") - if args_dict["step"] == "run-correlation": - args_dict = { - **args_dict, "target_db": form.get("target_db"), - "criteria": int(form.get("criteria", 500))} + with database_connection() as conn: + target_dbs = target_databases(conn, traits, threshold=0) + return render_template( + "partial_correlations/pcorrs_select_operations.html", + trait_list_str=form.get("trait_list"), + traits=traits, + target_dbs=target_dbs) - corr_results = pcorrelations(conn, args_dict) +def process_pcorrs_command_output(result): + if result["status"] == "success": + def __format_number(num): + if num is None or math.isnan(num): + return "" + if abs(num) <= 1.04E-4: + return f"{num:.2e}" + return f"{num:.5f}" - return render_template( - "partial_correlations.html", **args_dict, target_dbs=target_dbs, - corr_results=corr_results, part_corr_url=f"{GN_SERVER_URL}api/correlation/partial") + return render_template( + "partial_correlations/pcorrs_results_presentation.html", + primary=result["results"]["primary_trait"], + controls=result["results"]["control_traits"], + correlations=result["results"]["correlations"], + dataset_type=result["results"]["dataset_type"], + method=result["results"]["method"], + format_number=__format_number) + if result["status"] == "error": + return render_error( + "The partial correlations computation failed with an error") + +@app.route("/partial_correlations/", methods=["GET"]) +def poll_partial_correlation_results(command_id): + response = requests.get( + url=f"{GN_SERVER_URL}api/async_commands/state/{command_id}") + if response.status_code == 200: + data = response.json() + if data["status"] == "error": + return render_error(response["result"]) + if data["status"] == "success": + return process_pcorrs_command_output(json.loads(data["result"])) + return render_template( + "partial_correlations/pcorrs_poll_results.html", + command_id = command_id) diff --git a/wqflask/wqflask/static/new/javascript/partial_correlations.js b/wqflask/wqflask/static/new/javascript/partial_correlations.js index 3bf2acc4..5de1204c 100644 --- a/wqflask/wqflask/static/new/javascript/partial_correlations.js +++ b/wqflask/wqflask/static/new/javascript/partial_correlations.js @@ -1,336 +1,26 @@ -/** - * This is, hopefully, a short-term stop-gap measure to get the system working - * and to get some feedback, even as better optimisation is done in the - * background to get better response/performance for the partial correlation - * computations - */ - -function key_value(keys, values) { - if(!(keys.length == values.length)) { - Error("The 'keys' and 'values' objects MUST be the same length"); - return null; - } - return values.reduce(function(accumulator, item, index) { - accumulator[keys[index]] = item; - return accumulator; - }, {}); -} - -function trait(trait_str) { - return key_value( - ["name", "dataset", "symbol", "description", "location", "mean_expr", - "max_lrs", "data_hmac"], - trait_str.split(":::")); -} - -function primary_trait() { - trait_string = document.querySelector( - "#partial-correlations-form input[name=primary_trait]").value; - return trait(trait_string); -} - -function control_traits() { - return document.querySelector( - "#partial-correlations-form input[name=control_traits]").value.split( - "|||").map(trait).filter(trait => !(trait === null)); -} - -function correlation_method() { - return document.querySelector( - "#partial-correlations-form select[name=method]").value; -} - -function criteria() { - return document.querySelector( - "#partial-correlations-form select[name=criteria]").value; -} - -function target_db() { - return document.querySelector( - "#partial-correlations-form select[name=target_db]").value; -} - -function partial_corr_request_data() { - return { - "primary_trait": primary_trait(), - "control_traits": control_traits(), - "method": correlation_method(), - "criteria": criteria(), - "target_db": target_db() - } -} - -function rho_or_r(method) { - if (method === "spearman") { - return "rho"; - } - return "r"; -} - -function format_number(num) { - if(num === null) { - return NaN; - } - if(Math.abs(num) <= 1.04e-4) { - return num.toExponential(2); - } - return num.toFixed(5); -} - -function display_publish_results(primary, controls, correlations, method) { - table = document.getElementById("part-corr-results-publish"); - table.setAttribute("style", "display: block;"); - table_body = document.querySelector("#part-corr-results-publish tbody"); - template_row = document.querySelector( - "#part-corr-results-publish tr.template-publish-results-row"); - correlations.forEach(function(item, index, arr) { - new_row = template_row.cloneNode(true); - new_row.setAttribute("class", "results-row"); - new_row.querySelector( - 'td[data-column-heading="Record"]').innerHTML = item["trait_name"]; - new_row.querySelector( - 'td[data-column-heading="Phenotype"]').innerHTML = ( - item["post_publication_description"]); - new_row.querySelector( - 'td[data-column-heading="Authors"]').innerHTML = item["authors"]; - new_row.querySelector( - 'td[data-column-heading="Year"]').innerHTML = item["year"]; - new_row.querySelector( - 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; - new_row.querySelector( - `td[data-column-heading="Partial ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["partial_corr"]); - new_row.querySelector( - `td[data-column-heading="p(partial ${rho_or_r(method)})"]` - ).innerHTML = format_number(item["partial_corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="${rho_or_r(method)}"]` - ).innerHTML = format_number(item["corr"]); - new_row.querySelector( - `td[data-column-heading="p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="delta ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["delta"]); - table_body.appendChild(new_row); - }); - table_body.removeChild(template_row); -} - -function display_geno_results(primary, controls, correlations, method) { - table = document.getElementById("part-corr-results-geno"); - table.setAttribute("style", "display: block;"); - table_body = document.querySelector("#part-corr-results-geno tbody"); - template_row = document.querySelector( - "#part-corr-results-geno tr.template-geno-results-row"); - correlations.forEach(function(item, index, arr) { - new_row = template_row.cloneNode(true); - new_row.setAttribute("class", "results-row"); - new_row.querySelector( - 'td[data-column-heading="Locus"]').innerHTML = item["trait_name"]; - new_row.querySelector( - 'td[data-column-heading="Chr"]').innerHTML = item["chr"]; - new_row.querySelector( - 'td[data-column-heading="Megabase"]').innerHTML = item["mb"]; - new_row.querySelector( - 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; - new_row.querySelector( - `td[data-column-heading="Partial ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["partial_corr"]); - new_row.querySelector( - `td[data-column-heading="p(partial ${rho_or_r(method)})"]` - ).innerHTML = format_number(item["partial_corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="${rho_or_r(method)}"]` - ).innerHTML = format_number(item["corr"]); - new_row.querySelector( - `td[data-column-heading="p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["corr_p_value"]); - new_row.querySelector( - `td[data-column-heading="delta ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["delta"]); - table_body.appendChild(new_row); - }); - table_body.removeChild(template_row); -} - -function display_probeset_results(primary, controls, correlations, method) { - table = document.getElementById("part-corr-results-probeset"); - table.setAttribute("style", "display: block;"); - table_body = document.querySelector("#part-corr-results-probeset tbody"); - template_row = document.querySelector( - "#part-corr-results-probeset tr.template-probeset-results-row"); - correlations.forEach(function(item, index, arr) { - new_row = template_row.cloneNode(true); - new_row.setAttribute("class", "results-row"); - new_row.querySelector( - 'td[data-column-heading="Record"]').innerHTML = item["trait_name"]; - new_row.querySelector( - 'td[data-column-heading="Gene ID"]').innerHTML = item["geneid"]; - new_row.querySelector( - 'td[data-column-heading="Homologene ID"]').innerHTML = item["homologeneid"]; - new_row.querySelector( - 'td[data-column-heading="Symbol"]').innerHTML = item["symbol"]; - new_row.querySelector( - 'td[data-column-heading="Description"]').innerHTML = item["description"]; - new_row.querySelector( - 'td[data-column-heading="Chr"]').innerHTML = item["chr"]; - new_row.querySelector( - 'td[data-column-heading="Megabase"]').innerHTML = item["mb"]; - new_row.querySelector( - 'td[data-column-heading="Mean Expr"]').innerHTML = item["mean_expr"]; - new_row.querySelector( - 'td[data-column-heading="N"]').innerHTML = item["noverlap"]; - new_row.querySelector( - `td[data-column-heading="Sample Partial ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["partial_corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Sample p(partial ${rho_or_r(method)})"]` - ).innerHTML = format_number(item["partial_corr_p_value"] || NaN); - new_row.querySelector( - `td[data-column-heading="Sample ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Sample p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["corr_p_value"] || NaN); - new_row.querySelector( - `td[data-column-heading="delta ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["delta"] || NaN); - new_row.querySelector( - `td[data-column-heading="Lit Corr"]` - ).innerHTML = format_number(item["l_corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Tissue ${rho_or_r(method)}"]` - ).innerHTML = format_number(item["tissue_corr"] || NaN); - new_row.querySelector( - `td[data-column-heading="Tissue p(${rho_or_r(method)})"]` - ).innerHTML = format_number(item["tissue_p_value"] || NaN); - table_body.appendChild(new_row); - }); - template_row.setAttribute("display", "none"); - /*table_body.removeChild(template_row);*/ -} - -function replace_r_with_rho(method) { - /* Mostly utility: Replace `r` with `rho` in the appropriate places */ - pattern = /\br\b/; - if(method == "spearman") { - results_div = document.getElementById("partial-correlation-results"); - headers = results_div.getElementsByTagName("th"); - for(let header of headers) { - header.innerHTML = header.innerHTML.replace(pattern, "rho"); - } - - cells = results_div.getElementsByTagName("td"); - for(let cell of cells) { - cell.setAttribute( - "data-column-heading", - cell.getAttribute( - "data-column-heading").replace(pattern, "rho")); - } +function selected_traits() { + traits = $("#trait_table input:checked").map(function() { + return $(this).attr("data-trait-info"); + }).get(); + if (traits.length == 0){ + num_traits = $("#trait_table input").length + if (num_traits <= 100){ + traits = $("#trait_table input").map(function() { + return $(this).attr("data-trait-info"); + }).get(); + } } -} - -function display_partial_corr_results(data, status, xhr) { - progress_indicator = document.getElementById( - "partial-correlations-progress-indicator").style.display = "none"; - console.log(data); - - replace_r_with_rho(data["results"]["method"]); - - display_functions = { - "Publish": display_publish_results, - "Geno": display_geno_results, - "ProbeSet": display_probeset_results - } - - display_functions[data["results"]["dataset_type"]]( - data["results"]["primary_traits"], - data["results"]["control_traits"], - data["results"]["correlations"], - data["results"]["method"]); - - initializePcorrTable(data["results"]["dataset_type"]); -} - -function display_partial_corr_error(xhr, status, error) { - document.getElementById( - "partial-correlations-progress-indicator").style.display = "none"; - error_element = document.getElementById("part-corr-error"); - panel = document.createElement("div"); - panel.setAttribute("class", "panel panel-danger"); - error_element.appendChild(panel); - - panel_header = document.createElement("div"); - panel_header.setAttribute("class", "panel-heading"); - panel_header.textContent = "Error: " + xhr.status; - panel.appendChild(panel_header); - - panel_body = document.createElement("div"); - panel_body.setAttribute("class", "panel-body"); - panel_body.textContent = xhr.statusText; - panel.appendChild(panel_body); - console.log(xhr) -} - -function send_data_and_process_results( - remote_url, request_data, success_fn, error_fn, indicator_id) { - document.getElementById(indicator_id).style.display = "block"; - $.ajax({ - type: "POST", - url: remote_url, - contentType: "application/json", - data: JSON.stringify(request_data), - dataType: "JSON", - success: success_fn, - error: error_fn - }); -} - -function initializePcorrTable(dataType){ - tableId = "part-corr-results-" + dataType.toLowerCase(); - if (dataType == "Publish") { - orderCol = 8; - } else if (dataType == "ProbeSet") { - orderCol = 12; - } else { - orderCol = 7; - } - - $('#' + tableId).dataTable( { - 'drawCallback': function( settings ) { - $('#' + tableId + ' tr').off().on("click", function(event) { - if (event.target.type !== 'checkbox' && event.target.tagName.toLowerCase() !== 'a') { - var obj =$(this).find('input'); - obj.prop('checked', !obj.is(':checked')); - } - if ($(this).hasClass("selected") && event.target.tagName.toLowerCase() !== 'a'){ - $(this).removeClass("selected") - } else if (event.target.tagName.toLowerCase() !== 'a') { - $(this).addClass("selected") - } - }); - }, - "order": [[orderCol, "asc" ]], - "sDom": "itir", - "iDisplayLength": -1, - "autoWidth": false, - "bDeferRender": true, - "bSortClasses": false, - "paging": false - } ); -} - -$("#partial-correlations-form").submit(function(e) { - e.preventDefault(); -}); - -$("#run-partial-corr-button").click(function(evt) { - send_data_and_process_results( - document.getElementById( - "run-partial-corr-button").getAttribute("data-url"), - partial_corr_request_data(), - display_partial_corr_results, - display_partial_corr_error, - "partial-correlations-progress-indicator"); + return traits +} + +$("#partial-correlations").on("click", function() { + // Submit the form to the `partial_correlations` endpoint + url = $(this).data("url") + traits = selected_traits(); + $("#trait_list").val(traits.reduce(function(acc, str) { + return acc.concat(";;;".concat(str)); + })); + $("input[name=tool_used]").val("Partial Correlation") + $("input[name=form_url]").val(url) + return submit_special(url) }) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index f896471a..6f1a9680 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -67,22 +67,6 @@   -
- - - -
@@ -128,7 +112,13 @@
{% for this_trait in trait_obs %} - +