From a91c0c240e3534567f65b16578bcfaf72a9055c9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 12 Feb 2018 17:08:58 +0000 Subject: Quick parallel correlation should work, though I'm not sure why the code suddenly decided to start running it when it wasn't in the past. Switched correlation table to tabulator --- wqflask/wqflask/correlation/show_corr_results.py | 22 +++++++++- wqflask/wqflask/templates/correlation_page.html | 51 ++++++++++++++++++------ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 24432ad0..bb109f60 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -120,8 +120,15 @@ class CorrelationResults(object): self.min_expr = float(start_vars['min_expr']) else: self.min_expr = None - self.p_range_lower = float(start_vars['p_range_lower']) - self.p_range_upper = float(start_vars['p_range_upper']) + logger.debug("P RANGE:", start_vars['p_range_lower']) + try: + self.p_range_lower = float(start_vars['p_range_lower']) + except: + self.p_range_lower = -1.00 + try: + self.p_range_upper = float(start_vars['p_range_upper']) + except: + self.p_range_upper = 1.00 if ('loc_chr' in start_vars and 'min_loc_mb' in start_vars and @@ -1153,6 +1160,17 @@ class CorrelationResults(object): import math import reaper + def cmpOrder2(A,B): + try: + if A[-1] < B[-1]: + return -1 + elif A[-1] == B[-1]: + return 0 + else: + return 1 + except: + return 0 + def calCorrelation(dbdata,userdata,N): X = [] Y = [] diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index fa9e3585..c7fb7cc6 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -1,9 +1,10 @@ {% extends "base.html" %} {% block css %} + + {% endblock %} {% block content %} @@ -63,15 +64,14 @@
- +
- {% for header in target_dataset.header_fields %} {% if header == 'Year' %} {% elif header == 'Max LRS' %} - + {% elif header == 'Max LRS Location' %} {% elif header == 'Location' %} @@ -79,7 +79,7 @@ {% elif header == 'Mean' %} {% elif header == 'Additive Effect' %} - + {% elif header == 'Index' %} {% elif header == 'N' %} @@ -91,7 +91,7 @@ {% if target_dataset.type == "ProbeSet" %} {% if corr_method == 'pearson' %} - + @@ -129,8 +129,7 @@ {% for trait in correlation_results %} - - + {% for uc in collections %} + {% if g.user_session.user_ob %} + {% else %} + + {% endif %} diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 1254ea6a..0d305559 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -275,29 +275,32 @@ {% if target_dataset.type == "ProbeSet" %} - var json_array = new Array(); + var json_array = []; for (i=0; i < table_json.length; i++){ - json_array.push({ - checkbox: table_json[i]["checkbox"], - index: table_json[i]["index"], - trait_id: table_json[i]["trait_id"], - symbol: table_json[i]["symbol"], - description: table_json[i]["description"], - location: table_json[i]["location"], - mean: table_json[i]["mean"], - lrs_score: table_json[i]["lrs_score"], - lrs_location: table_json[i]["lrs_location"], - additive: table_json[i]["additive"], - sample_r: table_json[i]["sample_r"], - num_overlap: table_json[i]["num_overlap"], - sample_p: table_json[i]["sample_p"], - lit_corr: table_json[i]["lit_corr"], - tissue_corr: table_json[i]["tissue_corr"] - }); - console.log("JSON_DATA:", json_array); + this_array = json_array + + var row_dict = {}; + row_dict.checkbox = table_json[i]["checkbox"] + row_dict.index = table_json[i]["index"] + row_dict.trait_id = table_json[i]["trait_id"] + row_dict.symbol = table_json[i]["symbol"] + row_dict.description = table_json[i]["description"] + row_dict.location = table_json[i]["location"] + row_dict.mean = table_json[i]["mean"] + row_dict.lrs_score = table_json[i]["lrs_score"] + row_dict.lrs_location = table_json[i]["lrs_location"] + row_dict.additive = table_json[i]["additive"] + row_dict.sample_r = table_json[i]["sample_r"] + row_dict.num_overlap = table_json[i]["num_overlap"] + row_dict.sample_p = table_json[i]["sample_p"] + row_dict.lit_corr = table_json[i]["lit_corr"] + row_dict.tissue_corr = table_json[i]["tissue_corr"] + this_array.push(row_dict); + json_array.concat(this_array); + //console.log("JSON_ARRAY:", json_array); } - //console.log("JSON_DATA:", json_array) + console.log("JSON_DATA:", json_array) $("#trait_table").tabulator({ data: json_array, diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index dcec2b9e..ab2e2aae 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -1,6 +1,6 @@
{% if (use_pylmm_rqtl and dataset.group.species != "human") or use_plink_gemma %} -
+
{{header}}Max LRS ?Max LRS{{header}}{{header}}Additive Effect ?Additive Effect{{header}}Sample r  NN Sample p(r) Lit r Tissue r
{{ loop.index }}{{ loop.index }} + + + + + - {% endblock %} -- cgit v1.2.3 From 04240ba98c040887a318b9f19dff0be3f68f5e4d Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 26 Feb 2018 23:01:55 +0000 Subject: Committing some in-progress changes related to loading correlation page from JSON --- wqflask/wqflask/correlation/show_corr_results.py | 63 ++++++++++++++++++++++- wqflask/wqflask/static/dbdoc/TODO.md | 1 + wqflask/wqflask/templates/correlation_page.html | 65 ++++++++++++++++++------ wqflask/wqflask/templates/empty_collection.html | 2 +- wqflask/wqflask/views.py | 2 +- 5 files changed, 115 insertions(+), 18 deletions(-) create mode 100644 wqflask/wqflask/static/dbdoc/TODO.md diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 4d7fea61..9b048346 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -32,6 +32,7 @@ import pp import math import collections import resource +import json import scipy @@ -51,6 +52,7 @@ import utility.webqtlUtil #this is for parallel computing only. from wqflask.correlation import correlation_functions from utility.benchmark import Bench import utility.webqtlUtil +from wqflask import user_manager from MySQLdb import escape_string as escape @@ -248,7 +250,7 @@ class CorrelationResults(object): else: self.this_trait_vals.append("None") num_overlap = len(self.this_trait_vals) - + logger.debug("DOING PARALLEL") self.do_parallel_correlation(db_filename, num_overlap) else: for trait, values in self.target_dataset.trait_data.iteritems(): @@ -335,6 +337,7 @@ class CorrelationResults(object): #print("self.correlation_results: ", pf(self.correlation_results)) + self.json_results = generate_corr_json(self.correlation_results, self.this_trait, self.dataset, self.target_dataset) #XZ, 09/18/2008: get all information about the user selected database. #target_db_name = fd.corr_dataset @@ -1431,3 +1434,61 @@ class CorrelationResults(object): # for one_result in results: # for one_traitinfo in one_result: # allcorrelations.append( one_traitinfo ) + +def generate_corr_json(corr_results, this_trait, dataset, target_dataset): + results_list = [] + for i, trait in enumerate(corr_results): + results_dict = {} + results_dict['checkbox'] = "" + results_dict['index'] = i + 1 + results_dict['trait_id'] = ""+str(trait.name)+"" + if target_dataset.type == "ProbeSet": + results_dict['symbol'] = trait.symbol + results_dict['description'] = trait.description_display + results_dict['location'] = trait.location_repr + results_dict['mean'] = float(trait.mean) + if trait.LRS_score_repr != "N/A": + results_dict['lrs_score'] = "%.1f" % float(trait.LRS_score_repr) + else: + results_dict['lrs_score'] = "N/A" + results_dict['lrs_location'] = trait.LRS_location_repr + if trait.additive != "": + results_dict['additive'] = "%0.3f" % float(trait.additive) + else: + results_dict['additive'] = "N/A" + results_dict['sample_r'] = "" + "%0.3f" % float(trait.sample_r) + "" + results_dict['num_overlap'] = trait.num_overlap + results_dict['sample_p'] = "%0.3e" % float(trait.sample_p) + if trait.lit_corr == "" or trait.lit_corr == 0: + results_dict['lit_corr'] = "--" + else: + results_dict['lit_corr'] = "%0.3f" % float(trait.lit_corr) + if trait.tissue_corr == "" or trait.tissue_corr == 0: + results_dict['tissue_corr'] = "--" + else: + results_dict['tissue_corr'] = "%0.3f" % float(trait.tissue_corr) + elif target_dataset.type == "Publish": + results_dict['description'] = trait.description_display + results_dict['authors'] = trait.authors + if trait.pubmed_id: + results_dict['pubmed'] = " " + trait.pubmed_text + "" + else: + results_dict['pubmed'] = "N/A" + results_dict['lrs_score'] = trait.LRS_score_repr + results_dict['lrs_location'] = trait.LRS_location_repr + if trait.additive != "": + results_dict['additive'] = "%0.3f" % float(trait.additive) + else: + results_dict['additive'] = "N/A" + results_dict['sample_r'] = "" + "%0.3f" % trait.sample_r + "" + results_dict['num_overlap'] = trait.num_overlap + results_dict['sample_p'] = "%0.3e" % float(trait.sample_p) + else: + results_dict['lrs_location'] = trait.LRS_location_repr + results_dict['sample_r'] = "" + "%0.3f" % float(trait.sample_r) + "" + results_dict['num_overlap'] = trait.num_overlap + results_dict['sample_p'] = "%0.3e" % float(trait.sample_p) + + results_list.append(results_dict) + + return json.dumps(results_list) diff --git a/wqflask/wqflask/static/dbdoc/TODO.md b/wqflask/wqflask/static/dbdoc/TODO.md new file mode 100644 index 00000000..c0a8bab7 --- /dev/null +++ b/wqflask/wqflask/static/dbdoc/TODO.md @@ -0,0 +1 @@ +TODO: Add all database documentation into this folder diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index 629724c0..1254ea6a 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -65,8 +65,10 @@
+
@@ -267,24 +271,55 @@ $(document).ready( function () { + var table_json = {{ json_results | safe }} + + + {% if target_dataset.type == "ProbeSet" %} + var json_array = new Array(); + + for (i=0; i < table_json.length; i++){ + json_array.push({ + checkbox: table_json[i]["checkbox"], + index: table_json[i]["index"], + trait_id: table_json[i]["trait_id"], + symbol: table_json[i]["symbol"], + description: table_json[i]["description"], + location: table_json[i]["location"], + mean: table_json[i]["mean"], + lrs_score: table_json[i]["lrs_score"], + lrs_location: table_json[i]["lrs_location"], + additive: table_json[i]["additive"], + sample_r: table_json[i]["sample_r"], + num_overlap: table_json[i]["num_overlap"], + sample_p: table_json[i]["sample_p"], + lit_corr: table_json[i]["lit_corr"], + tissue_corr: table_json[i]["tissue_corr"] + }); + console.log("JSON_DATA:", json_array); + } + //console.log("JSON_DATA:", json_array) + $("#trait_table").tabulator({ + data: json_array, columns:[ - {title:"Index", formatter:"html"}, - {title:"Record", formatter:"html"}, - {title:"Symbol", formatter:"plaintext"}, - {title:"Description", formatter:"textarea", width:"25%"}, - {title:"Location", formatter:"plaintext"}, - {title:"Mean", formatter:"plaintext"}, - {title:"Max LRS", formatter:"plaintext"}, - {title:"Max LRS Location", formatter:"plaintext"}, - {title:"Additive Effect", formatter:"plaintext"}, - {title:"Sample r", formatter:"html"}, - {title:"N", formatter:"plaintext"}, - {title:"Sample p(r)", formatter:"plaintext"}, - {title:"Lit r", formatter:"plaintext"}, - {title:"Tissue r", formatter:"plaintext"} + {title:"", field:"checkbox", formatter:"html"}, + {title:"Index", field:"index", formatter:"plaintext"}, + {title:"Record", field:"trait_id", formatter:"html"}, + {title:"Symbol", field:"symbol", formatter:"plaintext"}, + {title:"Description", field:"description", formatter:"textarea", width:"25%"}, + {title:"Location", field:"location", formatter:"plaintext"}, + {title:"Mean", field:"mean", formatter:"plaintext"}, + {title:"Max LRS", field:"lrs_score", formatter:"plaintext"}, + {title:"Max LRS Location", field:"lrs_location", formatter:"plaintext"}, + {title:"Additive Effect", field:"additive", formatter:"plaintext"}, + {title:"Sample r", field:"sample_r", formatter:"html"}, + {title:"N", field:"num_overlap", formatter:"plaintext"}, + {title:"Sample p(r)", field:"sample_p", formatter:"plaintext"}, + {title:"Lit r", field:"lit_corr", formatter:"plaintext"}, + {title:"Tissue r", field:"tissue_corr", formatter:"plaintext"} ] }); + {% endif %} $('#trait_table tr').click(function(event) { if (event.target.type !== 'checkbox') { diff --git a/wqflask/wqflask/templates/empty_collection.html b/wqflask/wqflask/templates/empty_collection.html index 3f2b3786..d1b779ef 100644 --- a/wqflask/wqflask/templates/empty_collection.html +++ b/wqflask/wqflask/templates/empty_collection.html @@ -6,7 +6,7 @@
-

You must select at least one trait to use the {{ tool }}.

+

You must select at least two traits to use the {{ tool }}.

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4e81c29c..187b60dc 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -743,7 +743,7 @@ def corr_matrix_page(): start_vars = request.form traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] - if traits[0] != "": + if len(traits) > 1: template_vars = show_corr_matrix.CorrelationMatrix(start_vars) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, -- cgit v1.2.3 From 51383c96dc275f4462716398f9ec665fbd84802a Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 2 Mar 2018 17:59:28 +0000 Subject: Added option to display selected covariates for GEMMA mapping Changed layout/appearance of mapping options Partially fixed issue with deleting collections Changes to correlation table still in progress; having trouble getting tabulator to load JSON --- wqflask/wqflask/collect.py | 7 +- wqflask/wqflask/marker_regression/gemma_mapping.py | 4 +- .../javascript/get_covariates_from_collection.js | 5 + .../wqflask/static/new/javascript/show_trait.js | 3 +- wqflask/wqflask/templates/collections/list.html | 12 +- wqflask/wqflask/templates/correlation_page.html | 43 +++-- .../templates/show_trait_mapping_tools.html | 208 ++++++++++----------- 7 files changed, 146 insertions(+), 136 deletions(-) diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py index 2f6c3a96..0f72b29d 100644 --- a/wqflask/wqflask/collect.py +++ b/wqflask/wqflask/collect.py @@ -349,8 +349,11 @@ def delete_collection(): db_session.delete(uc) db_session.commit() else: - collection_name = params['collection_name'] - user_manager.AnonUser().delete_collection(collection_name) + if "collection_name" in params: + collection_name = params['collection_name'] + else: + for this_collection in params['uc_id'].split(":"): + user_manager.AnonUser().delete_collection(this_collection) flash("We've deleted the collection: {}.".format(collection_name), "alert-info") diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 233a5c76..7563eb2c 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -235,7 +235,7 @@ def parse_loco_output(this_dataset, gwa_output_filename): else: marker['chr'] = line.split("\t")[0] marker['Mb'] = float(line.split("\t")[2]) / 1000000 - marker['p_value'] = float(line.split("\t")[10]) + marker['p_value'] = float(line.split("\t")[11]) if math.isnan(marker['p_value']) or (marker['p_value'] <= 0): marker['lod_score'] = 0 #marker['lrs_value'] = 0 @@ -245,6 +245,6 @@ def parse_loco_output(this_dataset, gwa_output_filename): marker_obs.append(marker) included_markers.append(line.split("\t")[1]) - p_values.append(float(line.split("\t")[10])) + p_values.append(float(line.split("\t")[11])) return marker_obs \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js index 92e2b13b..f2e694d8 100644 --- a/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js +++ b/wqflask/wqflask/static/new/javascript/get_covariates_from_collection.js @@ -24,6 +24,7 @@ collection_click = function() { submit_click = function() { var covariates_string = ""; + var covariates_display_string = ""; $('#collections_holder').find('input[type=checkbox]:checked').each(function() { var this_dataset, this_trait; this_trait = $(this).parents('tr').find('.trait').text(); @@ -31,12 +32,15 @@ submit_click = function() { this_dataset = $(this).parents('tr').find('.dataset').text(); console.log("this_dataset is:", this_dataset); covariates_string += this_trait + ":" + this_dataset + "," + covariates_display_string += this_trait + "\n" }); // Trim the last comma covariates_string = covariates_string.substring(0, covariates_string.length - 1) + //covariates_display_string = covariates_display_string.substring(0, covariates_display_string.length - 2) console.log("COVARIATES:", covariates_string) $("input[name=covariates]").val(covariates_string) + $(".selected_covariates").val(covariates_display_string) return $.colorbox.close(); }; @@ -46,6 +50,7 @@ trait_click = function() { trait = $(this).parent().find('.trait').text(); dataset = $(this).parent().find('.dataset').text(); $("input[name=covariates]").val(trait + ":" + dataset) + $(".selected_covariates").text(trait) return $.colorbox.close(); }; diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 099d8010..df10c060 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -88,7 +88,8 @@ }; })(this)); $("#remove_covariates").click(function () { - $("input[name=covariates]").val("") + $("input[name=covariates]").val("") + $(".selected_covariates").val("") }); d3.select("#clear_compare_trait").on("click", (function(_this) { return function() { diff --git a/wqflask/wqflask/templates/collections/list.html b/wqflask/wqflask/templates/collections/list.html index ad72052e..e7f3229b 100644 --- a/wqflask/wqflask/templates/collections/list.html +++ b/wqflask/wqflask/templates/collections/list.html @@ -26,13 +26,13 @@ {% endif %}
-
- -
+
+ +
- +

@@ -51,7 +51,11 @@
{{ loop.index }} {% if g.user_session.user_ob %} {{ uc.name }}
- +{% endif %} {% endblock %} {% block js %} diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html index c5f815ce..ef233333 100644 --- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html +++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html @@ -1,9 +1,10 @@
+
- -
+ +
{% for tissue in corr_tools.dataset_menu %} {% if tissue.tissue %} @@ -37,8 +38,8 @@
- -
+ +
{% for group, pretty_group in sample_group_types.items() %} @@ -64,8 +65,8 @@
- -
+ +
- -
+ +
Chr:     Mb:  to  @@ -89,8 +90,8 @@
- -
+ +
@@ -103,48 +104,45 @@
- -
+ +
- -
- -
- - The Sample Correlation - is computed - between trait data and any - other traits in the sample database selected above. Use - Spearman - Rank - when the sample size is small (<20) or when there are influential outliers. - - - -
-
+
+
+ + The Sample Correlation + is computed + between trait data and any + other traits in the sample database selected above. Use + Spearman + Rank + when the sample size is small (<20) or when there are influential outliers. + + + +
\ No newline at end of file diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index ab2e2aae..03590c2c 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -1,5 +1,5 @@
- {% if (use_pylmm_rqtl and dataset.group.species != "human") or use_plink_gemma %} + {% if dataset.group.mapping_names|length > 0 %}
@@ -41,7 +41,7 @@
{% if genofiles and genofiles|length>0 %}
- +
- -
+ +
@@ -75,7 +75,7 @@
-->
- +
{% if g.user_session.user_ob and (g.user_session.user_ob.display_num_collections() == "") %} No collections available. Please add traits to a collection to use them as covariates. @@ -86,13 +86,13 @@
- + {% endif %}
- -
+ +
@@ -116,7 +116,7 @@
{% if genofiles and genofiles|length>0 %}
- +
- +
- -
+ +
{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} {% else %} @@ -184,8 +184,8 @@
- -
+ +
- -
+ +
@@ -212,7 +212,7 @@
{% if genofiles and genofiles|length>0 %}
- +
- -
+ +
{% if dataset.type == 'ProbeSet' and this_trait.locus_chr != "" %} {% else %} @@ -248,7 +248,7 @@
- +
@@ -289,8 +289,8 @@
-->
- -
+ +
- -
+ +
@@ -316,7 +316,7 @@
{% if genofiles and genofiles|length>0 %}
- +
{% for item in genofiles %} @@ -354,13 +354,13 @@
{% endif %}
- +
- +
{% if g.user_session.user_ob and (g.user_session.user_ob.display_num_collections() == "") %} No collections available. Please add traits to a collection to use them as covariates. @@ -376,8 +376,8 @@
- -
+ +
@@ -389,16 +389,16 @@