From 855a586f82fd1774358d8cfde28ce067f1c6c2ae Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Thu, 7 Mar 2013 00:34:51 +0000 Subject: Created file temp_data to store data related to progress of the marker regression calculations Storing progress of kinship matrix calculation in variable as portion of 45 (the rough percent of total marker regression calculation time --- wqflask/utility/temp_data.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 wqflask/utility/temp_data.py (limited to 'wqflask/utility/temp_data.py') diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py new file mode 100644 index 00000000..192dcc44 --- /dev/null +++ b/wqflask/utility/temp_data.py @@ -0,0 +1,27 @@ +from __future__ import print_function, division, absolute_import +from redis import Redis + +import simplejson as json + +class TempData(object): + + def __init__(self, temp_uuid): + self.temp_uuid = temp_uuid + self.redis = Redis() + self.key = "tempdata:{}".format(self.temp_uuid) + + def store(self, field, value): + print("Storing...") + self.redis.hset(self.key, field, value) + self.redis.expire(self.key, 60*15) # Expire in 15 minutes + + def get_all(self): + return self.redis.hgetall(self.key) + + + +if __name__ == "__main__": + redis = Redis() + for key in redis.keys(): + for field in redis.hkeys(key): + print("{}.{}={}".format(key, field, redis.hget(key, field))) -- cgit v1.2.3 From 3624c63e3373cb45ffcc8cfdbb8889765a3b5326 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Thu, 7 Mar 2013 21:56:27 +0000 Subject: Have first 45% of progress bar working Changed "threaded" to True in runserver.py; this fixed issue related to getting progress bar to work --- wqflask/runserver.py | 1 + wqflask/utility/temp_data.py | 1 - .../new/javascript/show_trait_mapping_tools.coffee | 29 ++++++--- .../new/javascript/show_trait_mapping_tools.js | 70 +++++++++++++++++++++- wqflask/wqflask/templates/show_trait.html | 2 +- .../wqflask/templates/show_trait_progress_bar.html | 2 +- 6 files changed, 94 insertions(+), 11 deletions(-) (limited to 'wqflask/utility/temp_data.py') diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 2a75faeb..1cf38f8e 100644 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -29,4 +29,5 @@ logging_tree.printout() app.run(host='0.0.0.0', use_debugger=False, + threaded=True, use_reloader=True) diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 192dcc44..0df79e54 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -19,7 +19,6 @@ class TempData(object): return self.redis.hgetall(self.key) - if __name__ == "__main__": redis = Redis() for key in redis.keys(): diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee index 50b97171..35572f67 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee @@ -11,14 +11,26 @@ $ -> $("#trait_data_form").attr("action", url); $("#trait_data_form").submit() - get_progress () => + get_progress = -> console.log("temp_uuid:", $("#temp_uuid").val()) - + temp_uuid = $("#temp_uuid").val() + params = { key:temp_uuid } + params_str = $.param(params) + url = "/get_temp_data?" + params_str + console.log("url:", url) + $.ajax( + type: "GET" + url: url + success: (progress_data) => + console.log("in get_progress data:", progress_data) + console.log(progress_data['percent_complete'] + "%") + $('#marker_regression_progress').css("width", progress_data['percent_complete'] + "%") + ) + return false + $("#marker_regression").click(() => $("#progress_bar_container").modal() - - get_progress() - + url = "/marker_regression" form_data = $('#trait_data_form').serialize() console.log("form_data is:", form_data) @@ -27,12 +39,15 @@ $ -> url: url data: form_data success: (data) => + clearInterval(this.my_timer) $('#progress_bar_container').modal('hide') $("body").html(data) ) + console.log("settingInterval") + this.my_timer = setInterval(get_progress, 1000) return false ) - + #$(".submit_special").click(submit_special) composite_mapping_fields = -> @@ -51,4 +66,4 @@ $ -> $("#display_all_lrs").change(-> toggle_enable_disable("#suggestive_lrs") - ) \ No newline at end of file + ); \ No newline at end of file diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index c8b0aa7b..78459692 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -1,6 +1,74 @@ // Generated by CoffeeScript 1.4.0 (function() { - + $(function() { + var composite_mapping_fields, get_progress, submit_special, toggle_enable_disable, + _this = this; + submit_special = function() { + var url; + console.log("In submit_special"); + console.log("this is:", this); + console.log("$(this) is:", $(this)); + url = $(this).data("url"); + console.log("url is:", url); + $("#trait_data_form").attr("action", url); + return $("#trait_data_form").submit(); + }; + get_progress = function() { + var params, params_str, temp_uuid, url, + _this = this; + console.log("temp_uuid:", $("#temp_uuid").val()); + temp_uuid = $("#temp_uuid").val(); + params = { + key: temp_uuid + }; + params_str = $.param(params); + url = "/get_temp_data?" + params_str; + console.log("url:", url); + $.ajax({ + type: "GET", + url: url, + success: function(progress_data) { + console.log("in get_progress data:", progress_data); + console.log(progress_data['percent_complete'] + "%"); + return $('#marker_regression_progress').css("width", progress_data['percent_complete'] + "%"); + } + }); + return false; + }; + $("#marker_regression").click(function() { + var form_data, url; + $("#progress_bar_container").modal(); + url = "/marker_regression"; + form_data = $('#trait_data_form').serialize(); + console.log("form_data is:", form_data); + $.ajax({ + type: "POST", + url: url, + data: form_data, + success: function(data) { + clearInterval(_this.my_timer); + $('#progress_bar_container').modal('hide'); + return $("body").html(data); + } + }); + console.log("settingInterval"); + _this.my_timer = setInterval(get_progress, 1000); + return false; + }); + composite_mapping_fields = function() { + return $(".composite_fields").toggle(); + }; + $("#use_composite_choice").change(composite_mapping_fields); + toggle_enable_disable = function(elem) { + return $(elem).prop("disabled", !$(elem).prop("disabled")); + }; + $("#choose_closet_control").change(function() { + return toggle_enable_disable("#control_locus"); + }); + return $("#display_all_lrs").change(function() { + return toggle_enable_disable("#suggestive_lrs"); + }); + }); }).call(this); diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 2d569b20..d8f44d53 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -22,7 +22,7 @@ {% endfor %} - +
diff --git a/wqflask/wqflask/templates/show_trait_progress_bar.html b/wqflask/wqflask/templates/show_trait_progress_bar.html index a7ee2195..2984cc02 100644 --- a/wqflask/wqflask/templates/show_trait_progress_bar.html +++ b/wqflask/wqflask/templates/show_trait_progress_bar.html @@ -4,7 +4,7 @@
-
+
\ No newline at end of file -- cgit v1.2.3 From 27d0c86528fdbeaed56a7f5eaa71a950c522f183 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Fri, 5 Apr 2013 23:17:02 +0000 Subject: Loading bar now correctly shows the time to completion and the code seems to run correctly --- wqflask/other_config/wqflask-nginx.conf | 2 +- wqflask/utility/temp_data.py | 1 - .../wqflask/marker_regression/marker_regression.py | 9 +++-- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 15 ++++---- .../new/javascript/show_trait_mapping_tools.coffee | 42 +++++++++++++++++++--- .../new/javascript/show_trait_mapping_tools.js | 36 ++++++++++++++++--- .../wqflask/templates/show_trait_progress_bar.html | 2 ++ 7 files changed, 86 insertions(+), 21 deletions(-) (limited to 'wqflask/utility/temp_data.py') diff --git a/wqflask/other_config/wqflask-nginx.conf b/wqflask/other_config/wqflask-nginx.conf index f0076f54..50f9d73c 100644 --- a/wqflask/other_config/wqflask-nginx.conf +++ b/wqflask/other_config/wqflask-nginx.conf @@ -36,7 +36,7 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_read_timeout 20m; + proxy_read_timeout 40m; } } diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 0df79e54..004d45c6 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -11,7 +11,6 @@ class TempData(object): self.key = "tempdata:{}".format(self.temp_uuid) def store(self, field, value): - print("Storing...") self.redis.hset(self.key, field, value) self.redis.expire(self.key, 60*15) # Expire in 15 minutes diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 5f4f9be8..545c8162 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -71,7 +71,7 @@ class MarkerRegression(object): p_values, t_stats = self.gen_human_results(pheno_vector, tempdata) else: genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] - + no_val_samples = self.identify_empty_samples() trimmed_genotype_data = self.trim_genotypes(genotype_data, no_value_samples=[]) pdb.set_trace() @@ -90,7 +90,12 @@ class MarkerRegression(object): self.dataset.group.markers.add_pvalues(p_values) - self.qtl_results = self.dataset.group.markers.markers + self.qtl_results = [] + for marker in self.dataset.group.markers.markers: + if marker['p_value'] < 0.2: + self.qtl_results.append(marker) + + #self.qtl_results = self.dataset.group.markers.markers def gen_human_results(self, pheno_vector, tempdata): diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 4de77173..0e965c8e 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -62,17 +62,18 @@ def run_human(pheno_vector, plink_input.getSNPIterator() total_snps = plink_input.numSNPs - - count = 0 + with Bench("snp iterator loop"): + count = 0 for snp, this_id in plink_input: - #if count > 1000: + #if count > 5000: # break count += 1 + percent_complete = (float(count) / total_snps) * 100 - print("percent_complete: ", pf(percent_complete)) - temp_data.store("percent_complete", percent_complete) + #print("percent_complete: ", percent_complete) + temp_data.store("percent_complete", percent_complete) x = snp[keep].reshape((n,1)) #x[[1,50,100,200,3000],:] = np.nan @@ -113,11 +114,9 @@ def run_human(pheno_vector, if refit: lmm_ob.fit(X=x) ts, ps, beta, betaVar = lmm_ob.association(x) - + p_values.append(ps) t_stats.append(ts) - - print("p_values: ", pf(p_values)) return p_values, t_stats diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee index 22427e4f..bc176ab9 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.coffee @@ -1,3 +1,6 @@ +# http://stackoverflow.com/a/4215132/1175849 +root = exports ? this + $ -> submit_special = -> # Add submit_special class plus a data-url field to any button @@ -10,7 +13,29 @@ $ -> console.log("url is:", url) $("#trait_data_form").attr("action", url); $("#trait_data_form").submit() + + #update_time_remaining = (percent_complete) -> + # seconds_remaining = 1 / (percent_complete - root.previous_percent) * (100 - percent_complete) + # minutes_remaining = seconds_remaining / 60 + # $('#time_remaining').text(Math.round(minutes_remaining) + " minutes remaining") + # root.previous_percent = percent_complete + update_time_remaining = (percent_complete) -> + now = new Date() + period = now.getTime() - root.start_time + console.log("period is:", period) + if period > 8000 + total_seconds_remaining = (period / percent_complete * (100 - percent_complete))/1000 + minutes_remaining = Math.round(total_seconds_remaining / 60) + #seconds_remaining = Math.round(total_seconds_remaining % 60) + #console.log("seconds_remaining:", seconds_remaining) + if minutes_remaining < 3 + $('#time_remaining').text(Math.round(total_seconds_remaining) + " seconds remaining") + else + $('#time_remaining').text(minutes_remaining + " minutes remaining") + #else + # $('#time_remaining').text("period too small") + get_progress = -> console.log("temp_uuid:", $("#temp_uuid").val()) temp_uuid = $("#temp_uuid").val() @@ -22,9 +47,16 @@ $ -> type: "GET" url: url success: (progress_data) => + percent_complete = progress_data['percent_complete'] console.log("in get_progress data:", progress_data) - console.log(progress_data['percent_complete'] + "%") - $('#marker_regression_progress').css("width", progress_data['percent_complete'] + "%") + + $('#marker_regression_progress').css("width", percent_complete + "%") + + if root.start_time + unless isNaN(percent_complete) + update_time_remaining(percent_complete) + else + root.start_time = new Date().getTime() ) return false @@ -38,17 +70,19 @@ $ -> url: url data: form_data error: (xhr, ajaxOptions, thrownError) => - alert("some error occurred") + alert("Sorry, an error occurred") console.log(xhr) clearInterval(this.my_timer) $('#progress_bar_container').modal('hide') - $("body").html("error") + $("body").html("We got an error.") success: (data) => clearInterval(this.my_timer) $('#progress_bar_container').modal('hide') $("body").html(data) ) console.log("settingInterval") + + #root.start_time = new Date().getTime() this.my_timer = setInterval(get_progress, 1000) return false ) diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index daf9b273..b017257a 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -1,8 +1,11 @@ // Generated by CoffeeScript 1.4.0 (function() { + var root; + + root = typeof exports !== "undefined" && exports !== null ? exports : this; $(function() { - var composite_mapping_fields, get_progress, submit_special, toggle_enable_disable, + var composite_mapping_fields, get_progress, submit_special, toggle_enable_disable, update_time_remaining, _this = this; submit_special = function() { var url; @@ -14,6 +17,21 @@ $("#trait_data_form").attr("action", url); return $("#trait_data_form").submit(); }; + update_time_remaining = function(percent_complete) { + var minutes_remaining, now, period, total_seconds_remaining; + now = new Date(); + period = now.getTime() - root.start_time; + console.log("period is:", period); + if (period > 8000) { + total_seconds_remaining = (period / percent_complete * (100 - percent_complete)) / 1000; + minutes_remaining = Math.round(total_seconds_remaining / 60); + if (minutes_remaining < 3) { + return $('#time_remaining').text(Math.round(total_seconds_remaining) + " seconds remaining"); + } else { + return $('#time_remaining').text(minutes_remaining + " minutes remaining"); + } + } + }; get_progress = function() { var params, params_str, temp_uuid, url, _this = this; @@ -29,9 +47,17 @@ type: "GET", url: url, success: function(progress_data) { + var percent_complete; + percent_complete = progress_data['percent_complete']; console.log("in get_progress data:", progress_data); - console.log(progress_data['percent_complete'] + "%"); - return $('#marker_regression_progress').css("width", progress_data['percent_complete'] + "%"); + $('#marker_regression_progress').css("width", percent_complete + "%"); + if (root.start_time) { + if (!isNaN(percent_complete)) { + return update_time_remaining(percent_complete); + } + } else { + return root.start_time = new Date().getTime(); + } } }); return false; @@ -47,11 +73,11 @@ url: url, data: form_data, error: function(xhr, ajaxOptions, thrownError) { - alert("some error occurred"); + alert("Sorry, an error occurred"); console.log(xhr); clearInterval(_this.my_timer); $('#progress_bar_container').modal('hide'); - return $("body").html("error"); + return $("body").html("We got an error."); }, success: function(data) { clearInterval(_this.my_timer); diff --git a/wqflask/wqflask/templates/show_trait_progress_bar.html b/wqflask/wqflask/templates/show_trait_progress_bar.html index eff5c391..0c3f0e6e 100644 --- a/wqflask/wqflask/templates/show_trait_progress_bar.html +++ b/wqflask/wqflask/templates/show_trait_progress_bar.html @@ -6,5 +6,7 @@
+
+
\ No newline at end of file -- cgit v1.2.3 From ea53a2f20d13130f3555967d57282b3c9562da5a Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Wed, 17 Apr 2013 23:49:47 +0000 Subject: Created file with pickled SNPIterator (from input.py) data for HLC datasets Still need to read in file --- wqflask/base/webqtlConfig.py | 1 + wqflask/utility/temp_data.py | 6 ++- .../wqflask/marker_regression/marker_regression.py | 48 ++++++++++++---------- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 27 ++++++++---- 4 files changed, 50 insertions(+), 32 deletions(-) (limited to 'wqflask/utility/temp_data.py') diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 1845c749..49afb631 100755 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -53,6 +53,7 @@ SECUREDIR = GNROOT + 'secure/' COMMON_LIB = GNROOT + 'support/admin' HTMLPATH = GNROOT + 'web/' PYLMM_PATH = HTMLPATH + 'plink/' +SNP_PATH = '/mnt/xvdf1/snps/' IMGDIR = HTMLPATH +'image/' IMAGESPATH = HTMLPATH + 'images/' UPLOADPATH = IMAGESPATH + 'upload/' diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 004d45c6..ddf2653c 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -5,14 +5,16 @@ import simplejson as json class TempData(object): - def __init__(self, temp_uuid): + def __init__(self, temp_uuid, part=None): self.temp_uuid = temp_uuid self.redis = Redis() self.key = "tempdata:{}".format(self.temp_uuid) + if part: + self.key += ":{}".format(part) def store(self, field, value): self.redis.hset(self.key, field, value) - self.redis.expire(self.key, 60*15) # Expire in 15 minutes + self.redis.expire(self.key, 60*60) # Expire in 60 minutes def get_all(self): return self.redis.hgetall(self.key) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 86d9fe06..2ede5660 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -9,11 +9,12 @@ import string import sys import os import collections -import pdb import numpy as np from scipy import linalg +import simplejson as json + #from redis import Redis @@ -41,8 +42,6 @@ class MarkerRegression(object): self.samples = [] # Want only ones with values self.vals = [] - print("start_vars: ", pf(start_vars)) - self.suggestive = float(start_vars['suggestive']) for sample in self.dataset.group.samplelist: value = start_vars['value:' + sample] @@ -52,13 +51,12 @@ class MarkerRegression(object): self.gen_data(tempdata) #Get chromosome lengths for drawing the manhattan plot - chromosomes = {} + chromosome_mb_lengths = {} for key in self.species.chromosomes.chromosomes.keys(): - this_chr = self.species.chromosomes.chromosomes[key] - chromosomes[key] = [this_chr.name, this_chr.mb_length] + chromosome_mb_lengths[key] = self.species.chromosomes.chromosomes[key].mb_length self.js_data = dict( - chromosomes = chromosomes, + chromosomes = chromosome_mb_lengths, qtl_results = self.qtl_results, ) @@ -74,10 +72,10 @@ class MarkerRegression(object): p_values, t_stats = self.gen_human_results(pheno_vector, tempdata) else: genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers] - + no_val_samples = self.identify_empty_samples() - trimmed_genotype_data = self.trim_genotypes(genotype_data, no_value_samples=[]) - pdb.set_trace() + trimmed_genotype_data = self.trim_genotypes(genotype_data, no_val_samples) + genotype_matrix = np.array(trimmed_genotype_data).T print("pheno_vector is: ", pf(pheno_vector)) @@ -90,23 +88,16 @@ class MarkerRegression(object): refit=False, temp_data=tempdata ) - + self.dataset.group.markers.add_pvalues(p_values) - self.qtl_results = [] - for marker in self.dataset.group.markers.markers: - if marker['lod_score'] >= self.suggestive: - self.qtl_results.append(marker) - - #self.qtl_results = self.dataset.group.markers.markers + self.qtl_results = self.dataset.group.markers.markers def gen_human_results(self, pheno_vector, tempdata): file_base = os.path.join(webqtlConfig.PYLMM_PATH, self.dataset.group.name) - - tempdata.store("percent_complete", 0) + plink_input = input.plink(file_base, type='b') - tempdata.store("percent_complete", 0.1) pheno_vector = pheno_vector.reshape((len(pheno_vector), 1)) covariate_matrix = np.ones((pheno_vector.shape[0],1)) @@ -118,11 +109,11 @@ class MarkerRegression(object): covariate_matrix, plink_input, kinship_matrix, - temp_data=tempdata + loading_progress=tempdata ) return p_values, t_stats - + def identify_empty_samples(self): no_val_samples = [] @@ -147,4 +138,17 @@ class MarkerRegression(object): trimmed_genotype_data.append(new_genotypes) return trimmed_genotype_data +def create_snp_iterator_file(group): + plink_file_base = os.path.join(webqtlConfig.PYLMM_PATH, group) + plink_input = input.plink(plink_file_base, type='b') + inputs = list(plink_input) + snp_file_base = os.path.join(webqtlConfig.SNP_PATH, group + ".snps") + + with open(snp_file_base, "w") as fh: + pickle.dump(inputs, fh) + + +if __name__ == '__main__': + import cPickle as pickle + create_snp_iterator_file("HLC") diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 24565af8..918f8200 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -19,15 +19,21 @@ from __future__ import absolute_import, print_function, division import sys import time +import uuid + import numpy as np from scipy import linalg from scipy import optimize from scipy import stats import pdb +#import cPickle as pickle +import simplejson as json + from pprint import pformat as pf from utility.benchmark import Bench +from utility import temp_data from wqflask.my_pylmm.pyLMM import chunks @@ -38,7 +44,7 @@ def run_human(pheno_vector, plink_input, kinship_matrix, refit=False, - temp_data=None): + loading_progress=None): v = np.isnan(pheno_vector) keep = True - v @@ -65,27 +71,32 @@ def run_human(pheno_vector, plink_input.getSNPIterator() total_snps = plink_input.numSNPs - number_chunks = 63 - with Bench("snp iterator loop"): count = 0 - - + with Bench("Create list of inputs"): inputs = list(plink_input) with Bench("Divide into chunks"): - results = chunks.divide_into_chunks(inputs, 63) + results = chunks.divide_into_chunks(inputs, 64) + result_store = [] + identifier = uuid.uuid4() + for part, result in enumerate(results): + data_store = temp_data.TempData(identifier, part) + + data_store.store(data=json.dumps(result.tolist())) + result_store.append(data_store) + for snp, this_id in plink_input: with Bench("part before association"): - if count > 500: + if count > 2000: break count += 1 percent_complete = (float(count) / total_snps) * 100 #print("percent_complete: ", percent_complete) - temp_data.store("percent_complete", percent_complete) + loading_progress.store("percent_complete", percent_complete) with Bench("actual association"): ps, ts = human_association(snp, -- cgit v1.2.3 From 5ffd0debd5ab7ee0e98def74374a8e996629f5c9 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Thu, 18 Apr 2013 22:36:39 +0000 Subject: The plink_input is split into chunks that are stored in temp_data, but we might decide to store it differently --- wqflask/utility/temp_data.py | 14 +++++++++----- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 8 ++++---- 2 files changed, 13 insertions(+), 9 deletions(-) (limited to 'wqflask/utility/temp_data.py') diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index ddf2653c..60f01167 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -1,14 +1,15 @@ from __future__ import print_function, division, absolute_import from redis import Redis +import redis import simplejson as json class TempData(object): - - def __init__(self, temp_uuid, part=None): + + def __init__(self, temp_uuid, preface="tempdata", part=None): self.temp_uuid = temp_uuid self.redis = Redis() - self.key = "tempdata:{}".format(self.temp_uuid) + self.key = "{}:{}".format(preface, self.temp_uuid) if part: self.key += ":{}".format(part) @@ -19,9 +20,12 @@ class TempData(object): def get_all(self): return self.redis.hgetall(self.key) - if __name__ == "__main__": redis = Redis() for key in redis.keys(): + print("key is:", key) + if "plink" not in key: + print(" Skipping...\n") + continue for field in redis.hkeys(key): - print("{}.{}={}".format(key, field, redis.hget(key, field))) + print(" {}.{}={}\n".format(key, field, len(redis.hget(key, field)))) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 8c0e0282..a6134fdd 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -89,13 +89,13 @@ def run_human(pheno_vector, with Bench("Divide into chunks"): results = chunks.divide_into_chunks(inputs, 64) - + result_store = [] identifier = uuid.uuid4() for part, result in enumerate(results): - data_store = temp_data.TempData(identifier, part) + data_store = temp_data.TempData(identifier, "plink", part) - data_store.store(data=pickle.dumps(result)) + data_store.store("data", pickle.dumps(result, pickle.HIGHEST_PROTOCOL)) result_store.append(data_store) for snp, this_id in plink_input: @@ -103,7 +103,7 @@ def run_human(pheno_vector, if count > 2000: break count += 1 - + percent_complete = (float(count) / total_snps) * 100 #print("percent_complete: ", percent_complete) loading_progress.store("percent_complete", percent_complete) -- cgit v1.2.3 From 8810c7735ed8a1bfa225449f7b388438e2ace890 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Tue, 23 Apr 2013 21:37:55 +0000 Subject: Created file correlation_plot.py for the correlation scatterplot Reverted temp_data.py to previous version that doesn't include the "part" input (for chunks) Made change to lmm related to splitting main iterator into chunks Deleted a bunch of unnecessary commented out code from show_trait.py --- misc/notes.txt | 12 + wqflask/maintenance/quick_search_table.py | 32 +- wqflask/utility/temp_data.py | 18 +- wqflask/wqflask/correlation/correlation_plot.py | 48 +++ wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 109 +++---- wqflask/wqflask/show_trait/show_trait.py | 377 +----------------------- 6 files changed, 139 insertions(+), 457 deletions(-) create mode 100644 wqflask/wqflask/correlation/correlation_plot.py (limited to 'wqflask/utility/temp_data.py') diff --git a/misc/notes.txt b/misc/notes.txt index a48ee5bf..6bdcccf3 100644 --- a/misc/notes.txt +++ b/misc/notes.txt @@ -107,6 +107,11 @@ sudo /etc/init.d/redis_6379 start =========================================== Start screen session: + +If "no option -R": +byobu-select-backend +2. screen + byobu -RD (to start) control-a then :multiuser on control-a then :acladd sam @@ -172,6 +177,13 @@ cp -a (archive; copies recursively and doesn't follow symbol links) mv (same as above, but with no -a) +=========================================== + +Add user: +sudo adduser +Edit /etc/sudoers to give user root privileges + + =========================================== tidyp - Improves/beautifies html code diff --git a/wqflask/maintenance/quick_search_table.py b/wqflask/maintenance/quick_search_table.py index d175e600..046a05c4 100644 --- a/wqflask/maintenance/quick_search_table.py +++ b/wqflask/maintenance/quick_search_table.py @@ -42,7 +42,7 @@ Metadata.bind = Engine class PublishXRef(Base): __tablename__ = 'PublishXRef' - + Id = sa.Column(sa.Integer, primary_key=True) InbredSetId = sa.Column(sa.Integer, primary_key=True) PhenotypeId = sa.Column(sa.Integer) @@ -53,7 +53,7 @@ class PublishXRef(Base): additive = sa.Column(sa.Float) Sequence = sa.Column(sa.Integer) comments = sa.Column(sa.Text) - + @classmethod def run(cls): conn = Engine.connect() @@ -69,7 +69,7 @@ class PublishXRef(Base): conn.execute(ins) counter += 1 print("Done:", counter) - + @staticmethod def get_unique_terms(publishxref_id, inbredset_id): results = Session.query( @@ -114,7 +114,7 @@ class PublishXRef(Base): continue unique.add(token) - print("\nUnique terms are: {}\n".format(unique)) + #print("\nUnique terms are: {}\n".format(unique)) return " ".join(unique) @staticmethod @@ -155,12 +155,12 @@ class PublishXRef(Base): #"Geno.SpeciesId = Species.Id and " #"Geno.Name = PublishXRef.Locus ").params(publishxref_id=publishxref_id, # inbredset_id=inbredset_id).all() - for result in results: - print("****", result) + #for result in results: + # print("****", result) assert len(set(result for result in results)) == 1, "Different results or no results" - print("results are:", results) + #print("results are:", results) result = results[0] result = row2dict(result) try: @@ -214,7 +214,7 @@ class GenoXRef(Base): "FROM Geno " "WHERE Geno.Id = :geno_id ").params(geno_id=geno_id).all() - print("results: ", pf(results)) + #print("results: ", pf(results)) unique = set() if len(results): @@ -234,7 +234,7 @@ class GenoXRef(Base): continue unique.add(token) - print("\nUnique terms are: {}\n".format(unique)) + #print("\nUnique terms are: {}\n".format(unique)) return " ".join(unique) @@ -271,11 +271,11 @@ class GenoXRef(Base): "InbredSet.Id = GenoFreeze.InbredSetId and " "InbredSet.SpeciesId = Species.Id ").params(geno_id=geno_id, dataset_id=dataset_id).all() - for result in results: - print(result) + #for result in results: + # print(result) assert len(set(result for result in results)) == 1, "Different results" - print("results are:", results) + #print("results are:", results) result = results[0] result = row2dict(result) try: @@ -366,7 +366,7 @@ class ProbeSetXRef(Base): continue unique.add(token) - print("\nUnique terms are: {}\n".format(unique)) + #print("\nUnique terms are: {}\n".format(unique)) return " ".join(unique) @@ -420,14 +420,14 @@ class ProbeSetXRef(Base): "ProbeFreeze.InbredSetId = InbredSet.Id and " "InbredSet.SpeciesId = Species.Id ").params(probeset_id=probeset_id, dataset_id=dataset_id).all() - for result in results: - print("-", result) + #for result in results: + # print("-", result) if len(set(result for result in results)) != 1: return None #assert len(set(result for result in results)) == 1, "Different results" - print("results are:", results) + #print("results are:", results) result = results[0] result = row2dict(result) try: diff --git a/wqflask/utility/temp_data.py b/wqflask/utility/temp_data.py index 60f01167..004d45c6 100644 --- a/wqflask/utility/temp_data.py +++ b/wqflask/utility/temp_data.py @@ -1,31 +1,25 @@ from __future__ import print_function, division, absolute_import from redis import Redis -import redis import simplejson as json class TempData(object): - - def __init__(self, temp_uuid, preface="tempdata", part=None): + + def __init__(self, temp_uuid): self.temp_uuid = temp_uuid self.redis = Redis() - self.key = "{}:{}".format(preface, self.temp_uuid) - if part: - self.key += ":{}".format(part) + self.key = "tempdata:{}".format(self.temp_uuid) def store(self, field, value): self.redis.hset(self.key, field, value) - self.redis.expire(self.key, 60*60) # Expire in 60 minutes + self.redis.expire(self.key, 60*15) # Expire in 15 minutes def get_all(self): return self.redis.hgetall(self.key) + if __name__ == "__main__": redis = Redis() for key in redis.keys(): - print("key is:", key) - if "plink" not in key: - print(" Skipping...\n") - continue for field in redis.hkeys(key): - print(" {}.{}={}\n".format(key, field, len(redis.hget(key, field)))) + print("{}.{}={}".format(key, field, redis.hget(key, field))) diff --git a/wqflask/wqflask/correlation/correlation_plot.py b/wqflask/wqflask/correlation/correlation_plot.py new file mode 100644 index 00000000..4b043fc3 --- /dev/null +++ b/wqflask/wqflask/correlation/correlation_plot.py @@ -0,0 +1,48 @@ +#!/usr/bin/python + +from __future__ import print_function, division + +from base.trait import GeneralTrait +from base import data_set +from wqflask.show_trait.SampleList import SampleList + +class CorrelationPlot(object): + """Page that displays a correlation scatterplot with a line fitted to it""" + + def __init__(self, start_vars): + self.dataset1 = data_set.create_dataset(start_vars['dataset1']) + self.trait1 = GeneralTrait(dataset=self.dataset1.name, + name=start_vars['trait1']) + + self.dataset2 = data_set.create_dataset(start_vars['dataset2']) + self.trait2 = GeneralTrait(dataset=self.dataset2.name, + name=start_vars['trait2']) + + sample_names_1 = self.get_sample_names(self.dataset1) + sample_names_2 = self.get_sample_names(self.dataset2) + + self.samples_1 = self.get_samples(self.dataset1, sample_names_1, self.trait1) + self.samples_2 = self.get_samples(self.dataset2, sample_names_2, self.trait2) + + + def get_sample_names(self, dataset): + if dataset.group.parlist: + sample_names = (dataset.group.parlist + + dataset.group.f1list + + dataset.group.samplelist) + elif dataset.group.f1list: + sample_names = dataset.group.f1list + dataset.group.samplelist + else: + sample_names = dataset.group.samplelist + + return sample_names + + + def get_samples(self, dataset, sample_names, trait): + samples = SampleList(dataset = dataset, + sample_names=sample_names, + this_trait=trait, + sample_group_type='primary', + header="%s Only" % (dataset.group.name)) + + return samples \ No newline at end of file diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 2e8f020d..a3ba8fdb 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -468,7 +468,7 @@ class LMM: the heritability or the proportion of the total variance attributed to genetics. The X is the covariate matrix. """ - + S = 1.0/(h*self.Kva + (1.0 - h)) Xt = X.T*S XX = matrixMult(Xt,X) @@ -487,67 +487,68 @@ class LMM: def LL(self,h,X=None,stack=True,REML=False): - """ - Computes the log-likelihood for a given heritability (h). If X==None, then the - default X0t will be used. If X is set and stack=True, then X0t will be matrix concatenated with - the input X. If stack is false, then X is used in place of X0t in the LL calculation. - REML is computed by adding additional terms to the standard LL and can be computed by setting REML=True. - """ - - if X == None: X = self.X0t - elif stack: - self.X0t_stack[:,(self.q)] = matrixMult(self.Kve.T,X)[:,0] - X = self.X0t_stack + """ + Computes the log-likelihood for a given heritability (h). If X==None, then the + default X0t will be used. If X is set and stack=True, then X0t will be matrix concatenated with + the input X. If stack is false, then X is used in place of X0t in the LL calculation. + REML is computed by adding additional terms to the standard LL and can be computed by setting REML=True. + """ - n = float(self.N) - q = float(X.shape[1]) - beta,sigma,Q,XX_i,XX = self.getMLSoln(h,X) - LL = n*np.log(2*np.pi) + np.log(h*self.Kva + (1.0-h)).sum() + n + n*np.log(1.0/n * Q) - LL = -0.5 * LL + if X == None: + X = self.X0t + elif stack: + self.X0t_stack[:,(self.q)] = matrixMult(self.Kve.T,X)[:,0] + X = self.X0t_stack - if REML: - LL_REML_part = q*np.log(2.0*np.pi*sigma) + np.log(linalg.det(matrixMult(X.T,X))) - np.log(linalg.det(XX)) - LL = LL + 0.5*LL_REML_part + n = float(self.N) + q = float(X.shape[1]) + beta,sigma,Q,XX_i,XX = self.getMLSoln(h,X) + LL = n*np.log(2*np.pi) + np.log(h*self.Kva + (1.0-h)).sum() + n + n*np.log(1.0/n * Q) + LL = -0.5 * LL - return LL,beta,sigma,XX_i + if REML: + LL_REML_part = q*np.log(2.0*np.pi*sigma) + np.log(linalg.det(matrixMult(X.T,X))) - np.log(linalg.det(XX)) + LL = LL + 0.5*LL_REML_part + + return LL,beta,sigma,XX_i def getMax(self,H, X=None,REML=False): - """ - Helper functions for .fit(...). - This function takes a set of LLs computed over a grid and finds possible regions - containing a maximum. Within these regions, a Brent search is performed to find the - optimum. - - """ - n = len(self.LLs) - HOpt = [] - for i in range(1,n-2): - if self.LLs[i-1] < self.LLs[i] and self.LLs[i] > self.LLs[i+1]: - HOpt.append(optimize.brent(self.LL_brent,args=(X,REML),brack=(H[i-1],H[i+1]))) - if np.isnan(HOpt[-1][0]): - HOpt[-1][0] = [self.LLs[i-1]] - - if len(HOpt) > 1: - if self.verbose: - sys.stderr.write("NOTE: Found multiple optima. Returning first...\n") - return HOpt[0] - elif len(HOpt) == 1: - return HOpt[0] - elif self.LLs[0] > self.LLs[n-1]: - return H[0] - else: - return H[n-1] + """ + Helper functions for .fit(...). + This function takes a set of LLs computed over a grid and finds possible regions + containing a maximum. Within these regions, a Brent search is performed to find the + optimum. + + """ + n = len(self.LLs) + HOpt = [] + for i in range(1,n-2): + if self.LLs[i-1] < self.LLs[i] and self.LLs[i] > self.LLs[i+1]: + HOpt.append(optimize.brent(self.LL_brent,args=(X,REML),brack=(H[i-1],H[i+1]))) + if np.isnan(HOpt[-1][0]): + HOpt[-1][0] = [self.LLs[i-1]] + + if len(HOpt) > 1: + if self.verbose: + sys.stderr.write("NOTE: Found multiple optima. Returning first...\n") + return HOpt[0] + elif len(HOpt) == 1: + return HOpt[0] + elif self.LLs[0] > self.LLs[n-1]: + return H[0] + else: + return H[n-1] def fit(self,X=None,ngrids=100,REML=True): """ - Finds the maximum-likelihood solution for the heritability (h) given the current parameters. - X can be passed and will transformed and concatenated to X0t. Otherwise, X0t is used as - the covariate matrix. - - This function calculates the LLs over a grid and then uses .getMax(...) to find the optimum. - Given this optimum, the function computes the LL and associated ML solutions. + Finds the maximum-likelihood solution for the heritability (h) given the current parameters. + X can be passed and will transformed and concatenated to X0t. Otherwise, X0t is used as + the covariate matrix. + + This function calculates the LLs over a grid and then uses .getMax(...) to find the optimum. + Given this optimum, the function computes the LL and associated ML solutions. """ if X == None: @@ -575,8 +576,8 @@ class LMM: def association(self,X, h = None, stack=True,REML=True, returnBeta=True): """ - Calculates association statitics for the SNPs encoded in the vector X of size n. - If h == None, the optimal h stored in optH is used. + Calculates association statitics for the SNPs encoded in the vector X of size n. + If h == None, the optimal h stored in optH is used. """ if stack: diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 85e33595..60e42afb 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -41,26 +41,11 @@ class ShowTrait(object): helper_functions.get_species_dataset_trait(self, kw) - #self.dataset = create_dataset(kw['dataset']) - # - ##self.cell_id = None - # - # - #this_trait = GeneralTrait(dataset=self.dataset.name, - # name=self.trait_id, - # cellid=None) - # - # self.dataset.group.read_genotype_file() - #if not self.dataset.group.genotype: - # self.read_data(include_f1=True) - - # Todo: Add back in the ones we actually need from below, as we discover we need them hddn = OrderedDict() - ## Some fields, like method, are defaulted to None; otherwise in IE the field can't be changed using jquery #hddn = OrderedDict( # FormID = fmID, @@ -101,14 +86,8 @@ class ShowTrait(object): # this_trait.mysqlid) # heritability = self.cursor.fetchone() - #hddn['mappingMethodId'] = webqtlDatabaseFunction.getMappingMethod (cursor=self.cursor, - # groupName=fd.group) - self.dispTraitInformation(kw, "", hddn, self.this_trait) #Display trait information + function buttons - #if this_trait == None: - # this_trait = webqtlTrait(data=kw['allTraitData'], dataset=None) - self.build_correlation_tools(self.this_trait) self.make_sample_lists(self.this_trait) @@ -134,29 +113,9 @@ class ShowTrait(object): sample_lists = sample_lists, attribute_names = self.sample_groups[0].attributes, temp_uuid = self.temp_uuid) - #print("js_data:", pf(js_data)) self.js_data = js_data - #def get_this_trait(self): - # this_trait = GeneralTrait(dataset=self.dataset.name, - # name=self.trait_id, - # cellid=self.cell_id) - # - # ###identification, etc. - # #self.identification = '%s : %s' % (self.dataset.shortname, self.trait_id) - # #this_trait.returnURL = webqtlConfig.CGIDIR + webqtlConfig.SCRIPTFILE + '?FormID=showDatabase&database=%s\ - # # &ProbeSetID=%s&group=%s&parentsf1=on' %(self.dataset, self.trait_id, self.dataset.group.name) - # # - # #if self.cell_id: - # # self.identification = '%s/%s'%(self.identification, self.cell_id) - # # this_trait.returnURL = '%s&CellID=%s' % (this_trait.returnURL, self.cell_id) - # - # this_trait.retrieve_info() - # this_trait.retrieve_sample_data() - # return this_trait - - def read_data(self, include_f1=False): '''read user input data or from trait data and analysis form''' @@ -327,13 +286,11 @@ class ShowTrait(object): if snpurl: snpBrowserButton = HT.Href(url="#redirect", onClick="openNewWin('%s')" % snpurl) snpBrowserButton_img = HT.Image("/images/snp_icon.jpg", name="snpbrowser", alt=" View SNPs and Indels ", title=" View SNPs and Indels ", style="border:none;") - #snpBrowserButton.append(snpBrowserButton_img) snpBrowserText = "SNPs" #XZ: Show GeneWiki for all species geneWikiButton = HT.Href(url="#redirect", onClick="openNewWin('%s')" % (os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE) + "?FormID=geneWiki&symbol=%s" % this_trait.symbol)) geneWikiButton_img = HT.Image("/images/genewiki_icon.jpg", name="genewiki", alt=" Write or review comments about this gene ", title=" Write or review comments about this gene ", style="border:none;") - #geneWikiButton.append(geneWikiButton_img) geneWikiText = 'GeneWiki' #XZ: display similar traits in other selected datasets @@ -342,15 +299,9 @@ class ShowTrait(object): similarUrl = "%s?cmd=sch&gene=%s&alias=1&species=%s" % (os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), this_trait.symbol, _Species) similarButton = HT.Href(url="#redirect", onClick="openNewWin('%s')" % similarUrl) similarButton_img = HT.Image("/images/find_icon.jpg", name="similar", alt=" Find similar expression data ", title=" Find similar expression data ", style="border:none;") - #similarButton.append(similarButton_img) similarText = "Find" else: pass - #tbl.append(HT.TR( - #HT.TD('Gene Symbol: ', Class="fwb fs13", valign="top", nowrap="on", width=90), - #HT.TD(width=10, valign="top"), - #HT.TD(HT.Span('%s' % this_trait.symbol, valign="top", Class="fs13 fsI"), valign="top", width=740) - #)) else: tbl.append(HT.TR( HT.TD('Gene Symbol: ', Class="fwb fs13", valign="top", nowrap="on"), @@ -396,11 +347,6 @@ class ShowTrait(object): for seqt in seqs: if int(seqt[1][-1]) %2 == 1: blatsequence += '%3EProbe_'+string.strip(seqt[1])+'%0A'+string.strip(seqt[0])+'%0A' - #-------- - #XZ, 07/16/2009: targetsequence is not used, so I comment out this block - #targetsequence = this_trait.targetseq - #if targetsequence==None: - # targetsequence = "" #XZ: Pay attention to the parameter of version (rn, mm, hg). They need to be changed if necessary. if _Species == "rat": @@ -449,55 +395,6 @@ class ShowTrait(object): #probeButton.append(probeButton_img) probeText = "Probes" - #tSpan = HT.Span(Class="fs13") - - #XZ: deal with blat score and blat specificity. - #if this_trait.probe_set_specificity or this_trait.probe_set_blat_score: - # if this_trait.probe_set_specificity: - # pass - # #tSpan.append(HT.Href(url="/blatInfo.html", target="_blank", title="Values higher than 2 for the specificity are good", text="BLAT specificity", Class="non_bold"),": %.1f" % float(this_trait.probe_set_specificity), " "*3) - # if this_trait.probe_set_blat_score: - # pass - # #tSpan.append("Score: %s" % int(this_trait.probe_set_blat_score), " "*2) - - #onClick="openNewWin('/blatInfo.html')" - - #tbl.append(HT.TR( - # HT.TD('Target Score: ', Class="fwb fs13", valign="top", nowrap="on"), - # HT.TD(width=10, valign="top"), - # HT.TD(tSpan, valign="top") - # )) - - #tSpan = HT.Span(Class="fs13") - #tSpan.append(str(_Species).capitalize(), ", ", fd.group) - # - #tbl.append(HT.TR( - # HT.TD('Species and Group: ', Class="fwb fs13", valign="top", nowrap="on"), - # HT.TD(width=10, valign="top"), - # HT.TD(tSpan, valign="top") - # )) - - #if this_trait.cellid: - # self.cursor.execute(""" - # select ProbeFreeze.Name from ProbeFreeze, ProbeSetFreeze - # where - # ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId AND - # ProbeSetFreeze.Id = %d""" % this_trait.dataset.id) - # probeDBName = self.cursor.fetchone()[0] - # tbl.append(HT.TR( - # HT.TD('Database: ', Class="fs13 fwb", valign="top", nowrap="on"), - # HT.TD(width=10, valign="top"), - # HT.TD(HT.Span('%s' % probeDBName, Class="non_bold"), valign="top") - # )) - #else: - #tbl.append(HT.TR( - # HT.TD('Database: ', Class="fs13 fwb", valign="top", nowrap="on"), - # HT.TD(width=10, valign="top"), - # HT.TD(HT.Href(text=this_trait.dataset.fullname, url = webqtlConfig.INFOPAGEHREF % this_trait.dataset.name, - # target='_blank', Class="fs13 fwn non_bold"), valign="top") - # )) - #pass - this_trait.species = _Species # We need this in the template, so we tuck it into this_trait this_trait.database = this_trait.get_database() @@ -532,14 +429,6 @@ class ShowTrait(object): url=webqtlConfig.HOMOLOGENE_ID % this_trait.homologeneid, Class="fs14 fwn", title="Find similar genes in other species") #tSpan.append(HT.Span(hurl, style=idStyle), " "*2) - #tbl.append( - # HT.TR(HT.TD(colspan=3,height=6)), - # HT.TR( - # HT.TD('Resource Links: ', Class="fwb fs13", valign="top", nowrap="on"), - # HT.TD(width=10, valign="top"), - # HT.TD(tSpan, valign="top") - # )) - #XZ: Resource Links: if this_trait.symbol: linkStyle = "background:#dddddd;padding:2" @@ -584,9 +473,7 @@ class ShowTrait(object): # txen), # Class="fs14 fwn"), style=linkStyle) # , " "*2) - #except: - # pass - + #XZ, 7/16/2009: The url for SymAtlas (renamed as BioGPS) has changed. We don't need this any more #tSpan.append(HT.Span(HT.Href(text= 'SymAtlas',target="mainFrame",\ # url="http://symatlas.gnf.org/SymAtlas/bioentry?querytext=%s&query=14&species=%s&type=Expression" \ @@ -655,28 +542,12 @@ class ShowTrait(object): # title="Allen Brain Atlas"), style=linkStyle), " "*2) pass - #tbl.append( - # HT.TR(HT.TD(colspan=3,height=6)), - # HT.TR( - # HT.TD(' '), - # HT.TD(width=10, valign="top"), - # HT.TD(tSpan, valign="top"))) - - #menuTable = HT.TableLite(cellpadding=2, Class="collap", width="620", id="target1") - #menuTable.append(HT.TR(HT.TD(addSelectionButton, align="center"),HT.TD(similarButton, align="center"),HT.TD(verifyButton, align="center"),HT.TD(geneWikiButton, align="center"),HT.TD(snpBrowserButton, align="center"),HT.TD(rnaseqButton, align="center"),HT.TD(probeButton, align="center"),HT.TD(updateButton, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - #menuTable.append(HT.TR(HT.TD(addSelectionText, align="center"),HT.TD(similarText, align="center"),HT.TD(verifyText, align="center"),HT.TD(geneWikiText, align="center"),HT.TD(snpBrowserText, align="center"),HT.TD(rnaseqText, align="center"),HT.TD(probeText, align="center"),HT.TD(updateText, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - - #for zhou mi's cliques, need to be removed #if self.database[:6] == 'BXDMic' and self.ProbeSetID in cliqueID: # Info2Disp.append(HT.Strong('Clique Search: '),HT.Href(text='Search',\ # url ="http://compbio1.utmem.edu/clique_go/results.php?pid=%s&pval_1=0&pval_2=0.001" \ # % self.ProbeSetID,target='_blank',Class="normalsize"),HT.BR()) - #linkTable.append(HT.TR(linkTD)) - #Info2Disp.append(linkTable) - #title1Body.append(tbl, HT.BR(), menuTable) - elif this_trait and this_trait.dataset and this_trait.dataset.type =='Publish': #Check if trait is phenotype #if this_trait.confidential: @@ -759,12 +630,6 @@ class ShowTrait(object): # )) pass - #menuTable = HT.TableLite(cellpadding=2, Class="collap", width="150", id="target1") - #menuTable.append(HT.TR(HT.TD(addSelectionButton, align="center"),HT.TD(updateButton, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - #menuTable.append(HT.TR(HT.TD(addSelectionText, align="center"),HT.TD(updateText, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - - #title1Body.append(tbl, HT.BR(), menuTable) - elif this_trait and this_trait.dataset and this_trait.dataset.type == 'Geno': #Check if trait is genotype if this_trait.chr and this_trait.mb: @@ -810,41 +675,10 @@ class ShowTrait(object): # valign="top", width=740) # )) - #menuTable = HT.TableLite(cellpadding=2, Class="collap", width="275", id="target1") - #menuTable.append(HT.TR(HT.TD(addSelectionButton, align="center"),HT.TD(verifyButton, align="center"),HT.TD(rnaseqButton, align="center"), HT.TD(updateButton, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - #menuTable.append(HT.TR(HT.TD(addSelectionText, align="center"),HT.TD(verifyText, align="center"),HT.TD(rnaseqText, align="center"), HT.TD(updateText, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - - #title1Body.append(tbl, HT.BR(), menuTable) - - elif (this_trait == None or this_trait.dataset.type == 'Temp'): #if temporary trait (user-submitted trait or PCA trait) - - #TempInfo = HT.Paragraph() - if this_trait != None: - if this_trait.description: - pass - #tbl.append(HT.TR(HT.TD(HT.Strong('Description: '),' %s ' % this_trait.description,HT.BR()), colspan=3, height=15)) - else: - tbl.append(HT.TR(HT.TD(HT.Strong('Description: '),'not available',HT.BR(),HT.BR()), colspan=3, height=15)) - - if (updateText == "Edit"): - menuTable = HT.TableLite(cellpadding=2, Class="collap", width="150", id="target1") - else: - menuTable = HT.TableLite(cellpadding=2, Class="collap", width="80", id="target1") - - #menuTable.append(HT.TR(HT.TD(addSelectionButton, align="right"),HT.TD(updateButton, align="right"), colspan=3, height=50, style="vertical-align:bottom;") ) - #menuTable.append(HT.TR(HT.TD(addSelectionText, align="center"),HT.TD(updateText, align="center"), colspan=3, height=50, style="vertical-align:bottom;")) - # - #title1Body.append(tbl, HT.BR(), menuTable) - - else: - pass - def dispBasicStatistics(self, fd, this_trait): #XZ, June 22, 2011: The definition and usage of primary_samples, other_samples, specialStrains, all_samples are not clear and hard to understand. But since they are only used in this function for draw graph purpose, they will not hurt the business logic outside. As of June 21, 2011, this function seems work fine, so no hurry to clean up. These parameters and code in this function should be cleaned along with fd.f1list, fd.parlist, fd.samplelist later. - #stats_row = HT.TR() - #stats_cell = HT.TD() # This should still be riset here - Sam - Nov. 2012 if fd.genotype.type == "riset": @@ -912,7 +746,6 @@ class ShowTrait(object): for sampleNameOrig in all_samples: sampleName = sampleNameOrig.replace("_2nd_", "") - #try: print("* type of this_trait:", type(this_trait)) print(" name:", this_trait.__class__.__name__) print(" this_trait:", this_trait) @@ -925,28 +758,16 @@ class ShowTrait(object): print(" thisvar:", thisvar) thisValFull = [sampleName, thisval, thisvar] print(" thisValFull:", thisValFull) - #except: - # continue vals1.append(thisValFull) - - #vals1 = [[sampleNameOrig.replace("_2nd_", ""), - # this_trait.data[sampleName].val, - # this_trait.data[sampleName].var] - # for sampleNameOrig in all_samples]] - # - #Using just the group sample for sampleNameOrig in primary_samples: sampleName = sampleNameOrig.replace("_2nd_", "") - #try: thisval = this_trait.data[sampleName].value thisvar = this_trait.data[sampleName].variance thisValFull = [sampleName,thisval,thisvar] - #except: - # continue vals2.append(thisValFull) @@ -954,12 +775,9 @@ class ShowTrait(object): for sampleNameOrig in other_samples: sampleName = sampleNameOrig.replace("_2nd_", "") - #try: thisval = this_trait.data[sampleName].value thisvar = this_trait.data[sampleName].variance thisValFull = [sampleName,thisval,thisvar] - #except: - # continue vals3.append(thisValFull) @@ -972,12 +790,9 @@ class ShowTrait(object): for sampleNameOrig in all_samples: sampleName = sampleNameOrig.replace("_2nd_", "") - #try: thisval = this_trait.data[sampleName].value thisvar = this_trait.data[sampleName].variance thisValFull = [sampleName,thisval,thisvar] - #except: - # continue vals.append(thisValFull) @@ -988,8 +803,6 @@ class ShowTrait(object): if i == 0 and len(vals) < 4: stats_container = HT.Div(id="stats_tabs", style="padding:10px;", Class="ui-tabs") #Needed for tabs; notice the "stats_script_text" below referring to this element stats_container.append(HT.Div(HT.Italic("Fewer than 4 case data were entered. No statistical analysis has been attempted."))) - #stats_script_text = """$(function() { $("#stats_tabs").tabs();});""" - #stats_cell.append(stats_container) break elif (i == 1 and len(primary_samples) < 4): stats_container = HT.Div(id="stats_tabs%s" % i, Class="ui-tabs") @@ -997,20 +810,12 @@ class ShowTrait(object): elif (i == 2 and len(other_samples) < 4): stats_container = HT.Div(id="stats_tabs%s" % i, Class="ui-tabs") stats_container.append(HT.Div(HT.Italic("Fewer than 4 non-" + fd.group + " case data were entered. No statistical analysis has been attempted."))) - #stats_script_text = """$(function() { $("#stats_tabs0").tabs(); $("#stats_tabs1").tabs(); $("#stats_tabs2").tabs();});""" else: continue if len(vals) > 4: stats_tab_list = [HT.Href(text="Basic Table", url="#statstabs-1", Class="stats_tab"),HT.Href(text="Probability Plot", url="#statstabs-5", Class="stats_tab"), HT.Href(text="Bar Graph (by name)", url="#statstabs-3", Class="stats_tab"), HT.Href(text="Bar Graph (by rank)", url="#statstabs-4", Class="stats_tab"), HT.Href(text="Box Plot", url="#statstabs-2", Class="stats_tab")] - #stats_tabs = HT.List(stats_tab_list) - #stats_container.append(stats_tabs) - # - #table_div = HT.Div(id="statstabs-1") - #table_container = HT.Paragraph() - # - #statsTable = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") if this_trait.dataset: if this_trait.cellid: @@ -1020,12 +825,6 @@ class ShowTrait(object): else: self.stats_data.append(BasicStatisticsFunctions.basicStatsTable(vals=vals)) - #statsTable.append(HT.TR(HT.TD(statsTableCell))) - - #table_container.append(statsTable) - #table_div.append(table_container) - #stats_container.append(table_div) - # #normalplot_div = HT.Div(id="statstabs-5") #normalplot_container = HT.Paragraph() #normalplot = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") @@ -1043,49 +842,22 @@ class ShowTrait(object): #normally distributed. Different symbols represent different groups.",HT.BR(),HT.BR(), #"More about ", HT.Href(url="http://en.wikipedia.org/wiki/Normal_probability_plot", # target="_blank", text="Normal Probability Plots"), " and more about interpreting these plots from the ", HT.Href(url="/glossary.html#normal_probability", target="_blank", text="glossary")))) - #normalplot_container.append(normalplot) - #normalplot_div.append(normalplot_container) - #stats_container.append(normalplot_div) #boxplot_div = HT.Div(id="statstabs-2") #boxplot_container = HT.Paragraph() #boxplot = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") #boxplot_img, boxplot_link = BasicStatisticsFunctions.plotBoxPlot(vals) #boxplot.append(HT.TR(HT.TD(boxplot_img, HT.P(), boxplot_link, align="left"))) - #boxplot_container.append(boxplot) - #boxplot_div.append(boxplot_container) - #stats_container.append(boxplot_div) - #barName_div = HT.Div(id="statstabs-3") #barName_container = HT.Paragraph() #barName = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") #barName_img = BasicStatisticsFunctions.plotBarGraph(identification=fd.identification, group=fd.group, vals=vals, type="name") - #barName.append(HT.TR(HT.TD(barName_img))) - #barName_container.append(barName) - #barName_div.append(barName_container) - #stats_container.append(barName_div) - # + #barRank_div = HT.Div(id="statstabs-4") #barRank_container = HT.Paragraph() #barRank = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") #barRank_img = BasicStatisticsFunctions.plotBarGraph(identification=fd.identification, group=fd.group, vals=vals, type="rank") - #barRank.append(HT.TR(HT.TD(barRank_img))) - #barRank_container.append(barRank) - #barRank_div.append(barRank_container) - #stats_container.append(barRank_div) - - # stats_cell.append(stats_container) - # - #stats_script.append(stats_script_text) - # - #submitTable = HT.TableLite(cellspacing=0, cellpadding=0, width="100%", Class="target2") - #stats_row.append(stats_cell) - - #submitTable.append(stats_row) - #submitTable.append(stats_script) - - #title2Body.append(submitTable) def build_correlation_tools(self, this_trait): @@ -1100,15 +872,6 @@ class ShowTrait(object): this_group = 'BXD' if this_group: - #sample_correlation = HT.Input(type='button',name='sample_corr', value=' Compute ', Class="button sample_corr") - #lit_correlation = HT.Input(type='button',name='lit_corr', value=' Compute ', Class="button lit_corr") - #tissue_correlation = HT.Input(type='button',name='tiss_corr', value=' Compute ', Class="button tiss_corr") - #methodText = HT.Span("Calculate:", Class="ffl fwb fs12") - # - #databaseText = HT.Span("Database:", Class="ffl fwb fs12") - #databaseMenu1 = HT.Select(name='database1') - #databaseMenu2 = HT.Select(name='database2') - #databaseMenu3 = HT.Select(name='database3') dataset_menu = [] print("[tape4] webqtlConfig.PUBLICTHRESH:", webqtlConfig.PUBLICTHRESH) @@ -1133,8 +896,6 @@ class ShowTrait(object): tissues = g.db.execute("SELECT Id, Name FROM Tissue order by Name") for item in tissues.fetchall(): tissue_id, tissue_name = item - #databaseMenuSub = HT.Optgroup(label = '%s ------' % tissue_name) - #dataset_sub_menu = [] data_sets = g.db.execute('''SELECT ProbeSetFreeze.FullName,ProbeSetFreeze.Name FROM ProbeSetFreeze, ProbeFreeze, InbredSet WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id and ProbeFreeze.TissueId = %s and ProbeSetFreeze.public > %s and ProbeFreeze.InbredSetId = InbredSet.Id and InbredSet.Name like %s @@ -1144,149 +905,15 @@ class ShowTrait(object): if dataset_sub_menu: dataset_menu.append(dict(tissue=tissue_name, datasets=dataset_sub_menu)) - # ("**heading**", tissue_name)) - #dataset_menu.append(dataset_sub_menu) dataset_menu_selected = None if len(dataset_menu): if this_trait and this_trait.dataset: dataset_menu_selected = this_trait.dataset.name - #criteriaText = HT.Span("Return:", Class="ffl fwb fs12") - - #criteriaMenu1 = HT.Select(name='criteria1', selected='500', onMouseOver="if (NS4 || IE4) activateEl('criterias', event);") - return_results_menu = (100, 200, 500, 1000, 2000, 5000, 10000, 15000, 20000) return_results_menu_selected = 500 - #criteriaMenu1.append(('top 100','100')) - #criteriaMenu1.append(('top 200','200')) - #criteriaMenu1.append(('top 500','500')) - #criteriaMenu1.append(('top 1000','1000')) - #criteriaMenu1.append(('top 2000','2000')) - #criteriaMenu1.append(('top 5000','5000')) - #criteriaMenu1.append(('top 10000','10000')) - #criteriaMenu1.append(('top 15000','15000')) - #criteriaMenu1.append(('top 20000','20000')) - - #self.MDPRow1 = HT.TR(Class='mdp1') - #self.MDPRow2 = HT.TR(Class='mdp2') - #self.MDPRow3 = HT.TR(Class='mdp3') - - # correlationMenus1 = HT.TableLite( - # HT.TR(HT.TD(databaseText), HT.TD(databaseMenu1, colspan="3")), - # HT.TR(HT.TD(criteriaText), HT.TD(criteriaMenu1)), - # self.MDPRow1, cellspacing=0, width="619px", cellpadding=2) - # correlationMenus1.append(HT.Input(name='orderBy', value='2', type='hidden')) # to replace the orderBy menu - # correlationMenus2 = HT.TableLite( - # HT.TR(HT.TD(databaseText), HT.TD(databaseMenu2, colspan="3")), - # HT.TR(HT.TD(criteriaText), HT.TD(criteriaMenu2)), - # self.MDPRow2, cellspacing=0, width="619px", cellpadding=2) - # correlationMenus2.append(HT.Input(name='orderBy', value='2', type='hidden')) - # correlationMenus3 = HT.TableLite( - # HT.TR(HT.TD(databaseText), HT.TD(databaseMenu3, colspan="3")), - # HT.TR(HT.TD(criteriaText), HT.TD(criteriaMenu3)), - # self.MDPRow3, cellspacing=0, width="619px", cellpadding=2) - # correlationMenus3.append(HT.Input(name='orderBy', value='2', type='hidden')) - # - #else: - # correlationMenus = "" - - - #corr_row = HT.TR() - #corr_container = HT.Div(id="corr_tabs", Class="ui-tabs") - # - #if (this_trait.dataset != None and this_trait.dataset.type =='ProbeSet'): - # corr_tab_list = [HT.Href(text='Sample r', url="#corrtabs-1"), - # HT.Href(text='Literature r', url="#corrtabs-2"), - # HT.Href(text='Tissue r', url="#corrtabs-3")] - #else: - # corr_tab_list = [HT.Href(text='Sample r', url="#corrtabs-1")] - # - #corr_tabs = HT.List(corr_tab_list) - #corr_container.append(corr_tabs) - - #if correlationMenus1 or correlationMenus2 or correlationMenus3: - #sample_div = HT.Div(id="corrtabs-1") - #sample_container = HT.Span() - # - #sample_type = HT.Input(type="radio", name="sample_method", value="1", checked="checked") - #sample_type2 = HT.Input(type="radio", name="sample_method", value="2") - # - #sampleTable = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") - #sampleTD = HT.TD(correlationMenus1, HT.BR(), - # "Pearson", sample_type, " "*3, "Spearman Rank", sample_type2, HT.BR(), HT.BR(), - # sample_correlation, HT.BR(), HT.BR()) - # - #sampleTD.append(HT.Span("The ", - # HT.Href(url="/correlationAnnotation.html#sample_r", target="_blank", - # text="Sample Correlation")," is computed between trait data and", - # " any ",HT.BR()," other traits in the sample database selected above. Use ", - # HT.Href(url="/glossary.html#Correlations", target="_blank", text="Spearman Rank"), - # HT.BR(),"when the sample size is small (<20) or when there are influential \ - # outliers.", HT.BR(),Class="fs12")) - - #sampleTable.append(sampleTD) - - #sample_container.append(sampleTable) - #sample_div.append(sample_container) - #corr_container.append(sample_div) - # - #literature_div = HT.Div(id="corrtabs-2") - #literature_container = HT.Span() - - #literatureTable = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") - #literatureTD = HT.TD(correlationMenus2,HT.BR(),lit_correlation, HT.BR(), HT.BR()) - #literatureTD.append(HT.Span("The ", HT.Href(url="/correlationAnnotation.html", target="_blank",text="Literature Correlation"), " (Lit r) between this gene and all other genes is computed",HT.BR(), - # "using the ", HT.Href(url="https://grits.eecs.utk.edu/sgo/sgo.html", target="_blank", text="Semantic Gene Organizer"), - # " and human, rat, and mouse data from PubMed. ", HT.BR(),"Values are ranked by Lit r, \ - # but Sample r and Tissue r are also displayed.", HT.BR(), HT.BR(), - # HT.Href(url="/glossary.html#Literature", target="_blank", text="More on using Lit r"), Class="fs12")) - #literatureTable.append(literatureTD) - # - #literature_container.append(literatureTable) - #literature_div.append(literature_container) - # - #if this_trait.dataset != None: - # if (this_trait.dataset.type =='ProbeSet'): - # corr_container.append(literature_div) - # - #tissue_div = HT.Div(id="corrtabs-3") - #tissue_container = HT.Span() - # - #tissue_type = HT.Input(type="radio", name="tissue_method", value="4", checked="checked") - #tissue_type2 = HT.Input(type="radio", name="tissue_method", value="5") - # - #tissueTable = HT.TableLite(cellspacing=0, cellpadding=0, width="100%") - #tissueTD = HT.TD(correlationMenus3,HT.BR(), - # "Pearson", tissue_type, " "*3, "Spearman Rank", tissue_type2, HT.BR(), HT.BR(), - # tissue_correlation, HT.BR(), HT.BR()) - #tissueTD.append(HT.Span("The ", HT.Href(url="/webqtl/main.py?FormID=tissueCorrelation", target="_blank", text="Tissue Correlation"), - #" (Tissue r) estimates the similarity of expression of two genes",HT.BR()," or \ - #transcripts across different cells, tissues, or organs (",HT.Href(url="/correlationAnnotation.html#tissue_r", target="_blank", text="glossary"),"). \ - #Tissue correlations",HT.BR()," are generated by analyzing expression in multiple samples usually taken from \ - #single cases.",HT.BR(),HT.Bold("Pearson")," and ",HT.Bold("Spearman Rank")," correlations have been computed for all pairs \ - #of genes",HT.BR()," using data from mouse samples.", - #HT.BR(), Class="fs12")) - #tissueTable.append(tissueTD) - # - #tissue_container.append(tissueTable) - #tissue_div.append(tissue_container) - #if this_trait.dataset != None: - # if (this_trait.dataset.type =='ProbeSet'): - # corr_container.append(tissue_div) - # - #corr_row.append(HT.TD(corr_container)) - # - #corr_script = HT.Script(language="Javascript") - #corr_script_text = """$(function() { $("#corr_tabs").tabs(); });""" - #corr_script.append(corr_script_text) - # - #submitTable = HT.TableLite(cellspacing=0, cellpadding=0, width="100%", Class="target4") - #submitTable.append(corr_row) - #submitTable.append(corr_script) - # - #title3Body.append(submitTable) self.corr_tools = dict(dataset_menu = dataset_menu, dataset_menu_selected = dataset_menu_selected, return_results_menu = return_results_menu, -- cgit v1.2.3