From 34b8f6bc739e99689f531eda22b30fb689bb38c9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 5 Mar 2015 17:37:08 +0300 Subject: Kinship 1.0 parser --- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 70 ++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 wqflask/wqflask/my_pylmm/pyLMM/runlmm.py diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py new file mode 100644 index 00000000..e7bb452f --- /dev/null +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -0,0 +1,70 @@ +# This is the LMM runner that calls the possible methods using command line +# switches. It acts as a multiplexer where all the invocation complexity +# is kept outside the main LMM routines. +# +# Copyright (C) 2015 Pjotr Prins (pjotr.prins@thebird.nl) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. + +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +from optparse import OptionParser +import sys +import os +import numpy as np +# from lmm import LMM, run_other +import csv + + +usage = """ +python runlmm.py [options] command + + runlmm.py processing multiplexer reads standard input types and calls the routines + + Current commands are: + + parse : only parse input files + + try --help for more information +""" + +parser = OptionParser(usage=usage) +# parser.add_option("-f", "--file", dest="input file", +# help="In", metavar="FILE") +parser.add_option("--kinship",dest="kinship", + help="Kinship file format") +parser.add_option("-q", "--quiet", + action="store_false", dest="verbose", default=True, + help="don't print status messages to stdout") + +(options, args) = parser.parse_args() + +if len(args) != 1: + print usage + sys.exit(1) + +cmd = args[0] +print "Command: ",cmd + +if options.kinship: + K1 = [] + print options.kinship + with open(options.kinship,'r') as tsvin: + if tsvin.readline().strip() != "# Kinship format version 1.0": + raise Exception("Expecting Kinship format version 1.0") + tsvin.readline() + tsvin.readline() + tsv = csv.reader(tsvin, delimiter='\t') + for row in tsv: + ns = np.genfromtxt(row[1:]) + K1.append(ns) # <--- slow + K = np.array(K1) -- cgit v1.2.3 From 3a7d9fd232200a5cd95563caf2f067328035dfd3 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 5 Mar 2015 17:47:47 +0300 Subject: Check command --- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index e7bb452f..ff53b4ea 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -55,6 +55,7 @@ if len(args) != 1: cmd = args[0] print "Command: ",cmd + if options.kinship: K1 = [] print options.kinship @@ -68,3 +69,8 @@ if options.kinship: ns = np.genfromtxt(row[1:]) K1.append(ns) # <--- slow K = np.array(K1) + +if cmd == 'parse': + pass +else: + raise Exception("Unknown command") -- cgit v1.2.3 From cb97cb560ceaa88ad0a4087a3004d18331e88267 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Thu, 19 Feb 2015 19:51:37 +0000 Subject: Just committing changes to search so I can pull to production branch --- wqflask/wqflask/do_search.py | 98 ++++++++++++++++++++++++++++++++------- wqflask/wqflask/search_results.py | 28 +++++++++++ 2 files changed, 110 insertions(+), 16 deletions(-) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 29f3de98..f00f0c20 100755 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -139,11 +139,8 @@ class MrnaAssaySearch(DoSearch): return query - def run(self): - """Generates and runs a simple search of an mRNA expression dataset""" - - print("Running ProbeSetSearch") - query = self.base_query + """WHERE (MATCH (ProbeSet.Name, + def get_where_clause(self): + where_clause = """(MATCH (ProbeSet.Name, ProbeSet.description, ProbeSet.symbol, alias, @@ -156,6 +153,15 @@ class MrnaAssaySearch(DoSearch): """ % (escape(self.search_term[0]), escape(str(self.dataset.id))) + return where_clause + + def run(self): + """Generates and runs a simple search of an mRNA expression dataset""" + + print("Running ProbeSetSearch") + where_clause = self.get_where_clause() + query = self.base_query + "WHERE " + where_clause + #print("final query is:", pf(query)) return self.execute(query) @@ -433,12 +439,9 @@ class LrsSearch(MrnaAssaySearch): DoSearch.search_types['LRS'] = 'LrsSearch' - def run(self): - + def get_where_clause(self): self.search_term = [float(value) for value in self.search_term] - self.from_clause = ", Geno" - if self.search_operator == "=": assert isinstance(self.search_term, (list, tuple)) self.lrs_min, self.lrs_max = self.search_term[:2] @@ -466,14 +469,56 @@ class LrsSearch(MrnaAssaySearch): self.search_operator, self.search_term[0]) - self.where_clause = self.sub_clause + """ %sXRef.Locus = Geno.name and + where_clause = self.sub_clause + """ %sXRef.Locus = Geno.name and Geno.SpeciesId = %s and %s.Chr = Geno.Chr """ % self.mescape(self.dataset.type, self.species_id, self.dataset.type) - print("where_clause is:", pf(self.where_clause)) + return where_clause + + def run(self): + + self.from_clause = ", Geno" + + #self.search_term = [float(value) for value in self.search_term] + # + #if self.search_operator == "=": + # assert isinstance(self.search_term, (list, tuple)) + # self.lrs_min, self.lrs_max = self.search_term[:2] + # + # self.sub_clause = """ %sXRef.LRS > %s and + # %sXRef.LRS < %s and """ % self.mescape(self.dataset.type, + # min(self.lrs_min, self.lrs_max), + # self.dataset.type, + # max(self.lrs_min, self.lrs_max)) + # + # if len(self.search_term) > 2: + # self.chr_num = self.search_term[2] + # self.sub_clause += """ Geno.Chr = %s and """ % (escape(self.chr_num)) + # if len(self.search_term) == 5: + # self.mb_low, self.mb_high = self.search_term[3:] + # self.sub_clause += """ Geno.Mb > %s and + # Geno.Mb < %s and + # """ % self.mescape(min(self.mb_low, self.mb_high), + # max(self.mb_low, self.mb_high)) + # print("self.sub_clause is:", pf(self.sub_clause)) + #else: + # # Deal with >, <, >=, and <= + # print("self.search_term is:", self.search_term) + # self.sub_clause = """ %sXRef.LRS %s %s and """ % self.mescape(self.dataset.type, + # self.search_operator, + # self.search_term[0]) + # + #self.where_clause = self.sub_clause + """ %sXRef.Locus = Geno.name and + # Geno.SpeciesId = %s and + # %s.Chr = Geno.Chr + # """ % self.mescape(self.dataset.type, + # self.species_id, + # self.dataset.type) + + self.where_clause = self.get_where_clause() self.query = self.compile_final_query(self.from_clause, self.where_clause) @@ -586,25 +631,46 @@ class MeanSearch(MrnaAssaySearch): DoSearch.search_types['MEAN'] = "MeanSearch" - def run(self): - + def get_where_clause(self): self.search_term = [float(value) for value in self.search_term] if self.search_operator == "=": assert isinstance(self.search_term, (list, tuple)) self.mean_min, self.mean_max = self.search_term[:2] - self.where_clause = """ %sXRef.mean > %s and + where_clause = """ %sXRef.mean > %s and %sXRef.mean < %s """ % self.mescape(self.dataset.type, min(self.mean_min, self.mean_max), self.dataset.type, max(self.mean_min, self.mean_max)) else: # Deal with >, <, >=, and <= - self.where_clause = """ %sXRef.mean %s %s """ % self.mescape(self.dataset.type, + where_clause = """ %sXRef.mean %s %s """ % self.mescape(self.dataset.type, self.search_operator, - self.search_term[0]) + self.search_term[0]) + + return where_clause + + def run(self): + #self.search_term = [float(value) for value in self.search_term] + # + #if self.search_operator == "=": + # assert isinstance(self.search_term, (list, tuple)) + # self.mean_min, self.mean_max = self.search_term[:2] + # + # self.where_clause = """ %sXRef.mean > %s and + # %sXRef.mean < %s """ % self.mescape(self.dataset.type, + # min(self.mean_min, self.mean_max), + # self.dataset.type, + # max(self.mean_min, self.mean_max)) + #else: + # # Deal with >, <, >=, and <= + # self.where_clause = """ %sXRef.mean %s %s """ % self.mescape(self.dataset.type, + # self.search_operator, + # self.search_term[0]) + + self.where_clause = self.get_where_clause() print("where_clause is:", pf(self.where_clause)) self.query = self.compile_final_query(where_clause = self.where_clause) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index c6521106..fdf04d44 100755 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -221,6 +221,34 @@ class SearchResultPage(object): self.search_terms = parser.parse(self.search_terms) print("After parsing:", self.search_terms) + if len(self.search_terms) > 1: + combined_where_clause = "" + for i, a_search in enumerate(self.search_terms): + print("[kodak] item is:", pf(a_search)) + search_term = a_search['search_term'] + search_operator = a_search['separator'] + if a_search['key']: + search_type = a_search['key'].upper() + else: + # We fall back to the dataset type as the key to get the right object + search_type = self.dataset.type + + print("search_type is:", pf(search_type)) + + search_ob = do_search.DoSearch.get_search(search_type) + search_class = getattr(do_search, search_ob) + the_search = search_class(search_term, + search_operator, + self.dataset, + ) + + where_clause = the_search.get_where_clause() + combined_where_clause += "(" + where_clause + ")" + if (i+1) < len(self.search_terms): + combined_where_clause += "AND" + print("combined_where_clause:", combined_where_clause) + + for a_search in self.search_terms: print("[kodak] item is:", pf(a_search)) search_term = a_search['search_term'] -- cgit v1.2.3 From e0bd0ab02d26fd8520ddfb0f69baa746ddee91fe Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Mon, 2 Mar 2015 20:09:16 +0000 Subject: Combined searches now work for most terms Fixed appearance of the scatterplot matrix Changed pylmm code back because of an error after I had switched to Pjotr's code --- wqflask/wqflask/do_search.py | 102 +++++++++++++++++---- .../wqflask/marker_regression/marker_regression.py | 1 + wqflask/wqflask/search_results.py | 55 ++++++----- .../static/new/javascript/scatter-matrix.js | 4 +- .../static/new/javascript/show_trait.coffee | 4 +- .../wqflask/static/new/javascript/show_trait.js | 4 +- .../templates/show_trait_mapping_tools.html | 2 +- .../templates/show_trait_statistics_new.html | 2 +- wqflask/wqflask/views.py | 2 +- 9 files changed, 127 insertions(+), 49 deletions(-) diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index f00f0c20..31dc11b5 100755 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -38,7 +38,6 @@ class DoSearch(object): query = self.normalize_spaces(query) print("in do_search query is:", pf(query)) results = g.db.execute(query, no_parameters=True).fetchall() - #results = self.cursor.fetchall() return results #def escape(self, stringy): @@ -155,6 +154,28 @@ class MrnaAssaySearch(DoSearch): return where_clause + def run_combined(self, from_clause, where_clause): + """Generates and runs a combined search of an mRNA expression dataset""" + + print("Running ProbeSetSearch") + #query = self.base_query + from_clause + " WHERE " + where_clause + + from_clause = self.normalize_spaces(from_clause) + + query = (self.base_query + + """%s + WHERE %s + and ProbeSet.Id = ProbeSetXRef.ProbeSetId + and ProbeSetXRef.ProbeSetFreezeId = %s + """ % (escape(from_clause), + where_clause, + escape(str(self.dataset.id)))) + + print("final query is:", pf(query)) + + + return self.execute(query) + def run(self): """Generates and runs a simple search of an mRNA expression dataset""" @@ -439,6 +460,12 @@ class LrsSearch(MrnaAssaySearch): DoSearch.search_types['LRS'] = 'LrsSearch' + def get_from_clause(self): + if self.search_operator == "=": + return ", Geno" + else: + return "" + def get_where_clause(self): self.search_term = [float(value) for value in self.search_term] @@ -454,7 +481,7 @@ class LrsSearch(MrnaAssaySearch): if len(self.search_term) > 2: self.chr_num = self.search_term[2] - self.sub_clause += """ Geno.Chr = %s and """ % (escape(self.chr_num)) + self.sub_clause += """ Geno.Chr = %s and """ % (self.chr_num) if len(self.search_term) == 5: self.mb_low, self.mb_high = self.search_term[3:] self.sub_clause += """ Geno.Mb > %s and @@ -462,25 +489,33 @@ class LrsSearch(MrnaAssaySearch): """ % self.mescape(min(self.mb_low, self.mb_high), max(self.mb_low, self.mb_high)) print("self.sub_clause is:", pf(self.sub_clause)) + + #%s.Chr = Geno.Chr + where_clause = self.sub_clause + """ %sXRef.Locus = Geno.name and + Geno.SpeciesId = %s + """ % self.mescape(self.dataset.type, + self.species_id) else: # Deal with >, <, >=, and <= print("self.search_term is:", self.search_term) - self.sub_clause = """ %sXRef.LRS %s %s and """ % self.mescape(self.dataset.type, + self.sub_clause = """ %sXRef.LRS %s %s """ % self.mescape(self.dataset.type, self.search_operator, self.search_term[0]) - - where_clause = self.sub_clause + """ %sXRef.Locus = Geno.name and - Geno.SpeciesId = %s and - %s.Chr = Geno.Chr - """ % self.mescape(self.dataset.type, - self.species_id, - self.dataset.type) + where_clause = self.sub_clause + return where_clause + def get_final_query(self): + self.from_clause = self.get_from_clause() + self.where_clause = self.get_where_clause() + self.query = self.compile_final_query(self.from_clause, self.where_clause) + + return self.query + def run(self): - self.from_clause = ", Geno" + self.from_clause = self.get_from_clause() #self.search_term = [float(value) for value in self.search_term] # @@ -651,6 +686,14 @@ class MeanSearch(MrnaAssaySearch): return where_clause + def get_final_query(self): + self.where_clause = self.get_where_clause() + print("where_clause is:", pf(self.where_clause)) + + self.query = self.compile_final_query(where_clause = self.where_clause) + + return self.query + def run(self): #self.search_term = [float(value) for value in self.search_term] @@ -682,14 +725,11 @@ class RangeSearch(MrnaAssaySearch): DoSearch.search_types['RANGE'] = "RangeSearch" - def run(self): - - self.search_term = [float(value) for value in self.search_term] - + def get_where_clause(self): if self.search_operator == "=": assert isinstance(self.search_term, (list, tuple)) self.range_min, self.range_max = self.search_term[:2] - self.where_clause = """ (SELECT Pow(2, max(value) -min(value)) + where_clause = """ (SELECT Pow(2, max(value) -min(value)) FROM ProbeSetData WHERE ProbeSetData.Id = ProbeSetXRef.dataId) > %s AND (SELECT Pow(2, max(value) -min(value)) @@ -699,12 +739,38 @@ class RangeSearch(MrnaAssaySearch): max(self.range_min, self.range_max)) else: # Deal with >, <, >=, and <= - self.where_clause = """ (SELECT Pow(2, max(value) -min(value)) + where_clause = """ (SELECT Pow(2, max(value) -min(value)) FROM ProbeSetData WHERE ProbeSetData.Id = ProbeSetXRef.dataId) > %s """ % (escape(self.search_term[0])) - print("where_clause is:", pf(self.where_clause)) + print("where_clause is:", pf(where_clause)) + + return where_clause + + def run(self): + + #self.search_term = [float(value) for value in self.search_term] + # + #if self.search_operator == "=": + # assert isinstance(self.search_term, (list, tuple)) + # self.range_min, self.range_max = self.search_term[:2] + # self.where_clause = """ (SELECT Pow(2, max(value) -min(value)) + # FROM ProbeSetData + # WHERE ProbeSetData.Id = ProbeSetXRef.dataId) > %s AND + # (SELECT Pow(2, max(value) -min(value)) + # FROM ProbeSetData + # WHERE ProbeSetData.Id = ProbeSetXRef.dataId) < %s + # """ % self.mescape(min(self.range_min, self.range_max), + # max(self.range_min, self.range_max)) + #else: + # # Deal with >, <, >=, and <= + # self.where_clause = """ (SELECT Pow(2, max(value) -min(value)) + # FROM ProbeSetData + # WHERE ProbeSetData.Id = ProbeSetXRef.dataId) > %s + # """ % (escape(self.search_term[0])) + + self.where_clause = self.get_where_clause() self.query = self.compile_final_query(where_clause = self.where_clause) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 60221c05..fa680f5f 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -720,6 +720,7 @@ class MarkerRegression(object): #def gen_human_results(self, pheno_vector, tempdata): def gen_human_results(self, pheno_vector, key, temp_uuid): file_base = os.path.join(webqtlConfig.PYLMM_PATH, self.dataset.group.name) + print("file_base:", file_base) plink_input = input.plink(file_base, type='b') input_file_name = os.path.join(webqtlConfig.SNP_PATH, self.dataset.group.name + ".snps.gz") diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index fdf04d44..a2958108 100755 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -222,7 +222,8 @@ class SearchResultPage(object): print("After parsing:", self.search_terms) if len(self.search_terms) > 1: - combined_where_clause = "" + combined_from_clause = "" + combined_where_clause = "" for i, a_search in enumerate(self.search_terms): print("[kodak] item is:", pf(a_search)) search_term = a_search['search_term'] @@ -242,33 +243,41 @@ class SearchResultPage(object): self.dataset, ) + #search_query = the_search.get_final_query() + + get_from_clause = getattr(the_search, "get_from_clause", None) + if callable(get_from_clause): + from_clause = the_search.get_from_clause() + combined_from_clause += from_clause where_clause = the_search.get_where_clause() combined_where_clause += "(" + where_clause + ")" if (i+1) < len(self.search_terms): combined_where_clause += "AND" - print("combined_where_clause:", combined_where_clause) - - for a_search in self.search_terms: - print("[kodak] item is:", pf(a_search)) - search_term = a_search['search_term'] - search_operator = a_search['separator'] - if a_search['key']: - search_type = a_search['key'].upper() - else: - # We fall back to the dataset type as the key to get the right object - search_type = self.dataset.type + results = the_search.run_combined(combined_from_clause, combined_where_clause) + self.results.extend(results) + + else: + for a_search in self.search_terms: + print("[kodak] item is:", pf(a_search)) + search_term = a_search['search_term'] + search_operator = a_search['separator'] + if a_search['key']: + search_type = a_search['key'].upper() + else: + # We fall back to the dataset type as the key to get the right object + search_type = self.dataset.type - print("search_type is:", pf(search_type)) - - search_ob = do_search.DoSearch.get_search(search_type) - search_class = getattr(do_search, search_ob) - print("search_class is: ", pf(search_class)) - the_search = search_class(search_term, - search_operator, - self.dataset, - ) - self.results.extend(the_search.run()) - #print("in the search results are:", self.results) + print("search_type is:", pf(search_type)) + + search_ob = do_search.DoSearch.get_search(search_type) + search_class = getattr(do_search, search_ob) + print("search_class is: ", pf(search_class)) + the_search = search_class(search_term, + search_operator, + self.dataset, + ) + self.results.extend(the_search.run()) + #print("in the search results are:", self.results) self.header_fields = the_search.header_fields diff --git a/wqflask/wqflask/static/new/javascript/scatter-matrix.js b/wqflask/wqflask/static/new/javascript/scatter-matrix.js index 38fd276e..38cdc75b 100644 --- a/wqflask/wqflask/static/new/javascript/scatter-matrix.js +++ b/wqflask/wqflask/static/new/javascript/scatter-matrix.js @@ -56,9 +56,11 @@ ScatterMatrix.prototype.render = function () { var container = d3.select('#scatterplot_container').append('div') .attr('class', 'scatter-matrix-container'); var control = container.append('div') - .attr('class', 'scatter-matrix-control'); + .attr('class', 'scatter-matrix-control') + .style({'float':'left'}) var svg = container.append('div') .attr('class', 'scatter-matrix-svg') + .style({'float':'right'}) .html('Loading data...'); this.onData(function() { diff --git a/wqflask/wqflask/static/new/javascript/show_trait.coffee b/wqflask/wqflask/static/new/javascript/show_trait.coffee index 2cdad5ee..d05ccbf7 100755 --- a/wqflask/wqflask/static/new/javascript/show_trait.coffee +++ b/wqflask/wqflask/static/new/javascript/show_trait.coffee @@ -95,12 +95,12 @@ $ -> new Box_Plot(all_samples) d3.select("#select_compare_trait").on("click", => - $('.qtlcharts').empty() + $('.scatter-matrix-container').remove() open_trait_selection() ) d3.select("#clear_compare_trait").on("click", => - $('.qtlcharts').empty() + $('.scatter-matrix-container').remove() ) open_trait_selection = () -> diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 40bc3d0a..159dafcb 100755 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -91,13 +91,13 @@ $(function() { }); d3.select("#select_compare_trait").on("click", (function(_this) { return function() { - $('.qtlcharts').empty(); + $('.scatter-matrix-container').remove(); return open_trait_selection(); }; })(this)); d3.select("#clear_compare_trait").on("click", (function(_this) { return function() { - return $('.qtlcharts').empty(); + return $('.scatter-matrix-container').remove(); }; })(this)); open_trait_selection = function() { diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 6f5fe237..d6c7b18e 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -84,7 +84,7 @@
- +
-->
-
+
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index cdf93147..9110c5a1 100755 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -293,7 +293,7 @@ def marker_regression_page(): if key in wanted or key.startswith(('value:')): start_vars[key] = value - version = "v4" + version = "v3" key = "marker_regression:{}:".format(version) + json.dumps(start_vars, sort_keys=True) print("key is:", pf(key)) with Bench("Loading cache"): -- cgit v1.2.3