diff options
| -rwxr-xr-x | wqflask/base/data_set.py | 6 | ||||
| -rwxr-xr-x | wqflask/base/trait.py | 2 | ||||
| -rw-r--r-- | wqflask/wqflask/do_search.py | 8 | ||||
| -rwxr-xr-x | wqflask/wqflask/marker_regression/marker_regression.py | 5 | ||||
| -rw-r--r-- | wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 26 | ||||
| -rw-r--r-- | wqflask/wqflask/search_results.py | 38 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/all_results.html | 134 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/index_page.html | 10 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/quick_search.html | 96 | ||||
| -rw-r--r-- | wqflask/wqflask/templates/show_trait_details.html | 4 | ||||
| -rw-r--r-- | wqflask/wqflask/views.py | 32 |
11 files changed, 259 insertions, 102 deletions
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 1520b180..9b0a3dcc 100755 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -385,7 +385,7 @@ class PhenotypeDataSet(DataSet): continue # for now if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=this_trait.authorized_users): description = this_trait.pre_publication_description - this_trait.description_display = description + this_trait.description_display = unicode(description, "utf8") if not this_trait.year.isdigit(): this_trait.pubmed_text = "N/A" @@ -735,8 +735,10 @@ class MrnaAssayDataSet(DataSet): trait_location_value = 1000000 if this_trait.chr and this_trait.mb: + print("this_trait.chr is:", this_trait.chr) + print("this_trait.mb is:", this_trait.mb) try: - trait_location_value = int(this_trait.chr)*1000 + this_trait.mb + trait_location_value = float(this_trait.chr)*1000 + float(this_trait.mb) except: if this_trait.chr.upper() == 'X': trait_location_value = 20*1000 + this_trait.mb diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index 7c1c035c..dde8b8d8 100755 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -302,7 +302,7 @@ class GeneralTrait: #XZ: assign SQL query result to trait attributes. for i, field in enumerate(self.dataset.display_fields): - setattr(self, field, traitInfo[i]) + setattr(self, field, str(traitInfo[i])) if self.dataset.type == 'Publish': self.confidential = 0 diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py index 1b1b56fb..7b3e0869 100644 --- a/wqflask/wqflask/do_search.py +++ b/wqflask/wqflask/do_search.py @@ -36,7 +36,7 @@ class DoSearch(object): def execute(self, query): """Executes query and returns results""" query = self.normalize_spaces(query) - print("in do_search query is:", pf(query)) + #print("in do_search query is:", pf(query)) results = g.db.execute(query, no_parameters=True).fetchall() #results = self.cursor.fetchall() return results @@ -90,7 +90,7 @@ class QuickMrnaAssaySearch(DoSearch): AGAINST ('%s' IN BOOLEAN MODE)) """ % (escape(self.search_term[0])) - print("final query is:", pf(query)) + #print("final query is:", pf(query)) return self.execute(query) @@ -134,7 +134,7 @@ class MrnaAssaySearch(DoSearch): where_clause, escape(self.dataset.id))) - print("query is:", pf(query)) + #print("query is:", pf(query)) return query @@ -155,7 +155,7 @@ class MrnaAssaySearch(DoSearch): """ % (escape(self.search_term[0]), escape(str(self.dataset.id))) - print("final query is:", pf(query)) + #print("final query is:", pf(query)) return self.execute(query) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 6ae1318e..334ce631 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -78,8 +78,9 @@ class MarkerRegression(object): genotype_matrix = np.array(trimmed_genotype_data).T - print("pheno_vector is: ", pf(pheno_vector)) - print("genotype_matrix is: ", pf(genotype_matrix)) + print("pheno_vector: ", pf(pheno_vector)) + print("genotype_matrix: ", pf(genotype_matrix)) + print("genotype_matrix.shape: ", pf(genotype_matrix.shape)) t_stats, p_values = lmm.run( pheno_vector, diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index fc021a0b..5bfc3a01 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -58,6 +58,10 @@ def run_human(pheno_vector, identifier = str(uuid.uuid4()) + print("pheno_vector: ", pf(pheno_vector)) + print("kinship_matrix: ", pf(kinship_matrix)) + print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) + lmm_vars = pickle.dumps(dict( pheno_vector = pheno_vector, covariate_matrix = covariate_matrix, @@ -70,12 +74,12 @@ def run_human(pheno_vector, pheno_vector = pheno_vector[keep] #print("pheno_vector shape is now: ", pf(pheno_vector.shape)) covariate_matrix = covariate_matrix[keep,:] - print("kinship_matrix shape is: ", pf(kinship_matrix.shape)) + #print("kinship_matrix shape is: ", pf(kinship_matrix.shape)) #print("len(keep) is: ", pf(keep.shape)) kinship_matrix = kinship_matrix[keep,:][:,keep] n = kinship_matrix.shape[0] - print("n is:", n) + #print("n is:", n) lmm_ob = LMM(pheno_vector, kinship_matrix, covariate_matrix) @@ -86,7 +90,7 @@ def run_human(pheno_vector, p_values = [] t_stats = [] - print("input_file: ", plink_input_file) + #print("input_file: ", plink_input_file) with Bench("Opening and loading pickle file"): with gzip.open(plink_input_file, "rb") as input_file: @@ -103,6 +107,8 @@ def run_human(pheno_vector, with Bench("Create list of inputs"): inputs = list(plink_input) + + print("len(genotypes): ", len(inputs)) with Bench("Divide into chunks"): results = chunks.divide_into_chunks(inputs, 64) @@ -116,7 +122,7 @@ def run_human(pheno_vector, timestamp = datetime.datetime.utcnow().isoformat() - print("Starting adding loop") + #print("Starting adding loop") for part, result in enumerate(results): #data = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) holder = pickle.dumps(dict( @@ -126,10 +132,10 @@ def run_human(pheno_vector, result = result ), pickle.HIGHEST_PROTOCOL) - print("Adding:", part) + #print("Adding:", part) Redis.rpush(key, zlib.compress(holder)) - print("End adding loop") - print("***** Added to {} queue *****".format(key)) + #print("End adding loop") + #print("***** Added to {} queue *****".format(key)) for snp, this_id in plink_input: #with Bench("part before association"): if count > 2000: @@ -157,6 +163,10 @@ def run_human(pheno_vector, return p_values, t_stats +#class HumanAssociation(object): +# def __init__(self): +# + def human_association(snp, n, keep, @@ -233,6 +243,8 @@ def run(pheno_vector, with Bench("LMM_ob fitting"): lmm_ob.fit() + print("genotype_matrix: ", genotype_matrix.shape) + with Bench("Doing GWAS"): t_stats, p_values = GWAS(pheno_vector, genotype_matrix, diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index 5f3c036f..63aa481e 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -73,8 +73,8 @@ class SearchResultPage(object): self.quick = True self.search_terms = kw['q'] print("self.search_terms is: ", self.search_terms) + self.trait_type = kw['trait_type'] self.quick_search() - self.get_group_species_tree() else: self.results = [] #self.quick_search = False @@ -124,7 +124,6 @@ class SearchResultPage(object): FROM QuickSearch WHERE MATCH (terms) AGAINST ('{}' IN BOOLEAN MODE) """.format(search_terms) - #print("query is: ", query) with Bench("Doing QuickSearch Query: "): dbresults = g.db.execute(query, no_parameters=True).fetchall() @@ -137,22 +136,33 @@ class SearchResultPage(object): 'GenoXRef': 'genotype'} self.species_groups = {} + for dbresult in dbresults: this_result = {} this_result['table_name'] = dbresult.table_name - this_result['key'] = dbresult.the_key - this_result['result_fields'] = json.loads(dbresult.result_fields) - this_species = this_result['result_fields']['species'] - this_group = this_result['result_fields']['group_name'] - if type_dict[dbresult.table_name] not in self.species_groups: - self.species_groups[type_dict[dbresult.table_name]] = {} - if this_species not in self.species_groups[type_dict[dbresult.table_name]]: - self.species_groups[type_dict[dbresult.table_name]][this_species] = collections.defaultdict(list) - if this_group not in self.species_groups[type_dict[dbresult.table_name]][this_species]: - self.species_groups[type_dict[dbresult.table_name]][this_species].append(this_group) - self.results[type_dict[dbresult.table_name]].append(this_result) + if self.trait_type == type_dict[dbresult.table_name] or self.trait_type == 'all': + this_result['key'] = dbresult.the_key + this_result['result_fields'] = json.loads(dbresult.result_fields) + this_species = this_result['result_fields']['species'] + this_group = this_result['result_fields']['group_name'] + if this_species not in self.species_groups: + self.species_groups[this_species] = {} + if type_dict[dbresult.table_name] not in self.species_groups[this_species]: + self.species_groups[this_species][type_dict[dbresult.table_name]] = [] + if this_group not in self.species_groups[this_species][type_dict[dbresult.table_name]]: + self.species_groups[this_species][type_dict[dbresult.table_name]].append(this_group) + #if type_dict[dbresult.table_name] not in self.species_groups: + # self.species_groups[type_dict[dbresult.table_name]] = {} + #if this_species not in self.species_groups[type_dict[dbresult.table_name]]: + # self.species_groups[type_dict[dbresult.table_name]][this_species] = [] + #if this_group not in self.species_groups[type_dict[dbresult.table_name]][this_species]: + # self.species_groups[type_dict[dbresult.table_name]][this_species].append(this_group) + self.results[type_dict[dbresult.table_name]].append(this_result) - #print("results: ", pf(self.results['phenotype'])) + import redis + Redis = redis.Redis() + + #def get_group_species_tree(self): # self.species_groups = collections.default_dict(list) diff --git a/wqflask/wqflask/templates/all_results.html b/wqflask/wqflask/templates/all_results.html new file mode 100644 index 00000000..a42e42d8 --- /dev/null +++ b/wqflask/wqflask/templates/all_results.html @@ -0,0 +1,134 @@ + <ul class="nav nav-tabs"> + {% for species in species_groups %} + <li> <a href="#tab{{ loop.index }}" data-toggle="tab">{{ species }}</a></li> + {% endfor %} + </ul> + <div class="tab-content"> + {% for species in species_groups %} + <div class="tab-pane active" id="tab{{ loop.index }}"> + <div class="tabbable tabs-left"> + <ul class="nav nav-tabs"> + {% if species_groups[species]['phenotype'] %} + <li class="active"> <a href="#tab{{ loop.index }}_1" data-toggle="tab">Phenotype</a></li> + {% endif %} + {% if species_groups[species]['mrna_assay'] %} + <li> <a href="#tab{{ loop.index }}_2" data-toggle="tab">mRNA Assay</a></li> + {% endif %} + {% if species_groups[species]['genotype'] %} + <li> <a href="#tab{{ loop.index }}_3" data-toggle="tab">Genotype</a></li> + {% endif %} + </ul> + <div class="tab-content"> + {% if species_groups[species]['phenotype'] %} + <div class="tab-pane active" id="tab{{ loop.index }}_1"> + <table id="pheno_results" class="table table-hover table-striped table-bordered"> + <thead> + <tr> + <th>Id</th> + <th>Species</th> + <th>Group</th> + <th>Description</th> + <th>LRS</th> + <th>Year</th> + <th>Authors</th> + </tr> + </thead> + <tbody> + {% for result in results.phenotype %} + {% if result.result_fields['species'] == species %} + <tr> + <td>{{ result.result_fields['phenotype_id'] }}</td> + <td>{{ result.result_fields['species'] }}</td> + <td>{{ result.result_fields['group_name'] }}</td> + <td>{{ result.result_fields['description'] }}</td> + <td>{{ result.result_fields['lrs'] }}</td> + <td> + <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ result.result_fields['pubmed_id'] }}&dopt=Abstract"> + {{ result.result_fields['year'] }} + </a> + </td> + <td>{{ result.result_fields['authors'] }}</td> + </tr> + {% endif %} + {% endfor %} + </tbody> + </table> + </div> + {% endif %} + {% if species_groups[species]['mrna_assay'] %} + <div class="tab-pane" id="tab{{ loop.index }}_2"> + <table id="mrna_assay_results" class="table table-hover table-striped table-bordered"> + <thead> + <tr> + <th>Record ID</th> + <th>Species</th> + <th>Group</th> + <th>Data Set</th> + <th>Symbol</th> + <th>Description</th> + <th>Location</th> + <th>Mean Expr</th> + <th>Max LRS</th> + </tr> + </thead> + <tbody> + {% for result in results.mrna_assay %} + {% if result.result_fields['species'] == species %} + <tr> + <td> + <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['name'] }}&dataset={{ result.result_fields['dataset'] }}" + {{ result.result_fields['name'] }} + </a> + </td> + <td>{{ result.result_fields['species'] }}</td> + <td>{{ result.result_fields['group_name'] }}</td> + <td>{{ result.result_fields['dataset_name'] }}</td> + <td>{{ result.result_fields['symbol'] }}</td> + <td>{{ result.result_fields['description'] }}</td> + <td>{{ result.result_fields['chr'] }} : {{ result['mb'] }}</td> + <td>{{ result.result_fields['mean'] }}</td> + <td>{{ result.result_fields['lrs'] }}</td> + </tr> + {% endif %} + {% endfor %} + </tbody> + </table> + </div> + {% endif %} + {% if species_groups[species]['genotype'] %} + <div class="tab-pane" id="tab{{ loop.index }}_3"> + <table id="geno_results" class="table table-hover table-striped table-bordered"> + <thead> + <tr> + <th>Marker</th> + <th>Species</th> + <th>Group</th> + <th>Data Set</th> + <th>Location</th> + </tr> + </thead> + <tbody> + {% for result in results.genotype %} + {% if result.result_fields['species'] == species %} + <tr> + <td> + <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['marker_name'] }}&dataset={{ result.result_fields['dataset'] }}"> + {{ result.result_fields['marker_name'] }} + </a> + </td> + <td>{{ result.result_fields['species'] }}</td> + <td>{{ result.result_fields['group_name'] }}</td> + <td>{{ result.result_fields['dataset_name'] }}</td> + <td>{{ result.result_fields['chr'] }} : {{ result.result_fields['mb'] }}</td> + </tr> + {% endif %} + {% endfor %} + </tbody> + </table> + </div> + {% endif %} + </div> + </div> + </div> + {% endfor %} + </div> \ No newline at end of file diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 5c88c923..7ab4cd3f 100644 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -32,7 +32,14 @@ </div> <form method="get" action="/search" name="SEARCHFORM"> <fieldset> - <label for="quick">Search:</label> + <div class="input-prepend"> + <select id="trait_type" name="trait_type" class="span2"> + <option value="all">All</option> + <option value="mrna_assay">Molecular Assays</option> + <option value="phenotype">Phenotypes</option> + <option value="genotype">Genotypes</option> + </select> + </div> <div class="input-append"> <input class="input-xlarge" id="quick" name="q" @@ -42,7 +49,6 @@ class="btn btn-primary" value="Search"> </div> </fieldset> - </form> </section> <section id="search"> diff --git a/wqflask/wqflask/templates/quick_search.html b/wqflask/wqflask/templates/quick_search.html index 9d5b0c74..b0e38708 100644 --- a/wqflask/wqflask/templates/quick_search.html +++ b/wqflask/wqflask/templates/quick_search.html @@ -25,33 +25,36 @@ </li> {% endif %} </ul> - - <p>To study a record, click on its ID below.<br /> - Check records below and click Add button to add to selection.</p> + <div class="tabbable"> <!-- Only required for left/right tabs --> + {% if trait_type == 'all' %} + {% include 'all_results.html' %} + {% else %} <ul class="nav nav-tabs"> - {% if results.phenotype %} - <li class="active"> <a href="#tab1" data-toggle="tab">Phenotype</a></li> - {% endif %} - {% if results.mrna_assay %} - <li> <a href="#tab2" data-toggle="tab">mRNA Assay</a></li> - {% endif %} - {% if results.genotype %} - <li> <a href="#tab3" data-toggle="tab">Genotype</a></li> - {% endif %} + {% for species in species_groups %} + <li> <a href="#tab{{ loop.index }}" data-toggle="tab">{{ species }}</a></li> + {% endfor %} </ul> <div class="tab-content"> - <div class="tab-pane active" id="tab1"> - <div class="tabbable tabs-left"> + {% for species in species_groups %} + <div class="tab-pane active" id="tab{{ loop.index }}"> + <div> + {# <ul class="nav nav-tabs"> - {% for species in species_groups.phenotype %} - <li> <a href="#tab1_{{ loop.index }}" data-toggle="tab">{{ species }}</a></li> - {% endfor %} + {% if species_groups[species]['phenotype'] %} + <li class="active"> <a href="#tab{{ loop.index }}_1" data-toggle="tab">Phenotype</a></li> + {% endif %} + {% if species_groups[species]['mrna_assay'] %} + <li> <a href="#tab{{ loop.index }}_2" data-toggle="tab">mRNA Assay</a></li> + {% endif %} + {% if species_groups[species]['genotype'] %} + <li> <a href="#tab{{ loop.index }}_3" data-toggle="tab">Genotype</a></li> + {% endif %} </ul> - <div class="tab-content"> - {% for species in species_groups.phenotype %} - <div class="tab-pane active" id="tab{{ loop.index }}"> + #} + <div> + {% if trait_type == 'phenotype' and species_groups[species]['phenotype'] %} <table id="pheno_results" class="table table-hover table-striped table-bordered"> <thead> <tr> @@ -84,21 +87,8 @@ {% endfor %} </tbody> </table> - </div> - {% endfor %} - </div> - </div> - </div> - <div class="tab-pane active" id="tab2"> - <div class="tabbable tabs-left"> - <ul class="nav nav-tabs"> - {% for species in species_groups.mrna_assay %} - <li> <a href="#tab2_{{ loop.index }}" data-toggle="tab">{{ species }}</a></li> - {% endfor %} - </ul> - <div class="tab-content"> - {% for species in species_groups.mrna_assay %} - <div class="tab-pane active" id="tab{{ loop.index }}"> + {% endif %} + {% if trait_type == 'mrna_assay' and species_groups[species]['mrna_assay'] %} <table id="mrna_assay_results" class="table table-hover table-striped table-bordered"> <thead> <tr> @@ -134,22 +124,9 @@ {% endif %} {% endfor %} </tbody> - </table> - </div> - {% endfor %} - </div> - </div> - </div> - <div class="tab-pane active" id="tab3"> - <div class="tabbable tabs-left"> - <ul class="nav nav-tabs"> - {% for species in species_groups.genotype %} - <li> <a href="#tab3_{{ loop.index }}" data-toggle="tab">{{ species }}</a></li> - {% endfor %} - </ul> - <div class="tab-content"> - {% for species in species_groups.genotype %} - <div class="tab-pane active" id="tab{{ loop.index }}"> + </table> + {% endif %} + {% if trait_type == 'genotype' and species_groups[species]['genotype'] %} <table id="geno_results" class="table table-hover table-striped table-bordered"> <thead> <tr> @@ -177,16 +154,21 @@ {% endif %} {% endfor %} </tbody> - </table> - </div> - {% endfor %} + </table> + {% endif %} </div> </div> </div> + {% endfor %} </div> + {% endif %} </div> </div> - + +<!-- End of body --> + +{% endblock %} + {# <table id="pheno_results" class="table table-hover table-striped table-bordered"> <thead> @@ -285,9 +267,6 @@ </div> #} -<!-- End of body --> - -{% endblock %} {% block js %} <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.js"></script> @@ -324,3 +303,4 @@ }); </script> {% endblock %} + diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index c3abfc9f..b57c3c21 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -19,11 +19,11 @@ BLAT Specifity </a> </dt> - <dd>{{ "%.1f" % (this_trait.probe_set_specificity) }}</dd> + <dd>{{ "%s" % (this_trait.probe_set_specificity) }}</dd> {% endif %} {% if this_trait.probe_set_blat_score %} <dt>BLAT Score</dt> - <dd>{{ "%i" % (this_trait.probe_set_blat_score) }}</dd> + <dd>{{ "%s" % (this_trait.probe_set_blat_score) }}</dd> {% endif %} </dl> diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index f92042bd..ffcb0b97 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -81,17 +81,28 @@ def search_page(): else: return render_template("data_sharing.html", **template_vars.__dict__) else: - print("calling search_results.SearchResultPage") - the_search = search_results.SearchResultPage(request.args) - print("template_vars is:", pf(the_search.__dict__)) - #print("trait_list is:", pf(the_search.__dict__['trait_list'][0].__dict__)) - #for trait in the_search.trait_list: - # print(" -", trait.description_display) - - if the_search.quick: - return render_template("quick_search.html", **the_search.__dict__) + key = "search_results:v2:" + json.dumps(request.args, sort_keys=True) + print("key is:", pf(key)) + with Bench("Loading cache"): + result = Redis.get(key) + + if result: + print("Cache hit!!!") + with Bench("Loading results"): + result = pickle.loads(result) else: - return render_template("search_result_page.html", **the_search.__dict__) + print("calling search_results.SearchResultPage") + the_search = search_results.SearchResultPage(request.args) + result = the_search.__dict__ + + print("result: ", pf(result)) + Redis.set(key, pickle.dumps(result)) + Redis.expire(key, 60*60) + + if result['quick']: + return render_template("quick_search.html", **result) + else: + return render_template("search_result_page.html", **result) @app.route("/whats_new") @@ -179,6 +190,7 @@ def marker_regression_page(): version = "v14" key = "marker_regression:{}:".format(version) + json.dumps(start_vars, sort_keys=True) + print("key is:", pf(key)) with Bench("Loading cache"): result = Redis.get(key) |