From be9c4a39500d7978b4cae7536a5f96c3818d211e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 24 Mar 2021 09:41:47 +0300 Subject: initial commit for gn3-correlation api integration --- .../wqflask/correlation/test_correlation_gn3.py | 14 ++++ wqflask/wqflask/correlation/correlation_gn3_api.py | 77 ++++++++++++++++++++++ 2 files changed, 91 insertions(+) create mode 100644 wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py create mode 100644 wqflask/wqflask/correlation/correlation_gn3_api.py diff --git a/wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py b/wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py new file mode 100644 index 00000000..e1bd6d86 --- /dev/null +++ b/wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py @@ -0,0 +1,14 @@ +"""this module contains tests for code used in integrating to gn3 api""" +from unittest import TestCase +from base.data_set import create_dataset + +class TestCorrelation(TestCase): + + def test_create_dataset(self): + """test for creating datasets""" + + pass + def test_fetch_dataset_info(self): + """test for fetching dataset info data""" + + pass diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py new file mode 100644 index 00000000..4cf6533c --- /dev/null +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -0,0 +1,77 @@ +"""module that calls the gn3 api's to do the correlation """ +from base import data_set +from base.trait import create_trait +from base.trait import retrieve_sample_data + + + + + + + +def compute_sample_r(start_vars,target_dataset, trait_data, target_samplelist, method="pearson"): + import requests + from wqflask.correlation.correlation_gn3_api import compute_correlation + + cor_results = compute_correlation(start_vars) + + data = { + "target_dataset": target_dataset, + "target_samplelist": target_samplelist, + "trait_data": { + "trait_sample_data": trait_data, + "trait_id": "HC_Q" + } + } + requests_url = f"http://127.0.0.1:8080/api/correlation/sample_x/{method}" + + results = requests.post(requests_url, json=data) + + data = results.json() + + print(data) + + return data + + +def process_samples(start_vars,sample_names,excluded_samples=None): + sample_data = {} + if not excluded_samples: + excluded_samples = () + + sample_vals_dict = json.loads(start_vars["sample_vals"]) + + for sample in sample_names: + if sample not in excluded_samples: + val = sample_val_dict[sample] + if not val.strip().lower() == "x": + sample_data[str(sample)]=float(value) + + return sample_data + + +def create_fetch_dataset_data(dataset_name): + this_dataset = data_set.create_dataset(dataset_name=dataset_name) + + this_dataset.get_trait_data() + + +def create_target_this_trait(start_vars): + """this function prefetch required data for correlation""" + + this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) + target_dataset = data_set.create_dataset( + dataset_name=start_vars['corr_dataset']) + + this_trait = create_trait(dataset=this_dataset, + name=start_vars['trait_id']) + + this_trait = retrieve_sample_data(this_trait, this_dataset) + + target_dataset.get_trait_data() + + return (this_dataset,this_trait,target_dataset) +def compute_correlation(start_vars): + + this_dataset, this_trait, target_dataset = create_target_this_trait( + start_vars=start_vars) -- cgit v1.2.3 From d913848572dd284ae7656e72dad199e99907871a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 24 Mar 2021 12:59:49 +0300 Subject: initial commit for integrating to gn3 api --- wqflask/wqflask/correlation/show_corr_results.py | 428 +++++++++++++---------- 1 file changed, 242 insertions(+), 186 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index fb4dc4f4..a817a4a4 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -1,4 +1,4 @@ -## Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. # # This program is free software: you can redistribute it and/or modify it # under the terms of the GNU Affero General Public License @@ -58,6 +58,31 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 +def compute_sample_r(start_vars,target_dataset, trait_data, target_samplelist, method="pearson"): + import requests + from wqflask.correlation.correlation_gn3_api import compute_correlation + + # cor_results = compute_correlation(start_vars) + + data = { + "target_dataset": target_dataset, + "target_samplelist": target_samplelist, + "trait_data": { + "trait_sample_data": trait_data, + "trait_id": "HC_Q" + } + } + requests_url = f"http://127.0.0.1:8080/api/correlation/sample_x/{method}" + + results = requests.post(requests_url, json=data) + + data = results.json() + + print(data) + + return data + + class CorrelationResults(object): def __init__(self, start_vars): # get trait list from db (database name) @@ -78,11 +103,12 @@ class CorrelationResults(object): with Bench("Doing correlations"): if start_vars['dataset'] == "Temp": - self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) + self.dataset = data_set.create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=start_vars['group']) self.trait_id = start_vars['trait_id'] self.this_trait = create_trait(dataset=self.dataset, - name=self.trait_id, - cellid=None) + name=self.trait_id, + cellid=None) else: helper_functions.get_species_dataset_trait(self, start_vars) @@ -97,7 +123,7 @@ class CorrelationResults(object): if ('loc_chr' in start_vars and 'min_loc_mb' in start_vars and - 'max_loc_mb' in start_vars): + 'max_loc_mb' in start_vars): self.location_type = get_string(start_vars, 'location_type') self.location_chr = get_string(start_vars, 'loc_chr') @@ -109,8 +135,8 @@ class CorrelationResults(object): self.get_formatted_corr_type() self.return_number = int(start_vars['corr_return_results']) - #The two if statements below append samples to the sample list based upon whether the user - #rselected Primary Samples Only, Other Samples Only, or All Samples + # The two if statements below append samples to the sample list based upon whether the user + # rselected Primary Samples Only, Other Samples Only, or All Samples primary_samples = self.dataset.group.samplelist if self.dataset.group.parlist != None: @@ -118,23 +144,26 @@ class CorrelationResults(object): if self.dataset.group.f1list != None: primary_samples += self.dataset.group.f1list - #If either BXD/whatever Only or All Samples, append all of that group's samplelist + # If either BXD/whatever Only or All Samples, append all of that group's samplelist if corr_samples_group != 'samples_other': self.process_samples(start_vars, primary_samples) - #If either Non-BXD/whatever or All Samples, get all samples from this_trait.data and - #exclude the primary samples (because they would have been added in the previous - #if statement if the user selected All Samples) + # If either Non-BXD/whatever or All Samples, get all samples from this_trait.data and + # exclude the primary samples (because they would have been added in the previous + # if statement if the user selected All Samples) if corr_samples_group != 'samples_primary': if corr_samples_group == 'samples_other': primary_samples = [x for x in primary_samples if x not in ( - self.dataset.group.parlist + self.dataset.group.f1list)] - self.process_samples(start_vars, list(self.this_trait.data.keys()), primary_samples) + self.dataset.group.parlist + self.dataset.group.f1list)] + self.process_samples(start_vars, list( + self.this_trait.data.keys()), primary_samples) - self.target_dataset = data_set.create_dataset(start_vars['corr_dataset']) + self.target_dataset = data_set.create_dataset( + start_vars['corr_dataset']) self.target_dataset.get_trait_data(list(self.sample_data.keys())) - self.header_fields = get_header_fields(self.target_dataset.type, self.corr_method) + self.header_fields = get_header_fields( + self.target_dataset.type, self.corr_method) if self.target_dataset.type == "ProbeSet": self.filter_cols = [7, 6] @@ -153,7 +182,8 @@ class CorrelationResults(object): tissue_corr_data = self.do_tissue_correlation_for_all_traits() if tissue_corr_data != None: for trait in list(tissue_corr_data.keys())[:self.return_number]: - self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) + self.get_sample_r_and_p_values( + trait, self.target_dataset.trait_data[trait]) else: for trait, values in list(self.target_dataset.trait_data.items()): self.get_sample_r_and_p_values(trait, values) @@ -163,80 +193,85 @@ class CorrelationResults(object): lit_corr_data = self.do_lit_correlation_for_all_traits() for trait in list(lit_corr_data.keys())[:self.return_number]: - self.get_sample_r_and_p_values(trait, self.target_dataset.trait_data[trait]) + self.get_sample_r_and_p_values( + trait, self.target_dataset.trait_data[trait]) elif self.corr_type == "sample": - for trait, values in list(self.target_dataset.trait_data.items()): - self.get_sample_r_and_p_values(trait, values) - - self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), - key=lambda t: -abs(t[1][0]))) - - - #ZS: Convert min/max chromosome to an int for the location range option - range_chr_as_int = None - for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): - if 'loc_chr' in start_vars: - if chr_info.name == self.location_chr: - range_chr_as_int = order_id - - for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): - trait_object = create_trait(dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) - if not trait_object: - continue - - chr_as_int = 0 - for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): - if self.location_type == "highest_lod": - if chr_info.name == trait_object.locus_chr: - chr_as_int = order_id - else: - if chr_info.name == trait_object.chr: - chr_as_int = order_id - - if (float(self.correlation_data[trait][0]) >= self.p_range_lower and - float(self.correlation_data[trait][0]) <= self.p_range_upper): - - if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean): - if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr): - continue - - if range_chr_as_int != None and (chr_as_int != range_chr_as_int): - continue - if self.location_type == "highest_lod": - if (self.min_location_mb != None) and (float(trait_object.locus_mb) < float(self.min_location_mb)): - continue - if (self.max_location_mb != None) and (float(trait_object.locus_mb) > float(self.max_location_mb)): - continue - else: - if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)): - continue - if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)): - continue - - (trait_object.sample_r, - trait_object.sample_p, - trait_object.num_overlap) = self.correlation_data[trait] - - # Set some sane defaults - trait_object.tissue_corr = 0 - trait_object.tissue_pvalue = 0 - trait_object.lit_corr = 0 - if self.corr_type == "tissue" and tissue_corr_data != None: - trait_object.tissue_corr = tissue_corr_data[trait][1] - trait_object.tissue_pvalue = tissue_corr_data[trait][2] - elif self.corr_type == "lit": - trait_object.lit_corr = lit_corr_data[trait][1] - - self.correlation_results.append(trait_object) - - if self.corr_type != "lit" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": - self.do_lit_correlation_for_trait_list() - - if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": - self.do_tissue_correlation_for_trait_list() - - self.json_results = generate_corr_json(self.correlation_results, self.this_trait, self.dataset, self.target_dataset) + + compute_sample_r(start_vars, + self.target_dataset.trait_data, self.sample_data, self.target_dataset.samplelist) + # for trait, values in list(self.target_dataset.trait_data.items()): + # self.get_sample_r_and_p_values(trait, values) + + # self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), + # key=lambda t: -abs(t[1][0]))) + + # # ZS: Convert min/max chromosome to an int for the location range option + # range_chr_as_int = None + # for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): + # if 'loc_chr' in start_vars: + # if chr_info.name == self.location_chr: + # range_chr_as_int = order_id + + # for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): + # trait_object = create_trait( + # dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) + # if not trait_object: + # continue + + # chr_as_int = 0 + # for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): + # if self.location_type == "highest_lod": + # if chr_info.name == trait_object.locus_chr: + # chr_as_int = order_id + # else: + # if chr_info.name == trait_object.chr: + # chr_as_int = order_id + + # if (float(self.correlation_data[trait][0]) >= self.p_range_lower and + # float(self.correlation_data[trait][0]) <= self.p_range_upper): + + # if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean): + # if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr): + # continue + + # if range_chr_as_int != None and (chr_as_int != range_chr_as_int): + # continue + # if self.location_type == "highest_lod": + # if (self.min_location_mb != None) and (float(trait_object.locus_mb) < float(self.min_location_mb)): + # continue + # if (self.max_location_mb != None) and (float(trait_object.locus_mb) > float(self.max_location_mb)): + # continue + # else: + # if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)): + # continue + # if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)): + # continue + + # (trait_object.sample_r, + # trait_object.sample_p, + # trait_object.num_overlap) = self.correlation_data[trait] + + # # Set some sane defaults + # trait_object.tissue_corr = 0 + # trait_object.tissue_pvalue = 0 + # trait_object.lit_corr = 0 + # if self.corr_type == "tissue" and tissue_corr_data != None: + # trait_object.tissue_corr = tissue_corr_data[trait][1] + # trait_object.tissue_pvalue = tissue_corr_data[trait][2] + # elif self.corr_type == "lit": + # trait_object.lit_corr = lit_corr_data[trait][1] + + # self.correlation_results.append(trait_object) + + # if self.corr_type != "lit" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": + # self.do_lit_correlation_for_trait_list() + + # if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": + # self.do_tissue_correlation_for_trait_list() + + # self.json_results = generate_corr_json( + # self.correlation_results, self.this_trait, self.dataset, self.target_dataset) ############################################################################################################################################ @@ -259,39 +294,43 @@ class CorrelationResults(object): def do_tissue_correlation_for_trait_list(self, tissue_dataset_id=1): """Given a list of correlation results (self.correlation_results), gets the tissue correlation value for each""" - #Gets tissue expression values for the primary trait + # Gets tissue expression values for the primary trait primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list = [self.this_trait.symbol]) + symbol_list=[self.this_trait.symbol]) if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower()] - gene_symbol_list = [trait.symbol for trait in self.correlation_results if trait.symbol] + primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower( + )] + gene_symbol_list = [ + trait.symbol for trait in self.correlation_results if trait.symbol] - corr_result_tissue_vals_dict= correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=gene_symbol_list) + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=gene_symbol_list) for trait in self.correlation_results: if trait.symbol and trait.symbol.lower() in corr_result_tissue_vals_dict: - this_trait_tissue_values = corr_result_tissue_vals_dict[trait.symbol.lower()] + this_trait_tissue_values = corr_result_tissue_vals_dict[trait.symbol.lower( + )] result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values, - this_trait_tissue_values, - self.corr_method) + this_trait_tissue_values, + self.corr_method) trait.tissue_corr = result[0] trait.tissue_pvalue = result[2] def do_tissue_correlation_for_all_traits(self, tissue_dataset_id=1): - #Gets tissue expression values for the primary trait + # Gets tissue expression values for the primary trait primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list = [self.this_trait.symbol]) + symbol_list=[self.this_trait.symbol]) if self.this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower()] + primary_trait_tissue_values = primary_trait_tissue_vals_dict[self.this_trait.symbol.lower( + )] #print("trait_gene_symbols: ", pf(trait_gene_symbols.values())) - corr_result_tissue_vals_dict= correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=list(self.trait_symbol_dict.values())) + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=list(self.trait_symbol_dict.values())) #print("corr_result_tissue_vals: ", pf(corr_result_tissue_vals_dict)) @@ -300,27 +339,30 @@ class CorrelationResults(object): tissue_corr_data = {} for trait, symbol in list(self.trait_symbol_dict.items()): if symbol and symbol.lower() in corr_result_tissue_vals_dict: - this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower()] + this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower( + )] result = correlation_functions.cal_zero_order_corr_for_tiss(primary_trait_tissue_values, - this_trait_tissue_values, - self.corr_method) + this_trait_tissue_values, + self.corr_method) tissue_corr_data[trait] = [symbol, result[0], result[2]] tissue_corr_data = collections.OrderedDict(sorted(list(tissue_corr_data.items()), - key=lambda t: -abs(t[1][1]))) + key=lambda t: -abs(t[1][1]))) return tissue_corr_data def do_lit_correlation_for_trait_list(self): - input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) + input_trait_mouse_gene_id = self.convert_to_mouse_gene_id( + self.dataset.group.species.lower(), self.this_trait.geneid) for trait in self.correlation_results: if trait.geneid: - trait.mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), trait.geneid) + trait.mouse_gene_id = self.convert_to_mouse_gene_id( + self.dataset.group.species.lower(), trait.geneid) else: trait.mouse_gene_id = None @@ -348,13 +390,14 @@ class CorrelationResults(object): else: trait.lit_corr = 0 - def do_lit_correlation_for_all_traits(self): - input_trait_mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), self.this_trait.geneid) + input_trait_mouse_gene_id = self.convert_to_mouse_gene_id( + self.dataset.group.species.lower(), self.this_trait.geneid) lit_corr_data = {} for trait, gene_id in list(self.trait_geneid_dict.items()): - mouse_gene_id = self.convert_to_mouse_gene_id(self.dataset.group.species.lower(), gene_id) + mouse_gene_id = self.convert_to_mouse_gene_id( + self.dataset.group.species.lower(), gene_id) if mouse_gene_id and str(mouse_gene_id).find(";") == -1: #print("gene_symbols:", input_trait_mouse_gene_id + " / " + mouse_gene_id) @@ -382,7 +425,7 @@ class CorrelationResults(object): lit_corr_data[trait] = [gene_id, 0] lit_corr_data = collections.OrderedDict(sorted(list(lit_corr_data.items()), - key=lambda t: -abs(t[1][1]))) + key=lambda t: -abs(t[1][1]))) return lit_corr_data @@ -422,6 +465,7 @@ class CorrelationResults(object): return mouse_gene_id + def get_sample_r_and_p_values(self, trait, target_samples): """Calculates the sample r (or rho) and p-value @@ -431,6 +475,9 @@ class CorrelationResults(object): """ + print("below here>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") + print(self.target_dataset.trait_data) + self.this_trait_vals = [] target_vals = [] for index, sample in enumerate(self.target_dataset.samplelist): @@ -440,21 +487,26 @@ class CorrelationResults(object): self.this_trait_vals.append(sample_value) target_vals.append(target_sample_value) - self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals) + self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values( + self.this_trait_vals, target_vals) if num_overlap > 5: - #ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/ + # ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/ if self.corr_method == 'bicor': - sample_r, sample_p = do_bicor(self.this_trait_vals, target_vals) + sample_r, sample_p = do_bicor( + self.this_trait_vals, target_vals) elif self.corr_method == 'pearson': - sample_r, sample_p = scipy.stats.pearsonr(self.this_trait_vals, target_vals) + sample_r, sample_p = scipy.stats.pearsonr( + self.this_trait_vals, target_vals) else: - sample_r, sample_p = scipy.stats.spearmanr(self.this_trait_vals, target_vals) + sample_r, sample_p = scipy.stats.spearmanr( + self.this_trait_vals, target_vals) if numpy.isnan(sample_r): pass else: - self.correlation_data[trait] = [sample_r, sample_p, num_overlap] + self.correlation_data[trait] = [ + sample_r, sample_p, num_overlap] def process_samples(self, start_vars, sample_names, excluded_samples=None): if not excluded_samples: @@ -475,16 +527,18 @@ def do_bicor(this_trait_vals, target_trait_vals): r_library("WGCNA") r_bicor = ro.r["bicorAndPvalue"] # Map the bicorAndPvalue function - r_options(stringsAsFactors = False) + r_options(stringsAsFactors=False) this_vals = ro.Vector(this_trait_vals) target_vals = ro.Vector(target_trait_vals) - the_r, the_p, _fisher_transform, _the_t, _n_obs = [numpy.asarray(x) for x in r_bicor(x = this_vals, y = target_vals)] + the_r, the_p, _fisher_transform, _the_t, _n_obs = [ + numpy.asarray(x) for x in r_bicor(x=this_vals, y=target_vals)] return the_r, the_p -def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_api = False): + +def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_api=False): results_list = [] for i, trait in enumerate(corr_results): if trait.view == False: @@ -493,7 +547,8 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap results_dict['index'] = i + 1 results_dict['trait_id'] = trait.name results_dict['dataset'] = trait.dataset.name - results_dict['hmac'] = hmac.data_hmac('{}:{}'.format(trait.name, trait.dataset.name)) + results_dict['hmac'] = hmac.data_hmac( + '{}:{}'.format(trait.name, trait.dataset.name)) if target_dataset.type == "ProbeSet": results_dict['symbol'] = trait.symbol results_dict['description'] = "N/A" @@ -544,7 +599,8 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap if bool(trait.authors): authors_list = trait.authors.split(',') if len(authors_list) > 6: - results_dict['authors_display'] = ", ".join(authors_list[:6]) + ", et al." + results_dict['authors_display'] = ", ".join( + authors_list[:6]) + ", et al." else: results_dict['authors_display'] = trait.authors if bool(trait.pubmed_id): @@ -574,85 +630,85 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap return json.dumps(results_list) + def get_header_fields(data_type, corr_method): if data_type == "ProbeSet": if corr_method == "spearman": header_fields = ['Index', - 'Record', - 'Symbol', - 'Description', - 'Location', - 'Mean', - 'Sample rho', - 'N', - 'Sample p(rho)', - 'Lit rho', - 'Tissue rho', - 'Tissue p(rho)', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Record', + 'Symbol', + 'Description', + 'Location', + 'Mean', + 'Sample rho', + 'N', + 'Sample p(rho)', + 'Lit rho', + 'Tissue rho', + 'Tissue p(rho)', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] else: header_fields = ['Index', - 'Record', - 'Symbol', - 'Description', - 'Location', - 'Mean', - 'Sample r', - 'N', - 'Sample p(r)', - 'Lit r', - 'Tissue r', - 'Tissue p(r)', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Record', + 'Symbol', + 'Description', + 'Location', + 'Mean', + 'Sample r', + 'N', + 'Sample p(r)', + 'Lit r', + 'Tissue r', + 'Tissue p(r)', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] elif data_type == "Publish": if corr_method == "spearman": header_fields = ['Index', - 'Record', - 'Abbreviation', - 'Description', - 'Mean', - 'Authors', - 'Year', - 'Sample rho', - 'N', - 'Sample p(rho)', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Record', + 'Abbreviation', + 'Description', + 'Mean', + 'Authors', + 'Year', + 'Sample rho', + 'N', + 'Sample p(rho)', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] else: header_fields = ['Index', - 'Record', - 'Abbreviation', - 'Description', - 'Mean', - 'Authors', - 'Year', - 'Sample r', - 'N', - 'Sample p(r)', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Record', + 'Abbreviation', + 'Description', + 'Mean', + 'Authors', + 'Year', + 'Sample r', + 'N', + 'Sample p(r)', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] else: if corr_method == "spearman": header_fields = ['Index', - 'ID', - 'Location', - 'Sample rho', - 'N', - 'Sample p(rho)'] + 'ID', + 'Location', + 'Sample rho', + 'N', + 'Sample p(rho)'] else: header_fields = ['Index', - 'ID', - 'Location', - 'Sample r', - 'N', - 'Sample p(r)'] + 'ID', + 'Location', + 'Sample r', + 'N', + 'Sample p(r)'] return header_fields - -- cgit v1.2.3 From e5d2ce8f29e43900977b967ec8cac715f544a2f0 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 25 Mar 2021 02:25:45 +0300 Subject: add code for calling gn3 correlation endpoint --- wqflask/wqflask/correlation/correlation_gn3_api.py | 131 ++++++++++++---- wqflask/wqflask/correlation/show_corr_results.py | 174 +++++++++------------ wqflask/wqflask/views.py | 5 +- 3 files changed, 177 insertions(+), 133 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 4cf6533c..7e269e41 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -1,20 +1,17 @@ """module that calls the gn3 api's to do the correlation """ -from base import data_set -from base.trait import create_trait -from base.trait import retrieve_sample_data +import json +import requests +from wqflask.wqflask.correlation import correlation_functions +from wqflask.base import data_set +from wqflask.base.trait import create_trait +from wqflask.base.trait import retrieve_sample_data +GN3_CORRELATION_API = "http://127.0.0.1:8080/api/correlation" - - - -def compute_sample_r(start_vars,target_dataset, trait_data, target_samplelist, method="pearson"): - import requests - from wqflask.correlation.correlation_gn3_api import compute_correlation - - cor_results = compute_correlation(start_vars) - +def compute_sample(target_dataset, trait_data, target_samplelist, method="pearson"): + """integration for integrating sample_r api correlation""" data = { "target_dataset": target_dataset, "target_samplelist": target_samplelist, @@ -29,33 +26,60 @@ def compute_sample_r(start_vars,target_dataset, trait_data, target_samplelist, m data = results.json() - print(data) - return data -def process_samples(start_vars,sample_names,excluded_samples=None): +def get_tissue_correlation_input(this_trait, trait_symbol_dict): + """Gets tissue expression values for the primary trait and target tissues values""" + primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=[this_trait.symbol]) + + if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: + primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( + )] + + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=list(trait_symbol_dict.values())) + + target_tissue_data = [] + for trait, symbol in list(trait_symbol_dict.items()): + if symbol and symbol.lower() in corr_result_tissue_vals_dict: + this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower( + )] + + this_trait_data = {"trait_id": trait, + "tissue_values": this_trait_tissue_values} + + target_tissue_data.append(this_trait_data) + + primary_tissue_data = { + "this_id": "TT", + "tissue_values": primary_trait_tissue_values + + } + + return (primary_tissue_data, target_tissue_data) + + return None + + +def process_samples(start_vars, sample_names, excluded_samples=None): + """process samples method""" sample_data = {} if not excluded_samples: excluded_samples = () - sample_vals_dict = json.loads(start_vars["sample_vals"]) + sample_vals_dict = json.loads(start_vars["sample_vals"]) for sample in sample_names: if sample not in excluded_samples: - val = sample_val_dict[sample] + val = sample_vals_dict[sample] if not val.strip().lower() == "x": - sample_data[str(sample)]=float(value) + sample_data[str(sample)] = float(val) return sample_data -def create_fetch_dataset_data(dataset_name): - this_dataset = data_set.create_dataset(dataset_name=dataset_name) - - this_dataset.get_trait_data() - - def create_target_this_trait(start_vars): """this function prefetch required data for correlation""" @@ -66,12 +90,61 @@ def create_target_this_trait(start_vars): this_trait = create_trait(dataset=this_dataset, name=start_vars['trait_id']) + sample_data = process_samples(start_vars, this_dataset.group.samplelist) + # target_dataset.get_trait_data(list(self.sample_data.keys())) + this_trait = retrieve_sample_data(this_trait, this_dataset) - target_dataset.get_trait_data() + target_dataset.get_trait_data(list(sample_data.keys())) + + return (this_dataset, this_trait, target_dataset, sample_data) + + +def compute_correlation(start_vars, method="pearson"): + """compute correlation for to call gn3 api""" + + corr_type = start_vars['corr_type'] + + (this_dataset, this_trait, target_dataset, + sample_data) = create_target_this_trait(start_vars) + + # cor_results = compute_correlation(start_vars) + + method = start_vars['corr_sample_method'] + + corr_input_data = {} - return (this_dataset,this_trait,target_dataset) -def compute_correlation(start_vars): + if corr_type == "sample": + corr_input_data = { + "target_dataset": target_dataset.trait_data, + "target_samplelist": target_dataset.samplelist, + "trait_data": { + "trait_sample_data": sample_data, + "trait_id": start_vars["trait_id"] + } + } + + requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" + + elif corr_type == "tissue": + trait_symbol_dict = this_dataset.retrieve_genes("Symbol") + primary_tissue_data, target_tissue_data = get_tissue_correlation_input( + this_trait, trait_symbol_dict) + + corr_input_data = { + "primary_tissue": primary_tissue_data, + "target_tissues": target_tissue_data + } - this_dataset, this_trait, target_dataset = create_target_this_trait( - start_vars=start_vars) + requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" + + else: + pass + # lit correlation/literature + # can fetch values in gn3 not set up in gn3 + + corr_results = requests.post(requests_url, json=corr_input_data) + + data = corr_results.json() + + return data diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index a817a4a4..50b3ba26 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -57,32 +57,6 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 - -def compute_sample_r(start_vars,target_dataset, trait_data, target_samplelist, method="pearson"): - import requests - from wqflask.correlation.correlation_gn3_api import compute_correlation - - # cor_results = compute_correlation(start_vars) - - data = { - "target_dataset": target_dataset, - "target_samplelist": target_samplelist, - "trait_data": { - "trait_sample_data": trait_data, - "trait_id": "HC_Q" - } - } - requests_url = f"http://127.0.0.1:8080/api/correlation/sample_x/{method}" - - results = requests.post(requests_url, json=data) - - data = results.json() - - print(data) - - return data - - class CorrelationResults(object): def __init__(self, start_vars): # get trait list from db (database name) @@ -197,81 +171,78 @@ class CorrelationResults(object): trait, self.target_dataset.trait_data[trait]) elif self.corr_type == "sample": - - compute_sample_r(start_vars, - self.target_dataset.trait_data, self.sample_data, self.target_dataset.samplelist) - # for trait, values in list(self.target_dataset.trait_data.items()): - # self.get_sample_r_and_p_values(trait, values) - - # self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), - # key=lambda t: -abs(t[1][0]))) - - # # ZS: Convert min/max chromosome to an int for the location range option - # range_chr_as_int = None - # for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): - # if 'loc_chr' in start_vars: - # if chr_info.name == self.location_chr: - # range_chr_as_int = order_id - - # for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): - # trait_object = create_trait( - # dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) - # if not trait_object: - # continue - - # chr_as_int = 0 - # for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): - # if self.location_type == "highest_lod": - # if chr_info.name == trait_object.locus_chr: - # chr_as_int = order_id - # else: - # if chr_info.name == trait_object.chr: - # chr_as_int = order_id - - # if (float(self.correlation_data[trait][0]) >= self.p_range_lower and - # float(self.correlation_data[trait][0]) <= self.p_range_upper): - - # if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean): - # if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr): - # continue - - # if range_chr_as_int != None and (chr_as_int != range_chr_as_int): - # continue - # if self.location_type == "highest_lod": - # if (self.min_location_mb != None) and (float(trait_object.locus_mb) < float(self.min_location_mb)): - # continue - # if (self.max_location_mb != None) and (float(trait_object.locus_mb) > float(self.max_location_mb)): - # continue - # else: - # if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)): - # continue - # if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)): - # continue - - # (trait_object.sample_r, - # trait_object.sample_p, - # trait_object.num_overlap) = self.correlation_data[trait] - - # # Set some sane defaults - # trait_object.tissue_corr = 0 - # trait_object.tissue_pvalue = 0 - # trait_object.lit_corr = 0 - # if self.corr_type == "tissue" and tissue_corr_data != None: - # trait_object.tissue_corr = tissue_corr_data[trait][1] - # trait_object.tissue_pvalue = tissue_corr_data[trait][2] - # elif self.corr_type == "lit": - # trait_object.lit_corr = lit_corr_data[trait][1] - - # self.correlation_results.append(trait_object) - - # if self.corr_type != "lit" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": - # self.do_lit_correlation_for_trait_list() - - # if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": - # self.do_tissue_correlation_for_trait_list() - - # self.json_results = generate_corr_json( - # self.correlation_results, self.this_trait, self.dataset, self.target_dataset) + for trait, values in list(self.target_dataset.trait_data.items()): + self.get_sample_r_and_p_values(trait, values) + + self.correlation_data = collections.OrderedDict(sorted(list(self.correlation_data.items()), + key=lambda t: -abs(t[1][0]))) + + # ZS: Convert min/max chromosome to an int for the location range option + range_chr_as_int = None + for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): + if 'loc_chr' in start_vars: + if chr_info.name == self.location_chr: + range_chr_as_int = order_id + + for _trait_counter, trait in enumerate(list(self.correlation_data.keys())[:self.return_number]): + trait_object = create_trait( + dataset=self.target_dataset, name=trait, get_qtl_info=True, get_sample_info=False) + if not trait_object: + continue + + chr_as_int = 0 + for order_id, chr_info in list(self.dataset.species.chromosomes.chromosomes.items()): + if self.location_type == "highest_lod": + if chr_info.name == trait_object.locus_chr: + chr_as_int = order_id + else: + if chr_info.name == trait_object.chr: + chr_as_int = order_id + + if (float(self.correlation_data[trait][0]) >= self.p_range_lower and + float(self.correlation_data[trait][0]) <= self.p_range_upper): + + if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean): + if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr): + continue + + if range_chr_as_int != None and (chr_as_int != range_chr_as_int): + continue + if self.location_type == "highest_lod": + if (self.min_location_mb != None) and (float(trait_object.locus_mb) < float(self.min_location_mb)): + continue + if (self.max_location_mb != None) and (float(trait_object.locus_mb) > float(self.max_location_mb)): + continue + else: + if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)): + continue + if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)): + continue + + (trait_object.sample_r, + trait_object.sample_p, + trait_object.num_overlap) = self.correlation_data[trait] + + # Set some sane defaults + trait_object.tissue_corr = 0 + trait_object.tissue_pvalue = 0 + trait_object.lit_corr = 0 + if self.corr_type == "tissue" and tissue_corr_data != None: + trait_object.tissue_corr = tissue_corr_data[trait][1] + trait_object.tissue_pvalue = tissue_corr_data[trait][2] + elif self.corr_type == "lit": + trait_object.lit_corr = lit_corr_data[trait][1] + + self.correlation_results.append(trait_object) + + if self.corr_type != "lit" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": + self.do_lit_correlation_for_trait_list() + + if self.corr_type != "tissue" and self.dataset.type == "ProbeSet" and self.target_dataset.type == "ProbeSet": + self.do_tissue_correlation_for_trait_list() + + self.json_results = generate_corr_json( + self.correlation_results, self.this_trait, self.dataset, self.target_dataset) ############################################################################################################################################ @@ -465,7 +436,6 @@ class CorrelationResults(object): return mouse_gene_id - def get_sample_r_and_p_values(self, trait, target_samples): """Calculates the sample r (or rho) and p-value diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 2c0ba586..6ca9b23f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -46,6 +46,7 @@ from wqflask.marker_regression import run_mapping from wqflask.marker_regression import display_mapping_results from wqflask.network_graph import network_graph from wqflask.correlation import show_corr_results +from wqflask.correlation.correlation_gn3_api import compute_correlation from wqflask.correlation_matrix import show_corr_matrix from wqflask.correlation import corr_scatter_plot from wqflask.wgcna import wgcna_analysis @@ -880,8 +881,8 @@ def network_graph_page(): def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) logger.info(request.url) - template_vars = show_corr_results.CorrelationResults(request.form) - return render_template("correlation_page.html", **template_vars.__dict__) + correlation_results = compute_correlation(request.form) + return render_template("demo_correlation_page.html",correlation_results=correlation_results) @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): -- cgit v1.2.3 From 7a1e84cafdf02a1bcef4ddeb653d072b80a8deba Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 25 Mar 2021 02:27:38 +0300 Subject: add initial demo template page --- .../wqflask/templates/demo_correlation_page.html | 36 ++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 wqflask/wqflask/templates/demo_correlation_page.html diff --git a/wqflask/wqflask/templates/demo_correlation_page.html b/wqflask/wqflask/templates/demo_correlation_page.html new file mode 100644 index 00000000..4d310051 --- /dev/null +++ b/wqflask/wqflask/templates/demo_correlation_page.html @@ -0,0 +1,36 @@ +{% extends "base.html" %} +{% block title %}Demo Correlation Results{% endblock %} +{% block css %} + + + + + + +{% endblock %} +{% block content %} +
+ {{correlation_results}} + + +
+{% endblock %} + +{% block js %} + + + + + + + + + + + + + +{% endblock %} -- cgit v1.2.3 From cf42f769ec4db2efaebca64c63454935cc28b2a3 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 25 Mar 2021 03:55:16 +0300 Subject: modify gn3 integration code --- wqflask/wqflask/correlation/correlation_gn3_api.py | 89 +++++++++------------- .../wqflask/templates/demo_correlation_page.html | 22 +----- 2 files changed, 38 insertions(+), 73 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 7e269e41..7e865bf3 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -10,59 +10,6 @@ from wqflask.base.trait import retrieve_sample_data GN3_CORRELATION_API = "http://127.0.0.1:8080/api/correlation" -def compute_sample(target_dataset, trait_data, target_samplelist, method="pearson"): - """integration for integrating sample_r api correlation""" - data = { - "target_dataset": target_dataset, - "target_samplelist": target_samplelist, - "trait_data": { - "trait_sample_data": trait_data, - "trait_id": "HC_Q" - } - } - requests_url = f"http://127.0.0.1:8080/api/correlation/sample_x/{method}" - - results = requests.post(requests_url, json=data) - - data = results.json() - - return data - - -def get_tissue_correlation_input(this_trait, trait_symbol_dict): - """Gets tissue expression values for the primary trait and target tissues values""" - primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=[this_trait.symbol]) - - if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( - )] - - corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=list(trait_symbol_dict.values())) - - target_tissue_data = [] - for trait, symbol in list(trait_symbol_dict.items()): - if symbol and symbol.lower() in corr_result_tissue_vals_dict: - this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower( - )] - - this_trait_data = {"trait_id": trait, - "tissue_values": this_trait_tissue_values} - - target_tissue_data.append(this_trait_data) - - primary_tissue_data = { - "this_id": "TT", - "tissue_values": primary_trait_tissue_values - - } - - return (primary_tissue_data, target_tissue_data) - - return None - - def process_samples(start_vars, sample_names, excluded_samples=None): """process samples method""" sample_data = {} @@ -81,7 +28,7 @@ def process_samples(start_vars, sample_names, excluded_samples=None): def create_target_this_trait(start_vars): - """this function prefetch required data for correlation""" + """this function creates the required trait and target dataset for correlation""" this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( @@ -148,3 +95,37 @@ def compute_correlation(start_vars, method="pearson"): data = corr_results.json() return data + + +def get_tissue_correlation_input(this_trait, trait_symbol_dict): + """Gets tissue expression values for the primary trait and target tissues values""" + primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=[this_trait.symbol]) + + if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: + primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( + )] + + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=list(trait_symbol_dict.values())) + + target_tissue_data = [] + for trait, symbol in list(trait_symbol_dict.items()): + if symbol and symbol.lower() in corr_result_tissue_vals_dict: + this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower( + )] + + this_trait_data = {"trait_id": trait, + "tissue_values": this_trait_tissue_values} + + target_tissue_data.append(this_trait_data) + + primary_tissue_data = { + "this_id": "TT", + "tissue_values": primary_trait_tissue_values + + } + + return (primary_tissue_data, target_tissue_data) + + return None diff --git a/wqflask/wqflask/templates/demo_correlation_page.html b/wqflask/wqflask/templates/demo_correlation_page.html index 4d310051..ddcdf38d 100644 --- a/wqflask/wqflask/templates/demo_correlation_page.html +++ b/wqflask/wqflask/templates/demo_correlation_page.html @@ -10,27 +10,11 @@ {% endblock %} {% block content %}
- {{correlation_results}} + - -
-{% endblock %} -{% block js %} - - - - - - - - - - - - - + {% endblock %} -- cgit v1.2.3 From 7de35627a6dc3fa48a039c932be005ffe6c175c4 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 25 Mar 2021 04:02:49 +0300 Subject: fix import error --- wqflask/wqflask/correlation/correlation_gn3_api.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 7e865bf3..479bb0d8 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -1,11 +1,11 @@ """module that calls the gn3 api's to do the correlation """ import json import requests -from wqflask.wqflask.correlation import correlation_functions +from wqflask.correlation import correlation_functions -from wqflask.base import data_set -from wqflask.base.trait import create_trait -from wqflask.base.trait import retrieve_sample_data +from base import data_set +from base.trait import create_trait +from base.trait import retrieve_sample_data GN3_CORRELATION_API = "http://127.0.0.1:8080/api/correlation" -- cgit v1.2.3 From da72efa86846179d8d2aa64cd7b06a894469dc85 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 25 Mar 2021 10:14:31 +0300 Subject: minor fix --- wqflask/wqflask/correlation/correlation_gn3_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 479bb0d8..f1137c0e 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -88,7 +88,7 @@ def compute_correlation(start_vars, method="pearson"): else: pass # lit correlation/literature - # can fetch values in gn3 not set up in gn3 + # to fetch values from the database corr_results = requests.post(requests_url, json=corr_input_data) -- cgit v1.2.3 From 08ddec9dcbaa1730d0b65b643aa5c99d1077d4d5 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 28 Mar 2021 21:13:52 +0300 Subject: refactor correlation integration code --- wqflask/wqflask/correlation/correlation_gn3_api.py | 42 ++++++++++++---------- .../wqflask/templates/demo_correlation_page.html | 6 ++-- 2 files changed, 27 insertions(+), 21 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index f1137c0e..1cd1b332 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -63,6 +63,7 @@ def compute_correlation(start_vars, method="pearson"): if corr_type == "sample": corr_input_data = { + "target": target_dataset, "target_dataset": target_dataset.trait_data, "target_samplelist": target_dataset.samplelist, "trait_data": { @@ -80,16 +81,17 @@ def compute_correlation(start_vars, method="pearson"): corr_input_data = { "primary_tissue": primary_tissue_data, - "target_tissues": target_tissue_data + "target_tissues_dict": target_tissue_data } requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" - else: - pass - # lit correlation/literature - # to fetch values from the database + elif corr_type == "lit": + (this_trait_geneid, geneid_dict, species) = do_lit_correlation( + this_trait, this_dataset, target_dataset) + requests_url = f"{GN3_CORRELATION_API}/lit_corr/{species}/{this_trait_geneid}" + corr_input_data = geneid_dict corr_results = requests.post(requests_url, json=corr_input_data) data = corr_results.json() @@ -97,6 +99,18 @@ def compute_correlation(start_vars, method="pearson"): return data +def do_lit_correlation(this_trait, this_dataset, target_dataset): + geneid_dict = this_dataset.retrieve_genes("GeneId") + species = this_dataset.group.species.lower() + + this_trait_geneid = this_trait.geneid + this_trait_gene_data = { + this_trait.name: this_trait_geneid + } + + return (this_trait_geneid, geneid_dict, species) + + def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( @@ -108,23 +122,15 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=list(trait_symbol_dict.values())) - - target_tissue_data = [] - for trait, symbol in list(trait_symbol_dict.items()): - if symbol and symbol.lower() in corr_result_tissue_vals_dict: - this_trait_tissue_values = corr_result_tissue_vals_dict[symbol.lower( - )] - - this_trait_data = {"trait_id": trait, - "tissue_values": this_trait_tissue_values} - - target_tissue_data.append(this_trait_data) - primary_tissue_data = { - "this_id": "TT", + "this_id": this_trait.name, "tissue_values": primary_trait_tissue_values } + target_tissue_data = { + "trait_symbol_dict": trait_symbol_dict, + "symbol_tissue_vals_dict": corr_result_tissue_vals_dict + } return (primary_tissue_data, target_tissue_data) diff --git a/wqflask/wqflask/templates/demo_correlation_page.html b/wqflask/wqflask/templates/demo_correlation_page.html index ddcdf38d..a8651067 100644 --- a/wqflask/wqflask/templates/demo_correlation_page.html +++ b/wqflask/wqflask/templates/demo_correlation_page.html @@ -10,11 +10,11 @@ {% endblock %} {% block content %}
- - + {{correlation_results}} +
{% endblock %} -- cgit v1.2.3 From fcb93bef5ab230b948f83e0e77a1ef54b017aca1 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 8 Apr 2021 23:59:13 +0300 Subject: minor fix --- wqflask/wqflask/correlation/correlation_gn3_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 1cd1b332..c8d5347c 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -63,7 +63,6 @@ def compute_correlation(start_vars, method="pearson"): if corr_type == "sample": corr_input_data = { - "target": target_dataset, "target_dataset": target_dataset.trait_data, "target_samplelist": target_dataset.samplelist, "trait_data": { -- cgit v1.2.3 From e7b589f05e1c13612ea2f7245d66cc3f054fa14b Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 9 Apr 2021 17:40:53 +0000 Subject: Added varaiable 'categorical_attr_exists' tracking whether there are any case attributes with fewer than 10 distinct values, since it currently throws a JS error if case attributes exist but none have fewer than 10 distinct values (specifically when we have RRID as a case attribute) --- wqflask/wqflask/show_trait/show_trait.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 878c41c0..d3267190 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -203,6 +203,13 @@ class ShowTrait(object): if sample.value < 0: self.negative_vals_exist = "true" + #ZS: Check whether any attributes have few enough distinct values to show the "Block samples by group" option + self.categorical_attr_exists = False + for attribute in self.sample_groups[0].attributes: + if len(self.sample_groups[0].attributes[attribute].distinct_values) <= 10: + self.categorical_attr_exists = True + break + sample_column_width = max_samplename_width * 8 self.stats_table_width, self.trait_table_width = get_table_widths(self.sample_groups, sample_column_width, self.has_num_cases) @@ -277,6 +284,7 @@ class ShowTrait(object): se_exists = self.sample_groups[0].se_exists, has_num_cases = self.has_num_cases, attributes = self.sample_groups[0].attributes, + categorical_attr_exists = self.categorical_attr_exists, categorical_vars = ",".join(categorical_var_list), num_values = self.num_values, qnorm_values = self.qnorm_vals, -- cgit v1.2.3 From 9d7da4653c8b0241af712043bb375e3f2bc52a3f Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 9 Apr 2021 17:43:30 +0000 Subject: Store categorical_attr_exists as a string instead of boolean since apparently the boolean doesn't get passed to the template properly --- wqflask/wqflask/show_trait/show_trait.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index d3267190..6892f02b 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -204,10 +204,10 @@ class ShowTrait(object): self.negative_vals_exist = "true" #ZS: Check whether any attributes have few enough distinct values to show the "Block samples by group" option - self.categorical_attr_exists = False + self.categorical_attr_exists = "false" for attribute in self.sample_groups[0].attributes: if len(self.sample_groups[0].attributes[attribute].distinct_values) <= 10: - self.categorical_attr_exists = True + self.categorical_attr_exists = "true" break sample_column_width = max_samplename_width * 8 -- cgit v1.2.3 From 9fa88673447ab13dcd1b899c0e6c2c5915dd0114 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 9 Apr 2021 17:44:03 +0000 Subject: Replaced the conditional for whether to show 'Block samples by group' to instead check categorical_attr_exists --- wqflask/wqflask/templates/show_trait_transform_and_filter.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_transform_and_filter.html b/wqflask/wqflask/templates/show_trait_transform_and_filter.html index b70ca590..e3f5ef81 100644 --- a/wqflask/wqflask/templates/show_trait_transform_and_filter.html +++ b/wqflask/wqflask/templates/show_trait_transform_and_filter.html @@ -20,7 +20,7 @@ - {% if sample_groups[0].attributes %} + {% if categorical_attr_exists == "true" %}
+

This collection has {{ '{}'.format(numify(trait_obs|count, "record", "records")) }}

@@ -231,6 +232,21 @@ $('#collection_name').css('display', 'inline'); } }); + + make_default = function() { + alert("The current collection is now your default collection.") + let uc_id = $('#uc_id').val(); + $.cookie('default_collection', uc_id, { + expires: 365, + path: '/' + }); + + let default_collection_id = $.cookie('default_collection'); + }; + + $("#make_default").on("click", function(){ + make_default(); + }); }); -- cgit v1.2.3 From 939a27cb0ede3102ce929e445690e1ba86d5870a Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 14 Apr 2021 20:26:34 +0000 Subject: Added JS that automatically selects the default collection if it's set --- wqflask/wqflask/templates/collections/add.html | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html index b4e5385b..0398c6e4 100644 --- a/wqflask/wqflask/templates/collections/add.html +++ b/wqflask/wqflask/templates/collections/add.html @@ -49,8 +49,20 @@
-- cgit v1.2.3 From 328b176628ed9db6c1c60590cb10f4cca212738a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 15 Apr 2021 06:10:28 +0300 Subject: change api port --- wqflask/wqflask/correlation/correlation_gn3_api.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index c8d5347c..8ee4a9b7 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -7,7 +7,7 @@ from base import data_set from base.trait import create_trait from base.trait import retrieve_sample_data -GN3_CORRELATION_API = "http://127.0.0.1:8080/api/correlation" +GN3_CORRELATION_API = "http://127.0.0.1:8202/api/correlation" def process_samples(start_vars, sample_names, excluded_samples=None): @@ -30,6 +30,12 @@ def process_samples(start_vars, sample_names, excluded_samples=None): def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" + + print("creating the dataset and trait") + import time + + initial_time = time.time() + this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) @@ -44,6 +50,11 @@ def create_target_this_trait(start_vars): target_dataset.get_trait_data(list(sample_data.keys())) + + time_taken = time.time() - initial_time + + print(f"the time taken to create dataset abnd trait is",time_taken) + return (this_dataset, this_trait, target_dataset, sample_data) @@ -91,6 +102,8 @@ def compute_correlation(start_vars, method="pearson"): requests_url = f"{GN3_CORRELATION_API}/lit_corr/{species}/{this_trait_geneid}" corr_input_data = geneid_dict + + print("Sending this request") corr_results = requests.post(requests_url, json=corr_input_data) data = corr_results.json() -- cgit v1.2.3 From 5a9a7a645510d1385def017adf2f956d61fa2329 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 16 Apr 2021 02:09:28 +0300 Subject: add demo template --- wqflask/wqflask/correlation/correlation_gn3_api.py | 3 + .../wqflask/templates/demo_correlation_page.html | 78 ++++++++++++++++++++-- wqflask/wqflask/views.py | 2 +- 3 files changed, 77 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 8ee4a9b7..b4480076 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -47,11 +47,14 @@ def create_target_this_trait(start_vars): # target_dataset.get_trait_data(list(self.sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) + print(f"Starting to creat the target dataset ") + dataset_start_time = time.time() target_dataset.get_trait_data(list(sample_data.keys())) time_taken = time.time() - initial_time + print(f"the time taken to create dataset is",time.time()-dataset_start_time) print(f"the time taken to create dataset abnd trait is",time_taken) diff --git a/wqflask/wqflask/templates/demo_correlation_page.html b/wqflask/wqflask/templates/demo_correlation_page.html index a8651067..1900a0bd 100644 --- a/wqflask/wqflask/templates/demo_correlation_page.html +++ b/wqflask/wqflask/templates/demo_correlation_page.html @@ -10,11 +10,79 @@ {% endblock %} {% block content %}
- {{correlation_results}} - +
CORRELATION RESULTS
+
+

Trait_Name

+

Rho value

+

Num overlap

+

P value

+
+
+ {% for corr_result in correlation_results %} + {% for key,value in corr_result.items()%} +
+

trait_name_here

+ {%for o_key,o_value in value.items()%} +

{{o_value}}

+ {%endfor%} + {% endfor %} +
+ {% endfor %} +
+ + {% endblock %} + diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 6ca9b23f..072db466 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -882,7 +882,7 @@ def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) logger.info(request.url) correlation_results = compute_correlation(request.form) - return render_template("demo_correlation_page.html",correlation_results=correlation_results) + return render_template("demo_correlation_page.html",correlation_results=correlation_results[1:20]) @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): -- cgit v1.2.3 From 33e03898ee733f18b29e54e202c217ba14921f48 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 17 Apr 2021 04:14:33 +0300 Subject: use gn3 lib --- bin/genenetwork2 | 3 +- wqflask/wqflask/correlation/correlation_gn3_api.py | 57 +++++++++++++++++----- .../wqflask/templates/demo_correlation_page.html | 2 +- 3 files changed, 47 insertions(+), 15 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5f4e0f9a..917d6549 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -154,7 +154,8 @@ if [ ! -d $R_LIBS_SITE ] ; then fi # We may change this one: -export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH +# export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH +PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$HOME/genenetwork3:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index b4480076..c1d6132b 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -1,11 +1,17 @@ """module that calls the gn3 api's to do the correlation """ import json import requests +import time from wqflask.correlation import correlation_functions from base import data_set from base.trait import create_trait from base.trait import retrieve_sample_data +# gn3 lib +from gn3.computations.correlations import compute_all_sample_correlation +from gn3.computations.correlations import benchmark_compute_all_sample +from gn3.computations.correlations import map_shared_keys_to_values +from gn3.computations.correlations import compute_all_tissue_correlation GN3_CORRELATION_API = "http://127.0.0.1:8202/api/correlation" @@ -30,7 +36,6 @@ def process_samples(start_vars, sample_names, excluded_samples=None): def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" - print("creating the dataset and trait") import time @@ -52,11 +57,10 @@ def create_target_this_trait(start_vars): target_dataset.get_trait_data(list(sample_data.keys())) - time_taken = time.time() - initial_time - print(f"the time taken to create dataset is",time.time()-dataset_start_time) + print(f"the time taken to create dataset is", time.time()-dataset_start_time) - print(f"the time taken to create dataset abnd trait is",time_taken) + print(f"the time taken to create dataset abnd trait is", time_taken) return (this_dataset, this_trait, target_dataset, sample_data) @@ -76,16 +80,34 @@ def compute_correlation(start_vars, method="pearson"): corr_input_data = {} if corr_type == "sample": - corr_input_data = { - "target_dataset": target_dataset.trait_data, - "target_samplelist": target_dataset.samplelist, - "trait_data": { - "trait_sample_data": sample_data, - "trait_id": start_vars["trait_id"] - } + # corr_input_data = { + # "target_dataset": target_dataset.trait_data, + # "target_samplelist": target_dataset.samplelist, + # "trait_data": { + # "trait_sample_data": sample_data, + # "trait_id": start_vars["trait_id"] + # } + # } + + + + this_trait_data = { + "trait_sample_data": sample_data, + "trait_id": start_vars["trait_id"] } - requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" + initial_time = time.time() + print("Calling sample correlation") + results = map_shared_keys_to_values( + target_dataset.samplelist, target_dataset.trait_data) + correlation_results = compute_all_sample_correlation(corr_method=method, + this_trait=this_trait_data, + target_dataset=results) + + print("Time taken is>>>>",time.time()-initial_time) + + # requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" + return correlation_results elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") @@ -96,8 +118,17 @@ def compute_correlation(start_vars, method="pearson"): "primary_tissue": primary_tissue_data, "target_tissues_dict": target_tissue_data } + print("Calling tissue correlation") + initial_time = time.time() + correlation_results = compute_all_tissue_correlation(primary_tissue_dict=corr_input_data["primary_tissue"], + target_tissues_data=corr_input_data["target_tissues_dict"], + corr_method=method) + + time_taken = time.time() + print("Time taken is ??????",time_taken-initial_time) - requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" + # requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" + return correlation_results elif corr_type == "lit": (this_trait_geneid, geneid_dict, species) = do_lit_correlation( diff --git a/wqflask/wqflask/templates/demo_correlation_page.html b/wqflask/wqflask/templates/demo_correlation_page.html index 1900a0bd..d2979f9d 100644 --- a/wqflask/wqflask/templates/demo_correlation_page.html +++ b/wqflask/wqflask/templates/demo_correlation_page.html @@ -21,7 +21,7 @@ {% for corr_result in correlation_results %} {% for key,value in corr_result.items()%}
-

trait_name_here

+

{{key}}

{%for o_key,o_value in value.items()%}

{{o_value}}

{%endfor%} -- cgit v1.2.3 From ba2fa2025bdc381346afc8ec3203f229ed3551d6 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 17 Apr 2021 13:43:44 +0300 Subject: refactoring fetching of data --- wqflask/wqflask/correlation/correlation_gn3_api.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index c1d6132b..75bd5561 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -48,20 +48,17 @@ def create_target_this_trait(start_vars): this_trait = create_trait(dataset=this_dataset, name=start_vars['trait_id']) - sample_data = process_samples(start_vars, this_dataset.group.samplelist) + # target_dataset.get_trait_data(list(self.sample_data.keys())) - this_trait = retrieve_sample_data(this_trait, this_dataset) + # this_trait = retrieve_sample_data(this_trait, this_dataset) print(f"Starting to creat the target dataset ") dataset_start_time = time.time() + sample_data = () - target_dataset.get_trait_data(list(sample_data.keys())) + time_taken = time.time() - initial_time - print(f"the time taken to create dataset is", time.time()-dataset_start_time) - - print(f"the time taken to create dataset abnd trait is", time_taken) - return (this_dataset, this_trait, target_dataset, sample_data) @@ -89,6 +86,10 @@ def compute_correlation(start_vars, method="pearson"): # } # } + sample_data = process_samples(start_vars, this_dataset.group.samplelist) + target_dataset.get_trait_data(list(sample_data.keys())) + this_trait = retrieve_sample_data(this_trait, this_dataset) + this_trait_data = { @@ -111,8 +112,10 @@ def compute_correlation(start_vars, method="pearson"): elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") + time_to_retrieve = time.time() primary_tissue_data, target_tissue_data = get_tissue_correlation_input( this_trait, trait_symbol_dict) + print("Time taken to retrieve this is",time.time()-time_to_retrieve) corr_input_data = { "primary_tissue": primary_tissue_data, -- cgit v1.2.3 From 50c0ee93a59eecd40a6fbd19139671c94003c21b Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 19 Apr 2021 00:24:36 +0300 Subject: fix for correlation_demo template --- wqflask/wqflask/correlation/correlation_gn3_api.py | 53 ++++++++++------------ .../wqflask/templates/demo_correlation_page.html | 23 +++++++++- 2 files changed, 44 insertions(+), 32 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 75bd5561..ba606b92 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -9,9 +9,10 @@ from base.trait import create_trait from base.trait import retrieve_sample_data # gn3 lib from gn3.computations.correlations import compute_all_sample_correlation -from gn3.computations.correlations import benchmark_compute_all_sample from gn3.computations.correlations import map_shared_keys_to_values from gn3.computations.correlations import compute_all_tissue_correlation +from gn3.computations.correlations import compute_all_lit_correlation +from gn3.db_utils import database_connector GN3_CORRELATION_API = "http://127.0.0.1:8202/api/correlation" @@ -36,10 +37,6 @@ def process_samples(start_vars, sample_names, excluded_samples=None): def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" - print("creating the dataset and trait") - import time - - initial_time = time.time() this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( @@ -48,17 +45,10 @@ def create_target_this_trait(start_vars): this_trait = create_trait(dataset=this_dataset, name=start_vars['trait_id']) - # target_dataset.get_trait_data(list(self.sample_data.keys())) # this_trait = retrieve_sample_data(this_trait, this_dataset) - print(f"Starting to creat the target dataset ") - dataset_start_time = time.time() sample_data = () - - - - time_taken = time.time() - initial_time return (this_dataset, this_trait, target_dataset, sample_data) @@ -77,6 +67,7 @@ def compute_correlation(start_vars, method="pearson"): corr_input_data = {} if corr_type == "sample": + initial_time = time.time() # corr_input_data = { # "target_dataset": target_dataset.trait_data, # "target_samplelist": target_dataset.samplelist, @@ -85,50 +76,44 @@ def compute_correlation(start_vars, method="pearson"): # "trait_id": start_vars["trait_id"] # } # } - - sample_data = process_samples(start_vars, this_dataset.group.samplelist) + sample_data = process_samples( + start_vars, this_dataset.group.samplelist) target_dataset.get_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) - + print("Creating dataset and trait took",time.time()-initial_time) this_trait_data = { "trait_sample_data": sample_data, "trait_id": start_vars["trait_id"] } - initial_time = time.time() - print("Calling sample correlation") results = map_shared_keys_to_values( target_dataset.samplelist, target_dataset.trait_data) correlation_results = compute_all_sample_correlation(corr_method=method, this_trait=this_trait_data, target_dataset=results) - print("Time taken is>>>>",time.time()-initial_time) + print("doing sample correlation took",time.time()-initial_time) # requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" return correlation_results elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") - time_to_retrieve = time.time() primary_tissue_data, target_tissue_data = get_tissue_correlation_input( this_trait, trait_symbol_dict) - print("Time taken to retrieve this is",time.time()-time_to_retrieve) corr_input_data = { "primary_tissue": primary_tissue_data, "target_tissues_dict": target_tissue_data } - print("Calling tissue correlation") initial_time = time.time() correlation_results = compute_all_tissue_correlation(primary_tissue_dict=corr_input_data["primary_tissue"], target_tissues_data=corr_input_data["target_tissues_dict"], corr_method=method) + print("time taken for compute tissue is",time.time()-initial_time) - time_taken = time.time() - print("Time taken is ??????",time_taken-initial_time) # requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" return correlation_results @@ -137,15 +122,23 @@ def compute_correlation(start_vars, method="pearson"): (this_trait_geneid, geneid_dict, species) = do_lit_correlation( this_trait, this_dataset, target_dataset) - requests_url = f"{GN3_CORRELATION_API}/lit_corr/{species}/{this_trait_geneid}" - corr_input_data = geneid_dict + conn, _cursor_object = database_connector() + initial_time = time.time() + with conn: - print("Sending this request") - corr_results = requests.post(requests_url, json=corr_input_data) + lit_corr_results = compute_all_lit_correlation( + conn=conn, trait_lists=list(geneid_dict.items()), + species=species, gene_id=this_trait_geneid) - data = corr_results.json() + return lit_corr_results + print("the time taken is",time.time()-initial_time) + # requests_url = f"{GN3_CORRELATION_API}/lit_corr/{species}/{this_trait_geneid}" + # corr_input_data = geneid_dict + # corr_results = requests.post(requests_url, json=corr_input_data) - return data + # data = corr_results.json() + + # return data def do_lit_correlation(this_trait, this_dataset, target_dataset): @@ -164,11 +157,11 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=[this_trait.symbol]) - if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] + time_to_to_fetch_all = time.time() corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=list(trait_symbol_dict.values())) primary_tissue_data = { diff --git a/wqflask/wqflask/templates/demo_correlation_page.html b/wqflask/wqflask/templates/demo_correlation_page.html index d2979f9d..67e3c57c 100644 --- a/wqflask/wqflask/templates/demo_correlation_page.html +++ b/wqflask/wqflask/templates/demo_correlation_page.html @@ -22,9 +22,28 @@ {% for key,value in corr_result.items()%}

{{key}}

- {%for o_key,o_value in value.items()%} + + {% if "corr_coeffient" in value %} +

{{value["corr_coeffient"]}}

+ {%elif "tissue_corr" in value %} +

{{value["tissue_corr"]}}

+ {%elif "lit_corr" in value %} + {{value["lit_corr"]}} + {% endif %} + {%if "tissue_number" in value %} +
{{value["tissue_number"]}}
+ {%elif "num_overlap" in value %} +

{{value["num_overlap"]}}

+ {% endif %} +

{{value["p_value"]}}

+ + + + + {% endfor %}
{% endfor %} -- cgit v1.2.3 From e6c21d96f45dc74de43db451383236f28e723847 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 19 Apr 2021 16:46:39 +0000 Subject: Removed unused code --- wqflask/base/data_set.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 178234fe..44325d5b 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -573,11 +573,6 @@ class DataSet(object): """Gets overridden later, at least for Temp...used by trait's get_given_name""" return None - # Delete this eventually - @property - def riset(): - Weve_Renamed_This_As_Group - def get_accession_id(self): if self.type == "Publish": results = g.db.execute("""select InfoFiles.GN_AccesionId from InfoFiles, PublishFreeze, InbredSet where -- cgit v1.2.3 From 34b6fbf0044fd950ecb95590b4b772abd2cbcb8e Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 19 Apr 2021 16:57:21 +0000 Subject: Fixed NIAAA link because apparently it was changed --- wqflask/wqflask/templates/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index ccb2ac5a..1fbaad51 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -191,7 +191,7 @@ Translational Systems Genetics of Mitochondria, Metabolism, and Aging (R01AG043930, 2013-2018)
  • - NIAAA + NIAAA Integrative Neuroscience Initiative on Alcoholism (U01 AA016662, U01 AA013499, U24 AA013513, U01 AA014425, 2006-2017)
  • -- cgit v1.2.3 From c8f9367bc12340bb8b8a7ce1a5f42789e311555a Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 19 Apr 2021 16:57:52 +0000 Subject: Removed commented out js file --- wqflask/wqflask/templates/base.html | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index 1fbaad51..e6f22deb 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -257,7 +257,6 @@ - -- cgit v1.2.3 From 789dd88a0b9a6a8cbe0413c47212c89c1681765c Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 19 Apr 2021 17:08:30 +0000 Subject: Removedu nused function from network_graph.py --- wqflask/wqflask/network_graph/network_graph.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/wqflask/wqflask/network_graph/network_graph.py b/wqflask/wqflask/network_graph/network_graph.py index 1d5316a2..132e1884 100644 --- a/wqflask/wqflask/network_graph/network_graph.py +++ b/wqflask/wqflask/network_graph/network_graph.py @@ -182,15 +182,3 @@ class NetworkGraph(object): samples=self.all_sample_list, sample_data=self.sample_data, elements=self.elements,) - - def get_trait_db_obs(self, trait_db_list): - self.trait_list = [] - for i, trait_db in enumerate(trait_db_list): - if i == (len(trait_db_list) - 1): - break - trait_name, dataset_name = trait_db.split(":") - dataset_ob = data_set.create_dataset(dataset_name) - trait_ob = create_trait(dataset=dataset_ob, - name=trait_name, - cellid=None) - self.trait_list.append((trait_ob, dataset_ob)) -- cgit v1.2.3 From 68ee0a995fceaf6aefdd3c8f780e46a83b51a0e8 Mon Sep 17 00:00:00 2001 From: zsloan Date: Mon, 19 Apr 2021 17:11:41 +0000 Subject: Specify only getting the first two items after splitting the trait/dataset input string --- wqflask/utility/helper_functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 7eb7f013..46eeb35d 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -40,7 +40,7 @@ def get_trait_db_obs(self, trait_db_list): data, _separator, hmac_string = trait.rpartition(':') data = data.strip() assert hmac_string==hmac.hmac_creation(data), "Data tampering?" - trait_name, dataset_name = data.split(":") + trait_name, dataset_name = data.split(":")[:2] if dataset_name == "Temp": dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2]) else: -- cgit v1.2.3 From 34e4933de5a1cd444abe618fcfd93b424bf3442e Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Tue, 20 Apr 2021 01:38:26 +0300 Subject: refactor code for iterating mrna tissue data --- wqflask/base/mrna_assay_tissue_data.py | 39 +++++++++++++++++++--- .../wqflask/correlation/correlation_functions.py | 6 ++-- wqflask/wqflask/correlation/correlation_gn3_api.py | 24 +++++++------ 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index f1929518..0220d73b 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -6,6 +6,7 @@ from utility import db_tools from utility import Bunch from utility.db_tools import escape +from gn3.db_utils import database_connector from utility.logger import getLogger @@ -44,16 +45,42 @@ class MrnaAssayTissueData(object): and t.Mean = x.maxmean; '''.format(in_clause) - results = g.db.execute(query).fetchall() - lower_symbols = [] + # lower_symbols = [] + lower_symbols = {} for gene_symbol in gene_symbols: + # lower_symbols[gene_symbol.lower()] = True if gene_symbol != None: - lower_symbols.append(gene_symbol.lower()) - + lower_symbols[gene_symbol.lower()] = True + + import time + # initial_time = time.time() + # conn,cursor = database_connector() + # cursor.execute(query) + # for result in cursor.fetchall(): + # symbol = result[0] + # self.data[symbol].gene_id = result[1] + # self.data[symbol].data_id = result[2] + # self.data[symbol].chr = result[3] + # self.data[symbol].mb = result[4] + # self.data[symbol].description = result[5] + # self.data[symbol].probe_target_description = result[6] + + + # print("my loop takes>>>>",time.time()-initial_time) + # conn.close() + # r + + # takes 5 seconds + initial_time = time.time() + results = list(g.db.execute(query).fetchall()) for result in results: symbol = result[0] - if symbol.lower() in lower_symbols: + # if symbol is not None + # exists = lower_symbols.get(symbol.lower()) + # if symbol.lower() in lower_symbols: + if symbol is not None and lower_symbols.get(symbol.lower()): + symbol = symbol.lower() self.data[symbol].gene_id = result.GeneId @@ -62,6 +89,7 @@ class MrnaAssayTissueData(object): self.data[symbol].mb = result.Mb self.data[symbol].description = result.description self.data[symbol].probe_target_description = result.Probe_Target_Description + print("time taken in the loop is",time.time()-initial_time) ########################################################################### #Input: cursor, symbolList (list), dataIdDict(Dict) @@ -82,6 +110,7 @@ class MrnaAssayTissueData(object): WHERE TissueProbeSetData.Id IN {} and TissueProbeSetXRef.DataId = TissueProbeSetData.Id""".format(db_tools.create_in_clause(id_list)) + results = g.db.execute(query).fetchall() for result in results: if result.Symbol.lower() not in symbol_values_dict: diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py index fd7691d4..af1d6060 100644 --- a/wqflask/wqflask/correlation/correlation_functions.py +++ b/wqflask/wqflask/correlation/correlation_functions.py @@ -82,6 +82,6 @@ def cal_zero_order_corr_for_tiss (primaryValue=[], targetValue=[], method='pears def get_trait_symbol_and_tissue_values(symbol_list=None): tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list) - - if len(tissue_data.gene_symbols): - return tissue_data.get_symbol_values_pairs() + if len(tissue_data.gene_symbols) >0: + results = tissue_data.get_symbol_values_pairs() + return results diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index ba606b92..e7394647 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -12,6 +12,7 @@ from gn3.computations.correlations import compute_all_sample_correlation from gn3.computations.correlations import map_shared_keys_to_values from gn3.computations.correlations import compute_all_tissue_correlation from gn3.computations.correlations import compute_all_lit_correlation +from gn3.computations.correlations import experimental_compute_all_tissue_correlation from gn3.db_utils import database_connector GN3_CORRELATION_API = "http://127.0.0.1:8202/api/correlation" @@ -37,7 +38,6 @@ def process_samples(start_vars, sample_names, excluded_samples=None): def create_target_this_trait(start_vars): """this function creates the required trait and target dataset for correlation""" - this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset']) target_dataset = data_set.create_dataset( dataset_name=start_vars['corr_dataset']) @@ -81,7 +81,7 @@ def compute_correlation(start_vars, method="pearson"): target_dataset.get_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) - print("Creating dataset and trait took",time.time()-initial_time) + print("Creating dataset and trait took", time.time()-initial_time) this_trait_data = { "trait_sample_data": sample_data, @@ -94,7 +94,7 @@ def compute_correlation(start_vars, method="pearson"): this_trait=this_trait_data, target_dataset=results) - print("doing sample correlation took",time.time()-initial_time) + print("doing sample correlation took", time.time()-initial_time) # requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" return correlation_results @@ -109,11 +109,16 @@ def compute_correlation(start_vars, method="pearson"): "target_tissues_dict": target_tissue_data } initial_time = time.time() - correlation_results = compute_all_tissue_correlation(primary_tissue_dict=corr_input_data["primary_tissue"], - target_tissues_data=corr_input_data["target_tissues_dict"], - corr_method=method) - print("time taken for compute tissue is",time.time()-initial_time) - + correlation_results = experimental_compute_all_tissue_correlation(primary_tissue_dict=corr_input_data["primary_tissue"], + target_tissues_data=corr_input_data[ + "target_tissues_dict"], + corr_method=method) + print("correlation y took", time.time()-initial_time) + # initial_time = time.time() + # correlation_results = compute_all_tissue_correlation(primary_tissue_dict=corr_input_data["primary_tissue"], + # target_tissues_data=corr_input_data["target_tissues_dict"], + # corr_method=method) + # print("time taken for compute tissue is", time.time()-initial_time) # requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" return correlation_results @@ -131,7 +136,7 @@ def compute_correlation(start_vars, method="pearson"): species=species, gene_id=this_trait_geneid) return lit_corr_results - print("the time taken is",time.time()-initial_time) + print("the time taken is", time.time()-initial_time) # requests_url = f"{GN3_CORRELATION_API}/lit_corr/{species}/{this_trait_geneid}" # corr_input_data = geneid_dict # corr_results = requests.post(requests_url, json=corr_input_data) @@ -161,7 +166,6 @@ def get_tissue_correlation_input(this_trait, trait_symbol_dict): primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] - time_to_to_fetch_all = time.time() corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( symbol_list=list(trait_symbol_dict.values())) primary_tissue_data = { -- cgit v1.2.3 From 315d7f13b254aa62e277805d7d9816b0c7042479 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 22 Apr 2021 18:23:50 +0000 Subject: Added JS that sets the 'add to collection' dropdown to default to the collection the use set as their default collection --- wqflask/wqflask/static/new/javascript/search_results.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/wqflask/wqflask/static/new/javascript/search_results.js b/wqflask/wqflask/static/new/javascript/search_results.js index ecb1220d..48b9b7be 100644 --- a/wqflask/wqflask/static/new/javascript/search_results.js +++ b/wqflask/wqflask/static/new/javascript/search_results.js @@ -326,4 +326,15 @@ $(function() { } ); } + apply_default = function() { + let default_collection_id = $.cookie('default_collection'); + if (default_collection_id) { + let the_option = $('[name=existing_collection] option').filter(function() { + return ($(this).text().split(":")[0] == default_collection_id); + }) + the_option.prop('selected', true); + } + } + apply_default(); + }); \ No newline at end of file -- cgit v1.2.3 From 1a7bb988ee360b3ef48e22e25b419c375dccb9fa Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 22 Apr 2021 18:26:01 +0000 Subject: Account for situations where the trait symbol is null for ProbeSet traits; previously it could throw an error --- wqflask/wqflask/search_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py index f23c0582..cb01a2af 100644 --- a/wqflask/wqflask/search_results.py +++ b/wqflask/wqflask/search_results.py @@ -119,7 +119,7 @@ class SearchResultPage(object): trait_dict['dataset'] = this_trait.dataset.name trait_dict['hmac'] = hmac.data_hmac('{}:{}'.format(this_trait.name, this_trait.dataset.name)) if this_trait.dataset.type == "ProbeSet": - trait_dict['symbol'] = this_trait.symbol + trait_dict['symbol'] = this_trait.symbol if this_trait.symbol else "N/A" trait_dict['description'] = "N/A" if this_trait.description_display: trait_dict['description'] = this_trait.description_display -- cgit v1.2.3 From b0ccb12682fed83bf72d22ff42f1f442a8e6176e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 19 Apr 2021 14:43:16 +0300 Subject: Remove stale comments --- wqflask/base/data_set.py | 11 ---- wqflask/utility/helper_functions.py | 4 -- wqflask/wqflask/show_trait/show_trait.py | 72 +++++++++++++++----------- wqflask/wqflask/templates/index_page_orig.html | 10 ---- wqflask/wqflask/templates/submit_trait.html | 12 ----- wqflask/wqflask/views.py | 23 ++------ 6 files changed, 46 insertions(+), 86 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 178234fe..cc5a428c 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -262,8 +262,6 @@ class Markers(object): elif isinstance(p_values, dict): filtered_markers = [] for marker in self.markers: - #logger.debug("marker[name]", marker['name']) - #logger.debug("p_values:", p_values) if marker['name'] in p_values: #logger.debug("marker {} IS in p_values".format(i)) marker['p_value'] = p_values[marker['name']] @@ -276,10 +274,6 @@ class Markers(object): marker['lrs_value'] = - \ math.log10(marker['p_value']) * 4.61 filtered_markers.append(marker) - # else: - #logger.debug("marker {} NOT in p_values".format(i)) - # self.markers.remove(marker) - #del self.markers[i] self.markers = filtered_markers @@ -306,7 +300,6 @@ class HumanMarkers(Markers): marker['Mb'] = float(splat[3]) / 1000000 self.markers.append(marker) - #logger.debug("markers is: ", pf(self.markers)) def add_pvalues(self, p_values): super(HumanMarkers, self).add_pvalues(p_values) @@ -520,7 +513,6 @@ def datasets(group_name, this_group=None): break if tissue_already_exists: - #logger.debug("dataset_menu:", dataset_menu[i]['datasets']) dataset_menu[i]['datasets'].append((dataset, dataset_short)) else: dataset_menu.append(dict(tissue=tissue_name, @@ -735,9 +727,6 @@ class PhenotypeDataSet(DataSet): DS_NAME_MAP['Publish'] = 'PhenotypeDataSet' def setup(self): - - #logger.debug("IS A PHENOTYPEDATASET") - # Fields in the database table self.search_fields = ['Phenotype.Post_publication_description', 'Phenotype.Pre_publication_description', diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 7eb7f013..15d5b3ab 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -10,7 +10,6 @@ import logging logger = logging.getLogger(__name__ ) def get_species_dataset_trait(self, start_vars): - #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype" if "temp_trait" in list(start_vars.keys()): if start_vars['temp_trait'] == "True": self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) @@ -27,9 +26,6 @@ def get_species_dataset_trait(self, start_vars): get_qtl_info=True) logger.debug("After creating trait") - #if read_genotype: - #self.dataset.group.read_genotype_file() - #self.genotype = self.dataset.group.genotype def get_trait_db_obs(self, trait_db_list): if isinstance(trait_db_list, str): diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 6892f02b..ed55d473 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -138,17 +138,12 @@ class ShowTrait(object): self.ncbi_summary = get_ncbi_summary(self.this_trait) - #Get nearest marker for composite mapping + # Get nearest marker for composite mapping if not self.temp_trait: if check_if_attr_exists(self.this_trait, 'locus_chr') and self.dataset.type != "Geno" and self.dataset.type != "Publish": self.nearest_marker = get_nearest_marker(self.this_trait, self.dataset) - #self.nearest_marker1 = get_nearest_marker(self.this_trait, self.dataset)[0] - #self.nearest_marker2 = get_nearest_marker(self.this_trait, self.dataset)[1] else: self.nearest_marker = "" - #self.nearest_marker1 = "" - #self.nearest_marker2 = "" - self.make_sample_lists() @@ -168,16 +163,19 @@ class ShowTrait(object): categorical_var_list = [] if not self.temp_trait: - categorical_var_list = get_categorical_variables(self.this_trait, self.sample_groups[0]) #ZS: Only using first samplelist, since I think mapping only uses those samples + # ZS: Only using first samplelist, since I think mapping only uses those samples + categorical_var_list = get_categorical_variables(self.this_trait, self.sample_groups[0]) - #ZS: Get list of chromosomes to select for mapping + # ZS: Get list of chromosomes to select for mapping self.chr_list = [["All", -1]] for i, this_chr in enumerate(self.dataset.species.chromosomes.chromosomes): self.chr_list.append([self.dataset.species.chromosomes.chromosomes[this_chr].name, i]) self.genofiles = self.dataset.group.get_genofiles() - if "QTLReaper" or "R/qtl" in dataset.group.mapping_names: #ZS: No need to grab scales from .geno file unless it's using a mapping method that reads .geno files + # ZS: No need to grab scales from .geno file unless it's using + # a mapping method that reads .geno files + if "QTLReaper" or "R/qtl" in dataset.group.mapping_names: if self.genofiles: self.scales_in_geno = get_genotype_scales(self.genofiles) else: @@ -187,10 +185,15 @@ class ShowTrait(object): self.has_num_cases = has_num_cases(self.this_trait) - #ZS: Needed to know whether to display bar chart + get max sample name length in order to set table column width + # ZS: Needed to know whether to display bar chart + get max + # sample name length in order to set table column width self.num_values = 0 - self.binary = "true" #ZS: So it knows whether to display the Binary R/qtl mapping method, which doesn't work unless all values are 0 or 1 - self.negative_vals_exist = "false" #ZS: Since we don't want to show log2 transform option for situations where it doesn't make sense + # ZS: So it knows whether to display the Binary R/qtl mapping + # method, which doesn't work unless all values are 0 or 1 + self.binary = "true" + # ZS: Since we don't want to show log2 transform option for + # situations where it doesn't make sense + self.negative_vals_exist = "false" max_samplename_width = 1 for group in self.sample_groups: for sample in group.sample_list: @@ -203,7 +206,8 @@ class ShowTrait(object): if sample.value < 0: self.negative_vals_exist = "true" - #ZS: Check whether any attributes have few enough distinct values to show the "Block samples by group" option + # ZS: Check whether any attributes have few enough distinct + # values to show the "Block samples by group" option self.categorical_attr_exists = "false" for attribute in self.sample_groups[0].attributes: if len(self.sample_groups[0].attributes[attribute].distinct_values) <= 10: @@ -258,7 +262,6 @@ class ShowTrait(object): if not self.temp_trait: if hasattr(self.this_trait, 'locus_chr') and self.this_trait.locus_chr != "" and self.dataset.type != "Geno" and self.dataset.type != "Publish": hddn['control_marker'] = self.nearest_marker - #hddn['control_marker'] = self.nearest_marker1+","+self.nearest_marker2 hddn['do_control'] = False hddn['maf'] = 0.05 hddn['mapping_scale'] = "physic" @@ -268,7 +271,8 @@ class ShowTrait(object): if len(self.scales_in_geno) < 2: hddn['mapping_scale'] = self.scales_in_geno[list(self.scales_in_geno.keys())[0]][0][0] - # We'll need access to this_trait and hddn in the Jinja2 Template, so we put it inside self + # We'll need access to this_trait and hddn in the Jinja2 + # Template, so we put it inside self self.hddn = hddn js_data = dict(trait_id = self.trait_id, @@ -294,7 +298,8 @@ class ShowTrait(object): self.js_data = js_data def get_external_links(self): - #ZS: There's some weirdness here because some fields don't exist while others are empty strings + # ZS: There's some weirdness here because some fields don't + # exist while others are empty strings self.pubmed_link = webqtlConfig.PUBMEDLINK_URL % self.this_trait.pubmed_id if check_if_attr_exists(self.this_trait, 'pubmed_id') else None self.ncbi_gene_link = webqtlConfig.NCBI_LOCUSID % self.this_trait.geneid if check_if_attr_exists(self.this_trait, 'geneid') else None self.omim_link = webqtlConfig.OMIM_ID % self.this_trait.omim if check_if_attr_exists(self.this_trait, 'omim') else None @@ -320,7 +325,6 @@ class ShowTrait(object): self.panther_link = webqtlConfig.PANTHER_URL % self.this_trait.symbol self.ebi_gwas_link = webqtlConfig.EBIGWAS_URL % self.this_trait.symbol self.protein_atlas_link = webqtlConfig.PROTEIN_ATLAS_URL % self.this_trait.symbol - #self.open_targets_link = webqtlConfig.OPEN_TARGETS_URL % self.this_trait.symbol if self.dataset.group.species == "mouse" or self.dataset.group.species == "human": self.rgd_link = webqtlConfig.RGD_URL % (self.this_trait.symbol, self.dataset.group.species.capitalize()) @@ -429,7 +433,9 @@ class ShowTrait(object): all_samples_ordered.append(sample) other_sample_names.append(sample) - #ZS: CFW is here because the .geno file doesn't properly contain its full list of samples. This should probably be fixed. + # ZS: CFW is here because the .geno file doesn't properly + # contain its full list of samples. This should probably + # be fixed. if self.dataset.group.species == "human" or (set(primary_sample_names) == set(parent_f1_samples)) or self.dataset.group.name == "CFW": primary_sample_names += other_sample_names other_sample_names = [] @@ -445,7 +451,8 @@ class ShowTrait(object): sample_group_type='primary', header=primary_header) - #if other_sample_names and self.dataset.group.species != "human" and self.dataset.group.name != "CFW": + # if other_sample_names and self.dataset.group.species != + # "human" and self.dataset.group.name != "CFW": if len(other_sample_names) > 0: other_sample_names.sort() #Sort other samples if parent_f1_samples: @@ -539,7 +546,8 @@ def get_z_scores(sample_groups): def get_nearest_marker(this_trait, this_db): this_chr = this_trait.locus_chr this_mb = this_trait.locus_mb - #One option is to take flanking markers, another is to take the two (or one) closest + # One option is to take flanking markers, another is to take the + # two (or one) closest query = """SELECT Geno.Name FROM Geno, GenoXRef, GenoFreeze WHERE Geno.Chr = '{}' AND @@ -552,7 +560,6 @@ def get_nearest_marker(this_trait, this_db): if result == []: return "" - #return "", "" else: return result[0][0] @@ -617,7 +624,8 @@ def check_if_attr_exists(the_trait, id_type): def get_ncbi_summary(this_trait): if check_if_attr_exists(this_trait, 'geneid'): - #ZS: Need to switch this try/except to something that checks the output later + # ZS: Need to switch this try/except to something that checks + # the output later try: response = requests.get("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gene&id=%s&retmode=json" % this_trait.geneid) summary = json.loads(response.content)['result'][this_trait.geneid]['summary'] @@ -661,8 +669,8 @@ def get_genotype_scales(genofiles): def get_scales_from_genofile(file_location): geno_path = locate_ignore_error(file_location, 'genotype') - - if not geno_path: #ZS: This is just to allow the code to run when + # ZS: This is just to allow the code to run when + if not geno_path: return [["physic", "Mb"]] cm_and_mb_cols_exist = True cm_column = None @@ -670,7 +678,9 @@ def get_scales_from_genofile(file_location): with open(geno_path, "r") as geno_fh: for i, line in enumerate(geno_fh): if line[0] == "#" or line[0] == "@": - if "@scale" in line: #ZS: If the scale is made explicit in the metadata, use that + # ZS: If the scale is made explicit in the metadata, + # use that + if "@scale" in line: scale = line.split(":")[1].strip() if scale == "morgan": return [["morgan", "cM"]] @@ -690,12 +700,16 @@ def get_scales_from_genofile(file_location): mb_column = 3 break - #ZS: This attempts to check whether the cM and Mb columns are 'real', since some .geno files have one column be a copy of the other column, or have one column that is all 0s + # ZS: This attempts to check whether the cM and Mb columns are + # 'real', since some .geno files have one column be a copy of + # the other column, or have one column that is all 0s cm_all_zero = True mb_all_zero = True cm_mb_all_equal = True for i, line in enumerate(geno_fh): - if first_marker_line <= i < first_marker_line + 10: #ZS: I'm assuming there won't be more than 10 markers where the position is listed as 0 + # ZS: I'm assuming there won't be more than 10 markers + # where the position is listed as 0 + if first_marker_line <= i < first_marker_line + 10: if cm_column: cm_val = line.split("\t")[cm_column].strip() if cm_val != "0": @@ -711,8 +725,8 @@ def get_scales_from_genofile(file_location): if i > first_marker_line + 10: break - - #ZS: This assumes that both won't be all zero, since if that's the case mapping shouldn't be an option to begin with + # ZS: This assumes that both won't be all zero, since if that's + # the case mapping shouldn't be an option to begin with if mb_all_zero: return [["morgan", "cM"]] elif cm_mb_all_equal: diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html index 7f82b35c..87cf1b45 100755 --- a/wqflask/wqflask/templates/index_page_orig.html +++ b/wqflask/wqflask/templates/index_page_orig.html @@ -7,16 +7,6 @@ {% endblock %} {% block content %} - - -
    {{ flash_me() }} diff --git a/wqflask/wqflask/templates/submit_trait.html b/wqflask/wqflask/templates/submit_trait.html index 68b06f55..334a608d 100644 --- a/wqflask/wqflask/templates/submit_trait.html +++ b/wqflask/wqflask/templates/submit_trait.html @@ -61,18 +61,6 @@
    -

    Paste or Type Multiple Values: You can enter data by pasting a series of numbers representing trait values into this area. diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 2c0ba586..c4b510d4 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -156,12 +156,6 @@ def index_page(): import_collections = params['import_collections'] if import_collections == "true": g.user_session.import_traits_to_user(params['anon_id']) - #if USE_GN_SERVER: - # # The menu is generated using GN_SERVER - # return render_template("index_page.html", gn_server_url = GN_SERVER_URL, version=GN_VERSION) - #else: - - # Old style static menu (OBSOLETE) return render_template("index_page_orig.html", version=GN_VERSION) @@ -343,14 +337,10 @@ def intro(): @app.route("/tutorials") def tutorials(): - #doc = Docs("links", request.args) - #return render_template("docs.html", **doc.__dict__) return render_template("tutorials.html") @app.route("/credits") def credits(): - #doc = Docs("links", request.args) - #return render_template("docs.html", **doc.__dict__) return render_template("credits.html") @app.route("/update_text", methods=('POST',)) @@ -368,12 +358,9 @@ def submit_trait_form(): @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) - - #template_vars = submit_trait.SubmitTrait(request.form) - doc = Docs("links") return render_template("links.html", **doc.__dict__) - #return render_template("show_trait.html", **template_vars.__dict__) + @app.route('/export_trait_excel', methods=('POST',)) def export_trait_excel(): @@ -487,21 +474,17 @@ def export_perm_data(): mimetype='text/csv', headers={"Content-Disposition":"attachment;filename=" + file_name + ".csv"}) + @app.route("/show_temp_trait", methods=('POST',)) def show_temp_trait_page(): logger.info(request.url) template_vars = show_trait.ShowTrait(request.form) - #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") - # Sorting the keys messes up the ordered dictionary, so don't do that - #sort_keys=True) - - #logger.info("js_data after dump:", template_vars.js_data) - #logger.info("show_trait template_vars:", pf(template_vars.__dict__)) return render_template("show_trait.html", **template_vars.__dict__) + @app.route("/show_trait") def show_trait_page(): logger.info(request.url) -- cgit v1.2.3 From 4534daa6fb07c23b90e024560ca64091fc330eed Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 19 Apr 2021 17:46:38 +0300 Subject: Move looped sql query into one statement in "get_species_groups" It's in-efficient to have a sql query executed in a loop. As data grows, the query becomes slower. It's better to let sql handle such queries. --- wqflask/utility/helper_functions.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 15d5b3ab..4ba92ed5 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -47,19 +47,18 @@ def get_trait_db_obs(self, trait_db_list): if trait_ob: self.trait_list.append((trait_ob, dataset_ob)) -def get_species_groups(): - - species_query = "SELECT SpeciesId, MenuName FROM Species" - species_ids_and_names = g.db.execute(species_query).fetchall() - - species_and_groups = [] - for species_id, species_name in species_ids_and_names: - this_species_groups = {} - this_species_groups['species'] = species_name - groups_query = "SELECT InbredSetName FROM InbredSet WHERE SpeciesId = %s" % (species_id) - groups = [group[0] for group in g.db.execute(groups_query).fetchall()] - this_species_groups['groups'] = groups - species_and_groups.append(this_species_groups) - - return species_and_groups +def get_species_groups(): + """Group each species into a group""" + _menu = {} + for species, group_name in g.db.execute( + "SELECT s.MenuName, i.InbredSetName FROM InbredSet i " + "INNER JOIN Species s ON s.SpeciesId = i.SpeciesId " + "ORDER BY i.SpeciesId ASC, i.Name ASC").fetchall(): + if _menu.get(species): + _menu = _menu[species].append(group_name) + else: + _menu[species] = [group_name] + return [{"species": key, + "groups": value} for key, value in + list(_menu.items())] -- cgit v1.2.3 From d2e2046a3ce1af0ca97ea1b6d9ccb3a4c9aecf7c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:21:12 +0300 Subject: Add full link to genetic data collected as part of WebQTL project --- wqflask/wqflask/templates/submit_trait.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/submit_trait.html b/wqflask/wqflask/templates/submit_trait.html index 334a608d..2cc18240 100644 --- a/wqflask/wqflask/templates/submit_trait.html +++ b/wqflask/wqflask/templates/submit_trait.html @@ -14,7 +14,7 @@

    Introduction

    The trait values that you enter are statistically compared with verified genotypes collected at a set of microsatellite markers in each RI set. The markers are drawn from a set of over 750, but for each set redundant markers have been removed, preferentially retaining those that are most informative.

    -

    These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.

    +

    These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.


    -- cgit v1.2.3 From 2114ad9e84ad7778e048b52cf865b5f031ceab88 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:27:31 +0300 Subject: Rename index_page_orig to index_page --- wqflask/wqflask/templates/index_page.html | 222 +++++++++------- wqflask/wqflask/templates/index_page_orig.html | 339 ------------------------- wqflask/wqflask/views.py | 2 +- 3 files changed, 136 insertions(+), 427 deletions(-) mode change 100644 => 100755 wqflask/wqflask/templates/index_page.html delete mode 100755 wqflask/wqflask/templates/index_page_orig.html diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html old mode 100644 new mode 100755 index 31846f87..87cf1b45 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -1,78 +1,73 @@ {% extends "base.html" %} {% block title %}GeneNetwork{% endblock %} +{% block css %} + +{% endblock %} {% block content %} - - - -
    +
    {{ flash_me() }} -
    +
    -
    +
    -
    - +
  • System Maintenance Code
    • + +
    + + + + -->
    -

    Affiliates and mirrors

    +

    Links

    -

    Websites affiliated with GeneNetwork

    - -

    GN1 Mirror and development sites

    - +

    GeneNetwork v2:

    + +

    GeneNetwork v1:

    -
    + + @@ -268,11 +303,7 @@ {%endblock%} {% block js %} - - - + + + + + + {% endblock %} diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html deleted file mode 100755 index 87cf1b45..00000000 --- a/wqflask/wqflask/templates/index_page_orig.html +++ /dev/null @@ -1,339 +0,0 @@ -{% extends "base.html" %} -{% block title %}GeneNetwork{% endblock %} -{% block css %} - -{% endblock %} -{% block content %} - -
    - - {{ flash_me() }} - -
    - -
    - -
    -
    -

    Advanced commands

    -
    - -

    You can also use advanced commands. Copy these simple examples into the Get Any field for single term searches and Combined for searches with multiple terms:

    - -
      -
    • POSITION=(chr1 25 30) finds genes, markers, or transcripts on - chromosome 1 between 25 and 30 Mb.
      • - -
    • MEAN=(15 16) in the Combined field finds - highly expressed genes (15 to 16 log2 units)
      • - -
    • RANGE=(1.5 2.5) in the Any field finds traits with values with a specified fold-range (minimum = 1). - Useful for finding "housekeeping genes" (1.0 1.2) or highly variable molecular assays (10 100).
      • - -
    • LRS=(15 1000) or LOD=(2 8) finds all traits with peak LRS or LOD scores between lower and upper limits.
      • - -
    • LRS=(9 999 Chr4 122 155) finds all traits on Chr 4 from 122 and 155 Mb with LRS scores between 9 and 999.
      • - -
    • cisLRS=(15 1000 5) or cisLOD=(2 8 5) finds all cis eQTLs with peak LRS or LOD scores between lower and upper limits, - with an inclusion zone of 5 Mb around the parent gene.
      • - -
    • transLRS=(15 1000 5) or transLOD=(2 8 5) finds all trans eQTLs with peak LRS or LOD scores between lower and upper limits, - with an exclusion zone of 5 Mb around the parent gene. You can also add a fourth term specifying which chromosome you want the transLRS to be on - (for example transLRS=(15 1000 5 7) would find all trans eQTLs with peak LRS on chromosome 7 that is also a trans eQTL with exclusionary zone of 5Mb).
      • - -
    • POSITION=(Chr4 122 130) cisLRS=(9 999 10) - finds all traits on Chr 4 from 122 and 155 Mb with cisLRS scores - between 9 and 999 and an inclusion zone of 10 Mb.
      • - -
    • RIF=mitochondrial searches RNA databases for - GeneRIF links.
      • - -
    • WIKI=nicotine searches - GeneWiki for genes that you or other users have annotated - with the word nicotine.
      • - -
    • GO:0045202 searches for synapse-associated genes listed in the - - Gene Ontology.
      • - -
    • RIF=diabetes LRS=(9 999 Chr2 100 105) transLRS=(9 999 10) - finds diabetes-associated transcripts with peak - trans eQTLs on Chr 2 between 100 and 105 Mb with LRS - scores between 9 and 999.
      • -
    -
    -
    - -
    -
    -
    -

    Affiliates

    - -
    -
    -
    -
    -

    News

    -
    - -
    -
    -
    -
    -

    Github

    - -
    -
    - -
    -
    -

    Links

    -
    -

    GeneNetwork v2:

    - -

    GeneNetwork v1:

    -
      -
    • Main website at UTHSC
      • -
    • Time Machine: Full GN versions from 2009 to 2016 (mm9)
      • - Cloud (EC2) -
    - -
    - - -
    -
    -
    - -{%endblock%} - -{% block js %} - - - - - - - - - -{% endblock %} diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index c4b510d4..156b4772 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -156,7 +156,7 @@ def index_page(): import_collections = params['import_collections'] if import_collections == "true": g.user_session.import_traits_to_user(params['anon_id']) - return render_template("index_page_orig.html", version=GN_VERSION) + return render_template("index_page.html", version=GN_VERSION) @app.route("/tmp/") -- cgit v1.2.3 From 8bad987466203baed948ba761d248e24f04ca49c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:32:35 +0300 Subject: Rewrite For Loop in a more Pythonic way --- wqflask/wqflask/api/gen_menu.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 18afc5ad..eaddecd7 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -23,12 +23,7 @@ def get_species(): """Build species list""" results = g.db.execute( "SELECT Name, MenuName FROM Species ORDER BY OrderId").fetchall() - - species = [] - for result in results: - species.append([str(result[0]), str(result[1])]) - - return species + return [[name, menu_name] for name, menu_name in results] def get_groups(species): -- cgit v1.2.3 From d66a8d2ccbf10b53b31ef094835c5ba8ec672a68 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 17:35:54 +0300 Subject: Apply PEP-8 --- wqflask/utility/helper_functions.py | 28 +++++++++++++++++----------- wqflask/wqflask/views.py | 11 +++++++++-- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index 4ba92ed5..e0ae3414 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -4,19 +4,23 @@ from base.species import TheSpecies from utility import hmac -from flask import Flask, g +from flask import g import logging -logger = logging.getLogger(__name__ ) +logger = logging.getLogger(__name__) + def get_species_dataset_trait(self, start_vars): if "temp_trait" in list(start_vars.keys()): - if start_vars['temp_trait'] == "True": - self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) - else: - self.dataset = data_set.create_dataset(start_vars['dataset']) + if start_vars['temp_trait'] == "True": + self.dataset = data_set.create_dataset( + dataset_name="Temp", + dataset_type="Temp", + group_name=start_vars['group']) + else: + self.dataset = data_set.create_dataset(start_vars['dataset']) else: - self.dataset = data_set.create_dataset(start_vars['dataset']) + self.dataset = data_set.create_dataset(start_vars['dataset']) logger.debug("After creating dataset") self.species = TheSpecies(dataset=self.dataset) logger.debug("After creating species") @@ -35,15 +39,17 @@ def get_trait_db_obs(self, trait_db_list): for trait in trait_db_list: data, _separator, hmac_string = trait.rpartition(':') data = data.strip() - assert hmac_string==hmac.hmac_creation(data), "Data tampering?" + assert hmac_string == hmac.hmac_creation(data), "Data tampering?" trait_name, dataset_name = data.split(":") if dataset_name == "Temp": - dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2]) + dataset_ob = data_set.create_dataset( + dataset_name=dataset_name, dataset_type="Temp", + group_name=trait_name.split("_")[2]) else: dataset_ob = data_set.create_dataset(dataset_name) trait_ob = create_trait(dataset=dataset_ob, - name=trait_name, - cellid=None) + name=trait_name, + cellid=None) if trait_ob: self.trait_list.append((trait_ob, dataset_ob)) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 156b4772..36033d80 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -334,26 +334,33 @@ def intro(): return render_template("docs.html", **doc.__dict__) - @app.route("/tutorials") def tutorials(): return render_template("tutorials.html") + @app.route("/credits") def credits(): return render_template("credits.html") + @app.route("/update_text", methods=('POST',)) def update_page(): update_text(request.form) doc = Docs(request.form['entry_type'], request.form) return render_template("docs.html", **doc.__dict__) + @app.route("/submit_trait") def submit_trait_form(): logger.info(request.url) species_and_groups = get_species_groups() - return render_template("submit_trait.html", **{'species_and_groups' : species_and_groups, 'gn_server_url' : GN_SERVER_URL, 'version' : GN_VERSION}) + return render_template( + "submit_trait.html", + **{'species_and_groups': species_and_groups, + 'gn_server_url': GN_SERVER_URL, + 'version': GN_VERSION}) + @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): -- cgit v1.2.3 From f97c298c1653c538be1753c87336f9f75c3a754c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 23 Apr 2021 20:19:37 +0300 Subject: Remove stale comments --- wqflask/wqflask/templates/index_page.html | 77 ------------------------------- wqflask/wqflask/views.py | 7 +-- 2 files changed, 1 insertion(+), 83 deletions(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 87cf1b45..942776e7 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -65,12 +65,6 @@ - - - - - -
    @@ -116,12 +110,10 @@
    -
    - @@ -207,40 +199,10 @@ -

    Links

    @@ -248,7 +210,6 @@

    GeneNetwork v2:

    GeneNetwork v1:

      @@ -258,44 +219,6 @@
    - - diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 36033d80..baaece2f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -496,17 +496,12 @@ def show_temp_trait_page(): def show_trait_page(): logger.info(request.url) template_vars = show_trait.ShowTrait(request.args) - #logger.info("js_data before dump:", template_vars.js_data) template_vars.js_data = json.dumps(template_vars.js_data, default=json_default_handler, indent=" ") - # Sorting the keys messes up the ordered dictionary, so don't do that - #sort_keys=True) - - #logger.info("js_data after dump:", template_vars.js_data) - #logger.info("show_trait template_vars:", pf(template_vars.__dict__)) return render_template("show_trait.html", **template_vars.__dict__) + @app.route("/heatmap", methods=('POST',)) def heatmap_page(): logger.info("In heatmap, request.form is:", pf(request.form)) -- cgit v1.2.3 From 1b0566d7c9779b979d20c350f66d5628fb55eba6 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 23 Apr 2021 23:22:46 +0300 Subject: debugging for fetching probe data --- wqflask/base/data_set.py | 51 ++++++++++++++++++++-- wqflask/wqflask/correlation/correlation_gn3_api.py | 2 +- wqflask/wqflask/views.py | 3 ++ 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 178234fe..468c4da0 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -115,7 +115,8 @@ Publish or ProbeSet. E.g. except: pass - self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) + self.redis_instance.set( + "dataset_structure", json.dumps(self.datasets)) def set_dataset_key(self, t, name): """If name is not in the object's dataset dictionary, set it, and update @@ -154,10 +155,12 @@ Publish or ProbeSet. E.g. if t in ['pheno', 'other_pheno']: group_name = name.replace("Publish", "") - results = g.db.execute(sql_query_mapping[t].format(group_name)).fetchone() + results = g.db.execute( + sql_query_mapping[t].format(group_name)).fetchone() if results: self.datasets[name] = dataset_name_mapping[t] - self.redis_instance.set("dataset_structure", json.dumps(self.datasets)) + self.redis_instance.set( + "dataset_structure", json.dumps(self.datasets)) return True return None @@ -169,7 +172,8 @@ Publish or ProbeSet. E.g. # This has side-effects, with the end result being a truth-y value if(self.set_dataset_key(t, name)): break - return self.datasets.get(name, None) # Return None if name has not been set + # Return None if name has not been set + return self.datasets.get(name, None) # Do the intensive work at startup one time only @@ -651,6 +655,43 @@ class DataSet(object): "Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass + def fetch_probe_trait_data(self, sample_list=None): + if sample_list: + self.samplelist = sample_list + else: + self.samplelist = self.group.samplelist + + if self.group.parlist != None and self.group.f1list != None: + if (self.group.parlist + self.group.f1list) in self.samplelist: + self.samplelist += self.group.parlist + self.group.f1list + + query = """ + SELECT Strain.Name, Strain.Id FROM Strain, Species + WHERE Strain.Name IN {} + and Strain.SpeciesId=Species.Id + and Species.name = '{}' + """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) + logger.sql(query) + results = dict(g.db.execute(query).fetchall()) + sample_ids = [results[item] for item in self.samplelist] + + query = """SELECT * from ProbeSetData WHERE Id in ( SELECT ProbeSetXRef.DataId FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id and ProbeSetFreeze.Name = 'HC_M2_0606_P' and ProbeSet.Id = ProbeSetXRef.ProbeSetId order by ProbeSet.Id ) and StrainId in ({})""".format( + ",".join(str(sample_id) for sample_id in sample_ids)) + + results = g.db.execute(query).fetchall() + + # with conn: + # cursor = conn.cursor() + # cursor.execute(query) + # results = cursor.fetchall() + trait_data = {} + for trait_id, StrainId, value in results: + if trait_id in trait_data: + trait_data[trait_id].append(value) + else: + trait_data[trait_id] = [value] + self.trait_data = trait_data + def get_trait_data(self, sample_list=None): if sample_list: self.samplelist = sample_list @@ -670,6 +711,7 @@ class DataSet(object): logger.sql(query) results = dict(g.db.execute(query).fetchall()) sample_ids = [results[item] for item in self.samplelist] + print("the number of sample ids are", len(sample_ids)) # MySQL limits the number of tables that can be used in a join to 61, # so we break the sample ids into smaller chunks @@ -720,6 +762,7 @@ class DataSet(object): trait_sample_data.append(results) trait_count = len(trait_sample_data[0]) + print("the trait count is >>>", trait_count) self.trait_data = collections.defaultdict(list) # put all of the separate data together into a dictionary where the keys are diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index e7394647..51bf5fb5 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -78,7 +78,7 @@ def compute_correlation(start_vars, method="pearson"): # } sample_data = process_samples( start_vars, this_dataset.group.samplelist) - target_dataset.get_trait_data(list(sample_data.keys())) + target_dataset.fetch_probe_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) print("Creating dataset and trait took", time.time()-initial_time) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 072db466..2c239425 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -881,7 +881,10 @@ def network_graph_page(): def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) logger.info(request.url) + import time + initial_time = time.time() correlation_results = compute_correlation(request.form) + print(">>>>Time taken by this endpoint",time.time()-initial_time) return render_template("demo_correlation_page.html",correlation_results=correlation_results[1:20]) @app.route("/corr_matrix", methods=('POST',)) -- cgit v1.2.3 From 7556f8a5dfc4c98bc0f0c8241592acec22b65102 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 26 Apr 2021 15:42:07 +0300 Subject: test for probe-type sample and tissue --- wqflask/wqflask/correlation/correlation_gn3_api.py | 71 +++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 51bf5fb5..c945f699 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -52,8 +52,64 @@ def create_target_this_trait(start_vars): return (this_dataset, this_trait, target_dataset, sample_data) +def sample_for_trait_lists(corr_results, target_dataset, this_trait, this_dataset, start_vars): + sample_data = process_samples( + start_vars, this_dataset.group.samplelist) + target_dataset.get_trait_data(list(sample_data.keys())) + + this_trait = retrieve_sample_data(this_trait, this_dataset) + + this_trait_data = { + "trait_sample_data": sample_data, + "trait_id": start_vars["trait_id"] + } + # trait_lists = dict([(list(corr_result)[0],True) for corr_result in corr_results]) + # target_dataset.trait_data =list(filter(lambda dict_obj: dict_obj.keys()[ + # 0] in corr_results_traits, target_dataset_data)) + results = map_shared_keys_to_values( + target_dataset.samplelist, target_dataset.trait_data) + correlation_results = compute_all_sample_correlation(corr_method="pearson", + this_trait=this_trait_data, + target_dataset=results) + + + return correlation_results + + +def tissue_for_trait_lists(corr_results, this_dataset, target_dataset, this_trait): + # # print(corr_results[0])-- + # [{"awsdsd_at": {'corr_coeffient': 0.49714692782257336, 'p_value': 1.872077762359228e-05, 'num_overlap': 67}}] + + print("creating trait_lists") + # corr_results = corr_results[0::] + trait_lists = dict([(list(corr_result)[0], True) + for corr_result in corr_results]) + print("finished creating trait_list") + + traits_symbol_dict = this_dataset.retrieve_genes("Symbol") + print("Retrieved symbol dict") + print("creating dict here>>>>>>>>>") + import time + init_time = time.time() + traits_symbol_dict = dict({trait_name: symbol for ( + trait_name, symbol) in traits_symbol_dict.items() if trait_lists.get(trait_name)}) + print("time taken to create this max dict is>>>>", time.time()-init_time) + print("finished creatinf the dict") + print("Fetching tissue datas") + primary_tissue_data, target_tissue_data = get_tissue_correlation_input( + this_trait, traits_symbol_dict) + print("finihsed>>>>>>>>>>>>>>>>>>") + print("Calling experimental_compute_all_tissue_correlation") + corr_results = experimental_compute_all_tissue_correlation( + primary_tissue_dict=primary_tissue_data, target_tissues_data=target_tissue_data, corr_method="pearson") + # print('finished calling this tissue reuslts',corr_results) + + return corr_results + + def compute_correlation(start_vars, method="pearson"): """compute correlation for to call gn3 api""" + import time corr_type = start_vars['corr_type'] @@ -67,6 +123,7 @@ def compute_correlation(start_vars, method="pearson"): corr_input_data = {} if corr_type == "sample": + import time initial_time = time.time() # corr_input_data = { # "target_dataset": target_dataset.trait_data, @@ -78,7 +135,7 @@ def compute_correlation(start_vars, method="pearson"): # } sample_data = process_samples( start_vars, this_dataset.group.samplelist) - target_dataset.fetch_probe_trait_data(list(sample_data.keys())) + target_dataset.get_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) print("Creating dataset and trait took", time.time()-initial_time) @@ -94,8 +151,15 @@ def compute_correlation(start_vars, method="pearson"): this_trait=this_trait_data, target_dataset=results) + print("computedd>>>>>>>>>>>>>") + print("doing sample correlation took", time.time()-initial_time) + other_results_time = time.time() + other_results = tissue_for_trait_lists( + correlation_results, this_dataset, target_dataset, this_trait) + print(">>>time taken for this is", time.time()-other_results_time) + # requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" return correlation_results @@ -121,6 +185,9 @@ def compute_correlation(start_vars, method="pearson"): # print("time taken for compute tissue is", time.time()-initial_time) # requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" + + sample_results = sample_for_trait_lists( + correlation_results, target_dataset, this_trait, this_dataset, start_vars) return correlation_results elif corr_type == "lit": @@ -148,6 +215,8 @@ def compute_correlation(start_vars, method="pearson"): def do_lit_correlation(this_trait, this_dataset, target_dataset): geneid_dict = this_dataset.retrieve_genes("GeneId") + # + print("CALLING THE LIT CORRELATION HERE") species = this_dataset.group.species.lower() this_trait_geneid = this_trait.geneid -- cgit v1.2.3 From 067d27460965aaf1ceaa863a315a0c7dbc47ae02 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 26 Apr 2021 17:05:06 +0300 Subject: fix:remove debug statements and commented code --- wqflask/base/mrna_assay_tissue_data.py | 25 --------- wqflask/wqflask/correlation/correlation_gn3_api.py | 60 +++------------------- 2 files changed, 8 insertions(+), 77 deletions(-) diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py index 0220d73b..5a64afb2 100644 --- a/wqflask/base/mrna_assay_tissue_data.py +++ b/wqflask/base/mrna_assay_tissue_data.py @@ -52,33 +52,9 @@ class MrnaAssayTissueData(object): # lower_symbols[gene_symbol.lower()] = True if gene_symbol != None: lower_symbols[gene_symbol.lower()] = True - - import time - # initial_time = time.time() - # conn,cursor = database_connector() - # cursor.execute(query) - # for result in cursor.fetchall(): - # symbol = result[0] - # self.data[symbol].gene_id = result[1] - # self.data[symbol].data_id = result[2] - # self.data[symbol].chr = result[3] - # self.data[symbol].mb = result[4] - # self.data[symbol].description = result[5] - # self.data[symbol].probe_target_description = result[6] - - - # print("my loop takes>>>>",time.time()-initial_time) - # conn.close() - # r - - # takes 5 seconds - initial_time = time.time() results = list(g.db.execute(query).fetchall()) for result in results: symbol = result[0] - # if symbol is not None - # exists = lower_symbols.get(symbol.lower()) - # if symbol.lower() in lower_symbols: if symbol is not None and lower_symbols.get(symbol.lower()): symbol = symbol.lower() @@ -89,7 +65,6 @@ class MrnaAssayTissueData(object): self.data[symbol].mb = result.Mb self.data[symbol].description = result.description self.data[symbol].probe_target_description = result.Probe_Target_Description - print("time taken in the loop is",time.time()-initial_time) ########################################################################### #Input: cursor, symbolList (list), dataIdDict(Dict) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index c945f699..3c21a850 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -63,9 +63,6 @@ def sample_for_trait_lists(corr_results, target_dataset, this_trait, this_datase "trait_sample_data": sample_data, "trait_id": start_vars["trait_id"] } - # trait_lists = dict([(list(corr_result)[0],True) for corr_result in corr_results]) - # target_dataset.trait_data =list(filter(lambda dict_obj: dict_obj.keys()[ - # 0] in corr_results_traits, target_dataset_data)) results = map_shared_keys_to_values( target_dataset.samplelist, target_dataset.trait_data) correlation_results = compute_all_sample_correlation(corr_method="pearson", @@ -77,33 +74,15 @@ def sample_for_trait_lists(corr_results, target_dataset, this_trait, this_datase def tissue_for_trait_lists(corr_results, this_dataset, target_dataset, this_trait): - # # print(corr_results[0])-- - # [{"awsdsd_at": {'corr_coeffient': 0.49714692782257336, 'p_value': 1.872077762359228e-05, 'num_overlap': 67}}] - - print("creating trait_lists") - # corr_results = corr_results[0::] trait_lists = dict([(list(corr_result)[0], True) for corr_result in corr_results]) - print("finished creating trait_list") - traits_symbol_dict = this_dataset.retrieve_genes("Symbol") - print("Retrieved symbol dict") - print("creating dict here>>>>>>>>>") - import time - init_time = time.time() traits_symbol_dict = dict({trait_name: symbol for ( trait_name, symbol) in traits_symbol_dict.items() if trait_lists.get(trait_name)}) - print("time taken to create this max dict is>>>>", time.time()-init_time) - print("finished creatinf the dict") - print("Fetching tissue datas") primary_tissue_data, target_tissue_data = get_tissue_correlation_input( this_trait, traits_symbol_dict) - print("finihsed>>>>>>>>>>>>>>>>>>") - print("Calling experimental_compute_all_tissue_correlation") corr_results = experimental_compute_all_tissue_correlation( primary_tissue_dict=primary_tissue_data, target_tissues_data=target_tissue_data, corr_method="pearson") - # print('finished calling this tissue reuslts',corr_results) - return corr_results @@ -123,22 +102,14 @@ def compute_correlation(start_vars, method="pearson"): corr_input_data = {} if corr_type == "sample": - import time - initial_time = time.time() - # corr_input_data = { - # "target_dataset": target_dataset.trait_data, - # "target_samplelist": target_dataset.samplelist, - # "trait_data": { - # "trait_sample_data": sample_data, - # "trait_id": start_vars["trait_id"] - # } - # } + sample_data = process_samples( start_vars, this_dataset.group.samplelist) + initial_time = time.time() target_dataset.get_trait_data(list(sample_data.keys())) this_trait = retrieve_sample_data(this_trait, this_dataset) + print("Creating target dataset and trait took", time.time()-initial_time) - print("Creating dataset and trait took", time.time()-initial_time) this_trait_data = { "trait_sample_data": sample_data, @@ -151,15 +122,9 @@ def compute_correlation(start_vars, method="pearson"): this_trait=this_trait_data, target_dataset=results) - print("computedd>>>>>>>>>>>>>") - print("doing sample correlation took", time.time()-initial_time) - - other_results_time = time.time() - other_results = tissue_for_trait_lists( - correlation_results, this_dataset, target_dataset, this_trait) - print(">>>time taken for this is", time.time()-other_results_time) - + # other_results = tissue_for_trait_lists( + # correlation_results, this_dataset, target_dataset, this_trait) # requests_url = f"{GN3_CORRELATION_API}/sample_x/{method}" return correlation_results @@ -177,17 +142,9 @@ def compute_correlation(start_vars, method="pearson"): target_tissues_data=corr_input_data[ "target_tissues_dict"], corr_method=method) - print("correlation y took", time.time()-initial_time) - # initial_time = time.time() - # correlation_results = compute_all_tissue_correlation(primary_tissue_dict=corr_input_data["primary_tissue"], - # target_tissues_data=corr_input_data["target_tissues_dict"], - # corr_method=method) - # print("time taken for compute tissue is", time.time()-initial_time) - - # requests_url = f"{GN3_CORRELATION_API}/tissue_corr/{method}" - - sample_results = sample_for_trait_lists( - correlation_results, target_dataset, this_trait, this_dataset, start_vars) + print("computing tissue took >>>>", time.time()-initial_time) + # sample_results = sample_for_trait_lists( + # correlation_results, target_dataset, this_trait, this_dataset, start_vars) return correlation_results elif corr_type == "lit": @@ -203,7 +160,6 @@ def compute_correlation(start_vars, method="pearson"): species=species, gene_id=this_trait_geneid) return lit_corr_results - print("the time taken is", time.time()-initial_time) # requests_url = f"{GN3_CORRELATION_API}/lit_corr/{species}/{this_trait_geneid}" # corr_input_data = geneid_dict # corr_results = requests.post(requests_url, json=corr_input_data) -- cgit v1.2.3 From 243f884b0f7e5e42aaab7d7c8076479818be8578 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 27 Apr 2021 14:35:37 +0300 Subject: templates: submit_trait.html: Add field for inputting trait name --- wqflask/wqflask/templates/submit_trait.html | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/submit_trait.html b/wqflask/wqflask/templates/submit_trait.html index 2cc18240..3572b0a9 100644 --- a/wqflask/wqflask/templates/submit_trait.html +++ b/wqflask/wqflask/templates/submit_trait.html @@ -53,7 +53,7 @@ -
    +

    2. Enter Trait Data:

    File uploading isn't enabled yet, but is coming soon.


    @@ -77,6 +77,24 @@
    +
    +

    3. Enable Use of Trait Variance:

    +
    + +
    +
    +
    +

    + Name Your Trait: (optional) +

    + +
    +
    +
    + + +
    +
    -- cgit v1.2.3 From 75bcab50abcf29c8a5f98a9f17b37457ac433d3b Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 27 Apr 2021 21:37:12 +0000 Subject: Stopped using the scanone function pointer when doing R/qtl mapping, since the results are not converted into a Python object in a way that preserves marker names (which is important because pseudomarkers can be added) Instead the marker names are extracted from the scanone results using R immediately after they're generated, and then passed to process_rqtl_results --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 39 ++++++++++++----------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 4117a0e5..f593ae91 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -78,6 +78,7 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec cross_object = calc_genoprob(cross_object) else: cross_object = calc_genoprob(cross_object, step=5, stepwidth="max") + logger.info("after calc_genoprob"); pheno_string = sanitize_rqtl_phenotype(vals) logger.info("phenostring done"); @@ -110,11 +111,13 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec return process_pair_scan_results(result_data_frame) else: if do_control == "true" or cofactors != "": - logger.info("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covars, model=model, method=method) - ro.r('save.image(file = "/home/zas1024/gn2-zach/itp_cofactor_test.RData")') + ro.r(f"qtl_results = scanone(the_cross, pheno='the_pheno', addcovar=all_covars, model='{model}', method='{method}')") + result_data_frame = ro.r("qtl_results") else: - logger.info("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method) + ro.r(f"qtl_results = scanone(the_cross, pheno='the_pheno', model='{model}', method='{method}')") + result_data_frame = ro.r("qtl_results") + marker_names = np.asarray(ro.r("row.names(qtl_results)")) if num_perm > 0 and permCheck == "ON": # Do permutation (if requested by user) if len(perm_strata_list) > 0: #ZS: The strata list would only be populated if "Stratified" was checked on before mapping cross_object, strata_ob = add_perm_strata(cross_object, perm_strata_list) @@ -129,9 +132,10 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method) perm_output, suggestive, significant = process_rqtl_perm_results(num_perm, perm_data_frame) # Functions that sets the thresholds for the webinterface - return perm_output, suggestive, significant, process_rqtl_results(result_data_frame, dataset.group.species) + + return perm_output, suggestive, significant, process_rqtl_results(marker_names, result_data_frame, dataset.group.species) else: - return process_rqtl_results(result_data_frame, dataset.group.species) + return process_rqtl_results(marker_names, result_data_frame, dataset.group.species) def generate_cross_from_rdata(dataset): rdata_location = locate(dataset.group.name + ".RData", "genotype/rdata") @@ -144,7 +148,6 @@ def generate_cross_from_rdata(dataset): """ % (rdata_location)) def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly - ro.r(""" trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) } getGenoCode <- function(header, name = 'unk'){ @@ -376,9 +379,8 @@ def process_pair_scan_results(result): def process_rqtl_perm_results(num_perm, results): perm_vals = [] - for line in str(results).split("\n")[1:(num_perm+1)]: - #print("R/qtl permutation line:", line.split()) - perm_vals.append(float(line.split()[1])) + for item in results: + perm_vals.append(item) perm_output = perm_vals suggestive = np.percentile(np.array(perm_vals), 67) @@ -386,20 +388,21 @@ def process_rqtl_perm_results(num_perm, results): return perm_output, suggestive, significant -def process_rqtl_results(result, species_name): # TODO: how to make this a one liner and not copy the stuff in a loop +def process_rqtl_results(marker_names, results, species_name): # TODO: how to make this a one liner and not copy the stuff in a loop qtl_results = [] - output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)] - for i, line in enumerate(result.iter_row()): + for i, line in enumerate(results): marker = {} - marker['name'] = result.rownames[i] - if species_name == "mouse" and output[i][0] == 20: #ZS: This is awkward, but I'm not sure how to change the 20s to Xs in the RData file + marker['name'] = marker_names[i] + if species_name == "mouse" and line[0] == 20: marker['chr'] = "X" else: - marker['chr'] = output[i][0] - marker['cM'] = output[i][1] - marker['Mb'] = output[i][1] - marker['lod_score'] = output[i][2] + try: + marker['chr'] = int(line[0]) + except: + marker['chr'] = line[0] + marker['cM'] = marker['Mb'] = line[1] + marker['lod_score'] = line[2] qtl_results.append(marker) return qtl_results \ No newline at end of file -- cgit v1.2.3 From 1845086220e427ed38d1bb2b216dc53de3760d64 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 27 Apr 2021 22:24:11 +0000 Subject: Randomized cross object filename, since I think it throws an error if the same file is being written to simultaneously (or being written to while it's being read) --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index f593ae91..c70fbbec 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -10,6 +10,7 @@ from base.trait import create_trait from base.data_set import create_dataset from utility import webqtlUtil from utility.tools import locate, TEMPDIR +from wqflask.marker_regression.gemma_mapping import generate_random_n_string from flask import g import utility.logger @@ -148,13 +149,17 @@ def generate_cross_from_rdata(dataset): """ % (rdata_location)) def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly + + cross_filename = (f"{str(dataset.group.name)}_" + f"{generate_random_n_string(6)}") + ro.r(""" trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) } getGenoCode <- function(header, name = 'unk'){ mat = which(unlist(lapply(header,function(x){ length(grep(paste('@',name,sep=''), x)) })) == 1) return(trim(strsplit(header[mat],':')[[1]][2])) } - GENOtoCSVR <- function(genotypes = '%s', out = 'cross.csvr', phenotype = NULL, sex = NULL, verbose = FALSE){ + GENOtoCSVR <- function(genotypes = '%s', out = '%s.csvr', phenotype = NULL, sex = NULL, verbose = FALSE){ header = readLines(genotypes, 40) # Assume a geno header is not longer than 40 lines toskip = which(unlist(lapply(header, function(x){ length(grep("Chr\t", x)) })) == 1)-1 # Major hack to skip the geno headers type <- getGenoCode(header, 'type') @@ -188,7 +193,7 @@ def generate_cross_from_geno(dataset, scale_units): # TODO: Need to figur } return(cross) } - """ % (dataset.group.genofile, scale_units)) + """ % (dataset.group.genofile, cross_filename, scale_units)) def add_perm_strata(cross, perm_strata): col_string = 'c("the_strata")' -- cgit v1.2.3 From 63ca33f75fcb67ba8c3851c60088b62becb5f735 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 27 Apr 2021 22:25:44 +0000 Subject: Convert to array and transpose R/qtl scanone results when not using cofactors. For some reason the rows/columns are inverted when converted to a python object when doing scanone with cofactors vs without cofactors --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index c70fbbec..cd43577e 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -116,7 +116,7 @@ def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permChec result_data_frame = ro.r("qtl_results") else: ro.r(f"qtl_results = scanone(the_cross, pheno='the_pheno', model='{model}', method='{method}')") - result_data_frame = ro.r("qtl_results") + result_data_frame = np.asarray(ro.r("qtl_results")).T marker_names = np.asarray(ro.r("row.names(qtl_results)")) if num_perm > 0 and permCheck == "ON": # Do permutation (if requested by user) -- cgit v1.2.3 From ac9be3f74e005e95a057f2c49baa7822d05f1ece Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Wed, 28 Apr 2021 08:46:53 +0300 Subject: minor fixes for correlation --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 917d6549..f73f235c 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -155,7 +155,7 @@ fi # We may change this one: # export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH -PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$HOME/genenetwork3:$PYTHONPATH +PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$HOME/project/genenetwork3:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then -- cgit v1.2.3 From f799477b49ef8c2bd8d8408b0c1b1ee6094cb8ca Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 20:53:13 +0300 Subject: wqflask: views.py: Clean up imports section Remove unused imports and break up long imports into shorter lines. --- wqflask/wqflask/views.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index baaece2f..91fb8df2 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -1,20 +1,17 @@ """Main routing table for GN2""" -import traceback # for error page +import traceback # for error page import os # for error gifs import random # for random error gif import datetime # for errors -import time # for errors import sys import csv import simplejson as json -import yaml import xlsxwriter import io # Todo: Use cStringIO? from zipfile import ZipFile, ZIP_DEFLATED -import gc import numpy as np import pickle as pickle import uuid @@ -24,23 +21,27 @@ import base64 import array import sqlalchemy from wqflask import app -from flask import g, Response, request, make_response, render_template, send_from_directory, jsonify, redirect, url_for, send_file +from flask import g +from flask import Response +from flask import request +from flask import make_response +from flask import render_template +from flask import send_from_directory +from flask import redirect +from flask import url_for +from flask import send_file -from wqflask import group_manager -from wqflask import resource_manager from wqflask import search_results -from wqflask import export_traits -from wqflask import gsearch -from wqflask import update_search_results -from wqflask import docs from wqflask import news from wqflask import server_side from wqflask.submit_bnw import get_bnw_input -from base.data_set import create_dataset, DataSet # Used by YAML in marker_regression +from base.data_set import create_dataset # Used by YAML in marker_regression from wqflask.show_trait import show_trait from wqflask.show_trait import export_trait_data from wqflask.heatmap import heatmap -from wqflask.external_tools import send_to_bnw, send_to_webgestalt, send_to_geneweaver +from wqflask.external_tools import send_to_bnw +from wqflask.external_tools import send_to_webgestalt +from wqflask.external_tools import send_to_geneweaver from wqflask.comparison_bar_chart import comparison_bar_chart from wqflask.marker_regression import run_mapping from wqflask.marker_regression import display_mapping_results @@ -59,24 +60,31 @@ from wqflask.docs import Docs, update_text from wqflask.db_info import InfoPage from utility import temp_data -from utility.tools import SQL_URI, TEMPDIR, USE_REDIS, USE_GN_SERVER, GN_SERVER_URL, GN_VERSION, JS_TWITTER_POST_FETCHER_PATH, JS_GUIX_PATH, CSS_PATH +from utility.tools import SQL_URI +from utility.tools import TEMPDIR +from utility.tools import USE_REDIS +from utility.tools import GN_SERVER_URL +from utility.tools import GN_VERSION +from utility.tools import JS_TWITTER_POST_FETCHER_PATH +from utility.tools import JS_GUIX_PATH from utility.helper_functions import get_species_groups from utility.authentication_tools import check_resource_availability from utility.redis_tools import get_redis_conn -Redis = get_redis_conn() + from base.webqtlConfig import GENERATED_IMAGE_DIR, DEFAULT_PRIVILEGES from utility.benchmark import Bench from pprint import pformat as pf -from wqflask import collect from wqflask.database import db_session -import werkzeug import utility.logger -logger = utility.logger.getLogger(__name__ ) + +Redis = get_redis_conn() + +logger = utility.logger.getLogger(__name__) @app.before_request -- cgit v1.2.3 From 5ae817e71a7232a747c988cd9f6abfa25a35971c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:17:01 +0300 Subject: wqflask: views: Remove news import This name clashes with "def news" which is defined later. --- wqflask/wqflask/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 91fb8df2..3aafd7d3 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -32,7 +32,6 @@ from flask import url_for from flask import send_file from wqflask import search_results -from wqflask import news from wqflask import server_side from wqflask.submit_bnw import get_bnw_input from base.data_set import create_dataset # Used by YAML in marker_regression -- cgit v1.2.3 From ed434e8fba9f807d94892d06d6191f5de6670bd9 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:20:32 +0300 Subject: Remove local variables that are unused --- wqflask/wqflask/views.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3aafd7d3..b1c2ed68 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -97,7 +97,6 @@ def connect_db(): @app.before_request def check_access_permissions(): logger.debug("@app.before_request check_access_permissions") - available = True if 'dataset' in request.args: permissions = DEFAULT_PRIVILEGES if request.args['dataset'] != "Temp": @@ -656,7 +655,6 @@ def loading_page(): dataset = create_dataset(start_vars['dataset'], group_name = start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) - genofile_samplelist = [] samples = start_vars['primary_samples'].split(",") if 'genofile' in start_vars: if start_vars['genofile'] != "": @@ -846,7 +844,6 @@ def export_pdf(): svg_xml = request.form.get("data", "Invalid data") logger.info("svg_xml:", svg_xml) filename = request.form.get("filename", "interval_map_pdf") - filepath = GENERATED_IMAGE_DIR+filename pdf_file = cairosvg.svg2pdf(bytestring=svg_xml) response = Response(pdf_file, mimetype="application/pdf") response.headers["Content-Disposition"] = "attachment; filename=%s"%filename @@ -942,7 +939,6 @@ def security_tutorial_page(): @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): logger.info(request.url) - template_vars = get_bnw_input(request.form) return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) # Take this out or secure it before putting into production -- cgit v1.2.3 From 41791eb9eb809c11b5bd953235ae98c4b4d82156 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:20:54 +0300 Subject: Remove stale comments --- wqflask/wqflask/views.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b1c2ed68..31ba8df0 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -972,9 +972,6 @@ def json_default_handler(obj): # Handle custom objects if hasattr(obj, '__dict__'): return obj.__dict__ - #elif type(obj) == "Dataset": - # logger.info("Not going to serialize Dataset") - # return None else: raise TypeError('Object of type %s with value of %s is not JSON serializable' % ( type(obj), repr(obj))) -- cgit v1.2.3 From ca1bc2a22f93591ac9c120bca1ba554ae891c9ab Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:21:03 +0300 Subject: Use right block level comments --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 31ba8df0..a33c64f1 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -962,7 +962,7 @@ def browser_inputs(): ########################################################################## def json_default_handler(obj): - '''Based on http://stackoverflow.com/a/2680060/1175849''' + """Based on http://stackoverflow.com/a/2680060/1175849""" # Handle datestamps if hasattr(obj, 'isoformat'): return obj.isoformat() -- cgit v1.2.3 From f23a198951e3aa3040b6ae8c28aba0fd34514ebd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:24:04 +0300 Subject: wqflask: views: Apply pep-8 --- wqflask/wqflask/views.py | 222 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 154 insertions(+), 68 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index a33c64f1..319f1270 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -91,9 +91,11 @@ def connect_db(): logger.info("@app.before_request connect_db") db = getattr(g, '_database', None) if db is None: - g.db = g._database = sqlalchemy.create_engine(SQL_URI, encoding="latin1") + g.db = g._database = sqlalchemy.create_engine( + SQL_URI, encoding="latin1") logger.debug(g.db) + @app.before_request def check_access_permissions(): logger.debug("@app.before_request check_access_permissions") @@ -105,7 +107,8 @@ def check_access_permissions(): if dataset.type == "Temp": permissions = DEFAULT_PRIVILEGES elif 'trait_id' in request.args: - permissions = check_resource_availability(dataset, request.args['trait_id']) + permissions = check_resource_availability( + dataset, request.args['trait_id']) elif dataset.type != "Publish": permissions = check_resource_availability(dataset) @@ -116,6 +119,7 @@ def check_access_permissions(): if permissions['data'] == 'no-access': return redirect(url_for("no_access_page")) + @app.teardown_appcontext def shutdown_session(exception=None): db = getattr(g, '_database', None) @@ -124,6 +128,7 @@ def shutdown_session(exception=None): db_session.remove() g.db = None + @app.errorhandler(Exception) def handle_bad_request(e): err_msg = str(e) @@ -134,25 +139,30 @@ def handle_bad_request(e): logger.error(traceback.format_exc()) now = datetime.datetime.utcnow() time_str = now.strftime('%l:%M%p UTC %b %d, %Y') - formatted_lines = [request.url + " ("+time_str+")"]+traceback.format_exc().splitlines() + formatted_lines = [request.url + + " ("+time_str+")"]+traceback.format_exc().splitlines() # Handle random animations # Use a cookie to have one animation on refresh animation = request.cookies.get(err_msg[:32]) if not animation: - list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ] + list = [fn for fn in os.listdir( + "./wqflask/static/gif/error") if fn.endswith(".gif")] animation = random.choice(list) - resp = make_response(render_template("error.html", message=err_msg, stack=formatted_lines, error_image=animation, version=GN_VERSION)) + resp = make_response(render_template("error.html", message=err_msg, + stack=formatted_lines, error_image=animation, version=GN_VERSION)) # logger.error("Set cookie %s with %s" % (err_msg, animation)) resp.set_cookie(err_msg[:32], animation) return resp + @app.route("/authentication_needed") def no_access_page(): return render_template("new_security/not_authenticated.html") + @app.route("/") def index_page(): logger.info("Sending index_page") @@ -177,7 +187,7 @@ def tmp_page(img_path): imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) return render_template("show_image.html", - img_base64 = bytesarray ) + img_base64=bytesarray) @app.route("/js/") @@ -189,6 +199,7 @@ def js(filename): name = name.replace('js_alt/', '') return send_from_directory(js_path, name) + @app.route("/css/") def css(filename): js_path = JS_GUIX_PATH @@ -198,10 +209,12 @@ def css(filename): name = name.replace('js_alt/', '') return send_from_directory(js_path, name) + @app.route("/twitter/") def twitter(filename): return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) + @app.route("/search", methods=('GET',)) def search_page(): logger.info("in search_page") @@ -209,7 +222,8 @@ def search_page(): result = None if USE_REDIS: with Bench("Trying Redis cache"): - key = "search_results:v1:" + json.dumps(request.args, sort_keys=True) + key = "search_results:v1:" + \ + json.dumps(request.args, sort_keys=True) logger.debug("key is:", pf(key)) result = Redis.get(key) if result: @@ -232,6 +246,7 @@ def search_page(): else: return render_template("search_error.html") + @app.route("/search_table", methods=('GET',)) def search_page_table(): logger.info("in search_page table") @@ -242,7 +257,7 @@ def search_page_table(): logger.info(type(the_search.trait_list)) logger.info(the_search.trait_list) - + current_page = server_side.ServerSideTable( len(the_search.trait_list), the_search.trait_list, @@ -252,6 +267,7 @@ def search_page_table(): return flask.jsonify(current_page) + @app.route("/gsearch", methods=('GET',)) def gsearchact(): logger.info(request.url) @@ -262,6 +278,7 @@ def gsearchact(): elif type == "phenotype": return render_template("gsearch_pheno.html", **result) + @app.route("/gsearch_updating", methods=('POST',)) def gsearch_updating(): logger.info("REQUEST ARGS:", request.values) @@ -292,41 +309,59 @@ def generated_file(filename): logger.info(request.url) return send_from_directory(GENERATED_IMAGE_DIR, filename) + @app.route("/help") def help(): logger.info(request.url) doc = Docs("help", request.args) return render_template("docs.html", **doc.__dict__) + @app.route("/wgcna_setup", methods=('POST',)) def wcgna_setup(): - logger.info("In wgcna, request.form is:", request.form) # We are going to get additional user input for the analysis + # We are going to get additional user input for the analysis + logger.info("In wgcna, request.form is:", request.form) logger.info(request.url) - return render_template("wgcna_setup.html", **request.form) # Display them using the template + # Display them using the template + return render_template("wgcna_setup.html", **request.form) + @app.route("/wgcna_results", methods=('POST',)) def wcgna_results(): logger.info("In wgcna, request.form is:", request.form) logger.info(request.url) - wgcna = wgcna_analysis.WGCNA() # Start R, load the package and pointers and create the analysis - wgcnaA = wgcna.run_analysis(request.form) # Start the analysis, a wgcnaA object should be a separate long running thread - result = wgcna.process_results(wgcnaA) # After the analysis is finished store the result - return render_template("wgcna_results.html", **result) # Display them using the template + # Start R, load the package and pointers and create the analysis + wgcna = wgcna_analysis.WGCNA() + # Start the analysis, a wgcnaA object should be a separate long running thread + wgcnaA = wgcna.run_analysis(request.form) + # After the analysis is finished store the result + result = wgcna.process_results(wgcnaA) + # Display them using the template + return render_template("wgcna_results.html", **result) + @app.route("/ctl_setup", methods=('POST',)) def ctl_setup(): - logger.info("In ctl, request.form is:", request.form) # We are going to get additional user input for the analysis + # We are going to get additional user input for the analysis + logger.info("In ctl, request.form is:", request.form) logger.info(request.url) - return render_template("ctl_setup.html", **request.form) # Display them using the template + # Display them using the template + return render_template("ctl_setup.html", **request.form) + @app.route("/ctl_results", methods=('POST',)) def ctl_results(): logger.info("In ctl, request.form is:", request.form) logger.info(request.url) - ctl = ctl_analysis.CTL() # Start R, load the package and pointers and create the analysis - ctlA = ctl.run_analysis(request.form) # Start the analysis, a ctlA object should be a separate long running thread - result = ctl.process_results(ctlA) # After the analysis is finished store the result - return render_template("ctl_results.html", **result) # Display them using the template + # Start R, load the package and pointers and create the analysis + ctl = ctl_analysis.CTL() + # Start the analysis, a ctlA object should be a separate long running thread + ctlA = ctl.run_analysis(request.form) + # After the analysis is finished store the result + result = ctl.process_results(ctlA) + # Display them using the template + return render_template("ctl_results.html", **result) + @app.route("/news") def news(): @@ -381,9 +416,11 @@ def export_trait_excel(): logger.info("In export_trait_excel") logger.info("request.form:", request.form) logger.info(request.url) - trait_name, sample_data = export_trait_data.export_sample_table(request.form) + trait_name, sample_data = export_trait_data.export_sample_table( + request.form) - logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) + logger.info("sample_data - type: %s -- size: %s" % + (type(sample_data), len(sample_data))) buff = io.BytesIO() workbook = xlsxwriter.Workbook(buff, {'in_memory': True}) @@ -397,7 +434,8 @@ def export_trait_excel(): return Response(excel_data, mimetype='application/vnd.ms-excel', - headers={"Content-Disposition":"attachment;filename="+ trait_name + ".xlsx"}) + headers={"Content-Disposition": "attachment;filename=" + trait_name + ".xlsx"}) + @app.route('/export_trait_csv', methods=('POST',)) def export_trait_csv(): @@ -405,9 +443,11 @@ def export_trait_csv(): logger.info("In export_trait_csv") logger.info("request.form:", request.form) logger.info(request.url) - trait_name, sample_data = export_trait_data.export_sample_table(request.form) + trait_name, sample_data = export_trait_data.export_sample_table( + request.form) - logger.info("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) + logger.info("sample_data - type: %s -- size: %s" % + (type(sample_data), len(sample_data))) buff = io.StringIO() writer = csv.writer(buff) @@ -418,7 +458,8 @@ def export_trait_csv(): return Response(csv_data, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename="+ trait_name + ".csv"}) + headers={"Content-Disposition": "attachment;filename=" + trait_name + ".csv"}) + @app.route('/export_traits_csv', methods=('POST',)) def export_traits_csv(): @@ -443,7 +484,8 @@ def export_traits_csv(): else: return Response(file_list[0][1], mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=" + file_list[0][0]}) + headers={"Content-Disposition": "attachment;filename=" + file_list[0][0]}) + @app.route('/export_perm_data', methods=('POST',)) def export_perm_data(): @@ -454,7 +496,8 @@ def export_perm_data(): now = datetime.datetime.now() time_str = now.strftime('%H:%M_%d%B%Y') - file_name = "Permutation_" + perm_info['num_perm'] + "_" + perm_info['trait_name'] + "_" + time_str + file_name = "Permutation_" + \ + perm_info['num_perm'] + "_" + perm_info['trait_name'] + "_" + time_str the_rows = [ ["#Permutation Test"], @@ -468,10 +511,14 @@ def export_perm_data(): ["#N_genotypes: " + str(perm_info['n_genotypes'])], ["#Genotype_file: " + perm_info['genofile']], ["#Units_linkage: " + perm_info['units_linkage']], - ["#Permutation_stratified_by: " + ", ".join([ str(cofactor) for cofactor in perm_info['strat_cofactors']])], - ["#RESULTS_1: Suggestive LRS(p=0.63) = " + str(np.percentile(np.array(perm_info['perm_data']), 67))], - ["#RESULTS_2: Significant LRS(p=0.05) = " + str(np.percentile(np.array(perm_info['perm_data']), 95))], - ["#RESULTS_3: Highly Significant LRS(p=0.01) = " + str(np.percentile(np.array(perm_info['perm_data']), 99))], + ["#Permutation_stratified_by: " + + ", ".join([str(cofactor) for cofactor in perm_info['strat_cofactors']])], + ["#RESULTS_1: Suggestive LRS(p=0.63) = " + + str(np.percentile(np.array(perm_info['perm_data']), 67))], + ["#RESULTS_2: Significant LRS(p=0.05) = " + str( + np.percentile(np.array(perm_info['perm_data']), 95))], + ["#RESULTS_3: Highly Significant LRS(p=0.01) = " + str( + np.percentile(np.array(perm_info['perm_data']), 99))], ["#Comment: Results sorted from low to high peak linkage"] ] @@ -485,7 +532,7 @@ def export_perm_data(): return Response(csv_data, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=" + file_name + ".csv"}) + headers={"Content-Disposition": "attachment;filename=" + file_name + ".csv"}) @app.route("/show_temp_trait", methods=('POST',)) @@ -519,7 +566,8 @@ def heatmap_page(): traits = [trait.strip() for trait in start_vars['trait_list'].split(',')] if traits[0] != "": version = "v5" - key = "heatmap:{}:".format(version) + json.dumps(start_vars, sort_keys=True) + key = "heatmap:{}:".format( + version) + json.dumps(start_vars, sort_keys=True) logger.info("key is:", pf(key)) with Bench("Loading cache"): result = Redis.get(key) @@ -540,7 +588,8 @@ def heatmap_page(): result = template_vars.__dict__ for item in list(template_vars.__dict__.keys()): - logger.info(" ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) + logger.info( + " ---**--- {}: {}".format(type(template_vars.__dict__[item]), item)) pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL) logger.info("pickled result length:", len(pickled_result)) @@ -551,10 +600,12 @@ def heatmap_page(): rendered_template = render_template("heatmap.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'Heatmap'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'Heatmap'}) return rendered_template + @app.route("/bnw_page", methods=('POST',)) def bnw_page(): logger.info("In run BNW, request.form is:", pf(request.form)) @@ -569,10 +620,12 @@ def bnw_page(): result = template_vars.__dict__ rendered_template = render_template("bnw_page.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'BNW'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'BNW'}) return rendered_template + @app.route("/webgestalt_page", methods=('POST',)) def webgestalt_page(): logger.info("In run WebGestalt, request.form is:", pf(request.form)) @@ -587,10 +640,12 @@ def webgestalt_page(): result = template_vars.__dict__ rendered_template = render_template("webgestalt_page.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'WebGestalt'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'WebGestalt'}) return rendered_template + @app.route("/geneweaver_page", methods=('POST',)) def geneweaver_page(): logger.info("In run WebGestalt, request.form is:", pf(request.form)) @@ -605,10 +660,12 @@ def geneweaver_page(): result = template_vars.__dict__ rendered_template = render_template("geneweaver_page.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'GeneWeaver'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'GeneWeaver'}) return rendered_template + @app.route("/comparison_bar_chart", methods=('POST',)) def comp_bar_chart_page(): logger.info("In comp bar chart, request.form is:", pf(request.form)) @@ -620,26 +677,30 @@ def comp_bar_chart_page(): if traits[0] != "": template_vars = comparison_bar_chart.ComparisonBarChart(request.form) template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + default=json_default_handler, + indent=" ") result = template_vars.__dict__ - rendered_template = render_template("comparison_bar_chart.html", **result) + rendered_template = render_template( + "comparison_bar_chart.html", **result) else: - rendered_template = render_template("empty_collection.html", **{'tool':'Comparison Bar Chart'}) + rendered_template = render_template( + "empty_collection.html", **{'tool': 'Comparison Bar Chart'}) return rendered_template + @app.route("/mapping_results_container") def mapping_results_container_page(): return render_template("mapping_results_container.html") + @app.route("/loading", methods=('POST',)) def loading_page(): logger.info(request.url) initial_start_vars = request.form start_vars_container = {} - n_samples = 0 #ZS: So it can be displayed on loading page + n_samples = 0 # ZS: So it can be displayed on loading page if 'wanted_inputs' in initial_start_vars: wanted = initial_start_vars['wanted_inputs'].split(",") start_vars = {} @@ -652,7 +713,8 @@ def loading_page(): else: sample_vals_dict = json.loads(start_vars['sample_vals']) if 'group' in start_vars: - dataset = create_dataset(start_vars['dataset'], group_name = start_vars['group']) + dataset = create_dataset( + start_vars['dataset'], group_name=start_vars['group']) else: dataset = create_dataset(start_vars['dataset']) samples = start_vars['primary_samples'].split(",") @@ -660,7 +722,8 @@ def loading_page(): if start_vars['genofile'] != "": genofile_string = start_vars['genofile'] dataset.group.genofile = genofile_string.split(":")[0] - genofile_samples = run_mapping.get_genofile_samplelist(dataset) + genofile_samples = run_mapping.get_genofile_samplelist( + dataset) if len(genofile_samples) > 1: samples = genofile_samples @@ -680,6 +743,7 @@ def loading_page(): return rendered_template + @app.route("/run_mapping", methods=('POST',)) def mapping_results_page(): initial_start_vars = request.form @@ -750,9 +814,10 @@ def mapping_results_page(): start_vars[key] = value version = "v3" - key = "mapping_results:{}:".format(version) + json.dumps(start_vars, sort_keys=True) + key = "mapping_results:{}:".format( + version) + json.dumps(start_vars, sort_keys=True) with Bench("Loading cache"): - result = None # Just for testing + result = None # Just for testing #result = Redis.get(key) #logger.info("************************ Starting result *****************") @@ -772,12 +837,12 @@ def mapping_results_page(): rendered_template = render_template("mapping_error.html") return rendered_template except: - rendered_template = render_template("mapping_error.html") - return rendered_template + rendered_template = render_template("mapping_error.html") + return rendered_template template_vars.js_data = json.dumps(template_vars.js_data, - default=json_default_handler, - indent=" ") + default=json_default_handler, + indent=" ") result = template_vars.__dict__ @@ -792,18 +857,22 @@ def mapping_results_page(): imgB64 = base64.b64encode(imgdata) bytesarray = array.array('B', imgB64) result['pair_scan_array'] = bytesarray - rendered_template = render_template("pair_scan_results.html", **result) + rendered_template = render_template( + "pair_scan_results.html", **result) else: - gn1_template_vars = display_mapping_results.DisplayMappingResults(result).__dict__ + gn1_template_vars = display_mapping_results.DisplayMappingResults( + result).__dict__ with Bench("Rendering template"): #if (gn1_template_vars['mapping_method'] == "gemma") or (gn1_template_vars['mapping_method'] == "plink"): #gn1_template_vars.pop('qtlresults', None) - rendered_template = render_template("mapping_results.html", **gn1_template_vars) + rendered_template = render_template( + "mapping_results.html", **gn1_template_vars) return rendered_template -@app.route("/export_mapping_results", methods = ('POST',)) + +@app.route("/export_mapping_results", methods=('POST',)) def export_mapping_results(): logger.info("request.form:", request.form) logger.info(request.url) @@ -811,32 +880,35 @@ def export_mapping_results(): results_csv = open(file_path, "r").read() response = Response(results_csv, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=mapping_results.csv"}) + headers={"Content-Disposition": "attachment;filename=mapping_results.csv"}) return response -@app.route("/export_corr_matrix", methods = ('POST',)) + +@app.route("/export_corr_matrix", methods=('POST',)) def export_corr_matrix(): file_path = request.form.get("export_filepath") file_name = request.form.get("export_filename") results_csv = open(file_path, "r").read() response = Response(results_csv, mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=" + file_name + ".csv"}) + headers={"Content-Disposition": "attachment;filename=" + file_name + ".csv"}) return response -@app.route("/export", methods = ('POST',)) + +@app.route("/export", methods=('POST',)) def export(): logger.info("request.form:", request.form) logger.info(request.url) svg_xml = request.form.get("data", "Invalid data") filename = request.form.get("filename", "manhattan_plot_snp") response = Response(svg_xml, mimetype="image/svg+xml") - response.headers["Content-Disposition"] = "attachment; filename=%s"%filename + response.headers["Content-Disposition"] = "attachment; filename=%s" % filename return response -@app.route("/export_pdf", methods = ('POST',)) + +@app.route("/export_pdf", methods=('POST',)) def export_pdf(): import cairosvg logger.info("request.form:", request.form) @@ -846,9 +918,10 @@ def export_pdf(): filename = request.form.get("filename", "interval_map_pdf") pdf_file = cairosvg.svg2pdf(bytestring=svg_xml) response = Response(pdf_file, mimetype="application/pdf") - response.headers["Content-Disposition"] = "attachment; filename=%s"%filename + response.headers["Content-Disposition"] = "attachment; filename=%s" % filename return response + @app.route("/network_graph", methods=('POST',)) def network_graph_page(): logger.info("In network_graph, request.form is:", pf(request.form)) @@ -863,7 +936,8 @@ def network_graph_page(): return render_template("network_graph.html", **template_vars.__dict__) else: - return render_template("empty_collection.html", **{'tool':'Network Graph'}) + return render_template("empty_collection.html", **{'tool': 'Network Graph'}) + @app.route("/corr_compute", methods=('POST',)) def corr_compute_page(): @@ -872,6 +946,7 @@ def corr_compute_page(): template_vars = show_corr_results.CorrelationResults(request.form) return render_template("correlation_page.html", **template_vars.__dict__) + @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): logger.info("In corr_matrix, request.form is:", pf(request.form)) @@ -887,7 +962,8 @@ def corr_matrix_page(): return render_template("correlation_matrix.html", **template_vars.__dict__) else: - return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) + return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'}) + @app.route("/corr_scatter_plot") def corr_scatter_plot_page(): @@ -898,6 +974,7 @@ def corr_scatter_plot_page(): indent=" ") return render_template("corr_scatterplot.html", **template_vars.__dict__) + @app.route("/snp_browser", methods=('GET',)) def snp_browser_page(): logger.info(request.url) @@ -905,12 +982,14 @@ def snp_browser_page(): return render_template("snp_browser.html", **template_vars.__dict__) + @app.route("/db_info", methods=('GET',)) def db_info_page(): template_vars = InfoPage(request.args) return render_template("info_page.html", **template_vars.__dict__) + @app.route("/snp_browser_table", methods=('GET',)) def snp_browser_table(): logger.info(request.url) @@ -924,30 +1003,36 @@ def snp_browser_table(): return flask.jsonify(current_page) + @app.route("/tutorial/WebQTLTour", methods=('GET',)) def tutorial_page(): - #ZS: Currently just links to GN1 + # ZS: Currently just links to GN1 logger.info(request.url) return redirect("http://gn1.genenetwork.org/tutorial/WebQTLTour/") + @app.route("/tutorial/security", methods=('GET',)) def security_tutorial_page(): - #ZS: Currently just links to GN1 + # ZS: Currently just links to GN1 logger.info(request.url) return render_template("admin/security_help.html") + @app.route("/submit_bnw", methods=('POST',)) def submit_bnw(): logger.info(request.url) - return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) + return render_template("empty_collection.html", **{'tool': 'Correlation Matrix'}) # Take this out or secure it before putting into production + + @app.route("/get_temp_data") def get_temp_data(): logger.info(request.url) temp_uuid = request.args['key'] return flask.jsonify(temp_data.TempData(temp_uuid).get_all()) + @app.route("/browser_input", methods=('GET',)) def browser_inputs(): """ Returns JSON from tmp directory for the purescript genome browser""" @@ -961,6 +1046,7 @@ def browser_inputs(): ########################################################################## + def json_default_handler(obj): """Based on http://stackoverflow.com/a/2680060/1175849""" # Handle datestamps -- cgit v1.2.3 From a68795a639a731e2bdae0a977bccb01f291fc9f7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 28 Apr 2021 21:25:01 +0300 Subject: wqflask: views: Delete stale comments --- wqflask/wqflask/views.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 319f1270..28ab630f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -285,11 +285,7 @@ def gsearch_updating(): logger.info(request.url) result = UpdateGSearch(request.args).__dict__ return result['results'] - # type = request.args['type'] - # if type == "gene": - # return render_template("gsearch_gene_updating.html", **result) - # elif type == "phenotype": - # return render_template("gsearch_pheno.html", **result) + @app.route("/docedit") def docedit(): @@ -864,8 +860,6 @@ def mapping_results_page(): result).__dict__ with Bench("Rendering template"): - #if (gn1_template_vars['mapping_method'] == "gemma") or (gn1_template_vars['mapping_method'] == "plink"): - #gn1_template_vars.pop('qtlresults', None) rendered_template = render_template( "mapping_results.html", **gn1_template_vars) -- cgit v1.2.3 From 6e8afe5a6871a680778bfaed6b79c934b33d0307 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 28 Apr 2021 19:40:58 +0000 Subject: Account for temp traits when adding covariates with GEMMA mapping --- wqflask/wqflask/marker_regression/gemma_mapping.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index 83ebcdf9..06c9300a 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -146,7 +146,12 @@ def gen_covariates_file(this_dataset, covariates, samples): for covariate in covariate_list: this_covariate_data = [] trait_name = covariate.split(":")[0] - dataset_ob = create_dataset(covariate.split(":")[1]) + dataset_name = covariate.split(":")[1] + if dataset_name == "Temp": + temp_group = trait_name.split("_")[2] + dataset_ob = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group) + else: + dataset_ob = create_dataset(covariate.split(":")[1]) trait_ob = create_trait(dataset=dataset_ob, name=trait_name, cellid=None) -- cgit v1.2.3 From 65a2c8c1f455c5fad48fbac1a3e75310d49ed844 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 28 Apr 2021 19:45:37 +0000 Subject: Added something to jsonable in trait.py to account for temp traits (this is necessary for them to show up correctly when selecting traits from collections in pop-up windows, like when selecting cofactors for mapping) --- wqflask/base/trait.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index df96d46e..b4b4452a 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -337,6 +337,10 @@ def jsonable(trait): dataset_name=dataset.shortname, location=trait.location_repr ) + elif dataset.name == "Temp": + return dict(name=trait.name, + dataset="Temp", + dataset_name="Temp") else: return dict() -- cgit v1.2.3 From 90a6a950fba0bb463cff9669f04a87cab8efb36c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 11:49:42 +0300 Subject: wqflask: views: Replace dict de-structuring with kw arguments * wqflask/wqflask/views.py (submit_trait_form): Use kw arguments when passing variables to the template. This is more readable. --- wqflask/wqflask/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 28ab630f..38c56b71 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -394,9 +394,9 @@ def submit_trait_form(): species_and_groups = get_species_groups() return render_template( "submit_trait.html", - **{'species_and_groups': species_and_groups, - 'gn_server_url': GN_SERVER_URL, - 'version': GN_VERSION}) + species_and_groups=species_and_groups, + gn_server_url=GN_SERVER_URL, + version=GN_VERSION) @app.route("/create_temp_trait", methods=('POST',)) -- cgit v1.2.3 From e048c6321b37a1c05adbfb6754513d7920b10dd8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 12:45:28 +0300 Subject: templates: edit_trait.html: New file Copy of submit_trait.html. This is a copy; that'll form the basis of the new edit page. --- wqflask/wqflask/templates/edit_trait.html | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 wqflask/wqflask/templates/edit_trait.html diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html new file mode 100644 index 00000000..3572b0a9 --- /dev/null +++ b/wqflask/wqflask/templates/edit_trait.html @@ -0,0 +1,111 @@ +{% extends "base.html" %} +{% block title %}Trait Submission{% endblock %} +{% block content %} + +
    +
    + + {{ flash_me() }} + +
    +
    +
    +
    +

    Introduction

    +
    +

    The trait values that you enter are statistically compared with verified genotypes collected at a set of microsatellite markers in each RI set. The markers are drawn from a set of over 750, but for each set redundant markers have been removed, preferentially retaining those that are most informative.

    +

    These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.

    +
    +
    +
    +
    +
    +

    About Your Data

    +
    +

    You can open a separate window giving the number of strains for each data set and sample data.

    +

    None of your submitted data is copied or stored by this system except during the actual processing of your submission. By the time the reply page displays in your browser, your submission has been cleared from this system.

    +
    +
    +
    +
    +
    +
    +

    Trait Submission Form

    +
    +
    +

    1. Choose Species and Group:

    +
    +
    + +
    +
    +
    + +
    + +
    +
    +
    + +
    + +
    +
    +
    +
    +
    +

    2. Enter Trait Data:

    +

    File uploading isn't enabled yet, but is coming soon.

    +
    +
    + +
    +
    +
    +

    + Paste or Type Multiple Values: You can enter data by pasting a series of numbers representing trait values into this area. + The values can be on one line separated by spaces or tabs, or they can be on separate lines. Include one value for each individual + or line. Use an "x" for missing values. If you have chosen a set of inbred strains, then your data will be displayed in a form in + which you can confirm and/or edit. If you enter a file name in the previous section, + any data that you paste here will be ignored. Check sample data for the correct format. +

    + +
    +
    +
    + + +
    +
    +
    +

    3. Enable Use of Trait Variance:

    +
    + +
    +
    +
    +

    + Name Your Trait: (optional) +

    + +
    +
    +
    + + +
    +
    +
    +
    +
    +
    +
    + +{%endblock%} + +{% block js %} + + +{% endblock %} -- cgit v1.2.3 From 442fc67ba1d66dd7931e20d161b40451b4e4b8c8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 13:17:34 +0300 Subject: wqflask: views: Add new function for selecting a dataset * wqflask/wqflask/views.py(edit_trait_page): New function. --- wqflask/wqflask/views.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 38c56b71..81ccaf7d 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -399,6 +399,16 @@ def submit_trait_form(): version=GN_VERSION) +@app.route("/edit_trait_form") +def edit_trait_page(): + species_and_groups = get_species_groups() + return render_template( + "edit_trait.html", + species_and_groups=species_and_groups, + gn_server_url=GN_SERVER_URL, + version=GN_VERSION) + + @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) -- cgit v1.2.3 From 1e2e5b23ec30937f200778524f446515a5f83fb3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 13:18:50 +0300 Subject: templates: edit_trait.html: Add comment headers The comment headers mark different sections. --- wqflask/wqflask/templates/edit_trait.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 3572b0a9..82d7120d 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -39,12 +39,14 @@
    +
    +
    -- cgit v1.2.3 From ba8cd37b5cd7e038e31c879fbbc38939fa093acf Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 13:19:29 +0300 Subject: templates: edit_trait.html: Add extra fields for the form Add form elements for selecting group, type, and dataset. This mimics the home page. --- wqflask/wqflask/templates/edit_trait.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 82d7120d..4c9b3ba1 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -53,6 +53,20 @@
    + +
    + +
    + +
    +
    + +
    + +
    + +
    +
    -- cgit v1.2.3 From 43ed9d7e5c22de2a457457e75b8153f6c2e1488c Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 14:57:03 +0300 Subject: templates: edit_trait.html: Trim form to only contain selection menu --- wqflask/wqflask/templates/edit_trait.html | 45 ++----------------------------- 1 file changed, 2 insertions(+), 43 deletions(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 4c9b3ba1..14f44698 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -30,14 +30,11 @@
    -

    Trait Submission Form

    +

    Edit Trait Form

    -

    1. Choose Species and Group:

    +

    Choose Dataset to Edit:


    -
    - -
    @@ -68,44 +65,6 @@
    -
    -
    -

    2. Enter Trait Data:

    -

    File uploading isn't enabled yet, but is coming soon.

    -
    -
    - -
    -
    -
    -

    - Paste or Type Multiple Values: You can enter data by pasting a series of numbers representing trait values into this area. - The values can be on one line separated by spaces or tabs, or they can be on separate lines. Include one value for each individual - or line. Use an "x" for missing values. If you have chosen a set of inbred strains, then your data will be displayed in a form in - which you can confirm and/or edit. If you enter a file name in the previous section, - any data that you paste here will be ignored. Check sample data for the correct format. -

    - -
    -
    -
    - - -
    -
    -
    -

    3. Enable Use of Trait Variance:

    -
    - -
    -
    -
    -

    - Name Your Trait: (optional) -

    - -
    -
    -- cgit v1.2.3 From 99f8349b8cda4a3b17380c913207fc8395481653 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 29 Apr 2021 15:25:18 +0300 Subject: wqflask: Add new js file for processing menu entries * wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js: New file. Copy of dataset_select_menu_orig.js that will later modified... * wqflask/wqflask/templates/edit_trait.html: Use :point_up:. --- .../javascript/dataset_select_menu_edit_trait.js | 253 +++++++++++++++++++++ wqflask/wqflask/templates/edit_trait.html | 4 +- 2 files changed, 255 insertions(+), 2 deletions(-) create mode 100644 wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js diff --git a/wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js b/wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js new file mode 100644 index 00000000..1d4a94d9 --- /dev/null +++ b/wqflask/wqflask/static/new/javascript/dataset_select_menu_edit_trait.js @@ -0,0 +1,253 @@ +var apply_default, check_search_term, dataset_info, group_info, make_default, open_window, populate_dataset, populate_group, populate_species, populate_type, process_json, redo_dropdown; +process_json = function(data) { + window.jdata = data; + populate_species(); + if ($('#type').length > 0) { //This is to determine if it's the index page or the submit_trait page (which only has species and group selection and no make default option) + return apply_default(); + } +}; + +$.ajax('/api/v_pre1/gen_dropdown', { + dataType: 'json', + success: process_json +}); + +populate_species = function() { + var species_list; + species_list = this.jdata.species; + redo_dropdown($('#species'), species_list); + return populate_group(); +}; +window.populate_species = populate_species; +populate_group = function() { + var group_list, species; + console.log("in populate group"); + species = $('#species').val(); + group_list = this.jdata.groups[species]; + for (_i = 0, _len = group_list.length; _i < (_len - 1); _i++) { + if (group_list[_i][0] == "BXD300"){ + group_list.splice(_i, 1) + } + } + redo_dropdown($('#group'), group_list); + if ($('#type').length > 0) { //This is to determine if it's the index page or the submit_trait page (which only has species and group selection and no make default option) + return populate_type(); + } +}; +window.populate_group = populate_group; +populate_type = function() { + var group, species, type_list; + console.log("in populate type"); + species = $('#species').val(); + group = $('#group').val(); + type_list = this.jdata.types[species][group]; + redo_dropdown($('#type'), type_list); + return populate_dataset(); +}; +window.populate_type = populate_type; +populate_dataset = function() { + var dataset_list, group, species, type; + console.log("in populate dataset"); + species = $('#species').val(); + group = $('#group').val(); + type = $('#type').val(); + console.log("sgt:", species, group, type); + dataset_list = this.jdata.datasets[species][group][type]; + console.log("pop_dataset:", dataset_list); + return redo_dropdown($('#dataset'), dataset_list); +}; +window.populate_dataset = populate_dataset; +redo_dropdown = function(dropdown, items) { + var item, _i, _len, _results; + console.log("in redo:", dropdown, items); + dropdown.empty(); + _results = []; + + if (dropdown.attr('id') == "group"){ + group_family_list = []; + for (_i = 0, _len = items.length; _i < _len; _i++) { + item = items[_i]; + group_family = item[2].toString().split(":")[1] + group_family_list.push([item[0], item[1], group_family]) + } + + current_family = "" + this_opt_group = null + for (_i = 0, _len = group_family_list.length; _i < _len; _i++) { + item = group_family_list[_i]; + if (item[2] != "None" && current_family == ""){ + current_family = item[2] + this_opt_group = $("") + this_opt_group.append($("") + this_opt_group.append($("") + this_opt_group.append($("") + this_opt_group.append($("
    @@ -247,6 +247,7 @@ {% if geno_db_exists == "True" %}{% endif %} +

    @@ -612,7 +613,7 @@ return $('#marker_regression_form').submit(); } - $('#export_mapping_results').click(export_mapping_results); + $('.export_mapping_results').click(export_mapping_results); $('#browser_tab').click(function() { $('#gn1_map_options').css("display", "none") -- cgit v1.2.3 From 1e11d0ea0fd68a7b16dfb208217a442ad95909e9 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:18:20 +0000 Subject: Added a number of new metadata items to the mapping loading page, such as covariates and permutation/bootstrap numbers --- wqflask/wqflask/templates/loading.html | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 926f258d..69610f34 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -15,12 +15,31 @@ n = {{ start_vars.n_samples }}
    Method = {% if start_vars.method == "gemma" %}GEMMA{% else %}{{ start_vars.method }}{% endif %} + {% if start_vars.num_perm | int > 0 %}
    + # Permutations = {{ start_vars.num_perm }} + {% endif %} + {% if start_vars.num_bootstrap | int > 0 %} +
    + # Bootstrap = {{ start_vars.num_bootstrap }} + {% endif %} {% if start_vars.transform != "" %} - transform = {{ start_vars.transform }}
    + transform = {{ start_vars.transform }} {% endif %} + {% if start_vars.maf != "" and start_vars.method != "reaper" %} +
    MAF >= {{ start_vars.maf }} + {% endif %} + {% if start_vars.covariates != "" and start_vars.method != "reaper" %} +
    + {% set covariate_list = start_vars.covariates.split(",") %} + Trait Covariates: {% for covariate in covariate_list %}{% set this_covariate = covariate.split(":")[0] %}{{ this_covariate }}{% if not loop.last %}, {% endif %}{% endfor %} + {% endif %} + {% if start_vars.control_marker != "" and start_vars.do_control == "true" and start_vars.method != "gemma" %} +
    + Marker Covariate: {{ start_vars.control_marker }} + {% endif %} {% else %}

    Loading {{ start_vars.tool_used }} Results...

    {% endif %} -- cgit v1.2.3 From 6919762a2e0bce1afea0e979cb60a5c4b1dceacf Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:19:41 +0000 Subject: Removed an unused item from mapping_input_list --- wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js | 2 +- wqflask/wqflask/templates/mapping_results.html | 2 +- wqflask/wqflask/views.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js index 53bcd1f6..09e9d024 100644 --- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js +++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js @@ -143,7 +143,7 @@ $('input[name=display_all]').change((function(_this) { //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', - 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'control_marker_db', 'do_control', 'genofile', + 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples'] diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index edc30164..f054506c 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -481,7 +481,7 @@ var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals', 'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results', - 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'color_scheme', 'manhattan_single_color', 'control_marker', 'control_marker_db', 'do_control', 'genofile', + 'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'color_scheme', 'manhattan_single_color', 'control_marker', 'do_control', 'genofile', 'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 'mapmethod_rqtl_geno', 'mapmodel_rqtl_geno', 'temp_trait', 'group', 'species', 'reaper_version', 'primary_samples', 'n_samples'] diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index f9b8f310..276d3019 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -794,7 +794,6 @@ def mapping_results_page(): 'color_scheme', 'manhattan_single_color', 'control_marker', - 'control_marker_db', 'do_control', 'genofile', 'genofile_string', -- cgit v1.2.3 From 1dd80ec2767c1de0028ab720c7ef4caae834a99c Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:23:09 +0000 Subject: Added genotype file to mapping loading page --- wqflask/wqflask/templates/loading.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 69610f34..7344b918 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -15,6 +15,9 @@ n = {{ start_vars.n_samples }}
    Method = {% if start_vars.method == "gemma" %}GEMMA{% else %}{{ start_vars.method }}{% endif %} +
    + {% set genofile_desc = start_vars.genofile.split(":")[1] %} + Genotype File = {{ genofile_desc }} {% if start_vars.num_perm | int > 0 %}
    # Permutations = {{ start_vars.num_perm }} -- cgit v1.2.3 From a25ca626b7a6e1ca75d1fa0a830705e1df4d1e6c Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 5 May 2021 18:32:18 +0000 Subject: Added whether GEMMA is using LOCO to mapping loading page --- wqflask/wqflask/templates/loading.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 7344b918..124f0608 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -14,7 +14,7 @@
    n = {{ start_vars.n_samples }}
    - Method = {% if start_vars.method == "gemma" %}GEMMA{% else %}{{ start_vars.method }}{% endif %} + Method = {% if start_vars.method == "gemma" %}GEMMA {% if start_vars.use_loco == "True" %}using LOCO {% endif %}{% else %}{{ start_vars.method }}{% endif %}
    {% set genofile_desc = start_vars.genofile.split(":")[1] %} Genotype File = {{ genofile_desc }} -- cgit v1.2.3 From 6202ef9c4016d08cecf75fd8217c9b53d18ed201 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 5 May 2021 15:58:41 +0300 Subject: wqflask: database: Remove logging --- wqflask/wqflask/database.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index e743c4b3..42fa1594 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -5,9 +5,6 @@ from sqlalchemy.orm import scoped_session, sessionmaker from sqlalchemy.ext.declarative import declarative_base from utility.tools import SQL_URI -import utility.logger -logger = utility.logger.getLogger(__name__) - engine = create_engine(SQL_URI, encoding="latin1") @@ -23,10 +20,8 @@ def init_db(): # they will be registered properly on the metadata. Otherwise # you will have to import them first before calling init_db() #import yourapplication.models - logger.info("Initializing database connection") import wqflask.model Base.metadata.create_all(bind=engine) - logger.info("Done creating all model metadata") init_db() -- cgit v1.2.3 From 773c758efb1a48f97f85ac1e43e91355f67bde72 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 5 May 2021 16:25:41 +0300 Subject: wqflask: user_manager: Delete usermanager.py and it's references This module is not used anywhere. --- test/requests/test_forgot_password.py | 4 +- test/requests/test_login_local.py | 3 - wqflask/wqflask/user_manager.py | 1095 --------------------------------- 3 files changed, 1 insertion(+), 1101 deletions(-) delete mode 100644 wqflask/wqflask/user_manager.py diff --git a/test/requests/test_forgot_password.py b/test/requests/test_forgot_password.py index 2bf34c5c..346524bc 100644 --- a/test/requests/test_forgot_password.py +++ b/test/requests/test_forgot_password.py @@ -1,5 +1,4 @@ import requests -from wqflask import user_manager from utility.elasticsearch_tools import get_user_by_unique_column from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -27,8 +26,7 @@ class TestForgotPassword(ParametrizedTest): "password": "test_password", "password_confirm": "test_password" } - user_manager.basic_info = lambda : { "basic_info": "basic" } - user_manager.RegisterUser(data) + def testWithoutEmail(self): data = {"email_address": ""} diff --git a/test/requests/test_login_local.py b/test/requests/test_login_local.py index 808649ca..6691d135 100644 --- a/test/requests/test_login_local.py +++ b/test/requests/test_login_local.py @@ -1,5 +1,4 @@ import requests -from wqflask import user_manager from parameterized import parameterized from parametrized_test import ParametrizedTest @@ -19,8 +18,6 @@ class TestLoginLocal(ParametrizedTest): "password": "test_password", "password_confirm": "test_password" } - user_manager.basic_info = lambda : { "basic_info": "basic" } - user_manager.RegisterUser(data) @parameterized.expand([ diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py deleted file mode 100644 index cf84ea73..00000000 --- a/wqflask/wqflask/user_manager.py +++ /dev/null @@ -1,1095 +0,0 @@ -import os -import hashlib -import datetime -import time -import uuid -import hmac -import base64 -import redis # used for collections -import simplejson as json -import requests - -from base.data_set import create_datasets_list - -from flask import g -from flask import render_template -from flask import url_for -from flask import request -from flask import make_response -from flask import redirect -from flask import flash - -from wqflask import app -from wqflask import pbkdf2 # password hashing -from wqflask.database import db_session -from wqflask import model - -from smtplib import SMTP - -from pprint import pformat as pf - -from utility import Bunch -from utility import Struct -from utility.logger import getLogger - -from utility.redis_tools import get_user_id -from utility.redis_tools import get_user_by_unique_column -from utility.redis_tools import set_user_attribute -from utility.redis_tools import save_user -from utility.redis_tools import save_verification_code -from utility.redis_tools import check_verification_code -from utility.redis_tools import get_user_collections -from utility.redis_tools import save_collections - -from utility.tools import SMTP_CONNECT -from utility.tools import SMTP_USERNAME -from utility.tools import SMTP_PASSWORD - - -logger = getLogger(__name__) - - -Redis = redis.StrictRedis() - -THREE_DAYS = 60 * 60 * 24 * 3 - - -def timestamp(): - return datetime.datetime.utcnow().isoformat() - - -class AnonUser: - """Anonymous user handling""" - cookie_name = 'anon_user_v1' - - def __init__(self): - self.cookie = request.cookies.get(self.cookie_name) - if self.cookie: - logger.debug("ANON COOKIE ALREADY EXISTS") - self.anon_id = verify_cookie(self.cookie) - else: - logger.debug("CREATING NEW ANON COOKIE") - self.anon_id, self.cookie = create_signed_cookie() - - self.key = "anon_collection:v1:{}".format(self.anon_id) - - def add_collection(self, new_collection): - collection_dict = dict(name=new_collection.name, - created_timestamp=datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - changed_timestamp=datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - num_members=new_collection.num_members, - members=new_collection.get_members()) - - Redis.set(self.key, json.dumps(collection_dict)) - Redis.expire(self.key, 60 * 60 * 24 * 365) - - def delete_collection(self, collection_name): - existing_collections = self.get_collections() - updated_collections = [] - for i, collection in enumerate(existing_collections): - if collection['name'] == collection_name: - continue - else: - this_collection = {} - this_collection['id'] = collection['id'] - this_collection['name'] = collection['name'] - this_collection['created_timestamp'] = collection['created_timestamp'].strftime( - '%b %d %Y %I:%M%p') - this_collection['changed_timestamp'] = collection['changed_timestamp'].strftime( - '%b %d %Y %I:%M%p') - this_collection['num_members'] = collection['num_members'] - this_collection['members'] = collection['members'] - updated_collections.append(this_collection) - - Redis.set(self.key, json.dumps(updated_collections)) - - def get_collections(self): - json_collections = Redis.get(self.key) - if json_collections == None or json_collections == "None": - return [] - else: - collections = json.loads(json_collections) - for collection in collections: - collection['created_timestamp'] = datetime.datetime.strptime( - collection['created_timestamp'], '%b %d %Y %I:%M%p') - collection['changed_timestamp'] = datetime.datetime.strptime( - collection['changed_timestamp'], '%b %d %Y %I:%M%p') - - collections = sorted( - collections, key=lambda i: i['changed_timestamp'], reverse=True) - return collections - - def import_traits_to_user(self): - result = Redis.get(self.key) - collections_list = json.loads(result if result else "[]") - for collection in collections_list: - collection_exists = g.user_session.get_collection_by_name( - collection['name']) - if collection_exists: - continue - else: - g.user_session.add_collection( - collection['name'], collection['members']) - - def display_num_collections(self): - """ - Returns the number of collections or a blank string if there are zero. - - Because this is so unimportant...we wrap the whole thing in a try/expect...last thing we - want is a webpage not to be displayed because of an error here - - Importand TODO: use redis to cache this, don't want to be constantly computing it - """ - try: - num = len(self.get_collections()) - if num > 0: - return num - else: - return "" - except Exception as why: - print("Couldn't display_num_collections:", why) - return "" - - -def verify_cookie(cookie): - the_uuid, separator, the_signature = cookie.partition(':') - assert len(the_uuid) == 36, "Is session_id a uuid?" - assert separator == ":", "Expected a : here" - assert the_signature == actual_hmac_creation( - the_uuid), "Uh-oh, someone tampering with the cookie?" - return the_uuid - - -def create_signed_cookie(): - the_uuid = str(uuid.uuid4()) - signature = actual_hmac_creation(the_uuid) - uuid_signed = the_uuid + ":" + signature - logger.debug("uuid_signed:", uuid_signed) - return the_uuid, uuid_signed - - -class UserSession: - """Logged in user handling""" - - cookie_name = 'session_id_v1' - - def __init__(self): - cookie = request.cookies.get(self.cookie_name) - if not cookie: - logger.debug("NO USER COOKIE") - self.logged_in = False - return - else: - session_id = verify_cookie(cookie) - - self.redis_key = self.cookie_name + ":" + session_id - logger.debug("self.redis_key is:", self.redis_key) - self.session_id = session_id - self.record = Redis.hgetall(self.redis_key) - - if not self.record: - # This will occur, for example, when the browser has been left open over a long - # weekend and the site hasn't been visited by the user - self.logged_in = False - - # Grrr...this won't work because of the way flask handles cookies - # Delete the cookie - #response = make_response(redirect(url_for('login'))) - #response.set_cookie(self.cookie_name, '', expires=0) - # flash( - # "Due to inactivity your session has expired. If you'd like please login again.") - # return response - return - - if Redis.ttl(self.redis_key) < THREE_DAYS: - # (Almost) everytime the user does something we extend the session_id in Redis... - logger.debug("Extending ttl...") - Redis.expire(self.redis_key, THREE_DAYS) - - logger.debug("record is:", self.record) - self.logged_in = True - - @property - def user_id(self): - """Shortcut to the user_id""" - if 'user_id' in self.record: - return self.record['user_id'] - else: - return '' - - @property - def redis_user_id(self): - """User id from ElasticSearch (need to check if this is the same as the id stored in self.records)""" - - user_email = self.record['user_email_address'] - - # ZS: Get user's collections if they exist - user_id = None - user_id = get_user_id("email_address", user_email) - return user_id - - @property - def user_name(self): - """Shortcut to the user_name""" - if 'user_name' in self.record: - return self.record['user_name'] - else: - return '' - - @property - def user_collections(self): - """List of user's collections""" - - # ZS: Get user's collections if they exist - collections = get_user_collections(self.redis_user_id) - return collections - - @property - def num_collections(self): - """Number of user's collections""" - - return len(self.user_collections) - - def add_collection(self, collection_name, traits): - """Add collection into ElasticSearch""" - - collection_dict = {'id': str(uuid.uuid4()), - 'name': collection_name, - 'created_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - 'changed_timestamp': datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), - 'num_members': len(traits), - 'members': list(traits)} - - current_collections = self.user_collections - current_collections.append(collection_dict) - self.update_collections(current_collections) - - return collection_dict['id'] - - def delete_collection(self, collection_id): - """Remove collection with given ID""" - - updated_collections = [] - for collection in self.user_collections: - if collection['id'] == collection_id: - continue - else: - updated_collections.append(collection) - - self.update_collections(updated_collections) - - return collection['name'] - - def add_traits_to_collection(self, collection_id, traits_to_add): - """Add specified traits to a collection""" - - this_collection = self.get_collection_by_id(collection_id) - - updated_collection = this_collection - updated_traits = this_collection['members'] + traits_to_add - - updated_collection['members'] = updated_traits - updated_collection['num_members'] = len(updated_traits) - updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime( - '%b %d %Y %I:%M%p') - - updated_collections = [] - for collection in self.user_collections: - if collection['id'] == collection_id: - updated_collections.append(updated_collection) - else: - updated_collections.append(collection) - - self.update_collections(updated_collections) - - def remove_traits_from_collection(self, collection_id, traits_to_remove): - """Remove specified traits from a collection""" - - this_collection = self.get_collection_by_id(collection_id) - - updated_collection = this_collection - updated_traits = [] - for trait in this_collection['members']: - if trait in traits_to_remove: - continue - else: - updated_traits.append(trait) - - updated_collection['members'] = updated_traits - updated_collection['num_members'] = len(updated_traits) - updated_collection['changed_timestamp'] = datetime.datetime.utcnow().strftime( - '%b %d %Y %I:%M%p') - - updated_collections = [] - for collection in self.user_collections: - if collection['id'] == collection_id: - updated_collections.append(updated_collection) - else: - updated_collections.append(collection) - - self.update_collections(updated_collections) - - return updated_traits - - def get_collection_by_id(self, collection_id): - for collection in self.user_collections: - if collection['id'] == collection_id: - return collection - - def get_collection_by_name(self, collection_name): - for collection in self.user_collections: - if collection['name'] == collection_name: - return collection - - return None - - def update_collections(self, updated_collections): - collection_body = json.dumps(updated_collections) - - save_collections(self.redis_user_id, collection_body) - - def delete_session(self): - # And more importantly delete the redis record - Redis.delete(self.cookie_name) - logger.debug("At end of delete_session") - - -@app.before_request -def get_cookie(): - logger.info("@app.before_request get cookie") - g.user_session = UserSession() - g.cookie_session = AnonUser() - -# @app.after_request - - -def set_cookie(response): - if not request.cookies.get(g.cookie_session.cookie_name): - response.set_cookie(g.cookie_session.cookie_name, - g.cookie_session.cookie) - return response - - -class UsersManager: - def __init__(self): - self.users = model.User.query.all() - logger.debug("Users are:", self.users) - - -class UserManager: - def __init__(self, kw): - self.user_id = kw['user_id'] - logger.debug("In UserManager locals are:", pf(locals())) - #self.user = model.User.get(user_id) - #logger.debug("user is:", user) - self.user = model.User.query.get(self.user_id) - logger.debug("user is:", self.user) - datasets = create_datasets_list() - for dataset in datasets: - if not dataset.check_confidentiality(): - continue - logger.debug("\n Name:", dataset.name) - logger.debug(" Type:", dataset.type) - logger.debug(" ID:", dataset.id) - logger.debug(" Confidential:", dataset.check_confidentiality()) - #logger.debug(" ---> self.datasets:", self.datasets) - - -class RegisterUser: - def __init__(self, kw): - self.thank_you_mode = False - self.errors = [] - self.user = Bunch() - - self.user.email_address = kw.get( - 'email_address', '').encode("utf-8").strip() - if not (5 <= len(self.user.email_address) <= 50): - self.errors.append( - 'Email Address needs to be between 5 and 50 characters.') - else: - email_exists = get_user_by_unique_column( - "email_address", self.user.email_address) - #email_exists = get_user_by_unique_column(es, "email_address", self.user.email_address) - if email_exists: - self.errors.append('User already exists with that email') - - self.user.full_name = kw.get('full_name', '').encode("utf-8").strip() - if not (5 <= len(self.user.full_name) <= 50): - self.errors.append( - 'Full Name needs to be between 5 and 50 characters.') - - self.user.organization = kw.get( - 'organization', '').encode("utf-8").strip() - if self.user.organization and not (5 <= len(self.user.organization) <= 50): - self.errors.append( - 'Organization needs to be empty or between 5 and 50 characters.') - - password = str(kw.get('password', '')) - if not (6 <= len(password)): - self.errors.append('Password needs to be at least 6 characters.') - - if kw.get('password_confirm') != password: - self.errors.append("Passwords don't match.") - - if self.errors: - return - - logger.debug("No errors!") - - set_password(password, self.user) - self.user.user_id = str(uuid.uuid4()) - self.user.confirmed = 1 - - self.user.registration_info = json.dumps(basic_info(), sort_keys=True) - save_user(self.user.__dict__, self.user.user_id) - - -def set_password(password, user): - pwfields = Bunch() - - pwfields.algorithm = "pbkdf2" - pwfields.hashfunc = "sha256" - #hashfunc = getattr(hashlib, pwfields.hashfunc) - - # Encoding it to base64 makes storing it in json much easier - pwfields.salt = base64.b64encode(os.urandom(32)) - - # https://forums.lastpass.com/viewtopic.php?t=84104 - pwfields.iterations = 100000 - pwfields.keylength = 32 - - pwfields.created_ts = timestamp() - # One more check on password length - assert len(password) >= 6, "Password shouldn't be so short here" - - logger.debug("pwfields:", vars(pwfields)) - logger.debug("locals:", locals()) - - enc_password = Password(password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) - - pwfields.password = enc_password.password - pwfields.encrypt_time = enc_password.encrypt_time - - user.password = json.dumps(pwfields.__dict__, - sort_keys=True, - ) - - -class VerificationEmail: - template_name = "email/verification.txt" - key_prefix = "verification_code" - subject = "GeneNetwork email verification" - - def __init__(self, user): - verification_code = str(uuid.uuid4()) - key = self.key_prefix + ":" + verification_code - - data = json.dumps(dict(id=user.user_id, - timestamp=timestamp()) - ) - - Redis.set(key, data) - #two_days = 60 * 60 * 24 * 2 - Redis.expire(key, THREE_DAYS) - to = user.email_address - subject = self.subject - body = render_template(self.template_name, - verification_code=verification_code) - send_email(to, subject, body) - - -class ForgotPasswordEmail(VerificationEmail): - template_name = "email/forgot_password.txt" - key_prefix = "forgot_password_code" - subject = "GeneNetwork password reset" - fromaddr = "no-reply@genenetwork.org" - - def __init__(self, toaddr): - from email.MIMEMultipart import MIMEMultipart - from email.MIMEText import MIMEText - verification_code = str(uuid.uuid4()) - key = self.key_prefix + ":" + verification_code - - data = { - "verification_code": verification_code, - "email_address": toaddr, - "timestamp": timestamp() - } - - save_verification_code(toaddr, verification_code) - - subject = self.subject - body = render_template( - self.template_name, - verification_code=verification_code) - - msg = MIMEMultipart() - msg["To"] = toaddr - msg["Subject"] = self.subject - msg["From"] = self.fromaddr - msg.attach(MIMEText(body, "plain")) - - send_email(toaddr, msg.as_string()) - - -class Password: - def __init__(self, unencrypted_password, salt, iterations, keylength, hashfunc): - hashfunc = getattr(hashlib, hashfunc) - logger.debug("hashfunc is:", hashfunc) - # On our computer it takes around 1.4 seconds in 2013 - start_time = time.time() - salt = base64.b64decode(salt) - self.password = pbkdf2.pbkdf2_hex(str(unencrypted_password), - salt, iterations, keylength, hashfunc) - self.encrypt_time = round(time.time() - start_time, 3) - logger.debug("Creating password took:", self.encrypt_time) - - -def basic_info(): - return dict(timestamp=timestamp(), - ip_address=request.remote_addr, - user_agent=request.headers.get('User-Agent')) - -# @app.route("/manage/verify_email") - - -def verify_email(): - user = DecodeUser(VerificationEmail.key_prefix).user - user.confirmed = json.dumps(basic_info(), sort_keys=True) - db_session.commit() - - # As long as they have access to the email account - # We might as well log them in - - session_id_signed = LoginUser().successful_login(user) - response = make_response(render_template("new_security/thank_you.html")) - response.set_cookie(UserSession.cookie_name, session_id_signed) - return response - -# @app.route("/n/password_reset", methods=['GET']) - - -def password_reset(): - """Entry point after user clicks link in E-mail""" - logger.debug("in password_reset request.url is:", request.url) - # We do this mainly just to assert that it's in proper form for displaying next page - # Really not necessary but doesn't hurt - # user_encode = DecodeUser(ForgotPasswordEmail.key_prefix).reencode_standalone() - verification_code = request.args.get('code') - hmac = request.args.get('hm') - - if verification_code: - user_email = check_verification_code(verification_code) - if user_email: - user_details = get_user_by_unique_column( - 'email_address', user_email) - if user_details: - return render_template( - "new_security/password_reset.html", user_encode=user_details["user_id"]) - else: - flash("Invalid code: User no longer exists!", "error") - else: - flash( - "Invalid code: Password reset code does not exist or might have expired!", "error") - else: - return redirect(url_for("login")) - -# @app.route("/n/password_reset_step2", methods=('POST',)) - - -def password_reset_step2(): - """Handle confirmation E-mail for password reset""" - logger.debug("in password_reset request.url is:", request.url) - - errors = [] - user_id = request.form['user_encode'] - - logger.debug("locals are:", locals()) - - user = Bunch() - password = request.form['password'] - set_password(password, user) - - set_user_attribute(user_id, "password", user.__dict__.get("password")) - - flash("Password changed successfully. You can now sign in.", "alert-info") - response = make_response(redirect(url_for('login'))) - - return response - - -class DecodeUser: - - def __init__(self, code_prefix): - verify_url_hmac(request.url) - - #params = urlparse.parse_qs(url) - - self.verification_code = request.args['code'] - self.user = self.actual_get_user(code_prefix, self.verification_code) - - def reencode_standalone(self): - hmac = actual_hmac_creation(self.verification_code) - return self.verification_code + ":" + hmac - - @staticmethod - def actual_get_user(code_prefix, verification_code): - data = Redis.get(code_prefix + ":" + verification_code) - logger.debug("in get_coded_user, data is:", data) - data = json.loads(data) - logger.debug("data is:", data) - return model.User.query.get(data['id']) - -# @app.route("/n/login", methods=('GET', 'POST')) - - -def login(): - lu = LoginUser() - login_type = request.args.get("type") - if login_type: - uid = request.args.get("uid") - return lu.oauth2_login(login_type, uid) - else: - return lu.standard_login() - -# @app.route("/n/login/github_oauth2", methods=('GET', 'POST')) - - -def github_oauth2(): - from utility.tools import GITHUB_CLIENT_ID, GITHUB_CLIENT_SECRET - code = request.args.get("code") - data = { - "client_id": GITHUB_CLIENT_ID, - "client_secret": GITHUB_CLIENT_SECRET, - "code": code - } - result = requests.post( - "https://github.com/login/oauth/access_token", json=data) - result_dict = {arr[0]: arr[1] for arr in [tok.split( - "=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]} - - github_user = get_github_user_details(result_dict["access_token"]) - - user_details = get_user_by_unique_column("github_id", github_user["id"]) - if user_details == None: - user_details = { - "user_id": str(uuid.uuid4()), "name": github_user["name"].encode("utf-8"), "github_id": github_user["id"], "user_url": github_user["html_url"].encode("utf-8"), "login_type": "github", "organization": "", "active": 1, "confirmed": 1 - } - save_user(user_details, user_details["user_id"]) - - url = "/n/login?type=github&uid=" + user_details["user_id"] - return redirect(url) - -# @app.route("/n/login/orcid_oauth2", methods=('GET', 'POST')) - - -def orcid_oauth2(): - from uuid import uuid4 - from utility.tools import ORCID_CLIENT_ID, ORCID_CLIENT_SECRET, ORCID_TOKEN_URL, ORCID_AUTH_URL - code = request.args.get("code") - error = request.args.get("error") - url = "/n/login" - if code: - data = { - "client_id": ORCID_CLIENT_ID, "client_secret": ORCID_CLIENT_SECRET, "grant_type": "authorization_code", "code": code - } - result = requests.post(ORCID_TOKEN_URL, data=data) - result_dict = json.loads(result.text.encode("utf-8")) - - user_details = get_user_by_unique_column("orcid", result_dict["orcid"]) - if user_details == None: - user_details = { - "user_id": str(uuid4()), "name": result_dict["name"], "orcid": result_dict["orcid"], "user_url": "%s/%s" % ( - "/".join(ORCID_AUTH_URL.split("/")[:-2]), - result_dict["orcid"]), "login_type": "orcid", "organization": "", "active": 1, "confirmed": 1 - } - save_user(user_details, user_details["user_id"]) - - url = "/n/login?type=orcid&uid=" + user_details["user_id"] - else: - flash("There was an error getting code from ORCID") - return redirect(url) - - -def get_github_user_details(access_token): - from utility.tools import GITHUB_API_URL - result = requests.get(GITHUB_API_URL, params={ - "access_token": access_token}) - return result.json() - - -class LoginUser: - remember_time = 60 * 60 * 24 * 30 # One month in seconds - - def __init__(self): - self.remember_me = False - self.logged_in = False - - def oauth2_login(self, login_type, user_id): - """Login via an OAuth2 provider""" - - user_details = get_user_by_unique_column("user_id", user_id) - if user_details: - user = model.User() - user.id = user_details["user_id"] if user_details["user_id"] == None else "N/A" - user.full_name = user_details["name"] - user.login_type = user_details["login_type"] - return self.actual_login(user) - else: - flash("Error logging in via OAuth2") - return make_response(redirect(url_for('login'))) - - def standard_login(self): - """Login through the normal form""" - params = request.form if request.form else request.args - logger.debug("in login params are:", params) - - if not params: - from utility.tools import GITHUB_AUTH_URL, GITHUB_CLIENT_ID, ORCID_AUTH_URL, ORCID_CLIENT_ID - external_login = {} - if GITHUB_AUTH_URL and GITHUB_CLIENT_ID != 'UNKNOWN': - external_login["github"] = GITHUB_AUTH_URL - if ORCID_AUTH_URL and ORCID_CLIENT_ID != 'UNKNOWN': - external_login["orcid"] = ORCID_AUTH_URL - - return render_template( - "new_security/login_user.html", external_login=external_login, redis_is_available=is_redis_available()) - else: - user_details = get_user_by_unique_column( - "email_address", params["email_address"]) - #user_details = get_user_by_unique_column(es, "email_address", params["email_address"]) - user = None - valid = None - if user_details: - user = model.User() - for key in user_details: - user.__dict__[key] = user_details[key] - valid = False - - submitted_password = params['password'] - pwfields = Struct(json.loads(user.password)) - encrypted = Password( - submitted_password, - pwfields.salt, - pwfields.iterations, - pwfields.keylength, - pwfields.hashfunc) - logger.debug("\n\nComparing:\n{}\n{}\n".format( - encrypted.password, pwfields.password)) - valid = pbkdf2.safe_str_cmp( - encrypted.password, pwfields.password) - logger.debug("valid is:", valid) - - if valid and not user.confirmed: - VerificationEmail(user) - return render_template("new_security/verification_still_needed.html", - subject=VerificationEmail.subject) - if valid: - if params.get('remember'): - logger.debug("I will remember you") - self.remember_me = True - - if 'import_collections' in params: - import_col = "true" - else: - import_col = "false" - - # g.cookie_session.import_traits_to_user() - - self.logged_in = True - - return self.actual_login(user, import_collections=import_col) - - else: - if user: - self.unsuccessful_login(user) - flash("Invalid email-address or password. Please try again.", - "alert-danger") - response = make_response(redirect(url_for('login'))) - - return response - - def actual_login(self, user, assumed_by=None, import_collections=None): - """The meat of the logging in process""" - session_id_signed = self.successful_login(user, assumed_by) - flash("Thank you for logging in {}.".format( - user.full_name), "alert-success") - response = make_response( - redirect(url_for('index_page', import_collections=import_collections))) - if self.remember_me: - max_age = self.remember_time - else: - max_age = None - - response.set_cookie(UserSession.cookie_name, - session_id_signed, max_age=max_age) - return response - - def successful_login(self, user, assumed_by=None): - login_rec = model.Login(user) - login_rec.successful = True - login_rec.session_id = str(uuid.uuid4()) - login_rec.assumed_by = assumed_by - #session_id = "session_id:{}".format(login_rec.session_id) - session_id_signature = actual_hmac_creation(login_rec.session_id) - session_id_signed = login_rec.session_id + ":" + session_id_signature - logger.debug("session_id_signed:", session_id_signed) - - if not user.id: - user.id = '' - - session = dict(login_time=time.time(), - user_id=user.id, - user_name=user.full_name, - user_email_address=user.email_address) - - key = UserSession.cookie_name + ":" + login_rec.session_id - logger.debug("Key when signing:", key) - Redis.hmset(key, session) - if self.remember_me: - expire_time = self.remember_time - else: - expire_time = THREE_DAYS - Redis.expire(key, expire_time) - - return session_id_signed - - def unsuccessful_login(self, user): - login_rec = model.Login(user) - login_rec.successful = False - db_session.add(login_rec) - db_session.commit() - -# @app.route("/n/logout") - - -def logout(): - logger.debug("Logging out...") - UserSession().delete_session() - flash("You are now logged out. We hope you come back soon!") - response = make_response(redirect(url_for('index_page'))) - # Delete the cookie - response.set_cookie(UserSession.cookie_name, '', expires=0) - return response - - -# @app.route("/n/forgot_password", methods=['GET']) -def forgot_password(): - """Entry point for forgotten password""" - print("ARGS: ", request.args) - errors = {"no-email": request.args.get("no-email")} - print("ERRORS: ", errors) - return render_template("new_security/forgot_password.html", errors=errors) - -# @app.route("/n/forgot_password_submit", methods=('POST',)) - - -def forgot_password_submit(): - """When a forgotten password form is submitted we get here""" - params = request.form - email_address = params['email_address'] - next_page = None - if email_address != "": - logger.debug("Wants to send password E-mail to ", email_address) - user_details = get_user_by_unique_column( - "email_address", email_address) - if user_details: - ForgotPasswordEmail(user_details["email_address"]) - return render_template("new_security/forgot_password_step2.html", - subject=ForgotPasswordEmail.subject) - else: - flash("The e-mail entered is not associated with an account.", - "alert-danger") - return redirect(url_for("forgot_password")) - - else: - flash("You MUST provide an email", "alert-danger") - return redirect(url_for("forgot_password")) - - -@app.errorhandler(401) -def unauthorized(error): - return redirect(url_for('login')) - - -def is_redis_available(): - try: - Redis.ping() - except: - return False - return True - -### -# ZS: The following 6 functions require the old MySQL User accounts; I'm leaving them commented out just in case we decide to reimplement them using ElasticSearch -### -# def super_only(): -# try: -# superuser = g.user_session.user_ob.superuser -# except AttributeError: -# superuser = False -# if not superuser: -# flash("You must be a superuser to access that page.", "alert-error") -# abort(401) - -# @app.route("/manage/users") -# def manage_users(): -# super_only() -# template_vars = UsersManager() -# return render_template("admin/user_manager.html", **template_vars.__dict__) - -# @app.route("/manage/user") -# def manage_user(): -# super_only() -# template_vars = UserManager(request.args) -# return render_template("admin/ind_user_manager.html", **template_vars.__dict__) - -# @app.route("/manage/groups") -# def manage_groups(): -# super_only() -# template_vars = GroupsManager(request.args) -# return render_template("admin/group_manager.html", **template_vars.__dict__) - -# @app.route("/manage/make_superuser") -# def make_superuser(): -# super_only() -# params = request.args -# user_id = params['user_id'] -# user = model.User.query.get(user_id) -# superuser_info = basic_info() -# superuser_info['crowned_by'] = g.user_session.user_id -# user.superuser = json.dumps(superuser_info, sort_keys=True) -# db_session.commit() -# flash("We've made {} a superuser!".format(user.name_and_org)) -# return redirect(url_for("manage_users")) - -# @app.route("/manage/assume_identity") -# def assume_identity(): -# super_only() -# params = request.args -# user_id = params['user_id'] -# user = model.User.query.get(user_id) -# assumed_by = g.user_session.user_id -# return LoginUser().actual_login(user, assumed_by=assumed_by) - - -# @app.route("/n/register", methods=('GET', 'POST')) -def register(): - params = None - errors = None - - params = request.form if request.form else request.args - params = params.to_dict(flat=True) - - if params: - logger.debug("Attempting to register the user...") - result = RegisterUser(params) - errors = result.errors - - if len(errors) == 0: - flash( - "Registration successful. You may login with your new account", "alert-info") - return redirect(url_for("login")) - - return render_template("new_security/register_user.html", values=params, errors=errors) - - -################################# Sign and unsign ##################################### - -def url_for_hmac(endpoint, **values): - """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with""" - - url = url_for(endpoint, **values) - - hm = actual_hmac_creation(url) - if '?' in url: - combiner = "&" - else: - combiner = "?" - return url + combiner + "hm=" + hm - - -def data_hmac(stringy): - """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" - return stringy + ":" + actual_hmac_creation(stringy) - - -def verify_url_hmac(url): - """Pass in a url that was created with url_hmac and this assures it hasn't been tampered with""" - logger.debug("url passed in to verify is:", url) - # Verify parts are correct at the end - we expect to see &hm= or ?hm= followed by an hmac - assert url[-23:-20] == "hm=", "Unexpected url (stage 1)" - assert url[-24] in ["?", "&"], "Unexpected url (stage 2)" - hmac = url[-20:] - url = url[:-24] # Url without any of the hmac stuff - - #logger.debug("before urlsplit, url is:", url) - #url = divide_up_url(url)[1] - #logger.debug("after urlsplit, url is:", url) - - hm = actual_hmac_creation(url) - - assert hm == hmac, "Unexpected url (stage 3)" - - -def actual_hmac_creation(stringy): - """Helper function to create the actual hmac""" - - secret = app.config['SECRET_HMAC_CODE'] - - hmaced = hmac.new(secret, stringy, hashlib.sha1) - hm = hmaced.hexdigest() - # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." - # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html - hm = hm[:20] - return hm - - -app.jinja_env.globals.update(url_for_hmac=url_for_hmac, - data_hmac=data_hmac) - -####################################################################################### - -# def send_email(to, subject, body): -# msg = json.dumps(dict(From="no-reply@genenetwork.org", -# To=to, -# Subject=subject, -# Body=body)) -# Redis.rpush("mail_queue", msg) - - -def send_email(toaddr, msg, fromaddr="no-reply@genenetwork.org"): - """Send an E-mail through SMTP_CONNECT host. If SMTP_USERNAME is not - 'UNKNOWN' TLS is used - - """ - if SMTP_USERNAME == 'UNKNOWN': - logger.debug("SMTP: connecting with host " + SMTP_CONNECT) - server = SMTP(SMTP_CONNECT) - server.sendmail(fromaddr, toaddr, msg) - else: - logger.debug("SMTP: connecting TLS with host " + SMTP_CONNECT) - server = SMTP(SMTP_CONNECT) - server.starttls() - logger.debug("SMTP: login with user " + SMTP_USERNAME) - server.login(SMTP_USERNAME, SMTP_PASSWORD) - logger.debug("SMTP: " + fromaddr) - logger.debug("SMTP: " + toaddr) - logger.debug("SMTP: " + msg) - server.sendmail(fromaddr, toaddr, msg) - server.quit() - logger.info("Successfully sent email to " + toaddr) - - -class GroupsManager: - def __init__(self, kw): - self.datasets = create_datasets_list() - - -class RolesManager: - def __init__(self): - self.roles = model.Role.query.all() - logger.debug("Roles are:", self.roles) -- cgit v1.2.3 From 5c4da47e6f4c7c61666b80dc5133aa584db38e5d Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 5 May 2021 16:26:09 +0300 Subject: Delete model.py and it's references * wqflask/wqflask/database.py (init_db): Remove references to "wqflask.model" * wqflask/wqflask/model.py: Delete it. --- wqflask/wqflask/database.py | 5 -- wqflask/wqflask/model.py | 182 -------------------------------------------- 2 files changed, 187 deletions(-) delete mode 100644 wqflask/wqflask/model.py diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 42fa1594..38c37d2c 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -16,11 +16,6 @@ Base.query = db_session.query_property() def init_db(): - # import all modules here that might define models so that - # they will be registered properly on the metadata. Otherwise - # you will have to import them first before calling init_db() - #import yourapplication.models - import wqflask.model Base.metadata.create_all(bind=engine) diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py deleted file mode 100644 index a222b87c..00000000 --- a/wqflask/wqflask/model.py +++ /dev/null @@ -1,182 +0,0 @@ -import uuid -import datetime - -import simplejson as json - -from flask import request - -from wqflask import app - -import sqlalchemy -from sqlalchemy import (Column, ForeignKey, Unicode, Boolean, DateTime, - Text, Index) -from sqlalchemy.orm import relationship - -from wqflask.database import Base, init_db - - -class User(Base): - __tablename__ = "user" - id = Column(Unicode(36), primary_key=True, - default=lambda: str(uuid.uuid4())) - email_address = Column(Unicode(50), unique=True, nullable=False) - - # Todo: Turn on strict mode for Mysql - password = Column(Text, nullable=False) - - full_name = Column(Unicode(50)) - organization = Column(Unicode(50)) - - active = Column(Boolean(), nullable=False, default=True) - - # json detailing when they were registered, etc. - registration_info = Column(Text) - - confirmed = Column(Text) # json detailing when they confirmed, etc. - - # json detailing when they became a superuser, otherwise empty - superuser = Column(Text) - # if not superuser - - logins = relationship("Login", - order_by="desc(Login.timestamp)", - lazy='dynamic', # Necessary for filter in login_count - foreign_keys="Login.user", - ) - - user_collections = relationship("UserCollection", - order_by="asc(UserCollection.name)", - lazy='dynamic', - ) - - def display_num_collections(self): - """ - Returns the number of collections or a blank string if there are zero. - - - Because this is so unimportant...we wrap the whole thing in a try/expect...last thing we - want is a webpage not to be displayed because of an error here - - Importand TODO: use redis to cache this, don't want to be constantly computing it - - """ - try: - num = len(list(self.user_collections)) - return display_collapsible(num) - except Exception as why: - print("Couldn't display_num_collections:", why) - return "" - - def get_collection_by_name(self, collection_name): - try: - collect = self.user_collections.filter_by( - name=collection_name).first() - except sqlalchemy.orm.exc.NoResultFound: - collect = None - return collect - - @property - def name_and_org(self): - """Nice shortcut for printing out who the user is""" - if self.organization: - return "{} from {}".format(self.full_name, self.organization) - else: - return self.full_name - - @property - def login_count(self): - return self.logins.filter_by(successful=True).count() - - @property - def confirmed_at(self): - if self.confirmed: - confirmed_info = json.loads(self.confirmed) - return confirmed_info['timestamp'] - else: - return None - - @property - def superuser_info(self): - if self.superuser: - return json.loads(self.superuser) - else: - return None - - @property - def crowner(self): - """If made superuser, returns object of person who did the crowning""" - if self.superuser: - superuser_info = json.loads(self.superuser) - crowner = User.query.get(superuser_info['crowned_by']) - return crowner - else: - return None - - @property - def most_recent_login(self): - try: - return self.logins[0] - except IndexError: - return None - - -class Login(Base): - __tablename__ = "login" - id = Column(Unicode(36), primary_key=True, - default=lambda: str(uuid.uuid4())) - user = Column(Unicode(36), ForeignKey('user.id')) - timestamp = Column(DateTime(), default=lambda: datetime.datetime.utcnow()) - ip_address = Column(Unicode(39)) - # False if wrong password was entered - successful = Column(Boolean(), nullable=False) - # Set only if successfully logged in, otherwise should be blank - session_id = Column(Text) - - # Set to user who assumes identity if this was a login for debugging purposes by a superuser - assumed_by = Column(Unicode(36), ForeignKey('user.id')) - - def __init__(self, user): - self.user = user.id - self.ip_address = request.remote_addr - -################################################################################################## - - -class UserCollection(Base): - __tablename__ = "user_collection" - id = Column(Unicode(36), primary_key=True, - default=lambda: str(uuid.uuid4())) - user = Column(Unicode(36), ForeignKey('user.id')) - - # I'd prefer this to not have a length, but for the index below it needs one - name = Column(Unicode(50)) - created_timestamp = Column( - DateTime(), default=lambda: datetime.datetime.utcnow()) - changed_timestamp = Column( - DateTime(), default=lambda: datetime.datetime.utcnow()) - members = Column(Text) # We're going to store them as a json list - - # This index ensures a user doesn't have more than one collection with the same name - __table_args__ = (Index('usercollection_index', "user", "name"), ) - - @property - def num_members(self): - try: - return len(json.loads(self.members)) - except: - return 0 - - def members_as_set(self): - return set(json.loads(self.members)) - - -def display_collapsible(number): - if number: - return number - else: - return "" - - -def user_uuid(): - """Unique cookie for a user""" - user_uuid = request.cookies.get('user_uuid') -- cgit v1.2.3 From d60d30ca54106ff052d6953917f29465ae6e764b Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 19:45:37 +0000 Subject: Changed the way metadata is displayed on the mapping loading page --- wqflask/wqflask/templates/loading.html | 98 ++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 47 deletions(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 124f0608..95cecf76 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -7,56 +7,60 @@ {% endfor %}
    -
    +
    - {% if start_vars.tool_used == "Mapping" %} -

    Computing the Maps

    -
    - n = {{ start_vars.n_samples }} -
    - Method = {% if start_vars.method == "gemma" %}GEMMA {% if start_vars.use_loco == "True" %}using LOCO {% endif %}{% else %}{{ start_vars.method }}{% endif %} -
    - {% set genofile_desc = start_vars.genofile.split(":")[1] %} - Genotype File = {{ genofile_desc }} - {% if start_vars.num_perm | int > 0 %} -
    - # Permutations = {{ start_vars.num_perm }} - {% endif %} - {% if start_vars.num_bootstrap | int > 0 %} -
    - # Bootstrap = {{ start_vars.num_bootstrap }} - {% endif %} - {% if start_vars.transform != "" %} -
    - transform = {{ start_vars.transform }} - {% endif %} - {% if start_vars.maf != "" and start_vars.method != "reaper" %} -
    - MAF >= {{ start_vars.maf }} - {% endif %} - {% if start_vars.covariates != "" and start_vars.method != "reaper" %} -
    - {% set covariate_list = start_vars.covariates.split(",") %} - Trait Covariates: {% for covariate in covariate_list %}{% set this_covariate = covariate.split(":")[0] %}{{ this_covariate }}{% if not loop.last %}, {% endif %}{% endfor %} - {% endif %} - {% if start_vars.control_marker != "" and start_vars.do_control == "true" and start_vars.method != "gemma" %} -
    - Marker Covariate: {{ start_vars.control_marker }} - {% endif %} - {% else %} -

    Loading {{ start_vars.tool_used }} Results...

    - {% endif %} -

    - -
    - -
    -- cgit v1.2.3 From 6dc0c82ed93978be67357e0e84b6e5b3531054e6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 19:45:57 +0000 Subject: Include group/species in form parameters on trait page so they can be displayed on mapping loading page --- wqflask/wqflask/show_trait/show_trait.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index fcebbc4d..b6fcbcb8 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -263,6 +263,9 @@ class ShowTrait: hddn['temp_trait'] = True hddn['group'] = self.temp_group hddn['species'] = self.temp_species + else: + hddn['group'] = self.dataset.group.name + hddn['species'] = self.dataset.group.species hddn['use_outliers'] = False hddn['method'] = "gemma" hddn['selected_chr'] = -1 -- cgit v1.2.3 From 70735035fe7b59bae13fe955e4f32595055b3940 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 19:54:48 +0000 Subject: Include group and species in form parameters for mapping results page, so they can be included in the loading page if the map is reloaded --- wqflask/wqflask/templates/mapping_results.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html index f054506c..73d7501b 100644 --- a/wqflask/wqflask/templates/mapping_results.html +++ b/wqflask/wqflask/templates/mapping_results.html @@ -17,8 +17,9 @@ {% if temp_trait is defined %} - {% endif %} + + -- cgit v1.2.3 From 3c430082b767a29c3e35cb03e68c1b22373ad353 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 20:02:35 +0000 Subject: Fixed vertical position of loading text so that it should be positioned in the center vertically --- wqflask/wqflask/templates/loading.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 95cecf76..d384a087 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -8,7 +8,7 @@
    -
    +
    {% if start_vars.tool_used == "Mapping" %}

    Computing the Maps


    -- cgit v1.2.3 From 2444d60a93ef7c9900ed9a52877bff0ee08fbfb6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 22:54:05 +0000 Subject: Because each permutations value is returned as an array, need to just take the first item to convert to numbers --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index cd43577e..b9cb99cc 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -385,7 +385,7 @@ def process_pair_scan_results(result): def process_rqtl_perm_results(num_perm, results): perm_vals = [] for item in results: - perm_vals.append(item) + perm_vals.append(item[0]) perm_output = perm_vals suggestive = np.percentile(np.array(perm_vals), 67) -- cgit v1.2.3 From cec8c8870f12a52ac20ffcc2f4a8a05003a13a9f Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 23:14:22 +0000 Subject: Used a list comprehension for perm_vals as mentioned in Bonface's comment --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 072e4530..06eceada 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -431,9 +431,7 @@ def process_pair_scan_results(result): def process_rqtl_perm_results(num_perm, results): - perm_vals = [] - for item in results: - perm_vals.append(item[0]) + perm_vals = item[0] for item in results perm_output = perm_vals suggestive = np.percentile(np.array(perm_vals), 67) -- cgit v1.2.3 From a2627777d51b969869d8647624ce008fc9454c7b Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 6 May 2021 23:18:08 +0000 Subject: Forgot to add brackets around list comprehension, so added them --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 06eceada..1c8477bf 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -431,7 +431,7 @@ def process_pair_scan_results(result): def process_rqtl_perm_results(num_perm, results): - perm_vals = item[0] for item in results + perm_vals = [item[0] for item in results] perm_output = perm_vals suggestive = np.percentile(np.array(perm_vals), 67) -- cgit v1.2.3 From 3e34d0a5f964bb7931cccdd58b7a360187d35a4d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 6 May 2021 09:32:11 +0300 Subject: add blogs blueprint --- wqflask/wqflask/__init__.py | 2 ++ wqflask/wqflask/markdown_routes.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index a2bf3085..bf5e094b 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -12,6 +12,7 @@ from wqflask.markdown_routes import links_blueprint from wqflask.markdown_routes import policies_blueprint from wqflask.markdown_routes import environments_blueprint from wqflask.markdown_routes import facilities_blueprint +from wqflask.markdown_routes import blogs_blueprint app = Flask(__name__) @@ -29,6 +30,7 @@ app.register_blueprint(links_blueprint, url_prefix="/links") app.register_blueprint(policies_blueprint, url_prefix="/policies") app.register_blueprint(environments_blueprint, url_prefix="/environments") app.register_blueprint(facilities_blueprint, url_prefix="/facilities") +app.register_blueprint(blogs_blueprint, url_prefix="/blogs") @app.before_request diff --git a/wqflask/wqflask/markdown_routes.py b/wqflask/wqflask/markdown_routes.py index c27ff143..b9a6f8ce 100644 --- a/wqflask/wqflask/markdown_routes.py +++ b/wqflask/wqflask/markdown_routes.py @@ -20,6 +20,8 @@ links_blueprint = Blueprint("links_blueprint", __name__) policies_blueprint = Blueprint("policies_blueprint", __name__) facilities_blueprint = Blueprint("facilities_blueprint", __name__) +blogs_blueprint = Blueprint("blogs_blueprint", __name__) + def render_markdown(file_name, is_remote_file=True): """Try to fetch the file name from Github and if that fails, try to @@ -124,3 +126,36 @@ def policies(): @facilities_blueprint.route("/") def facilities(): return render_template("facilities.html", rendered_markdown=render_markdown("general/help/facilities.md")), 200 + + +@blogs_blueprint.route("/") +def display_blog(blog_title): + # should use the blog title path + + return render_template("blogs.html", rendered_markdown=render_markdown("blog/2021/proteome/Wang_WIlliams_Rat_Brain_Proteome_For_Blog.md")) + + +@blogs_blueprint.route("/") +def blogs_list(): + + # should fetch this from github + + blogs = {"2021": [ + { + "title": "proteome", + "subtitle": "Wang_WIlliams_Rat_Brain_Proteome_For_Blog" + }, + { + "title":"xxx", + "subtitle":"blog 2" + } + ], + "2020": [ + { + "title": "other", + "subtitle": "other" + } + ] + } + + return render_template("blogs_list.html", blogs=blogs) -- cgit v1.2.3 From 2e2e4601f4cebc767854e66af2d2ca8c3e8123c9 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 6 May 2021 09:32:35 +0300 Subject: add template for blogs --- wqflask/wqflask/templates/base.html | 1 + wqflask/wqflask/templates/blogs.html | 24 ++++++++++++ wqflask/wqflask/templates/blogs_list.html | 61 +++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 wqflask/wqflask/templates/blogs.html create mode 100644 wqflask/wqflask/templates/blogs_list.html diff --git a/wqflask/wqflask/templates/base.html b/wqflask/wqflask/templates/base.html index e6f22deb..ddb1d272 100644 --- a/wqflask/wqflask/templates/base.html +++ b/wqflask/wqflask/templates/base.html @@ -69,6 +69,7 @@
    • References
    • Tutorials/Primers
      • +
    • Blogs
    • Glossary of Term
    • FAQ
    • Policies
      • diff --git a/wqflask/wqflask/templates/blogs.html b/wqflask/wqflask/templates/blogs.html new file mode 100644 index 00000000..c3d42e13 --- /dev/null +++ b/wqflask/wqflask/templates/blogs.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} + +{% block title %}Blogs {% endblock %} + +{% block css %} + +{% endblock %} + +{% block content %} + +
      + +
      +
      + {{ rendered_markdown|safe }} + +
      + +{% endblock %} \ No newline at end of file diff --git a/wqflask/wqflask/templates/blogs_list.html b/wqflask/wqflask/templates/blogs_list.html new file mode 100644 index 00000000..93cb3d89 --- /dev/null +++ b/wqflask/wqflask/templates/blogs_list.html @@ -0,0 +1,61 @@ +{% extends "base.html" %} + +{% block title %}Blogs {% endblock %} + +{% block css %} + + + + +{% endblock %} + +{% block content %} + +
      +
      + {% for year, year_blogs in blogs.items() %} +

      {{year}}

      + {%for blog in year_blogs%} + +
      {{blog['subtitle']}} +
      + {% endfor %} + + {%endfor%} +
      + + +
      + +{% endblock %} \ No newline at end of file -- cgit v1.2.3 From 19771ba2c54b50313675c7a29ee8a53dcb0802ce Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 6 May 2021 21:39:16 +0300 Subject: add function for fetching blogs --- wqflask/wqflask/markdown_routes.py | 62 +++++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/wqflask/wqflask/markdown_routes.py b/wqflask/wqflask/markdown_routes.py index b9a6f8ce..28bd6e4b 100644 --- a/wqflask/wqflask/markdown_routes.py +++ b/wqflask/wqflask/markdown_routes.py @@ -57,6 +57,39 @@ def get_file_from_python_search_path(pathname_suffix): return None +def get_blogs(user="genenetwork", repo_name="gn-docs"): + + blogs = {} + github_url = f"https://api.github.com/repos/{user}/{repo_name}/git/trees/master?recursive=1" + + repo_tree = requests.get(github_url).json()["tree"] + + for data in repo_tree: + path_name = data["path"] + if path_name.startswith("blog") and path_name.endswith(".md"): + split_path = path_name.split("/")[1:] + try: + year, title, file_name = split_path + except Exception as e: + year, file_name = split_path + title = "" + + subtitle = os.path.splitext(file_name)[0] + + blog = { + "title": title, + "subtitle": subtitle, + "full_path": path_name + } + + if year in blogs: + blogs[int(year)].append(blog) + else: + blogs[int(year)] = [blog] + + return dict(sorted(blogs.items(), key=lambda x: x[0], reverse=True)) + + @glossary_blueprint.route('/') def glossary(): return render_template( @@ -128,34 +161,13 @@ def facilities(): return render_template("facilities.html", rendered_markdown=render_markdown("general/help/facilities.md")), 200 -@blogs_blueprint.route("/") -def display_blog(blog_title): - # should use the blog title path - - return render_template("blogs.html", rendered_markdown=render_markdown("blog/2021/proteome/Wang_WIlliams_Rat_Brain_Proteome_For_Blog.md")) +@blogs_blueprint.route("/") +def display_blog(blog_path): + return render_template("blogs.html", rendered_markdown=render_markdown(blog_path)) @blogs_blueprint.route("/") def blogs_list(): - - # should fetch this from github - - blogs = {"2021": [ - { - "title": "proteome", - "subtitle": "Wang_WIlliams_Rat_Brain_Proteome_For_Blog" - }, - { - "title":"xxx", - "subtitle":"blog 2" - } - ], - "2020": [ - { - "title": "other", - "subtitle": "other" - } - ] - } + blogs = get_blogs() return render_template("blogs_list.html", blogs=blogs) -- cgit v1.2.3 From 92246a53e63e51a9d54938a5c1a65553545853c2 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 6 May 2021 21:39:37 +0300 Subject: modify blog list ui --- wqflask/wqflask/templates/blogs_list.html | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/wqflask/wqflask/templates/blogs_list.html b/wqflask/wqflask/templates/blogs_list.html index 93cb3d89..4e2bfb82 100644 --- a/wqflask/wqflask/templates/blogs_list.html +++ b/wqflask/wqflask/templates/blogs_list.html @@ -11,11 +11,8 @@ } .blog_year { font-weight: bold; - opacity: 0.8; - /*letter-spacing: 0.2em;*/ font-size: 48px; padding:12px 10px; - color:#0F75CD; } .blog_title { @@ -26,14 +23,10 @@ .blog_title a{ - text-decoration: none; - font-size:16px; - color:#333; - /*margin: 10px 15px;*/ - padding-left: 30px; - opacity: 0.8; - - /*font-weight: bold;*/ + font-size:1.3em; + padding-left: 10px; + letter-spacing: 0.07em; + text-decoration: underline; } @@ -48,8 +41,16 @@

      {{year}}

      {%for blog in year_blogs%} -
      {{blog['subtitle']}} +
      +
      + +
      +
      {% endfor %} {%endfor%} -- cgit v1.2.3 From 45ba9ad0b88824fa0aafb7d9378a7f68a2c7ee6c Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 7 May 2021 00:40:24 +0300 Subject: fix tweets width --- wqflask/wqflask/templates/blogs_list.html | 1 - wqflask/wqflask/templates/index_page.html | 27 ++++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/templates/blogs_list.html b/wqflask/wqflask/templates/blogs_list.html index 4e2bfb82..e9255114 100644 --- a/wqflask/wqflask/templates/blogs_list.html +++ b/wqflask/wqflask/templates/blogs_list.html @@ -40,7 +40,6 @@ {% for year, year_blogs in blogs.items() %}

      {{year}}

      {%for blog in year_blogs%} -
        diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 338c8e2f..0a26aede 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -3,6 +3,31 @@ {% block css %} {% endblock %} {% block content %} @@ -254,7 +279,7 @@ "showImages": true, "lang": 'en' }; - twitterFetcher.fetch(configProfile); + twitterFetcher.fetch(configProfile) -- cgit v1.2.3 From 17bb8042eff1be9d7762e30499c1bf58d3505ea8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 7 May 2021 00:49:13 +0300 Subject: minor fix --- wqflask/wqflask/templates/index_page.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html index 0a26aede..7b103305 100755 --- a/wqflask/wqflask/templates/index_page.html +++ b/wqflask/wqflask/templates/index_page.html @@ -279,7 +279,7 @@ "showImages": true, "lang": 'en' }; - twitterFetcher.fetch(configProfile) + twitterFetcher.fetch(configProfile); -- cgit v1.2.3 From b5db22e76677a85055d2f9b22214e9eeea876071 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 7 May 2021 08:02:36 +0300 Subject: remove comments prettify html --- wqflask/wqflask/templates/blogs.html | 16 +------ wqflask/wqflask/templates/blogs_list.html | 73 ++++++++++++++----------------- 2 files changed, 34 insertions(+), 55 deletions(-) diff --git a/wqflask/wqflask/templates/blogs.html b/wqflask/wqflask/templates/blogs.html index c3d42e13..314eb733 100644 --- a/wqflask/wqflask/templates/blogs.html +++ b/wqflask/wqflask/templates/blogs.html @@ -1,24 +1,12 @@ {% extends "base.html" %} - {% block title %}Blogs {% endblock %} - {% block css %} {% endblock %} - {% block content %} - -
        - +
        - {{ rendered_markdown|safe }} - + {{ rendered_markdown|safe }}
        - {% endblock %} \ No newline at end of file diff --git a/wqflask/wqflask/templates/blogs_list.html b/wqflask/wqflask/templates/blogs_list.html index e9255114..6bad4628 100644 --- a/wqflask/wqflask/templates/blogs_list.html +++ b/wqflask/wqflask/templates/blogs_list.html @@ -1,61 +1,52 @@ {% extends "base.html" %} - {% block title %}Blogs {% endblock %} - {% block css %} - - {% endblock %} - {% block content %} -
        -
        - {% for year, year_blogs in blogs.items() %} -

        {{year}}

        - {%for blog in year_blogs%} +
        + {% for year, year_blogs in blogs.items() %} +
        +

        {{year}}

        +
        + {%for blog in year_blogs%}
        -
        - - +
        + +
        + {% endfor %} + {%endfor%}
        - {% endfor %} - - {%endfor%} -
        - -
        - {% endblock %} \ No newline at end of file -- cgit v1.2.3 From 5c474e24fa43cdffacd698cd3a1e9e5182a49016 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Fri, 7 May 2021 08:27:02 +0300 Subject: Add typing + autopep8 formatting --- wqflask/wqflask/markdown_routes.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/markdown_routes.py b/wqflask/wqflask/markdown_routes.py index 28bd6e4b..b81bfb55 100644 --- a/wqflask/wqflask/markdown_routes.py +++ b/wqflask/wqflask/markdown_routes.py @@ -2,17 +2,21 @@ Render pages from github, or if they are unavailable, look for it else where """ + import requests import markdown import os import sys -from bs4 import BeautifulSoup +from bs4 import BeautifulSoup # type: ignore from flask import send_from_directory from flask import Blueprint from flask import render_template +from typing import Dict +from typing import List + glossary_blueprint = Blueprint('glossary_blueprint', __name__) references_blueprint = Blueprint("references_blueprint", __name__) environments_blueprint = Blueprint("environments_blueprint", __name__) @@ -57,9 +61,10 @@ def get_file_from_python_search_path(pathname_suffix): return None -def get_blogs(user="genenetwork", repo_name="gn-docs"): +def get_blogs(user: str = "genenetwork", + repo_name: str = "gn-docs") -> dict: - blogs = {} + blogs: Dict[int, List] = {} github_url = f"https://api.github.com/repos/{user}/{repo_name}/git/trees/master?recursive=1" repo_tree = requests.get(github_url).json()["tree"] -- cgit v1.2.3 From 1895e953747bb5d9d41f249408590e187027b266 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 7 May 2021 15:57:05 +0000 Subject: Fixed issue that caused /submit_trait page to not load properly --- wqflask/utility/helper_functions.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index ecc075ae..50e00421 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -57,14 +57,16 @@ def get_trait_db_obs(self, trait_db_list): def get_species_groups(): """Group each species into a group""" _menu = {} + for species, group_name in g.db.execute( "SELECT s.MenuName, i.InbredSetName FROM InbredSet i " "INNER JOIN Species s ON s.SpeciesId = i.SpeciesId " "ORDER BY i.SpeciesId ASC, i.Name ASC").fetchall(): - if _menu.get(species): - _menu = _menu[species].append(group_name) - else: - _menu[species] = [group_name] + if species in _menu: + if _menu.get(species): + _menu = _menu[species].append(group_name) + else: + _menu[species] = [group_name] return [{"species": key, "groups": value} for key, value in list(_menu.items())] -- cgit v1.2.3 From c8bd4e95c5e83d9f3d4cad2e1ab63dfdde0975f7 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 7 May 2021 16:08:42 +0000 Subject: Check if num_bootstrap is in start_vars, since it won't exist for non-qtlreaper methods --- wqflask/wqflask/templates/loading.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index d384a087..1adeb2bd 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -49,10 +49,12 @@
        n of permutations = {{ start_vars.num_perm }} {% endif %} + {% if num_bootstrap in start_vars %} {% if start_vars.num_bootstrap | int > 0 and start_vars.method == "reaper" %}
        n of bootstrap = {{ start_vars.num_bootstrap }} {% endif %} + {% endif %} {% else %}

        Loading {{ start_vars.tool_used }} Results...

        {% endif %} -- cgit v1.2.3 From 90c35a2a9285be21882a6db03e5aeb26f329f602 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 7 May 2021 19:45:29 +0000 Subject: Display cofactors = None on the loading page if there are no cofactors --- wqflask/wqflask/templates/loading.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/loading.html b/wqflask/wqflask/templates/loading.html index 1adeb2bd..3c443690 100644 --- a/wqflask/wqflask/templates/loading.html +++ b/wqflask/wqflask/templates/loading.html @@ -33,10 +33,12 @@
        minor allele frequency lower limit = {{ start_vars.maf }} {% endif %} - {% if start_vars.covariates != "" and start_vars.method != "reaper" %}
        + {% if start_vars.covariates != "" and start_vars.method != "reaper" %} {% set covariate_list = start_vars.covariates.split(",") %} cofactors = {% for covariate in covariate_list %}{% set this_covariate = covariate.split(":")[0] %}{{ this_covariate }}{% if not loop.last %}, {% endif %}{% endfor %} + {% else %} + cofactors = None {% endif %} {% if start_vars.control_marker != "" and start_vars.do_control == "true" and start_vars.method != "gemma" %}
        -- cgit v1.2.3 From 074d724bffff4a579fe0c862ce817651cfe59910 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 8 May 2021 13:55:47 +0300 Subject: tests: test_gen_menu: Update failing tests Update tests to use (and mock) the injected MySQL connection object. --- wqflask/tests/unit/wqflask/api/test_gen_menu.py | 402 +++++++++++++----------- 1 file changed, 216 insertions(+), 186 deletions(-) diff --git a/wqflask/tests/unit/wqflask/api/test_gen_menu.py b/wqflask/tests/unit/wqflask/api/test_gen_menu.py index fd0fe52e..d0cd647a 100644 --- a/wqflask/tests/unit/wqflask/api/test_gen_menu.py +++ b/wqflask/tests/unit/wqflask/api/test_gen_menu.py @@ -73,209 +73,239 @@ class TestGenMenu(unittest.TestCase): def tearDown(self): self.app_context.pop() - @mock.patch('wqflask.api.gen_menu.g') - def test_get_species(self, db_mock): + def test_get_species(self): """Test that assertion is raised when dataset and dataset_name are defined""" - db_mock.db.execute.return_value.fetchall.return_value = ( - ('human', 'Human'), - ('mouse', 'Mouse')) - self.assertEqual(get_species(), - [['human', 'Human'], ['mouse', 'Mouse']]) - db_mock.db.execute.assert_called_once_with( - "SELECT Name, MenuName FROM Species ORDER BY OrderId" - ) - - @mock.patch('wqflask.api.gen_menu.g') - def test_get_groups(self, db_mock): - """Test that species groups are grouped correctly""" - db_mock.db.execute.return_value.fetchall.side_effect = [ - # Mouse - (('BXD', 'BXD', None), - ('HLC', 'Liver: Normal Gene Expression with Genotypes (Merck)', - 'Test')), - # Human - (('H_T1', "H_T", "DescriptionA"), - ('H_T2', "H_T'", None)) - ] - - self.assertEqual(get_groups([["human", "Human"], ["mouse", "Mouse"]]), - self.test_group) - - for name in ["mouse", "human"]: - db_mock.db.execute.assert_any_call( - ("SELECT InbredSet.Name, InbredSet.FullName, " + - "IFNULL(InbredSet.Family, 'None') " - + "FROM InbredSet, Species WHERE Species.Name " - + "= '{}' AND InbredSet.SpeciesId = Species.Id GROUP by " - + "InbredSet.Name ORDER BY IFNULL(InbredSet.FamilyOrder, " - + "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " - + "InbredSet.FullName) ASC, InbredSet.FullName ASC, " - + "InbredSet.MenuOrderId ASC").format(name) + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchall.return_value = ( + ('human', 'Human'), + ('mouse', 'Mouse')) + self.assertEqual(get_species(db_mock), + [['human', 'Human'], ['mouse', 'Mouse']]) + cursor.execute.assert_called_once_with( + "SELECT Name, MenuName FROM Species ORDER BY OrderId" ) - @mock.patch('wqflask.api.gen_menu.g') - def test_phenotypes_exist_called_with_correct_query(self, db_mock): + def test_get_groups(self): + """Test that species groups are grouped correctly""" + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchall.side_effect = [ + # Mouse + (('BXD', 'BXD', None), + ('HLC', ('Liver: Normal Gene Expression ' + 'with Genotypes (Merck)'), + 'Test')), + # Human + (('H_T1', "H_T", "DescriptionA"), + ('H_T2', "H_T'", None)) + ] + self.assertEqual(get_groups([["human", "Human"], + ["mouse", "Mouse"]], + db_mock), + self.test_group) + + for name in ["mouse", "human"]: + cursor.execute.assert_any_call( + ("SELECT InbredSet.Name, InbredSet.FullName, " + "IFNULL(InbredSet.Family, 'None') " + "FROM InbredSet, Species WHERE Species.Name " + "= '{}' AND InbredSet.SpeciesId = Species.Id GROUP by " + "InbredSet.Name ORDER BY IFNULL(InbredSet.FamilyOrder, " + "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " + "InbredSet.FullName) ASC, InbredSet.FullName ASC, " + "InbredSet.MenuOrderId ASC").format(name) + ) + + def test_phenotypes_exist_called_with_correct_query(self): """Test that phenotypes_exist is called with the correct query""" - db_mock.db.execute.return_value.fetchone.return_value = None - phenotypes_exist("test") - db_mock.db.execute.assert_called_with( - "SELECT Name FROM PublishFreeze " - "WHERE PublishFreeze.Name = 'testPublish'" - ) - - @mock.patch('wqflask.api.gen_menu.g') - def test_phenotypes_exist_with_falsy_values(self, db_mock): + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchone.return_value = None + phenotypes_exist("test", db_mock) + cursor.execute.assert_called_with( + "SELECT Name FROM PublishFreeze " + "WHERE PublishFreeze.Name = 'testPublish'" + ) + + def test_phenotypes_exist_with_falsy_values(self): """Test that phenotype check returns correctly when given a None value""" - for x in [None, False, (), [], ""]: - db_mock.db.execute.return_value.fetchone.return_value = x - self.assertFalse(phenotypes_exist("test")) - - @mock.patch('wqflask.api.gen_menu.g') - def test_phenotypes_exist_with_truthy_value(self, db_mock): - """Test that phenotype check returns correctly when given Truthy """ - for x in ["x", ("result"), ["result"], [1]]: - db_mock.db.execute.return_value.fetchone.return_value = (x) - self.assertTrue(phenotypes_exist("test")) - - @mock.patch('wqflask.api.gen_menu.g') - def test_genotypes_exist_called_with_correct_query(self, db_mock): + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + for x in [None, False, (), [], ""]: + cursor.fetchone.return_value = x + self.assertFalse(phenotypes_exist("test", db_mock)) + + def test_phenotypes_exist_with_truthy_value(self): + """Test that phenotype check returns correctly when given Truthy""" + db_mock = mock.MagicMock() + with db_mock.cursor() as conn: + with conn.cursor() as cursor: + for x in ["x", ("result"), ["result"], [1]]: + cursor.fetchone.return_value = (x) + self.assertTrue(phenotypes_exist("test", db_mock)) + + def test_genotypes_exist_called_with_correct_query(self): """Test that genotypes_exist is called with the correct query""" - db_mock.db.execute.return_value.fetchone.return_value = None - genotypes_exist("test") - db_mock.db.execute.assert_called_with( - "SELECT Name FROM GenoFreeze WHERE GenoFreeze.Name = 'testGeno'" - ) - - @mock.patch('wqflask.api.gen_menu.g') - def test_genotypes_exist_with_falsy_values(self, db_mock): - """Test that genotype check returns correctly when given - a None value""" - for x in [None, False, (), [], ""]: - db_mock.db.execute.return_value.fetchone.return_value = x - self.assertFalse(genotypes_exist("test")) + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchone.return_value = None + genotypes_exist("test", db_mock) + cursor.execute.assert_called_with( + "SELECT Name FROM GenoFreeze WHERE " + "GenoFreeze.Name = 'testGeno'" + ) + + def test_genotypes_exist_with_falsy_values(self): + """Test that genotype check returns correctly when given a None value + + """ + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + for x in [None, False, (), [], ""]: + cursor.fetchone.return_value = x + self.assertFalse(genotypes_exist("test", db_mock)) - @mock.patch('wqflask.api.gen_menu.g') - def test_genotypes_exist_with_truthy_value(self, db_mock): + def test_genotypes_exist_with_truthy_value(self): """Test that genotype check returns correctly when given Truthy """ - for x in ["x", ("result"), ["result"], [1]]: - db_mock.db.execute.return_value.fetchone.return_value = (x) - self.assertTrue(phenotypes_exist("test")) + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + for x in ["x", ("result"), ["result"], [1]]: + cursor.fetchone.return_value = (x) + self.assertTrue(phenotypes_exist("test", db_mock)) - @mock.patch('wqflask.api.gen_menu.g') - def test_build_datasets_with_type_phenotypes(self, db_mock): + def test_build_datasets_with_type_phenotypes(self): """Test that correct dataset is returned for a phenotype type""" - db_mock.db.execute.return_value.fetchall.return_value = ( - (602, "BXDPublish", "BXD Published Phenotypes"), - ) - self.assertEqual(build_datasets("Mouse", "BXD", "Phenotypes"), - [['602', "BXDPublish", "BXD Published Phenotypes"]]) - db_mock.db.execute.assert_called_with( - "SELECT InfoFiles.GN_AccesionId, PublishFreeze.Name, " - + "PublishFreeze.FullName FROM InfoFiles, PublishFreeze, " - + "InbredSet WHERE InbredSet.Name = 'BXD' AND " - + "PublishFreeze.InbredSetId = InbredSet.Id AND " - + "InfoFiles.InfoPageName = PublishFreeze.Name " - + "ORDER BY PublishFreeze.CreateTime ASC" - ) - self.assertEqual(build_datasets("Mouse", "MDP", "Phenotypes"), - [['602', "BXDPublish", "Mouse Phenome Database"]]) - - db_mock.db.execute.return_value.fetchall.return_value = () - db_mock.db.execute.return_value.fetchone.return_value = ( - "BXDPublish", "Mouse Phenome Database" - ) - self.assertEqual(build_datasets("Mouse", "MDP", "Phenotypes"), - [["None", "BXDPublish", "Mouse Phenome Database"]]) - - @mock.patch('wqflask.api.gen_menu.g') - def test_build_datasets_with_type_phenotypes_and_no_results(self, db_mock): + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchall.return_value = ( + (602, "BXDPublish", "BXD Published Phenotypes"), + ) + self.assertEqual(build_datasets("Mouse", "BXD", + "Phenotypes", db_mock), + [['602', "BXDPublish", + "BXD Published Phenotypes"]]) + cursor.execute.assert_called_with( + "SELECT InfoFiles.GN_AccesionId, PublishFreeze.Name, " + + "PublishFreeze.FullName FROM InfoFiles, PublishFreeze, " + + "InbredSet WHERE InbredSet.Name = 'BXD' AND " + + "PublishFreeze.InbredSetId = InbredSet.Id AND " + + "InfoFiles.InfoPageName = PublishFreeze.Name " + + "ORDER BY PublishFreeze.CreateTime ASC" + ) + self.assertEqual(build_datasets("Mouse", "MDP", + "Phenotypes", db_mock), + [['602', "BXDPublish", + "Mouse Phenome Database"]]) + + cursor.fetchall.return_value = () + cursor.fetchone.return_value = ( + "BXDPublish", "Mouse Phenome Database" + ) + self.assertEqual(build_datasets("Mouse", "MDP", + "Phenotypes", db_mock), + [["None", "BXDPublish", + "Mouse Phenome Database"]]) + + def test_build_datasets_with_type_phenotypes_and_no_results(self): """Test that correct dataset is returned for a phenotype type with no results """ - db_mock.db.execute.return_value.fetchall.return_value = None - db_mock.db.execute.return_value.fetchone.return_value = (121, - "text value") - self.assertEqual(build_datasets("Mouse", "BXD", "Phenotypes"), - [["None", "121", "text value"]]) - db_mock.db.execute.assert_called_with( - "SELECT PublishFreeze.Name, PublishFreeze.FullName " - "FROM PublishFreeze, InbredSet " - "WHERE InbredSet.Name = 'BXD' AND " - "PublishFreeze.InbredSetId = InbredSet.Id " - "ORDER BY PublishFreeze.CreateTime ASC" - ) - - @mock.patch('wqflask.api.gen_menu.g') - def test_build_datasets_with_type_genotypes(self, db_mock): + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchall.return_value = None + cursor.fetchone.return_value = (121, + "text value") + self.assertEqual(build_datasets("Mouse", "BXD", + "Phenotypes", db_mock), + [["None", "121", + "text value"]]) + cursor.execute.assert_called_with( + "SELECT PublishFreeze.Name, PublishFreeze.FullName " + "FROM PublishFreeze, InbredSet " + "WHERE InbredSet.Name = 'BXD' AND " + "PublishFreeze.InbredSetId = InbredSet.Id " + "ORDER BY PublishFreeze.CreateTime ASC" + ) + + def test_build_datasets_with_type_genotypes(self): """Test that correct dataset is returned for a phenotype type""" - db_mock.db.execute.return_value.fetchone.return_value = ( - 635, "HLCPublish", "HLC Published Genotypes" - ) - - self.assertEqual(build_datasets("Mouse", "HLC", "Genotypes"), - [["635", "HLCGeno", "HLC Genotypes"]]) - db_mock.db.execute.assert_called_with( - "SELECT InfoFiles.GN_AccesionId FROM InfoFiles, " - "GenoFreeze, InbredSet WHERE InbredSet.Name = 'HLC' AND " - "GenoFreeze.InbredSetId = InbredSet.Id AND " - "InfoFiles.InfoPageName = GenoFreeze.ShortName " - + "ORDER BY GenoFreeze.CreateTime DESC" - ) - db_mock.db.execute.return_value.fetchone.return_value = () - self.assertEqual(build_datasets("Mouse", "HLC", "Genotypes"), - [["None", "HLCGeno", "HLC Genotypes"]]) - - @mock.patch('wqflask.api.gen_menu.g') - def test_build_datasets_with_type_mrna(self, db_mock): + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchone.return_value = ( + 635, "HLCPublish", "HLC Published Genotypes" + ) + self.assertEqual(build_datasets("Mouse", "HLC", + "Genotypes", db_mock), + [["635", "HLCGeno", "HLC Genotypes"]]) + cursor.execute.assert_called_with( + "SELECT InfoFiles.GN_AccesionId FROM InfoFiles, " + "GenoFreeze, InbredSet WHERE InbredSet.Name = 'HLC' AND " + "GenoFreeze.InbredSetId = InbredSet.Id AND " + "InfoFiles.InfoPageName = GenoFreeze.ShortName " + "ORDER BY GenoFreeze.CreateTime DESC" + ) + cursor.fetchone.return_value = () + self.assertEqual(build_datasets("Mouse", "HLC", + "Genotypes", db_mock), + [["None", "HLCGeno", "HLC Genotypes"]]) + + def test_build_datasets_with_type_mrna(self): """Test that correct dataset is returned for a mRNA expression/ Probeset""" - db_mock.db.execute.return_value.fetchall.return_value = ( - (112, "HC_M2_0606_P", - "Hippocampus Consortium M430v2 (Jun06) PDNN"), ) - self.assertEqual(build_datasets("Mouse", "HLC", "mRNA"), [[ - "112", 'HC_M2_0606_P', "Hippocampus Consortium M430v2 (Jun06) PDNN" - ]]) - db_mock.db.execute.assert_called_once_with( - "SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, " - + "ProbeSetFreeze.FullName FROM ProbeSetFreeze, " - + "ProbeFreeze, InbredSet, Tissue, Species WHERE " - + "Species.Name = 'Mouse' AND Species.Id = " - + "InbredSet.SpeciesId AND InbredSet.Name = 'HLC' AND " - + "ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id AND " - + "Tissue.Name = 'mRNA' AND ProbeFreeze.TissueId = " - + "Tissue.Id AND ProbeFreeze.InbredSetId = InbredSet.Id AND " - + "ProbeSetFreeze.public > 0 " - + "ORDER BY -ProbeSetFreeze.OrderList DESC, ProbeSetFreeze.CreateTime DESC") + db_mock = mock.MagicMock() + with db_mock.cursor() as cursor: + cursor.fetchall.return_value = ( + (112, "HC_M2_0606_P", + "Hippocampus Consortium M430v2 (Jun06) PDNN"), ) + self.assertEqual(build_datasets("Mouse", + "HLC", "mRNA", db_mock), + [["112", 'HC_M2_0606_P', + "Hippocampus Consortium M430v2 (Jun06) PDNN" + ]]) + cursor.execute.assert_called_once_with( + "SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, " + "ProbeSetFreeze.FullName FROM ProbeSetFreeze, " + "ProbeFreeze, InbredSet, Tissue, Species WHERE " + "Species.Name = 'Mouse' AND Species.Id = " + "InbredSet.SpeciesId AND InbredSet.Name = 'HLC' AND " + "ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id AND " + "Tissue.Name = 'mRNA' AND ProbeFreeze.TissueId = " + "Tissue.Id AND ProbeFreeze.InbredSetId = InbredSet.Id AND " + "ProbeSetFreeze.public > 0 " + "ORDER BY -ProbeSetFreeze.OrderList DESC, " + "ProbeSetFreeze.CreateTime DESC") @mock.patch('wqflask.api.gen_menu.build_datasets') - @mock.patch('wqflask.api.gen_menu.g') - def test_build_types(self, db_mock, datasets_mock): + def test_build_types(self, datasets_mock): """Test that correct tissue metadata is returned""" + db_mock = mock.MagicMock() datasets_mock.return_value = [ ["112", 'HC_M2_0606_P', "Hippocampus Consortium M430v2 (Jun06) PDNN"] ] - db_mock.db.execute.return_value.fetchall.return_value = ( - ('Mouse Tissue'), ('Human Tissue'), ('Rat Tissue') - ) - self.assertEqual(build_types('mouse', 'random group'), - [['M', 'M', 'Molecular Traits'], - ['H', 'H', 'Molecular Traits'], - ['R', 'R', 'Molecular Traits']]) - db_mock.db.execute.assert_called_once_with( - "SELECT DISTINCT Tissue.Name " - + "FROM ProbeFreeze, ProbeSetFreeze, InbredSet, " - + "Tissue, Species WHERE Species.Name = 'mouse' " - + "AND Species.Id = InbredSet.SpeciesId AND " - + "InbredSet.Name = 'random group' AND " - + "ProbeFreeze.TissueId = Tissue.Id AND " - + "ProbeFreeze.InbredSetId = InbredSet.Id AND " - + "ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " - + "ORDER BY Tissue.Name" - ) + with db_mock.cursor() as cursor: + cursor.fetchall.return_value = ( + ('Mouse Tissue'), ('Human Tissue'), ('Rat Tissue') + ) + self.assertEqual(build_types('mouse', 'random group', db_mock), + [['M', 'M', 'Molecular Traits'], + ['H', 'H', 'Molecular Traits'], + ['R', 'R', 'Molecular Traits']]) + cursor.execute.assert_called_once_with( + "SELECT DISTINCT Tissue.Name " + "FROM ProbeFreeze, ProbeSetFreeze, InbredSet, " + "Tissue, Species WHERE Species.Name = 'mouse' " + "AND Species.Id = InbredSet.SpeciesId AND " + "InbredSet.Name = 'random group' AND " + "ProbeFreeze.TissueId = Tissue.Id AND " + "ProbeFreeze.InbredSetId = InbredSet.Id AND " + "ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "ORDER BY Tissue.Name" + ) @mock.patch('wqflask.api.gen_menu.build_types') @mock.patch('wqflask.api.gen_menu.genotypes_exist') @@ -297,7 +327,9 @@ class TestGenMenu(unittest.TestCase): build_types_mock.return_value = [ ['M', 'M', 'Molecular Trait Datasets'] ] - self.assertEqual(get_types(self.test_group), expected_result) + self.assertEqual(get_types(self.test_group, + mock.MagicMock()), + expected_result) @mock.patch('wqflask.api.gen_menu.build_types') @mock.patch('wqflask.api.gen_menu.genotypes_exist') @@ -315,10 +347,8 @@ class TestGenMenu(unittest.TestCase): genotypes_exist_mock.return_value = False build_types_mock.return_value = [] - self.assertEqual(get_types(self.test_group), { - 'mouse': {}, - 'human': {} - }) + self.assertEqual(get_types(self.test_group, mock.MagicMock()), + {'mouse': {}, 'human': {}}) @mock.patch('wqflask.api.gen_menu.build_types') @mock.patch('wqflask.api.gen_menu.genotypes_exist') @@ -345,7 +375,7 @@ class TestGenMenu(unittest.TestCase): 'human': { 'HLC': [['M', 'M', 'Molecular Trait Datasets']], 'BXD': [['M', 'M', 'Molecular Trait Datasets']]}} - self.assertEqual(get_types(self.test_group), + self.assertEqual(get_types(self.test_group, mock.MagicMock()), expected_result) @mock.patch('wqflask.api.gen_menu.build_datasets') @@ -367,7 +397,7 @@ class TestGenMenu(unittest.TestCase): 'BXD': {'Genotypes': 'Test', 'M': 'Test', 'Phenotypes': 'Test'}}} - self.assertEqual(get_datasets(self.test_type), + self.assertEqual(get_datasets(self.test_type, mock.MagicMock()), expected_result) @mock.patch('wqflask.api.gen_menu.build_datasets') @@ -381,7 +411,7 @@ class TestGenMenu(unittest.TestCase): 'H_T1': {}}, 'human': {'HLC': {}, 'BXD': {}}} - self.assertEqual(get_datasets(self.test_type), + self.assertEqual(get_datasets(self.test_type, mock.MagicMock()), expected_result) @mock.patch('wqflask.api.gen_menu.get_datasets') @@ -411,4 +441,4 @@ class TestGenMenu(unittest.TestCase): 'groups': ['groupA', 'groupB', 'groupC', 'groupD'], 'species': ['speciesA', 'speciesB', 'speciesC', 'speciesD']} - self.assertEqual(gen_dropdown_json(), expected_result) + self.assertEqual(gen_dropdown_json(mock.MagicMock()), expected_result) -- cgit v1.2.3 From 7940e0fa662fae04fb9b267923ed1ffd2ccb7a2f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 8 May 2021 06:52:26 +0300 Subject: api: router: Inject MySQL object when fetching menu items --- wqflask/wqflask/api/router.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/api/router.py b/wqflask/wqflask/api/router.py index aec74c9e..a739e5a9 100644 --- a/wqflask/wqflask/api/router.py +++ b/wqflask/wqflask/api/router.py @@ -6,11 +6,13 @@ import csv import json import datetime import requests +import MySQLdb from zipfile import ZipFile, ZIP_DEFLATED import flask +from flask import current_app from flask import g from flask import request from flask import make_response @@ -845,7 +847,11 @@ def get_genotypes(group_name, file_format="csv", dataset_name=None): @app.route("/api/v_{}/gen_dropdown".format(version), methods=("GET",)) def gen_dropdown_menu(): - results = gen_menu.gen_dropdown_json() + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + results = gen_menu.gen_dropdown_json(conn) if len(results) > 0: return flask.jsonify(results) -- cgit v1.2.3 From 6f1812df49422387c3a72d04405d688fb3604b81 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 8 May 2021 06:51:17 +0300 Subject: api: gen_menu: Inject MySQL conn object and use that for queries --- wqflask/wqflask/api/gen_menu.py | 251 +++++++++++++++++++++------------------- 1 file changed, 132 insertions(+), 119 deletions(-) diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 7f4ec1bc..0c1120ab 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,15 +1,12 @@ -from flask import g - - -def gen_dropdown_json(): +def gen_dropdown_json(conn): """Generates and outputs (as json file) the data for the main dropdown menus on the home page """ - species = get_species() - groups = get_groups(species) - types = get_types(groups) - datasets = get_datasets(types) + species = get_species(conn) + groups = get_groups(species, conn) + types = get_types(groups, conn) + datasets = get_datasets(types, conn) data = dict(species=species, groups=groups, @@ -19,48 +16,50 @@ def gen_dropdown_json(): return data -def get_species(): +def get_species(conn): """Build species list""" - results = g.db.execute( - "SELECT Name, MenuName FROM Species ORDER BY OrderId").fetchall() - return [[name, menu_name] for name, menu_name in results] + with conn.cursor() as cursor: + cursor.execute("SELECT Name, MenuName FROM Species " + "ORDER BY OrderId") + results = cursor.fetchall() + return [[name, menu_name] for name, menu_name in results] -def get_groups(species): +def get_groups(species, conn): """Build groups list""" groups = {} - for species_name, _species_full_name in species: - groups[species_name] = [] - - results = g.db.execute( - ("SELECT InbredSet.Name, InbredSet.FullName, " - "IFNULL(InbredSet.Family, 'None') " - "FROM InbredSet, Species WHERE Species.Name = '{}' " - "AND InbredSet.SpeciesId = Species.Id GROUP by InbredSet.Name " - "ORDER BY IFNULL(InbredSet.FamilyOrder, InbredSet.FullName) " - "ASC, IFNULL(InbredSet.Family, InbredSet.FullName) ASC, " - "InbredSet.FullName ASC, InbredSet.MenuOrderId ASC") - .format(species_name)).fetchall() - - for result in results: - family_name = "Family:" + str(result[2]) - groups[species_name].append( - [str(result[0]), str(result[1]), family_name]) - + with conn.cursor() as cursor: + for species_name, _species_full_name in species: + groups[species_name] = [] + cursor.execute( + ("SELECT InbredSet.Name, InbredSet.FullName, " + "IFNULL(InbredSet.Family, 'None') " + "FROM InbredSet, Species WHERE Species.Name = '{}' " + "AND InbredSet.SpeciesId = Species.Id GROUP by " + "InbredSet.Name ORDER BY IFNULL(InbredSet.FamilyOrder, " + "InbredSet.FullName) ASC, IFNULL(InbredSet.Family, " + "InbredSet.FullName) ASC, InbredSet.FullName ASC, " + "InbredSet.MenuOrderId ASC") + .format(species_name)) + results = cursor.fetchall() + for result in results: + family_name = "Family:" + str(result[2]) + groups[species_name].append( + [str(result[0]), str(result[1]), family_name]) return groups -def get_types(groups): +def get_types(groups, conn): """Build types list""" types = {} for species, group_dict in list(groups.items()): types[species] = {} for group_name, _group_full_name, _family_name in group_dict: - if phenotypes_exist(group_name): + if phenotypes_exist(group_name, conn): types[species][group_name] = [ ("Phenotypes", "Traits and Cofactors", "Phenotypes")] - if genotypes_exist(group_name): + if genotypes_exist(group_name, conn): if group_name in types[species]: types[species][group_name] += [ ("Genotypes", "DNA Markers and SNPs", "Genotypes")] @@ -68,11 +67,11 @@ def get_types(groups): types[species][group_name] = [ ("Genotypes", "DNA Markers and SNPs", "Genotypes")] if group_name in types[species]: - types_list = build_types(species, group_name) + types_list = build_types(species, group_name, conn) if len(types_list) > 0: types[species][group_name] += types_list else: - types_list = build_types(species, group_name) + types_list = build_types(species, group_name, conn) if len(types_list) > 0: types[species][group_name] = types_list else: @@ -83,22 +82,28 @@ def get_types(groups): return types -def phenotypes_exist(group_name): - results = g.db.execute( - ("SELECT Name FROM PublishFreeze " - "WHERE PublishFreeze.Name = " - "'{}'").format(group_name + "Publish")).fetchone() +def phenotypes_exist(group_name, conn): + results = [] + with conn.cursor() as cursor: + cursor.execute( + ("SELECT Name FROM PublishFreeze " + "WHERE PublishFreeze.Name = " + "'{}'").format(group_name + "Publish")) + results = cursor.fetchone() return bool(results) -def genotypes_exist(group_name): - results = g.db.execute( - ("SELECT Name FROM GenoFreeze " + - "WHERE GenoFreeze.Name = '{}'").format(group_name + "Geno")).fetchone() - return bool(results) +def genotypes_exist(group_name, conn): + with conn.cursor() as cursor: + cursor.execute( + ("SELECT Name FROM GenoFreeze " + + "WHERE GenoFreeze.Name = '{}'").format( + group_name + "Geno")) + results = cursor.fetchone() + return bool(results) -def build_types(species, group): +def build_types(species, group, conn): """Fetches tissues Gets the tissues with data for this species/group @@ -116,17 +121,19 @@ def build_types(species, group): "ORDER BY Tissue.Name").format(species, group) results = [] - for result in g.db.execute(query).fetchall(): - if bool(result): - these_datasets = build_datasets(species, group, result[0]) - if len(these_datasets) > 0: - results.append([str(result[0]), str(result[0]), - "Molecular Traits"]) - + with conn.cursor() as cursor: + cursor.execute(query) + for result in cursor.fetchall(): + if bool(result): + these_datasets = build_datasets(species, + group, result[0], conn) + if len(these_datasets) > 0: + results.append([str(result[0]), str(result[0]), + "Molecular Traits"]) return results -def get_datasets(types): +def get_datasets(types, conn): """Build datasets list""" datasets = {} for species, group_dict in list(types.items()): @@ -134,83 +141,89 @@ def get_datasets(types): for group, type_list in list(group_dict.items()): datasets[species][group] = {} for type_name in type_list: - these_datasets = build_datasets(species, group, type_name[0]) + these_datasets = build_datasets(species, group, + type_name[0], conn) if bool(these_datasets): datasets[species][group][type_name[0]] = these_datasets return datasets -def build_datasets(species, group, type_name): +def build_datasets(species, group, type_name, conn): """Gets dataset names from database""" dataset_text = dataset_value = None datasets = [] - if type_name == "Phenotypes": - results = g.db.execute( - ("SELECT InfoFiles.GN_AccesionId, PublishFreeze.Name, " - "PublishFreeze.FullName FROM InfoFiles, PublishFreeze, " - "InbredSet WHERE InbredSet.Name = '{}' AND " - "PublishFreeze.InbredSetId = InbredSet.Id AND " - "InfoFiles.InfoPageName = PublishFreeze.Name " - "ORDER BY PublishFreeze.CreateTime ASC").format(group)).fetchall() - if bool(results): - for result in results: - dataset_id = str(result[0]) - dataset_value = str(result[1]) - dataset_text = str(result[2]) - if group == 'MDP': - dataset_text = "Mouse Phenome Database" - + with conn.cursor() as cursor: + if type_name == "Phenotypes": + cursor.execute( + ("SELECT InfoFiles.GN_AccesionId, PublishFreeze.Name, " + "PublishFreeze.FullName FROM InfoFiles, PublishFreeze, " + "InbredSet WHERE InbredSet.Name = '{}' AND " + "PublishFreeze.InbredSetId = InbredSet.Id AND " + "InfoFiles.InfoPageName = PublishFreeze.Name " + "ORDER BY PublishFreeze.CreateTime ASC").format(group)) + results = cursor.fetchall() + if bool(results): + for result in results: + dataset_id = str(result[0]) + dataset_value = str(result[1]) + dataset_text = str(result[2]) + if group == 'MDP': + dataset_text = "Mouse Phenome Database" + + datasets.append([dataset_id, dataset_value, dataset_text]) + else: + cursor.execute( + ("SELECT PublishFreeze.Name, PublishFreeze.FullName " + "FROM PublishFreeze, InbredSet " + "WHERE InbredSet.Name = '{}' AND " + "PublishFreeze.InbredSetId = InbredSet.Id " + "ORDER BY PublishFreeze.CreateTime ASC") + .format(group)) + result = cursor.fetchone() + dataset_id = "None" + dataset_value = str(result[0]) + dataset_text = str(result[1]) datasets.append([dataset_id, dataset_value, dataset_text]) - else: - result = g.db.execute( - ("SELECT PublishFreeze.Name, PublishFreeze.FullName " - "FROM PublishFreeze, InbredSet " - "WHERE InbredSet.Name = '{}' AND " - "PublishFreeze.InbredSetId = InbredSet.Id " - "ORDER BY PublishFreeze.CreateTime ASC") - .format(group)).fetchone() + elif type_name == "Genotypes": + cursor.execute( + ("SELECT InfoFiles.GN_AccesionId " + "FROM InfoFiles, GenoFreeze, InbredSet " + "WHERE InbredSet.Name = '{}' AND " + "GenoFreeze.InbredSetId = InbredSet.Id AND " + "InfoFiles.InfoPageName = GenoFreeze.ShortName " + "ORDER BY GenoFreeze.CreateTime " + "DESC").format(group)) + results = cursor.fetchone() dataset_id = "None" - dataset_value = str(result[0]) - dataset_text = str(result[1]) + if bool(results): + dataset_id = str(results[0]) + + dataset_value = "%sGeno" % group + dataset_text = "%s Genotypes" % group datasets.append([dataset_id, dataset_value, dataset_text]) - elif type_name == "Genotypes": - results = g.db.execute( - ("SELECT InfoFiles.GN_AccesionId " - "FROM InfoFiles, GenoFreeze, InbredSet " - "WHERE InbredSet.Name = '{}' AND " - "GenoFreeze.InbredSetId = InbredSet.Id AND " - "InfoFiles.InfoPageName = GenoFreeze.ShortName " - "ORDER BY GenoFreeze.CreateTime DESC").format(group)).fetchone() - - dataset_id = "None" - if bool(results): - dataset_id = str(results[0]) - - dataset_value = "%sGeno" % group - dataset_text = "%s Genotypes" % group - datasets.append([dataset_id, dataset_value, dataset_text]) - - else: # for mRNA expression/ProbeSet - results = g.db.execute( - ("SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, " - "ProbeSetFreeze.FullName FROM ProbeSetFreeze, " - "ProbeFreeze, InbredSet, Tissue, Species WHERE " - "Species.Name = '{0}' AND Species.Id = " - "InbredSet.SpeciesId AND InbredSet.Name = '{1}' " - "AND ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " - "AND Tissue.Name = '{2}' AND ProbeFreeze.TissueId = " - "Tissue.Id AND ProbeFreeze.InbredSetId = InbredSet.Id " - "AND ProbeSetFreeze.public > 0 " - "ORDER BY -ProbeSetFreeze.OrderList DESC, ProbeSetFreeze.CreateTime DESC").format(species, group, type_name)).fetchall() - - datasets = [] - for dataset_info in results: - this_dataset_info = [] - for info in dataset_info: - this_dataset_info.append(str(info)) - datasets.append(this_dataset_info) + else: # for mRNA expression/ProbeSet + cursor.execute( + ("SELECT ProbeSetFreeze.Id, ProbeSetFreeze.Name, " + "ProbeSetFreeze.FullName FROM ProbeSetFreeze, " + "ProbeFreeze, InbredSet, Tissue, Species WHERE " + "Species.Name = '{0}' AND Species.Id = " + "InbredSet.SpeciesId AND InbredSet.Name = '{1}' " + "AND ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND Tissue.Name = '{2}' AND ProbeFreeze.TissueId = " + "Tissue.Id AND ProbeFreeze.InbredSetId = InbredSet.Id " + "AND ProbeSetFreeze.public > 0 " + "ORDER BY -ProbeSetFreeze.OrderList DESC, " + "ProbeSetFreeze.CreateTime " + "DESC").format(species, group, type_name)) + results = cursor.fetchall() + datasets = [] + for dataset_info in results: + this_dataset_info = [] + for info in dataset_info: + this_dataset_info.append(str(info)) + datasets.append(this_dataset_info) return datasets -- cgit v1.2.3 From 2b375d11300d00ed426f7143014a4993da48d708 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 7 May 2021 15:47:09 +0300 Subject: wqflask: Process MySQL URI and store the result in the global scope --- wqflask/wqflask/__init__.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index bf5e094b..2e94dd01 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -5,6 +5,8 @@ import jinja2 from flask import g from flask import Flask +from typing import Tuple +from urllib.parse import urlparse from utility import formatting from wqflask.markdown_routes import glossary_blueprint from wqflask.markdown_routes import references_blueprint @@ -16,9 +18,28 @@ from wqflask.markdown_routes import blogs_blueprint app = Flask(__name__) + +# Helper function for getting the SQL objects +def parse_db_url(sql_uri: str) -> Tuple: + """Parse SQL_URI env variable from an sql URI + e.g. 'mysql://user:pass@host_name/db_name' + + """ + parsed_db = urlparse(sql_uri) + return (parsed_db.hostname, parsed_db.username, + parsed_db.password, parsed_db.path[1:]) + + # See http://flask.pocoo.org/docs/config/#configuring-from-files # Note no longer use the badly named WQFLASK_OVERRIDES (nyi) app.config.from_envvar('GN2_SETTINGS') + +DB_HOST, DB_USER, DB_PASS, DB_NAME = parse_db_url(app.config.get('SQL_URI')) +app.config["DB_HOST"] = DB_HOST +app.config["DB_USER"] = DB_USER +app.config["DB_PASS"] = DB_PASS +app.config["DB_NAME"] = DB_NAME + app.jinja_env.globals.update( undefined=jinja2.StrictUndefined, numify=formatting.numify) -- cgit v1.2.3 From 395135fcebae0a2bca476b058f78f9e1ad8b13ee Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 7 May 2021 13:00:12 +0300 Subject: utility: startup_config: Remove dead comments --- wqflask/utility/startup_config.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 6ef759e0..07322a62 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -16,12 +16,10 @@ def app_config(): if not app.config.get('SECRET_KEY'): import os app.config['SECRET_KEY'] = str(os.urandom(24)) - mode = WEBSERVER_MODE if mode == "DEV" or mode == "DEBUG": app.config['TEMPLATES_AUTO_RELOAD'] = True - # if mode == "DEBUG": - # app.config['EXPLAIN_TEMPLATE_LOADING'] = True <--- use overriding app param instead + print("==========================================") show_settings() @@ -34,9 +32,5 @@ def app_config(): page = requests.get(get_setting("GN_SERVER_URL")) if page.status_code != 200: raise Exception("API server not found!") - - # import utility.elasticsearch_tools as es - # es.test_elasticsearch_connection() - print(("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE, str(port), ENDC, get_setting("WEBSERVER_URL")))) -- cgit v1.2.3 From 74befd19d3d24a89d7d382625648ddade74099f7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 7 May 2021 13:06:06 +0300 Subject: utility: startup_config: Break up long import statement --- wqflask/utility/startup_config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index 07322a62..d705a0ff 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -1,9 +1,12 @@ from wqflask import app -from utility.tools import WEBSERVER_MODE, show_settings, get_setting_int, get_setting, get_setting_bool -import utility.logger -logger = utility.logger.getLogger(__name__) +from utility.tools import WEBSERVER_MODE +from utility.tools import show_settings +from utility.tools import get_setting_int +from utility.tools import get_setting +from utility.tools import get_setting_bool + BLUE = '\033[94m' GREEN = '\033[92m' -- cgit v1.2.3 From 809351ae0021f48b8a138212764aab4396f564ef Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 7 May 2021 13:06:31 +0300 Subject: utility: startup_config: Use python3 f-strings --- wqflask/utility/startup_config.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py index d705a0ff..56d0af6f 100644 --- a/wqflask/utility/startup_config.py +++ b/wqflask/utility/startup_config.py @@ -24,16 +24,17 @@ def app_config(): app.config['TEMPLATES_AUTO_RELOAD'] = True print("==========================================") + show_settings() port = get_setting_int("SERVER_PORT") if get_setting_bool("USE_GN_SERVER"): - print( - ("GN2 API server URL is [" + BLUE + get_setting("GN_SERVER_URL") + ENDC + "]")) + print(f"GN2 API server URL is [{BLUE}GN_SERVER_URL{ENDC}]") import requests page = requests.get(get_setting("GN_SERVER_URL")) if page.status_code != 200: raise Exception("API server not found!") - print(("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % - (BLUE, str(port), ENDC, get_setting("WEBSERVER_URL")))) + print(f"GN2 is running. Visit {BLUE}" + f"[http://localhost:{str(port)}/{ENDC}]" + f"({get_setting('WEBSERVER_URL')})") -- cgit v1.2.3 From 1afece5464520700901cbde19599ac45222ea58f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Sat, 8 May 2021 16:11:20 +0300 Subject: tests: test_gen_menu: Remove app_context initialisation This is no longer required since the global object "Flask.g.db" is no longer coupled to the gen_menu module. --- wqflask/tests/unit/wqflask/api/test_gen_menu.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/wqflask/tests/unit/wqflask/api/test_gen_menu.py b/wqflask/tests/unit/wqflask/api/test_gen_menu.py index d0cd647a..743b3bde 100644 --- a/wqflask/tests/unit/wqflask/api/test_gen_menu.py +++ b/wqflask/tests/unit/wqflask/api/test_gen_menu.py @@ -2,7 +2,6 @@ import unittest from unittest import mock -from wqflask import app from wqflask.api.gen_menu import gen_dropdown_json from wqflask.api.gen_menu import get_species from wqflask.api.gen_menu import get_groups @@ -18,8 +17,6 @@ class TestGenMenu(unittest.TestCase): """Tests for the gen_menu module""" def setUp(self): - self.app_context = app.app_context() - self.app_context.push() self.test_group = { 'mouse': [ ['H_T1', @@ -70,9 +67,6 @@ class TestGenMenu(unittest.TestCase): } } - def tearDown(self): - self.app_context.pop() - def test_get_species(self): """Test that assertion is raised when dataset and dataset_name are defined""" -- cgit v1.2.3 From 049a438741d19b1cf6c8c290ec01343b51690b30 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sun, 9 May 2021 00:17:33 +0300 Subject: fix:add lit for trait list --- wqflask/wqflask/correlation/correlation_gn3_api.py | 39 ++++++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 4949bbe2..d68bb604 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -83,6 +83,23 @@ def tissue_for_trait_lists(corr_results, this_dataset, this_trait): return corr_results +def lit_for_trait_list(corr_results, this_dataset, this_trait): + (this_trait_geneid, geneid_dict, species) = do_lit_correlation( + this_trait, this_dataset) + + trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results} + + geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict if + trait_lists.get(trait_name)} + + conn, _cursor_object = database_connector() + correlation_results = compute_all_lit_correlation( + conn=conn, trait_lists=list(geneid_dict.items()), + species=species, gene_id=this_trait_geneid) + + return correlation_results[0:corr_return_results] + + def compute_correlation(start_vars, method="pearson"): """compute correlation for to call gn3 api""" # pylint: disable-msg=too-many-locals @@ -136,30 +153,30 @@ def compute_correlation(start_vars, method="pearson"): conn, _cursor_object = database_connector() with conn: correlation_results = compute_all_lit_correlation( - conn=conn, trait_lists=list(geneid_dict.items()), - species=species, gene_id=this_trait_geneid) + conn = conn, trait_lists = list(geneid_dict.items()), + species = species, gene_id = this_trait_geneid) return correlation_results[0:corr_return_results] def do_lit_correlation(this_trait, this_dataset): """function for fetching lit inputs""" - geneid_dict = this_dataset.retrieve_genes("GeneId") - species = this_dataset.group.species.lower() - trait_geneid = this_trait.geneid + geneid_dict=this_dataset.retrieve_genes("GeneId") + species=this_dataset.group.species.lower() + trait_geneid=this_trait.geneid return (trait_geneid, geneid_dict, species) def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" - primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=[this_trait.symbol]) + primary_trait_tissue_vals_dict=correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list = [this_trait.symbol]) if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( + primary_trait_tissue_values=primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] - corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list=list(trait_symbol_dict.values())) - primary_tissue_data = { + corr_result_tissue_vals_dict=correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list = list(trait_symbol_dict.values())) + primary_tissue_data={ "this_id": this_trait.name, "tissue_values": primary_trait_tissue_values -- cgit v1.2.3 From 27538980f93c1d72b0b2d76151312f3fbce4c9a5 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 10 May 2021 08:24:42 +0300 Subject: add previous endpoint for correlation --- wqflask/base/data_set.py | 37 ------------------------------------- wqflask/wqflask/views.py | 13 ++++++++----- 2 files changed, 8 insertions(+), 42 deletions(-) diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py index 468c4da0..d0f5e6f2 100644 --- a/wqflask/base/data_set.py +++ b/wqflask/base/data_set.py @@ -655,42 +655,7 @@ class DataSet(object): "Dataset {} is not yet available in GeneNetwork.".format(self.name)) pass - def fetch_probe_trait_data(self, sample_list=None): - if sample_list: - self.samplelist = sample_list - else: - self.samplelist = self.group.samplelist - - if self.group.parlist != None and self.group.f1list != None: - if (self.group.parlist + self.group.f1list) in self.samplelist: - self.samplelist += self.group.parlist + self.group.f1list - - query = """ - SELECT Strain.Name, Strain.Id FROM Strain, Species - WHERE Strain.Name IN {} - and Strain.SpeciesId=Species.Id - and Species.name = '{}' - """.format(create_in_clause(self.samplelist), *mescape(self.group.species)) - logger.sql(query) - results = dict(g.db.execute(query).fetchall()) - sample_ids = [results[item] for item in self.samplelist] - - query = """SELECT * from ProbeSetData WHERE Id in ( SELECT ProbeSetXRef.DataId FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze) WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id and ProbeSetFreeze.Name = 'HC_M2_0606_P' and ProbeSet.Id = ProbeSetXRef.ProbeSetId order by ProbeSet.Id ) and StrainId in ({})""".format( - ",".join(str(sample_id) for sample_id in sample_ids)) - results = g.db.execute(query).fetchall() - - # with conn: - # cursor = conn.cursor() - # cursor.execute(query) - # results = cursor.fetchall() - trait_data = {} - for trait_id, StrainId, value in results: - if trait_id in trait_data: - trait_data[trait_id].append(value) - else: - trait_data[trait_id] = [value] - self.trait_data = trait_data def get_trait_data(self, sample_list=None): if sample_list: @@ -711,7 +676,6 @@ class DataSet(object): logger.sql(query) results = dict(g.db.execute(query).fetchall()) sample_ids = [results[item] for item in self.samplelist] - print("the number of sample ids are", len(sample_ids)) # MySQL limits the number of tables that can be used in a join to 61, # so we break the sample ids into smaller chunks @@ -762,7 +726,6 @@ class DataSet(object): trait_sample_data.append(results) trait_count = len(trait_sample_data[0]) - print("the trait count is >>>", trait_count) self.trait_data = collections.defaultdict(list) # put all of the separate data together into a dictionary where the keys are diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index b042a211..19779651 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -881,11 +881,14 @@ def network_graph_page(): def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) logger.info(request.url) - import time - initial_time = time.time() - correlation_results = compute_correlation(request.form) - print(">>>>Time taken by this endpoint",time.time()-initial_time) - return render_template("test_correlation_page.html",correlation_results=correlation_results) + template_vars = show_corr_results.CorrelationResults(request.form) + return render_template("correlation_page.html", **template_vars.__dict__) + + # to test the new correlation api uncomment these lines + + # correlation_results = compute_correlation(request.form) + # print(">>>>Time taken by this endpoint",time.time()-initial_time) + # return render_template("test_correlation_page.html",correlation_results=correlation_results) @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): -- cgit v1.2.3 From aebbbc9240ab63f684dae2451cfc335681c049db Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 10 May 2021 08:25:21 +0300 Subject: remove debug statements --- wqflask/wqflask/correlation/show_corr_results.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py index 50b3ba26..7f69807b 100644 --- a/wqflask/wqflask/correlation/show_corr_results.py +++ b/wqflask/wqflask/correlation/show_corr_results.py @@ -57,6 +57,7 @@ TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] TISSUE_MOUSE_DB = 1 + class CorrelationResults(object): def __init__(self, start_vars): # get trait list from db (database name) @@ -445,9 +446,6 @@ class CorrelationResults(object): """ - print("below here>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>") - print(self.target_dataset.trait_data) - self.this_trait_vals = [] target_vals = [] for index, sample in enumerate(self.target_dataset.samplelist): -- cgit v1.2.3 From bc29c9dd907c8c1b024231fa713040b3c4092f3a Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 10 May 2021 08:29:46 +0300 Subject: revert python path --- bin/genenetwork2 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/genenetwork2 b/bin/genenetwork2 index f73f235c..5f4e0f9a 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -154,8 +154,7 @@ if [ ! -d $R_LIBS_SITE ] ; then fi # We may change this one: -# export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH -PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$HOME/project/genenetwork3:$PYTHONPATH +export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then -- cgit v1.2.3 From 93baf59aca12fb85e668315e67d36137c854f12d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 10 May 2021 08:33:22 +0300 Subject: minor changes to interface --- wqflask/wqflask/correlation/correlation_gn3_api.py | 60 ++++++++++++++-------- 1 file changed, 38 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index d68bb604..98d52591 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -68,9 +68,9 @@ def sample_for_trait_lists(corr_results, target_dataset, def tissue_for_trait_lists(corr_results, this_dataset, this_trait): """interface function for doing tissue corr_results on trait_list""" - # trait_lists = dict([(list(corr_result)[0], True) - # for corr_result in corr_results]) - trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results} + trait_lists = dict([(list(corr_result)[0], True) + for corr_result in corr_results]) + # trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results} traits_symbol_dict = this_dataset.retrieve_genes("Symbol") traits_symbol_dict = dict({trait_name: symbol for ( trait_name, symbol) in traits_symbol_dict.items() if trait_lists.get(trait_name)}) @@ -85,19 +85,24 @@ def tissue_for_trait_lists(corr_results, this_dataset, this_trait): def lit_for_trait_list(corr_results, this_dataset, this_trait): (this_trait_geneid, geneid_dict, species) = do_lit_correlation( - this_trait, this_dataset) + this_trait, this_dataset) - trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results} + # trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results} + trait_lists = dict([(list(corr_result)[0], True) + for corr_result in corr_results]) - geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict if - trait_lists.get(trait_name)} + geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if + trait_lists.get(trait_name)} conn, _cursor_object = database_connector() - correlation_results = compute_all_lit_correlation( - conn=conn, trait_lists=list(geneid_dict.items()), - species=species, gene_id=this_trait_geneid) - return correlation_results[0:corr_return_results] + with conn: + + correlation_results = compute_all_lit_correlation( + conn=conn, trait_lists=list(geneid_dict.items()), + species=species, gene_id=this_trait_geneid) + + return correlation_results def compute_correlation(start_vars, method="pearson"): @@ -129,6 +134,17 @@ def compute_correlation(start_vars, method="pearson"): this_trait=this_trait_data, target_dataset=results) + # do tissue correaltion + + # code to be use later + + # tissue_result = tissue_for_trait_lists( + # correlation_results, this_dataset, this_trait) + # # lit spoils the party so slow + # lit_result = lit_for_trait_list( + # correlation_results, this_dataset, this_trait) + + elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") primary_tissue_data, target_tissue_data = get_tissue_correlation_input( @@ -153,30 +169,30 @@ def compute_correlation(start_vars, method="pearson"): conn, _cursor_object = database_connector() with conn: correlation_results = compute_all_lit_correlation( - conn = conn, trait_lists = list(geneid_dict.items()), - species = species, gene_id = this_trait_geneid) + conn=conn, trait_lists=list(geneid_dict.items()), + species=species, gene_id=this_trait_geneid) return correlation_results[0:corr_return_results] def do_lit_correlation(this_trait, this_dataset): """function for fetching lit inputs""" - geneid_dict=this_dataset.retrieve_genes("GeneId") - species=this_dataset.group.species.lower() - trait_geneid=this_trait.geneid + geneid_dict = this_dataset.retrieve_genes("GeneId") + species = this_dataset.group.species.lower() + trait_geneid = this_trait.geneid return (trait_geneid, geneid_dict, species) def get_tissue_correlation_input(this_trait, trait_symbol_dict): """Gets tissue expression values for the primary trait and target tissues values""" - primary_trait_tissue_vals_dict=correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list = [this_trait.symbol]) + primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=[this_trait.symbol]) if this_trait.symbol.lower() in primary_trait_tissue_vals_dict: - primary_trait_tissue_values=primary_trait_tissue_vals_dict[this_trait.symbol.lower( + primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower( )] - corr_result_tissue_vals_dict=correlation_functions.get_trait_symbol_and_tissue_values( - symbol_list = list(trait_symbol_dict.values())) - primary_tissue_data={ + corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values( + symbol_list=list(trait_symbol_dict.values())) + primary_tissue_data = { "this_id": this_trait.name, "tissue_values": primary_trait_tissue_values -- cgit v1.2.3 From 0b723720f7b1b9802b2f5453b747c7e48b693817 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 10 May 2021 08:34:10 +0300 Subject: use new correlation code in endpoint --- wqflask/wqflask/views.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 19779651..a470a0c9 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -881,14 +881,14 @@ def network_graph_page(): def corr_compute_page(): logger.info("In corr_compute, request.form is:", pf(request.form)) logger.info(request.url) - template_vars = show_corr_results.CorrelationResults(request.form) - return render_template("correlation_page.html", **template_vars.__dict__) + # template_vars = show_corr_results.CorrelationResults(request.form) + # return render_template("correlation_page.html", **template_vars.__dict__) # to test the new correlation api uncomment these lines - # correlation_results = compute_correlation(request.form) + correlation_results = compute_correlation(request.form) # print(">>>>Time taken by this endpoint",time.time()-initial_time) - # return render_template("test_correlation_page.html",correlation_results=correlation_results) + return render_template("test_correlation_page.html",correlation_results=correlation_results) @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): -- cgit v1.2.3 From 848fc236461384e5352798e19d710480ead379c4 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Mon, 10 May 2021 08:50:20 +0300 Subject: minor-fix:remove debug statements --- wqflask/wqflask/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 317c5f63..c2e403a0 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -951,10 +951,9 @@ def corr_compute_page(): # template_vars = show_corr_results.CorrelationResults(request.form) # return render_template("correlation_page.html", **template_vars.__dict__) - # to test the new correlation api uncomment these lines + # to test/disable the new correlation api uncomment these lines correlation_results = compute_correlation(request.form) - # print(">>>>Time taken by this endpoint",time.time()-initial_time) return render_template("test_correlation_page.html",correlation_results=correlation_results) -- cgit v1.2.3 From a9fb861de6834f7a3351c08bf2bde00810921ac7 Mon Sep 17 00:00:00 2001 From: uditgulati Date: Sat, 6 Mar 2021 15:59:18 -0600 Subject: update gsearch pheno search --- wqflask/wqflask/gsearch.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index fb8bdc55..262ba3c2 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -177,16 +177,8 @@ class GSearch: {0} AND PublishXRef.`PhenotypeId`=Phenotype.`Id` AND PublishXRef.`PublicationId`=Publication.`Id` - AND (Phenotype.Post_publication_description REGEXP "[[:<:]]{1}[[:>:]]" - OR Phenotype.Pre_publication_description REGEXP "[[:<:]]{1}[[:>:]]" - OR Phenotype.Pre_publication_abbreviation REGEXP "[[:<:]]{1}[[:>:]]" - OR Phenotype.Post_publication_abbreviation REGEXP "[[:<:]]{1}[[:>:]]" - OR Phenotype.Lab_code REGEXP "[[:<:]]{1}[[:>:]]" - OR Publication.PubMed_ID REGEXP "[[:<:]]{1}[[:>:]]" - OR Publication.Abstract REGEXP "[[:<:]]{1}[[:>:]]" - OR Publication.Title REGEXP "[[:<:]]{1}[[:>:]]" - OR Publication.Authors REGEXP "[[:<:]]{1}[[:>:]]" - OR PublishXRef.Id REGEXP "[[:<:]]{1}[[:>:]]") + AND (MATCH (Phenotype.Post_publication_description, Phenotype.Pre_publication_description, Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, Phenotype.Lab_code) AGAINST ('{1}' IN BOOLEAN MODE) ) + AND (MATCH (Publication.Abstract, Publication.Title, Publication.Authors) AGAINST ('{1}' IN BOOLEAN MODE) ) ORDER BY Species.`Name`, InbredSet.`Name`, PublishXRef.`Id` LIMIT 6000 """.format(group_clause, search_term) -- cgit v1.2.3 From 8c8cec89a03b642b671f191ce8a6079883fe5500 Mon Sep 17 00:00:00 2001 From: uditgulati Date: Sat, 6 Mar 2021 16:47:26 -0600 Subject: move geno chr, mb to gsearch main query for phenotype search --- wqflask/wqflask/gsearch.py | 47 +++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 262ba3c2..44a8f7f7 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -169,16 +169,23 @@ class GSearch: PublishXRef.`LRS`, PublishXRef.`additive`, InbredSet.`InbredSetCode`, - PublishXRef.`mean` - FROM Species,InbredSet,PublishFreeze,PublishXRef,Phenotype,Publication - WHERE PublishXRef.`InbredSetId`=InbredSet.`Id` - AND PublishFreeze.`InbredSetId`=InbredSet.`Id` - AND InbredSet.`SpeciesId`=Species.`Id` + PublishXRef.`mean`, + PublishFreeze.Id, + Geno.Chr as geno_chr, + Geno.Mb as geno_mb + FROM Species + INNER JOIN InbredSet ON InbredSet.`SpeciesId`=Species.`Id` + INNER JOIN PublishFreeze ON PublishFreeze.`InbredSetId`=InbredSet.`Id` + INNER JOIN PublishXRef ON PublishXRef.`InbredSetId`=InbredSet.`Id` + INNER JOIN Phenotype ON PublishXRef.`PhenotypeId`=Phenotype.`Id` + INNER JOIN Publication ON PublishXRef.`PublicationId`=Publication.`Id` + LEFT JOIN Geno ON PublishXRef.Locus = Geno.Name AND Geno.SpeciesId = Species.Id + WHERE + ( + (MATCH (Phenotype.Post_publication_description, Phenotype.Pre_publication_description, Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, Phenotype.Lab_code) AGAINST ('{1}' IN BOOLEAN MODE) ) + OR (MATCH (Publication.Abstract, Publication.Title, Publication.Authors) AGAINST ('{1}' IN BOOLEAN MODE) ) + ) {0} - AND PublishXRef.`PhenotypeId`=Phenotype.`Id` - AND PublishXRef.`PublicationId`=Publication.`Id` - AND (MATCH (Phenotype.Post_publication_description, Phenotype.Pre_publication_description, Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, Phenotype.Lab_code) AGAINST ('{1}' IN BOOLEAN MODE) ) - AND (MATCH (Publication.Abstract, Publication.Title, Publication.Authors) AGAINST ('{1}' IN BOOLEAN MODE) ) ORDER BY Species.`Name`, InbredSet.`Name`, PublishXRef.`Id` LIMIT 6000 """.format(group_clause, search_term) @@ -213,6 +220,9 @@ class GSearch: this_trait['mean'] = f"{line[13]:.3f}" else: this_trait['mean'] = "N/A" + this_trait['dataset_id'] = line[14] + this_trait['locus_chr'] = line[15] + this_trait['locus_mb'] = line[16] this_trait['authors'] = line[7] this_trait['year'] = line[8] if this_trait['year'].isdigit(): @@ -233,17 +243,20 @@ class GSearch: if line[11] != "" and line[11] != None: this_trait['additive'] = '%.3f' % line[11] + dataset_ob = SimpleNamespace(id=this_trait["dataset_id"], type="Publish", species=this_trait["species"]) + permissions = check_resource_availability(dataset_ob, this_trait['name']) + if type(permissions['data']) is list: + if "view" not in permissions['data']: + continue + else: + if permissions['data'] == 'no-access': + continue + this_trait['max_lrs_text'] = "N/A" - trait_ob = create_trait( - dataset_name=this_trait['dataset'], name=this_trait['name'], get_qtl_info=True, get_sample_info=False) - if not trait_ob: - continue if this_trait['dataset'] == this_trait['group'] + "Publish": try: - if trait_ob.locus_chr != "" and trait_ob.locus_mb != "": - this_trait['max_lrs_text'] = "Chr" + \ - str(trait_ob.locus_chr) + \ - ": " + str(trait_ob.locus_mb) + if this_trait['locus_chr'] != None and this_trait['locus_mb'] != None: + this_trait['max_lrs_text'] = "Chr" + str(this_trait['locus_chr']) + ": " + str(this_trait['locus_mb']) except: this_trait['max_lrs_text'] = "N/A" -- cgit v1.2.3 From ff5ab1875025efdfa23209e58c651c7816c6b916 Mon Sep 17 00:00:00 2001 From: uditgulati Date: Mon, 10 May 2021 03:34:26 -0500 Subject: add ServerSideTable to gsearch geno and pheno tables --- wqflask/wqflask/gsearch.py | 54 ++++++++++++++++++++++------ wqflask/wqflask/templates/gsearch_gene.html | 15 ++++++-- wqflask/wqflask/templates/gsearch_pheno.html | 15 ++++++-- wqflask/wqflask/views.py | 13 +++++++ 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/wqflask/wqflask/gsearch.py b/wqflask/wqflask/gsearch.py index 44a8f7f7..4024fa7a 100644 --- a/wqflask/wqflask/gsearch.py +++ b/wqflask/wqflask/gsearch.py @@ -130,7 +130,7 @@ class GSearch: trait_list.append(this_trait) self.trait_count = len(trait_list) - self.trait_list = json.dumps(trait_list) + self.trait_list = trait_list self.header_fields = ['Index', 'Record', @@ -146,6 +146,22 @@ class GSearch: 'Max LRS Location', 'Additive Effect'] + self.header_data_names = [ + 'index', + 'name', + 'species', + 'group', + 'tissue', + 'dataset_fullname', + 'symbol', + 'description', + 'location_repr', + 'mean', + 'LRS_score_repr', + 'max_lrs_text', + 'additive', + ] + elif self.type == "phenotype": search_term = self.terms group_clause = "" @@ -263,15 +279,31 @@ class GSearch: trait_list.append(this_trait) self.trait_count = len(trait_list) - self.trait_list = json.dumps(trait_list) + self.trait_list = trait_list self.header_fields = ['Index', - 'Species', - 'Group', - 'Record', - 'Description', - 'Authors', - 'Year', - 'Max LRS', - 'Max LRS Location', - 'Additive Effect'] + 'Species', + 'Group', + 'Record', + 'Description', + 'Authors', + 'Year', + 'Max LRS', + 'Max LRS Location', + 'Additive Effect'] + + self.header_data_names = [ + 'index', + 'name', + 'species', + 'group', + 'tissue', + 'dataset_fullname', + 'symbol', + 'description', + 'location_repr', + 'mean', + 'LRS_score_repr', + 'max_lrs_text', + 'additive', + ] diff --git a/wqflask/wqflask/templates/gsearch_gene.html b/wqflask/wqflask/templates/gsearch_gene.html index 6fd0abe8..48953d98 100644 --- a/wqflask/wqflask/templates/gsearch_gene.html +++ b/wqflask/wqflask/templates/gsearch_gene.html @@ -55,7 +55,15 @@ - + -- cgit v1.2.3 From 92afd94c14891bec381b6de5ecf9926032bab908 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Feb 2021 04:37:11 +0300 Subject: replace datatable scroller cdn --- wqflask/wqflask/templates/show_trait.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index 09ecb7b6..fc14822c 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -148,7 +148,7 @@ - + -- cgit v1.2.3 From 4f826611242080089856ccb4e3a7cda398e57b0d Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 11 Feb 2021 04:46:20 +0300 Subject: replace font-awesome --- wqflask/wqflask/templates/correlation_page.html | 4 ++-- wqflask/wqflask/templates/search_result_page.html | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html index a9a3e1a0..4cad2749 100644 --- a/wqflask/wqflask/templates/correlation_page.html +++ b/wqflask/wqflask/templates/correlation_page.html @@ -4,7 +4,7 @@ - + {% endblock %} @@ -141,7 +141,7 @@ - + diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html index e7a7bc51..827bad98 100644 --- a/wqflask/wqflask/templates/search_result_page.html +++ b/wqflask/wqflask/templates/search_result_page.html @@ -4,7 +4,7 @@ - + {% endblock %} @@ -150,7 +150,7 @@ - + -- cgit v1.2.3 From 45fc1da136b78bac906aad013686a9530f68bd5e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 10:33:49 +0300 Subject: doc: docker-container: Default to python3-genenetwork2 for examples --- doc/docker-container.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/docker-container.org b/doc/docker-container.org index ec91824a..c894c4b5 100644 --- a/doc/docker-container.org +++ b/doc/docker-container.org @@ -59,11 +59,11 @@ RUN tar -xzf /tmp/gn2.tar.gz -C / && rm -f /tmp/gn2.tar.gz && \ Build the image(Note the fullstop at the end): -: sudo docker build -t python2-genenetwork2:latest -f Dockerfile . +: sudo docker build -t python3-genenetwork2:latest -f Dockerfile . To load the image interactively you've just created: -: docker run -ti "python2-genenetwork2:latest" bash +: docker run -ti "python3-genenetwork2:latest" bash Assuming you have a docker instance running, you could always run commands in it e.g: @@ -78,7 +78,7 @@ CI environment using Github Actions. To push to dockerhub, first get the image name by running =docker images=. Push to dockerhub using a command similar to: -: docker push bonfacekilz/python2-genenetwork2:latest +: docker push bonfacekilz/python3-genenetwork2:latest Right now, we have 2 images on DockerHub: -- cgit v1.2.3 From a3ea9cb840bc8dbc9e5e0940ec72a36712e13a2a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 10:47:51 +0300 Subject: workflows: main.yml: Update container image --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a36abc0a..64c0fdb2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ on: jobs: unittest: runs-on: ubuntu-latest - container: bonfacekilz/python3-genenetwork2:ad741c1 + container: bonfacekilz/python3-genenetwork2:00ba1f8 steps: # First start with mariadb set then checkout. The checkout gives -- cgit v1.2.3 From 5933991c3a82e847ebf0a86583aa5461d8c3f937 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 11:06:22 +0300 Subject: links_scraper: genelinks: Remove unused import --- test/requests/links_scraper/genelinks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/requests/links_scraper/genelinks.py b/test/requests/links_scraper/genelinks.py index 12300f4a..13aee7c8 100644 --- a/test/requests/links_scraper/genelinks.py +++ b/test/requests/links_scraper/genelinks.py @@ -2,7 +2,6 @@ import re import requests import urllib3 import os -import logging from urllib.request import urlopen as uReq from bs4 import BeautifulSoup as soup -- cgit v1.2.3 From d32b7f3a877f33ca90af2a4206d8e0d5e04c36de Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 11:08:57 +0300 Subject: links_scraper: genelinks: Apply pep-8 formatting --- test/requests/links_scraper/genelinks.py | 29 +++++++++-------------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/test/requests/links_scraper/genelinks.py b/test/requests/links_scraper/genelinks.py index 13aee7c8..52c13489 100644 --- a/test/requests/links_scraper/genelinks.py +++ b/test/requests/links_scraper/genelinks.py @@ -25,7 +25,6 @@ def search_templates(): parsed_page = soup( open(file_path, encoding="utf8"), "html.parser") html_parsed_pages.append(parsed_page) - return html_parsed_pages @@ -33,7 +32,7 @@ def is_valid_link(url_link): try: result = urlparse(url_link) return all([result.scheme, result.netloc, result.path]) - except Exception as e: + except Exception: return False @@ -41,13 +40,10 @@ def test_link(link): print(f'Checking -->{link}') results = None try: - results = requests.get(link, verify=False, timeout=10) status_code = results.status_code - - except Exception as e: + except Exception: status_code = 408 - return int(status_code) > 403 @@ -55,14 +51,11 @@ def fetch_css_links(parsed_page): print("fetching css links") for link in parsed_page.findAll("link"): full_path = None - link_url = link.attrs.get("href") if is_valid_link(link_url): full_path = link_url - elif re.match(r"^/css", link_url) or re.match(r"^/js", link_url): full_path = urljoin('http://localhost:5004/', link_url) - if full_path is not None: if test_link(full_path): BROKEN_LINKS.add(full_path) @@ -70,16 +63,13 @@ def fetch_css_links(parsed_page): def fetch_html_links(parsed_page): print("fetching a tags ") - for link in parsed_page.findAll("a"): full_path = None link_url = link.attrs.get("href") if re.match(r"^/", link_url): full_path = urljoin('http://localhost:5004/', link_url) - elif is_valid_link(link_url): full_path = link_url - if full_path is not None: if test_link(full_path): BROKEN_LINKS.add(full_path) @@ -91,8 +81,11 @@ def fetch_script_tags(parsed_page): js_link = link.attrs.get("src") if js_link is not None: if is_valid_link(js_link): - raise SystemExit("Failed,the library should be packaged in guix.\ - Please contact,http://genenetwork.org/ for more details") + raise SystemExit("Failed,the library should be " + "packaged in guix. " + "Please contact, " + "http://genenetwork.org/ " + "for more details") elif re.match(r"^/css", js_link) or re.match(r"^/js", js_link): full_path = urljoin('http://localhost:5004/', js_link) @@ -101,11 +94,9 @@ def fetch_script_tags(parsed_page): def fetch_page_links(page_url): - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) html_page = uReq(page_url) parsed_page = soup(html_page, "html.parser") - fetch_script_tags(parsed_page=parsed_page) fetch_css_links(parsed_page=parsed_page) fetch_html_links(parsed_page=parsed_page) @@ -113,13 +104,10 @@ def fetch_page_links(page_url): def webpages_to_check(): pages = [f"http://localhost:{PORT}/"] - return pages if __name__ == '__main__': - # results = search_templates() - for page in webpages_to_check(): fetch_page_links(page) if len(BROKEN_LINKS) > 0: @@ -129,4 +117,5 @@ if __name__ == '__main__': if len(BROKEN_LINKS) > 0: raise SystemExit( - "The links Above are broken.Please contact genenetwork.org<<<<<<<<") + "The links Above are broken. " + "Please contact genenetwork.org<<<<<<<<") -- cgit v1.2.3 From 5e3e8e5847fe994652e3c6675ce96312fd9dc16a Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 11:09:36 +0300 Subject: workflows: main.yml: Remove hard-coded path --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 64c0fdb2..2b7c3b16 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -57,10 +57,10 @@ jobs: - name: Test for Broken Links run: | - env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp\ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - etc/default_settings.py -c /__w/genenetwork2/genenetwork2/test/requests/links_scraper/genelinks.py + etc/default_settings.py -c \ + $PWD/test/requests/links_scraper/genelinks.py -- cgit v1.2.3 From 232148b1304ae531df6d9157f6d574d5c944830e Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 11:30:53 +0300 Subject: workflows: main.yml: Use screen to run gn2 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2b7c3b16..f9a3fa13 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,11 +49,11 @@ jobs: - name: Start Genenetwork as a Background Task run: | - env GN2_PROFILE=/gn2-profile \ + /gn2-profile/bin/screen -dmL bash -c "env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - etc/default_settings.py& + etc/default_settings.py" - name: Test for Broken Links run: | -- cgit v1.2.3 From 4538c0ad693ae94ebda5f0bf39678d776c0c8297 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 11:47:20 +0300 Subject: workflows: main.yml: Set up gn2 before running the unit tests --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f9a3fa13..b09f2f1d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -39,21 +39,21 @@ jobs: mysql -u root -e "CREATE USER 'gn2'@'localhost' IDENTIFIED BY 'mysql_password';" mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'gn2'@'localhost';FLUSH PRIVILEGES;" - - name: Run the unit tests + - name: Start Genenetwork as a Background Task run: | - env GN2_PROFILE=/gn2-profile \ + /gn2-profile/bin/screen -dmL bash -c "env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - etc/default_settings.py -c -m unittest discover -v + etc/default_settings.py" - - name: Start Genenetwork as a Background Task + - name: Run the unit tests run: | - /gn2-profile/bin/screen -dmL bash -c "env GN2_PROFILE=/gn2-profile \ + env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ - etc/default_settings.py" + etc/default_settings.py -c -m unittest discover -v - name: Test for Broken Links run: | -- cgit v1.2.3 From 75b378f757979af1d6fdaff3f3abde0599c5f744 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 12:10:37 +0300 Subject: workflows: main.yml: Use mariadb as a service --- .github/workflows/main.yml | 30 +++++++++++++----------------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b09f2f1d..a2925e08 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,16 +12,19 @@ jobs: unittest: runs-on: ubuntu-latest container: bonfacekilz/python3-genenetwork2:00ba1f8 + services: + mysql: + image: mariadb:15.1 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + MYSQL_USER: gn2 + MYSQL_PASSWORD: mysql_password + MYSQL_DATABASE: db_webqtl_s + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - # First start with mariadb set then checkout. The checkout gives - # the mysqld enough time to start - - name: Set up mariadb - run: | - mysql_install_db --user=mysql --datadir=/usr/local/mysql - # Wait for the mysqld_safe process to start - mysqld_safe --user=mysql --datadir=/usr/local/mysql & - # Use v1 of checkout since v2 fails - name: Checkout Project uses: actions/checkout@v1 @@ -31,17 +34,10 @@ jobs: run: | /gn2-profile/bin/screen -dmLS redisconn /gn2-profile/bin/redis-server - # Redis is required by some of the tests 6379 - - name: Bootstrap tables - run: | - mysql -u root -e "SHOW DATABASES;" - mysql -u root -e "CREATE DATABASE db_webqtl_s;" - mysql -u root -e "CREATE USER 'gn2'@'localhost' IDENTIFIED BY 'mysql_password';" - mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'gn2'@'localhost';FLUSH PRIVILEGES;" - - name: Start Genenetwork as a Background Task run: | - /gn2-profile/bin/screen -dmL bash -c "env GN2_PROFILE=/gn2-profile \ + /gn2-profile/bin/screen -dmLS gn2conn + bash -c "env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ -- cgit v1.2.3 From c4ec08c1b544c05cce1c1247b8bc1afb741643e4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 11 May 2021 13:38:47 +0300 Subject: workflows: main.yml: Remove mysql service and update gn2 b/g task --- .github/workflows/main.yml | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a2925e08..c6b56a81 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,19 +12,16 @@ jobs: unittest: runs-on: ubuntu-latest container: bonfacekilz/python3-genenetwork2:00ba1f8 - services: - mysql: - image: mariadb:15.1 - env: - MYSQL_ALLOW_EMPTY_PASSWORD: yes - MYSQL_USER: gn2 - MYSQL_PASSWORD: mysql_password - MYSQL_DATABASE: db_webqtl_s - ports: - - 3306:3306 - options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: + # First start with mariadb set then checkout. The checkout gives + # the mysqld enough time to start + - name: Set up mariadb + run: | + mysql_install_db --user=mysql --datadir=/usr/local/mysql + # Wait for the mysqld_safe process to start + mysqld_safe --user=mysql --datadir=/usr/local/mysql & + # Use v1 of checkout since v2 fails - name: Checkout Project uses: actions/checkout@v1 @@ -34,10 +31,17 @@ jobs: run: | /gn2-profile/bin/screen -dmLS redisconn /gn2-profile/bin/redis-server + # Initialise the tables + - name: Bootstrap tables + run: | + mysql -u root -e "SHOW DATABASES;" + mysql -u root -e "CREATE DATABASE db_webqtl_s;" + mysql -u root -e "CREATE USER 'gn2'@'localhost' IDENTIFIED BY 'mysql_password';" + mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'gn2'@'localhost';FLUSH PRIVILEGES;" + - name: Start Genenetwork as a Background Task run: | - /gn2-profile/bin/screen -dmLS gn2conn - bash -c "env GN2_PROFILE=/gn2-profile \ + /gn2-profile/bin/screen -dm bash -c "env GN2_PROFILE=/gn2-profile \ TMPDIR=/tmp SERVER_PORT=5004 \ WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \ GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ @@ -59,4 +63,3 @@ jobs: GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \ etc/default_settings.py -c \ $PWD/test/requests/links_scraper/genelinks.py - -- cgit v1.2.3 From 55f23731f66a8445635c8c3195b18369aa740626 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 12 May 2021 18:12:14 +0000 Subject: Get max digits for each sample group's vals and create a function for getting a sample group's vals so the same logic isn't repeated several times --- wqflask/wqflask/show_trait/show_trait.py | 56 +++++++++++++++++++------------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index b6fcbcb8..424b8ee5 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -156,8 +156,14 @@ class ShowTrait: self.make_sample_lists() - self.qnorm_vals = quantile_normalize_vals(self.sample_groups) - self.z_scores = get_z_scores(self.sample_groups) + trait_vals_by_group = [] + for sample_type in self.sample_groups: + trait_vals_by_group.append(get_trait_vals(sample_type.sample_list)) + + self.max_digits_by_group = get_max_digits(trait_vals_by_group) + + self.qnorm_vals = quantile_normalize_vals(self.sample_groups, trait_vals_by_group) + self.z_scores = get_z_scores(self.sample_groups, trait_vals_by_group) self.temp_uuid = uuid.uuid4() @@ -294,6 +300,7 @@ class ShowTrait: js_data = dict(trait_id=self.trait_id, trait_symbol=trait_symbol, + max_digits = self.max_digits_by_group, short_description=short_description, unit_type=trait_units, dataset_type=self.dataset.type, @@ -513,7 +520,26 @@ class ShowTrait: self.dataset.group.allsamples = all_samples_ordered -def quantile_normalize_vals(sample_groups): +def get_trait_vals(sample_list): + trait_vals = [] + for sample in sample_list: + try: + trait_vals.append(float(sample.value)) + except: + continue + + return trait_vals + +def get_max_digits(trait_vals): + max_digits = [] + for these_vals in trait_vals: + max_val = max(these_vals) + digits = len(str(max_val)) + max_digits.append(digits - 1) + + return max_digits + +def quantile_normalize_vals(sample_groups, trait_vals): def normf(trait_vals): ranked_vals = ss.rankdata(trait_vals) p_list = [] @@ -529,15 +555,8 @@ def quantile_normalize_vals(sample_groups): return normed_vals qnorm_by_group = [] - for sample_type in sample_groups: - trait_vals = [] - for sample in sample_type.sample_list: - try: - trait_vals.append(float(sample.value)) - except: - continue - - qnorm_vals = normf(trait_vals) + for i, sample_type in enumerate(sample_groups): + qnorm_vals = normf(trait_vals[i]) qnorm_vals_with_x = [] counter = 0 for sample in sample_type.sample_list: @@ -552,17 +571,10 @@ def quantile_normalize_vals(sample_groups): return qnorm_by_group -def get_z_scores(sample_groups): +def get_z_scores(sample_groups, trait_vals): zscore_by_group = [] - for sample_type in sample_groups: - trait_vals = [] - for sample in sample_type.sample_list: - try: - trait_vals.append(float(sample.value)) - except: - continue - - zscores = ss.mstats.zscore(np.array(trait_vals)).tolist() + for i, sample_type in enumerate(sample_groups): + zscores = ss.mstats.zscore(np.array(trait_vals[i])).tolist() zscores_with_x = [] counter = 0 for sample in sample_type.sample_list: -- cgit v1.2.3 From 1efc014dd5efc34d95b536f51f8a1de914069e20 Mon Sep 17 00:00:00 2001 From: zsloan Date: Wed, 12 May 2021 18:12:52 +0000 Subject: Remove maxlength from value/SE input fields and base the width off of the max digits for that sample group's vals --- .../wqflask/static/new/javascript/initialize_show_trait_tables.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js index 05e4d547..f9369fb9 100644 --- a/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js +++ b/wqflask/wqflask/static/new/javascript/initialize_show_trait_tables.js @@ -40,9 +40,9 @@ build_columns = function() { 'data': null, 'render': function(data, type, row, meta) { if (data.value == null) { - return '' + return '' } else { - return '' + return '' } } } @@ -66,9 +66,9 @@ build_columns = function() { 'data': null, 'render': function(data, type, row, meta) { if (data.variance == null) { - return '' + return '' } else { - return '' + return '' } } } -- cgit v1.2.3 From 0dd04c5ca0521f59f93b45663bdfd67c916a9f3c Mon Sep 17 00:00:00 2001 From: Arthur Centeno Date: Thu, 13 May 2021 08:05:30 +0000 Subject: Fix load_phenotypes to run with GN2 latest --- scripts/maintenance/datastructure.py | 177 +++++++++++++++++++++++++++++++ scripts/maintenance/load_phenotypes.py | 43 ++++---- scripts/maintenance/utilities.py | 89 ++++++++++++++++ wqflask/wqflask/templates/tutorials.html | 1 + 4 files changed, 292 insertions(+), 18 deletions(-) create mode 100755 scripts/maintenance/datastructure.py create mode 100644 scripts/maintenance/utilities.py diff --git a/scripts/maintenance/datastructure.py b/scripts/maintenance/datastructure.py new file mode 100755 index 00000000..9f3e8b1e --- /dev/null +++ b/scripts/maintenance/datastructure.py @@ -0,0 +1,177 @@ +import utilities + +def get_probesetfreezes(inbredsetid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT ProbeSetFreeze.`Id`, ProbeSetFreeze.`Name`, ProbeSetFreeze.`FullName` + FROM ProbeSetFreeze, ProbeFreeze + WHERE ProbeSetFreeze.`ProbeFreezeId`=ProbeFreeze.`Id` + AND ProbeFreeze.`InbredSetId`=%s + """ + cursor.execute(sql, (inbredsetid)) + return cursor.fetchall() + +def get_probesetfreeze(probesetfreezeid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT ProbeSetFreeze.`Id`, ProbeSetFreeze.`Name`, ProbeSetFreeze.`FullName` + FROM ProbeSetFreeze + WHERE ProbeSetFreeze.`Id`=%s + """ + cursor.execute(sql, (probesetfreezeid)) + return cursor.fetchone() + +def get_strains(inbredsetid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT Strain.`Id`, Strain.`Name` + FROM StrainXRef, Strain + WHERE StrainXRef.`InbredSetId`=%s + AND StrainXRef.`StrainId`=Strain.`Id` + ORDER BY StrainXRef.`OrderId` + """ + cursor.execute(sql, (inbredsetid)) + return cursor.fetchall() + +def get_inbredset(probesetfreezeid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT InbredSet.`Id`, InbredSet.`Name`, InbredSet.`FullName` + FROM InbredSet, ProbeFreeze, ProbeSetFreeze + WHERE InbredSet.`Id`=ProbeFreeze.`InbredSetId` + AND ProbeFreeze.`Id`=ProbeSetFreeze.`ProbeFreezeId` + AND ProbeSetFreeze.`Id`=%s + """ + cursor.execute(sql, (probesetfreezeid)) + return cursor.fetchone() + +def get_species(inbredsetid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT Species.`Id`, Species.`Name`, Species.`MenuName`, Species.`FullName` + FROM InbredSet, Species + WHERE InbredSet.`Id`=%s + AND InbredSet.`SpeciesId`=Species.`Id` + """ + cursor.execute(sql, (inbredsetid)) + return cursor.fetchone() + +def get_genofreeze_byinbredsetid(inbredsetid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT GenoFreeze.`Id`, GenoFreeze.`Name`, GenoFreeze.`FullName`, GenoFreeze.`InbredSetId` + FROM GenoFreeze + WHERE GenoFreeze.`InbredSetId`=%s + """ + cursor.execute(sql, (inbredsetid)) + return cursor.fetchone() + +def get_nextdataid_genotype(): + cursor, con = utilities.get_cursor() + sql = """ + SELECT GenoData.`Id` + FROM GenoData + ORDER BY GenoData.`Id` DESC + LIMIT 1 + """ + cursor.execute(sql) + re = cursor.fetchone() + dataid = re[0] + dataid += 1 + return dataid + +def get_nextdataid_phenotype(): + cursor, con = utilities.get_cursor() + sql = """ + SELECT PublishData.`Id` + FROM PublishData + ORDER BY PublishData.`Id` DESC + LIMIT 1 + """ + cursor.execute(sql) + re = cursor.fetchone() + dataid = re[0] + dataid += 1 + return dataid + +def get_nextorderid_strainxref(inbredsetid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT StrainXRef.`OrderId` + FROM StrainXRef + WHERE StrainXRef.`InbredSetId`=%s + ORDER BY StrainXRef.`OrderId` DESC + LIMIT 1 + """ + cursor.execute(sql, (inbredsetid)) + re = cursor.fetchone() + if re: + orderid = re[0] + 1 + else: + orderid = 1 + return orderid + +def insert_strain(inbredsetid, strainname): + speciesid = get_species(inbredsetid)[0] + cursor, con = utilities.get_cursor() + sql = """ + INSERT INTO Strain + SET + Strain.`Name`=%s, + Strain.`Name2`=%s, + Strain.`SpeciesId`=%s + """ + cursor.execute(sql, (strainname, strainname, speciesid)) + +def insert_strainxref(inbredsetid, strainid): + orderid = get_nextorderid_strainxref(inbredsetid) + cursor, con = utilities.get_cursor() + sql = """ + INSERT INTO StrainXRef + SET + StrainXRef.`InbredSetId`=%s, + StrainXRef.`StrainId`=%s, + StrainXRef.`OrderId`=%s, + StrainXRef.`Used_for_mapping`=%s, + StrainXRef.`PedigreeStatus`=%s + """ + cursor.execute(sql, (inbredsetid, strainid, orderid, "N", None)) + +def get_strain(inbredsetid, strainname): + speciesid = get_species(inbredsetid)[0] + cursor, con = utilities.get_cursor() + sql = """ + SELECT Strain.`Id`, Strain.`Name` + FROM Strain + WHERE Strain.`SpeciesId`=%s + AND Strain.`Name` LIKE %s + """ + cursor.execute(sql, (speciesid, strainname)) + return cursor.fetchone() + +def get_strainxref(inbredsetid, strainid): + cursor, con = utilities.get_cursor() + sql = """ + SELECT StrainXRef.`StrainId` + FROM StrainXRef + WHERE StrainXRef.`InbredSetId`=%s + AND StrainXRef.`StrainId`=%s + """ + cursor.execute(sql, (inbredsetid, strainid)) + return cursor.fetchone() + +def get_strain_sure(inbredsetid, strainname, updatestrainxref=None): + strain = get_strain(inbredsetid, strainname) + if not strain: + insert_strain(inbredsetid, strainname) + strain = get_strain(inbredsetid, strainname) + strainxref = get_strainxref(inbredsetid, strain[0]) + if not strainxref and updatestrainxref: + insert_strainxref(inbredsetid, strain[0]) + return strain + +def get_strains_bynames(inbredsetid, strainnames, updatestrainxref=None): + strains = [] + for strainname in strainnames: + strains.append(get_strain_sure(inbredsetid, strainname, updatestrainxref)) + return strains diff --git a/scripts/maintenance/load_phenotypes.py b/scripts/maintenance/load_phenotypes.py index 759d2eec..aa02d0cd 100755 --- a/scripts/maintenance/load_phenotypes.py +++ b/scripts/maintenance/load_phenotypes.py @@ -1,3 +1,11 @@ +# Load Python3 environment with GN2 utilities: +# +# source /usr/local/guix-profiles/gn-latest-20210512/etc/profile +# +# and run +# +# python load_phenotypes.py [args...] + import sys import csv @@ -9,35 +17,34 @@ def main(argv): config = utilities.get_config(argv[1]) print("config:") for item in config.items('config'): - print(("\t%s" % (str(item)))) + print("\t%s" % (str(item))) # var inbredsetid = config.get('config', 'inbredsetid') - print(("inbredsetid: %s" % inbredsetid)) + print("inbredsetid: %s" % inbredsetid) species = datastructure.get_species(inbredsetid) speciesid = species[0] - print(("speciesid: %s" % speciesid)) + print("speciesid: %s" % speciesid) dataid = datastructure.get_nextdataid_phenotype() - print(("next data id: %s" % dataid)) + print("next data id: %s" % dataid) cursor, con = utilities.get_cursor() # datafile datafile = open(config.get('config', 'datafile'), 'r') phenotypedata = csv.reader(datafile, delimiter='\t', quotechar='"') - phenotypedata_head = next(phenotypedata) - print(("phenotypedata head:\n\t%s" % phenotypedata_head)) + phenotypedata_head = phenotypedata.next() + print("phenotypedata head:\n\t%s" % phenotypedata_head) strainnames = phenotypedata_head[1:] strains = datastructure.get_strains_bynames(inbredsetid=inbredsetid, strainnames=strainnames, updatestrainxref="yes") # metafile metafile = open(config.get('config', 'metafile'), 'r') phenotypemeta = csv.reader(metafile, delimiter='\t', quotechar='"') - phenotypemeta_head = next(phenotypemeta) - print(("phenotypemeta head:\n\t%s" % phenotypemeta_head)) - print() + phenotypemeta_head = phenotypemeta.next() + print("phenotypemeta head:\n\t%s" % phenotypemeta_head) # load for metarow in phenotypemeta: # - datarow_value = next(phenotypedata) - datarow_se = next(phenotypedata) - datarow_n = next(phenotypedata) + datarow_value = phenotypedata.next() + datarow_se = phenotypedata.next() + datarow_n = phenotypedata.next() # Phenotype sql = """ INSERT INTO Phenotype @@ -67,7 +74,7 @@ def main(argv): )) rowcount = cursor.rowcount phenotypeid = con.insert_id() - print(("INSERT INTO Phenotype: %d record: %d" % (rowcount, phenotypeid))) + print("INSERT INTO Phenotype: %d record: %d" % (rowcount, phenotypeid)) # Publication publicationid = None # reset pubmed_id = utilities.to_db_string(metarow[0], None) @@ -81,7 +88,7 @@ def main(argv): re = cursor.fetchone() if re: publicationid = re[0] - print(("get Publication record: %d" % publicationid)) + print("get Publication record: %d" % publicationid) if not publicationid: sql = """ INSERT INTO Publication @@ -109,7 +116,7 @@ def main(argv): )) rowcount = cursor.rowcount publicationid = con.insert_id() - print(("INSERT INTO Publication: %d record: %d" % (rowcount, publicationid))) + print("INSERT INTO Publication: %d record: %d" % (rowcount, publicationid)) # data for index, strain in enumerate(strains): # @@ -158,14 +165,14 @@ def main(argv): cursor.execute(sql, (inbredsetid, phenotypeid, publicationid, dataid, "")) rowcount = cursor.rowcount publishxrefid = con.insert_id() - print(("INSERT INTO PublishXRef: %d record: %d" % (rowcount, publishxrefid))) + print("INSERT INTO PublishXRef: %d record: %d" % (rowcount, publishxrefid)) # for loop next dataid += 1 - print() + print # release con.close() if __name__ == "__main__": - print(("command line arguments:\n\t%s" % sys.argv)) + print("command line arguments:\n\t%s" % sys.argv) main(sys.argv) print("exit successfully") diff --git a/scripts/maintenance/utilities.py b/scripts/maintenance/utilities.py new file mode 100644 index 00000000..886410c2 --- /dev/null +++ b/scripts/maintenance/utilities.py @@ -0,0 +1,89 @@ +import MySQLdb +import re +import configparser + +def get_cursor(): + host = 'tux.uthsc.edu' + user = 'webqtlout' + passwd = 'webqtlout' + db = 'db_webqtl' + con = MySQLdb.Connect(db=db, host=host, user=user, passwd=passwd) + cursor = con.cursor() + return cursor, con + +def clearspaces(s, default=None): + if s: + s = re.sub('\s+', ' ', s) + s = s.strip() + return s + else: + return default + +def to_dic(keys, values): + dic = {} + for i in range(len(keys)): + key = keys[i] + value = values[i] + dic[key] = value + return dic + +def overlap(dic1, dic2): + keys = [] + values1 = [] + values2 = [] + for key in dic1.keys(): + if key in dic2: + value1 = dic1[key] + value2 = dic2[key] + if value1 and value2: + keys.append(key) + values1.append(value1) + values2.append(value2) + return keys, values1, values2 + +def to_db_string(s, default): + if s: + s = s.strip() + if len(s) == 0: + return default + elif s == 'x': + return default + else: + return s + else: + return default + +def to_db_float(s, default): + if s: + s = s.strip() + if len(s) == 0: + return default + elif s == 'x': + return default + else: + try: + return float(s) + except: + return default + else: + return default + +def to_db_int(s, default): + if s: + s = s.strip() + if len(s) == 0: + return default + elif s == 'x': + return default + else: + try: + return int(s) + except: + return default + else: + return default + +def get_config(configfile): + config = configparser.ConfigParser() + config.read(configfile) + return config diff --git a/wqflask/wqflask/templates/tutorials.html b/wqflask/wqflask/templates/tutorials.html index ce5d0e3d..ed1b6f3a 100644 --- a/wqflask/wqflask/templates/tutorials.html +++ b/wqflask/wqflask/templates/tutorials.html @@ -10,6 +10,7 @@
      • Statistical and genetic functions, and initial mapping results for Rat GWAS P50 as implemented in GeneNetwork.org
      • Webinar Series - Quantitative Genetics Tools for Mapping Trait Variation to Mechanisms, Therapeutics, and Interventions
        • +
      • TESTING

      -- cgit v1.2.3 From 413425b8ed3c69947b49ca366b210d69d2173805 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 13 May 2021 07:43:10 +0300 Subject: add dataset and trait-name in ui template --- wqflask/wqflask/correlation/correlation_gn3_api.py | 103 +++++++++++++-------- .../wqflask/templates/test_correlation_page.html | 5 +- wqflask/wqflask/views.py | 4 +- 3 files changed, 68 insertions(+), 44 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 98d52591..e5638b5a 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -41,27 +41,33 @@ def process_samples(start_vars, sample_names, excluded_samples=None): return sample_data +def merge_correlation_results(correlation_results, target_correlation_results): + + corr_dict = {} + + for trait_dict in target_correlation_results: + for trait_name, values in trait_dict.items(): + + corr_dict[trait_name] = values + for trait_dict in correlation_results: + for trait_name, values in trait_dict.items(): + + if corr_dict.get(trait_name): + + trait_dict[trait_name].update(corr_dict.get(trait_name)) + + return correlation_results + + def sample_for_trait_lists(corr_results, target_dataset, this_trait, this_dataset, start_vars): """interface function for correlation on top results""" - sample_data = process_samples( - start_vars, this_dataset.group.samplelist) - target_dataset.get_trait_data(list(sample_data.keys())) - # should filter target traits from here - _corr_results = corr_results - - this_trait = retrieve_sample_data(this_trait, this_dataset) - - this_trait_data = { - "trait_sample_data": sample_data, - "trait_id": start_vars["trait_id"] - } - results = map_shared_keys_to_values( - target_dataset.samplelist, target_dataset.trait_data) + (this_trait_data, target_dataset) = fetch_sample_data( + start_vars, this_trait, this_dataset, target_dataset) correlation_results = compute_all_sample_correlation(corr_method="pearson", this_trait=this_trait_data, - target_dataset=results) + target_dataset=target_dataset) return correlation_results @@ -105,6 +111,23 @@ def lit_for_trait_list(corr_results, this_dataset, this_trait): return correlation_results +def fetch_sample_data(start_vars, this_trait, this_dataset, target_dataset): + + sample_data = process_samples( + start_vars, this_dataset.group.samplelist) + target_dataset.get_trait_data(list(sample_data.keys())) + this_trait = retrieve_sample_data(this_trait, this_dataset) + this_trait_data = { + "trait_sample_data": sample_data, + "trait_id": start_vars["trait_id"] + } + + results = map_shared_keys_to_values( + target_dataset.samplelist, target_dataset.trait_data) + + return (this_trait_data, results) + + def compute_correlation(start_vars, method="pearson"): """compute correlation for to call gn3 api""" # pylint: disable-msg=too-many-locals @@ -119,31 +142,11 @@ def compute_correlation(start_vars, method="pearson"): corr_input_data = {} if corr_type == "sample": - - sample_data = process_samples( - start_vars, this_dataset.group.samplelist) - target_dataset.get_trait_data(list(sample_data.keys())) - this_trait = retrieve_sample_data(this_trait, this_dataset) - this_trait_data = { - "trait_sample_data": sample_data, - "trait_id": start_vars["trait_id"] - } - results = map_shared_keys_to_values( - target_dataset.samplelist, target_dataset.trait_data) + (this_trait_data, target_dataset) = fetch_sample_data( + start_vars, this_trait, this_dataset, target_dataset) correlation_results = compute_all_sample_correlation(corr_method=method, this_trait=this_trait_data, - target_dataset=results) - - # do tissue correaltion - - # code to be use later - - # tissue_result = tissue_for_trait_lists( - # correlation_results, this_dataset, this_trait) - # # lit spoils the party so slow - # lit_result = lit_for_trait_list( - # correlation_results, this_dataset, this_trait) - + target_dataset=target_dataset) elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") @@ -172,7 +175,29 @@ def compute_correlation(start_vars, method="pearson"): conn=conn, trait_lists=list(geneid_dict.items()), species=species, gene_id=this_trait_geneid) - return correlation_results[0:corr_return_results] + # correlation_results = correlation_results[0:corr_return_results] + # if corr_type != "tissue" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": + # pass + # # tissue_result = tissue_for_trait_lists( + # # correlation_results, this_dataset, this_trait) + + # # correlation_results = merge_correlation_results( + # # correlation_results, tissue_result) + + # if corr_type != "lit" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": + # pass + # # lit is very slow + # # lit_result = lit_for_trait_list(correlation_results, this_dataset, this_trait) + # # correlation_results = merge_correlation_results(correlation_results,lit_result) + # if corr_type != "sample": + # # do sample correlation + # pass + correlation_results = correlation_results[0:corr_return_results] + correlation_data = {"correlation_results": correlation_results, + "this_trait": this_trait.name, + "target_dataset": start_vars['corr_dataset']} + + return correlation_data def do_lit_correlation(this_trait, this_dataset): diff --git a/wqflask/wqflask/templates/test_correlation_page.html b/wqflask/wqflask/templates/test_correlation_page.html index 037e9735..186de4b7 100644 --- a/wqflask/wqflask/templates/test_correlation_page.html +++ b/wqflask/wqflask/templates/test_correlation_page.html @@ -42,7 +42,7 @@ {% block content %}
      -

      Correlation Results for Dataset_name against trait_name for the top all Results

      +

      Correlation Results for {{target_dataset}} against {{this_trait}} for the top all Results

      Toggle Columns

      @@ -84,7 +84,6 @@ diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3c875163..4834ee63 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -972,8 +972,8 @@ def corr_compute_page(): @app.route("/test_corr_compute", methods=["POST"]) def test_corr_compute_page(): - correlation_results = compute_correlation(request.form) - return render_template("test_correlation_page.html", correlation_results=correlation_results) + correlation_data = compute_correlation(request.form) + return render_template("test_correlation_page.html", **correlation_data) @app.route("/corr_matrix", methods=('POST',)) def corr_matrix_page(): -- cgit v1.2.3 From 75b4e74dede48c59e07d8f15d2db4b5d6289c064 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 13 May 2021 08:25:42 +0300 Subject: integrate comute corr for top results in ui --- wqflask/wqflask/correlation/correlation_gn3_api.py | 68 ++++++++++++++++------ .../wqflask/templates/test_correlation_page.html | 5 +- 2 files changed, 54 insertions(+), 19 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index e5638b5a..dec84728 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -137,16 +137,19 @@ def compute_correlation(start_vars, method="pearson"): (this_dataset, this_trait, target_dataset, sample_data) = create_target_this_trait(start_vars) + target_dataset_type = target_dataset.type + this_dataset_type = this_dataset.type + method = start_vars['corr_sample_method'] corr_return_results = int(start_vars.get("corr_return_results", 100)) corr_input_data = {} if corr_type == "sample": - (this_trait_data, target_dataset) = fetch_sample_data( + (this_trait_data, target_dataset_data) = fetch_sample_data( start_vars, this_trait, this_dataset, target_dataset) correlation_results = compute_all_sample_correlation(corr_method=method, this_trait=this_trait_data, - target_dataset=target_dataset) + target_dataset=target_dataset_data) elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") @@ -175,31 +178,62 @@ def compute_correlation(start_vars, method="pearson"): conn=conn, trait_lists=list(geneid_dict.items()), species=species, gene_id=this_trait_geneid) - # correlation_results = correlation_results[0:corr_return_results] - # if corr_type != "tissue" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": - # pass - # # tissue_result = tissue_for_trait_lists( - # # correlation_results, this_dataset, this_trait) + correlation_results = correlation_results[0:corr_return_results] - # # correlation_results = merge_correlation_results( - # # correlation_results, tissue_result) + compute_all = True + + correlation_results = compute_corr_for_top_results(correlation_results, + this_trait, + this_dataset, + target_dataset, + corr_type) + + # if corr_type != "tissue" and this_dataset_type == "ProbeSet" and target_dataset_type == "ProbeSet": + + # tissue_result = tissue_for_trait_lists( + # correlation_results, this_dataset, this_trait) + + # if corr_type != "lit" and this_dataset_type == "ProbeSet" and target_dataset_type == "ProbeSet": + # lit_result = lit_for_trait_list( + # correlation_results, this_dataset, this_trait) - # if corr_type != "lit" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": - # pass - # # lit is very slow - # # lit_result = lit_for_trait_list(correlation_results, this_dataset, this_trait) - # # correlation_results = merge_correlation_results(correlation_results,lit_result) # if corr_type != "sample": - # # do sample correlation # pass - correlation_results = correlation_results[0:corr_return_results] + correlation_data = {"correlation_results": correlation_results, "this_trait": this_trait.name, - "target_dataset": start_vars['corr_dataset']} + "target_dataset": start_vars['corr_dataset'], + "return_results": corr_return_results} return correlation_data +def compute_corr_for_top_results(correlation_results, + this_trait, + this_dataset, + target_dataset, + corr_type): + if corr_type != "tissue" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": + + tissue_result = tissue_for_trait_lists( + correlation_results, this_dataset, this_trait) + + correlation_results = merge_correlation_results( + correlation_results, tissue_result) + + if corr_type != "lit" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": + lit_result = lit_for_trait_list( + correlation_results, this_dataset, this_trait) + + correlation_results = merge_correlation_results( + correlation_results, lit_result) + + if corr_type != "sample": + pass + + return correlation_results + + def do_lit_correlation(this_trait, this_dataset): """function for fetching lit inputs""" geneid_dict = this_dataset.retrieve_genes("GeneId") diff --git a/wqflask/wqflask/templates/test_correlation_page.html b/wqflask/wqflask/templates/test_correlation_page.html index 186de4b7..0809b65e 100644 --- a/wqflask/wqflask/templates/test_correlation_page.html +++ b/wqflask/wqflask/templates/test_correlation_page.html @@ -42,7 +42,7 @@ {% block content %}
      -

      Correlation Results for {{target_dataset}} against {{this_trait}} for the top all Results

      +

      Correlation Results for {{target_dataset}} against {{this_trait}} for the top {{return_results}} Results

      Toggle Columns

      @@ -137,7 +137,8 @@ console.log(correlationResults) { "targets":2, "render":(data,type,row)=>{ - let urlLink = "/show_trait?trait_id=1453207_at&dataset=HC_M2_0606_P" + // should use a dynamic dataset name + let urlLink = `/show_trait?trait_id=${data}&dataset=HC_M2_0606_P` let traitLink = `${data}` return traitLink }, -- cgit v1.2.3 From 01582f853420c6775bb891675fd2039a7fd5b3b8 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Thu, 13 May 2021 08:28:15 +0300 Subject: pep8 formatting & delete comments --- wqflask/wqflask/correlation/correlation_gn3_api.py | 24 +++++++--------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index dec84728..46202ca3 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -180,25 +180,15 @@ def compute_correlation(start_vars, method="pearson"): correlation_results = correlation_results[0:corr_return_results] - compute_all = True + compute_all = True # later to be passed as argument - correlation_results = compute_corr_for_top_results(correlation_results, - this_trait, - this_dataset, - target_dataset, - corr_type) + if (compute_all): - # if corr_type != "tissue" and this_dataset_type == "ProbeSet" and target_dataset_type == "ProbeSet": - - # tissue_result = tissue_for_trait_lists( - # correlation_results, this_dataset, this_trait) - - # if corr_type != "lit" and this_dataset_type == "ProbeSet" and target_dataset_type == "ProbeSet": - # lit_result = lit_for_trait_list( - # correlation_results, this_dataset, this_trait) - - # if corr_type != "sample": - # pass + correlation_results = compute_corr_for_top_results(correlation_results, + this_trait, + this_dataset, + target_dataset, + corr_type) correlation_data = {"correlation_results": correlation_results, "this_trait": this_trait.name, -- cgit v1.2.3 From 1c2417fb3a8d8eb7087ab77a811aa790a9e2eb5b Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 14 May 2021 18:36:20 +0000 Subject: Changed logic in has_num_cases to ignore N=1 --- wqflask/wqflask/show_trait/show_trait.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 424b8ee5..837c7a54 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -631,7 +631,7 @@ def has_num_cases(this_trait): has_n = False if this_trait.dataset.type != "ProbeSet" and this_trait.dataset.type != "Geno": for name, sample in list(this_trait.data.items()): - if sample.num_cases: + if sample.num_cases and sample.num_cases != "1": has_n = True break -- cgit v1.2.3 From 95c585eaba3101033143370f536e840228e114f0 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 14 May 2021 18:37:09 +0000 Subject: Changed logic in SampleList to set num_cases_exists to False if all existing N are 1 --- wqflask/wqflask/show_trait/SampleList.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index f9d30dba..67768308 100644 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -8,7 +8,6 @@ from pprint import pformat as pf from utility import Plot from utility import Bunch - class SampleList: def __init__(self, dataset, @@ -92,8 +91,10 @@ class SampleList: self.sample_list.append(sample) self.se_exists = any(sample.variance for sample in self.sample_list) - self.num_cases_exists = any( - sample.num_cases for sample in self.sample_list) + self.num_cases_exists = False + if (any(sample.num_cases for sample in self.sample_list) and + any((sample.num_cases and sample.num_cases != "1") for sample in self.sample_list)): + self.num_cases_exists = True first_attr_col = self.get_first_attr_col() for sample in self.sample_list: -- cgit v1.2.3 From 2ddc4c1d3216e400a4bed191b1a2a469512c2535 Mon Sep 17 00:00:00 2001 From: Alexander Kabui Date: Sat, 15 May 2021 00:57:19 +0300 Subject: handle None for tissue input data --- wqflask/wqflask/correlation/correlation_gn3_api.py | 31 ++++++++++++++-------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py index 46202ca3..6974dbd5 100644 --- a/wqflask/wqflask/correlation/correlation_gn3_api.py +++ b/wqflask/wqflask/correlation/correlation_gn3_api.py @@ -80,13 +80,16 @@ def tissue_for_trait_lists(corr_results, this_dataset, this_trait): traits_symbol_dict = this_dataset.retrieve_genes("Symbol") traits_symbol_dict = dict({trait_name: symbol for ( trait_name, symbol) in traits_symbol_dict.items() if trait_lists.get(trait_name)}) - primary_tissue_data, target_tissue_data = get_tissue_correlation_input( + tissue_input = get_tissue_correlation_input( this_trait, traits_symbol_dict) - corr_results = compute_tissue_correlation( - primary_tissue_dict=primary_tissue_data, - target_tissues_data=target_tissue_data, - corr_method="pearson") - return corr_results + + if tissue_input is not None: + (primary_tissue_data, target_tissue_data) = tissue_input + corr_results = compute_tissue_correlation( + primary_tissue_dict=primary_tissue_data, + target_tissues_data=target_tissue_data, + corr_method="pearson") + return corr_results def lit_for_trait_list(corr_results, this_dataset, this_trait): @@ -153,9 +156,12 @@ def compute_correlation(start_vars, method="pearson"): elif corr_type == "tissue": trait_symbol_dict = this_dataset.retrieve_genes("Symbol") - primary_tissue_data, target_tissue_data = get_tissue_correlation_input( + tissue_input = get_tissue_correlation_input( this_trait, trait_symbol_dict) + if tissue_input is not None: + (primary_tissue_data, target_tissue_data) = tissue_input + corr_input_data = { "primary_tissue": primary_tissue_data, "target_tissues_dict": target_tissue_data @@ -208,15 +214,18 @@ def compute_corr_for_top_results(correlation_results, tissue_result = tissue_for_trait_lists( correlation_results, this_dataset, this_trait) - correlation_results = merge_correlation_results( - correlation_results, tissue_result) + if tissue_result: + + correlation_results = merge_correlation_results( + correlation_results, tissue_result) if corr_type != "lit" and this_dataset.type == "ProbeSet" and target_dataset.type == "ProbeSet": lit_result = lit_for_trait_list( correlation_results, this_dataset, this_trait) - correlation_results = merge_correlation_results( - correlation_results, lit_result) + if lit_result: + correlation_results = merge_correlation_results( + correlation_results, lit_result) if corr_type != "sample": pass -- cgit v1.2.3 From 687754de205f127bf8a5eeb2204974d0462475b4 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 May 2021 23:22:07 +0000 Subject: Fixed issue that caused the sample values in the Haplotype Analyst to be off by 1 --- wqflask/wqflask/marker_regression/display_mapping_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 5bf8822a..48f6f308 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -1771,7 +1771,7 @@ class DisplayMappingResults: if (plotbxd == 1): ind = [item.name for item in smd].index( - samplelist[j]) - 1 + samplelist[j]) expr = smd[ind].value # Place where font is hardcoded -- cgit v1.2.3 From 0149b5da461945350be2fe9b95d3c42cf4211f5b Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 18 May 2021 23:28:46 +0000 Subject: Last commit failed to correctly change Y-axis position; this should fix that --- wqflask/wqflask/marker_regression/display_mapping_results.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py index 48f6f308..ec17d3b0 100644 --- a/wqflask/wqflask/marker_regression/display_mapping_results.py +++ b/wqflask/wqflask/marker_regression/display_mapping_results.py @@ -1771,8 +1771,8 @@ class DisplayMappingResults: if (plotbxd == 1): ind = [item.name for item in smd].index( - samplelist[j]) - expr = smd[ind].value + samplelist[j]) - 1 + expr = smd[ind+1].value # Place where font is hardcoded im_drawer.text( -- cgit v1.2.3 From c2fac869f5ac1a398677d5646d34a1a0704e29e4 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 20 May 2021 21:44:12 +0000 Subject: Re-implemented R/qtl to get the results from the GN3 API --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 569 +++++----------------- wqflask/wqflask/marker_regression/run_mapping.py | 6 +- 2 files changed, 135 insertions(+), 440 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 1c8477bf..3f4899b0 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -1,460 +1,155 @@ -import rpy2.robjects as ro -import rpy2.robjects.numpy2ri as np2r -import numpy as np -import json - -from flask import g +import csv +import hashlib +import io +import requests +import shutil +from typing import Dict +from typing import List +from typing import Optional +from typing import TextIO from base.webqtlConfig import TMPDIR from base.trait import create_trait -from base.data_set import create_dataset -from utility import webqtlUtil -from utility.tools import locate, TEMPDIR -from wqflask.marker_regression.gemma_mapping import generate_random_n_string -from flask import g +from utility.tools import locate import utility.logger logger = utility.logger.getLogger(__name__) -# Get a trait's type (numeric, categorical, etc) from the DB - - -def get_trait_data_type(trait_db_string): - logger.info("get_trait_data_type") - the_query = "SELECT value FROM TraitMetadata WHERE type='trait_data_type'" - logger.info("the_query done") - results_json = g.db.execute(the_query).fetchone() - logger.info("the_query executed") - results_ob = json.loads(results_json[0]) - logger.info("json results loaded") - if trait_db_string in results_ob: - logger.info("found") - return results_ob[trait_db_string] - else: - logger.info("not found") - return "numeric" - +GN3_RQTL_URL = "http://localhost:8086/api/rqtl/compute" +GN3_TMP_PATH = "/export/local/home/zas1024/genenetwork3/tmp" -# Run qtl mapping using R/qtl -def run_rqtl_geno(vals, samples, dataset, mapping_scale, method, model, permCheck, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, pair_scan, cofactors): - logger.info("Start run_rqtl_geno") - # Get pointers to some common R functions - r_library = ro.r["library"] # Map the library function - r_c = ro.r["c"] # Map the c function - plot = ro.r["plot"] # Map the plot function - png = ro.r["png"] # Map the png function - dev_off = ro.r["dev.off"] # Map the device off function +def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, method, model, permCheck, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, pair_scan, cofactors): + """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" - print((r_library("qtl"))) # Load R/qtl - - logger.info("QTL library loaded") - - # Get pointers to some R/qtl functions - scanone = ro.r["scanone"] # Map the scanone function - scantwo = ro.r["scantwo"] # Map the scantwo function - # Map the calc.genoprob function - calc_genoprob = ro.r["calc.genoprob"] - - crossname = dataset.group.name - # try: - # generate_cross_from_rdata(dataset) - # read_cross_from_rdata = ro.r["generate_cross_from_rdata"] # Map the local read_cross_from_rdata function - # genofilelocation = locate(crossname + ".RData", "genotype/rdata") - # cross_object = read_cross_from_rdata(genofilelocation) # Map the local GENOtoCSVR function - # except: - - if mapping_scale == "morgan": - scale_units = "cM" - else: - scale_units = "Mb" - - generate_cross_from_geno(dataset, scale_units) - # Map the local GENOtoCSVR function - GENOtoCSVR = ro.r["GENOtoCSVR"] - crossfilelocation = TMPDIR + crossname + ".cross" + pheno_file = write_phenotype_file(trait_name, samples, vals, cofactors) if dataset.group.genofile: - genofilelocation = locate(dataset.group.genofile, "genotype") - else: - genofilelocation = locate(dataset.group.name + ".geno", "genotype") - logger.info("Going to create a cross from geno") - # TODO: Add the SEX if that is available - cross_object = GENOtoCSVR(genofilelocation, crossfilelocation) - logger.info("before calc_genoprob") - if manhattan_plot: - cross_object = calc_genoprob(cross_object) + geno_file = locate(dataset.group.genofile, "genotype") else: - cross_object = calc_genoprob(cross_object, step=5, stepwidth="max") - logger.info("after calc_genoprob") - - pheno_string = sanitize_rqtl_phenotype(vals) - logger.info("phenostring done") - names_string = sanitize_rqtl_names(samples) - logger.info("sanitized pheno and names") - # Add the phenotype - cross_object = add_phenotype(cross_object, pheno_string, "the_pheno") - # Add the phenotype - cross_object = add_names(cross_object, names_string, "the_names") - logger.info("Added pheno and names") - # Create the additive covariate markers - marker_covars = create_marker_covariates(control_marker, cross_object) - logger.info("Marker covars done") - if cofactors != "": - logger.info("Cofactors: " + cofactors) - # Create the covariates from selected traits - cross_object, trait_covars = add_cofactors( - cross_object, dataset, cofactors, samples) - ro.r('all_covars <- cbind(marker_covars, trait_covars)') - else: - ro.r('all_covars <- marker_covars') - covars = ro.r['all_covars'] - # DEBUG to save the session object to file - if pair_scan: - if do_control == "true": - logger.info("Using covariate") - result_data_frame = scantwo( - cross_object, pheno="the_pheno", addcovar=covars, model=model, method=method, n_cluster=16) - else: - logger.info("No covariates") - result_data_frame = scantwo( - cross_object, pheno="the_pheno", model=model, method=method, n_cluster=16) - - pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png" - png(file=TEMPDIR + pair_scan_filename) - plot(result_data_frame) - dev_off() - - return process_pair_scan_results(result_data_frame) - else: - if do_control == "true" or cofactors != "": - logger.info("Using covariate") - ro.r(f"qtl_results = scanone(the_cross, pheno='the_pheno', addcovar=all_covars, model='{model}', method='{method}')") - result_data_frame = ro.r("qtl_results") - else: - ro.r(f"qtl_results = scanone(the_cross, pheno='the_pheno', model='{model}', method='{method}')") - result_data_frame = np.asarray(ro.r("qtl_results")).T + geno_file = locate(dataset.group.name + ".geno", "genotype") - marker_names = np.asarray(ro.r("row.names(qtl_results)")) + post_data = { + "pheno_file": pheno_file, + "geno_file": geno_file, + "model": model, + "method": method, + "nperm": num_perm, + "scale": mapping_scale + } - # Do permutation (if requested by user) - if num_perm > 0 and permCheck == "ON": - # ZS: The strata list would only be populated if "Stratified" was checked on before mapping - if len(perm_strata_list) > 0: - cross_object, strata_ob = add_perm_strata( - cross_object, perm_strata_list) - - if do_control == "true" or cofactors != "": - perm_data_frame = scanone(cross_object, pheno_col="the_pheno", addcovar=covars, n_perm=int( - num_perm), perm_strata=strata_ob, model=model, method=method) - else: - perm_data_frame = scanone( - cross_object, pheno_col="the_pheno", n_perm=num_perm, perm_strata=strata_ob, model=model, method=method) - else: - if do_control == "true" or cofactors != "": - perm_data_frame = scanone(cross_object, pheno_col="the_pheno", addcovar=covars, n_perm=int( - num_perm), model=model, method=method) - else: - perm_data_frame = scanone( - cross_object, pheno_col="the_pheno", n_perm=num_perm, model=model, method=method) - - # Functions that sets the thresholds for the webinterface - perm_output, suggestive, significant = process_rqtl_perm_results( - num_perm, perm_data_frame) - return perm_output, suggestive, significant, process_rqtl_results(marker_names, result_data_frame, dataset.group.species) - else: - return process_rqtl_results(marker_names, result_data_frame, dataset.group.species) - - -def generate_cross_from_rdata(dataset): - rdata_location = locate(dataset.group.name + ".RData", "genotype/rdata") - ro.r(""" - generate_cross_from_rdata <- function(filename = '%s') { - load(file=filename) - cross = cunique - return(cross) - } - """ % (rdata_location)) - - -# TODO: Need to figure out why some genofiles have the wrong format and don't convert properly -def generate_cross_from_geno(dataset, scale_units): - - cross_filename = (f"{str(dataset.group.name)}_" - f"{generate_random_n_string(6)}") - - ro.r(""" - trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) } - getGenoCode <- function(header, name = 'unk'){ - mat = which(unlist(lapply(header,function(x){ length(grep(paste('@',name,sep=''), x)) })) == 1) - return(trim(strsplit(header[mat],':')[[1]][2])) - } - GENOtoCSVR <- function(genotypes = '%s', out = '%s.csvr', phenotype = NULL, sex = NULL, verbose = FALSE){ - header = readLines(genotypes, 40) # Assume a geno header is not longer than 40 lines - toskip = which(unlist(lapply(header, function(x){ length(grep("Chr\t", x)) })) == 1)-1 # Major hack to skip the geno headers - type <- getGenoCode(header, 'type') - if(type == '4-way'){ - genocodes <- NULL - } else { - genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat')) # Get the genotype codes - } - genodata <- read.csv(genotypes, sep='\t', skip=toskip, header=TRUE, na.strings=getGenoCode(header,'unk'), colClasses='character', comment.char = '#') - cat('Genodata:', toskip, " ", dim(genodata), genocodes, '\n') - if(is.null(phenotype)) phenotype <- runif((ncol(genodata)-4)) # If there isn't a phenotype, generate a random one - if(is.null(sex)) sex <- rep('m', (ncol(genodata)-4)) # If there isn't a sex phenotype, treat all as males - outCSVR <- rbind(c('Pheno', '', '', phenotype), # Phenotype - c('sex', '', '', sex), # Sex phenotype for the mice - cbind(genodata[,c('Locus','Chr', '%s')], genodata[, 5:ncol(genodata)])) # Genotypes - write.table(outCSVR, file = out, row.names=FALSE, col.names=FALSE,quote=FALSE, sep=',') # Save it to a file - require(qtl) - if(type == '4-way'){ - cat('Loading in as 4-WAY\n') - cross = read.cross(file=out, 'csvr', genotypes=NULL, crosstype="4way") # Load the created cross file using R/qtl read.cross - }else if(type == 'f2'){ - cat('Loading in as F2\n') - cross = read.cross(file=out, 'csvr', genotypes=genocodes, crosstype="f2") # Load the created cross file using R/qtl read.cross - }else{ - cat('Loading in as normal\n') - cross = read.cross(file=out, 'csvr', genotypes=genocodes) # Load the created cross file using R/qtl read.cross - } - if(type == 'riset'){ - cat('Converting to RISELF\n') - cross <- convert2riself(cross) # If its a RIL, convert to a RIL in R/qtl - } - return(cross) - } - """ % (dataset.group.genofile, cross_filename, scale_units)) + if do_control == "true" and control_marker: + post_data["control_marker"] = control_marker + if not manhattan_plot: + post_data["interval"] = True + if cofactors: + post_data["addcovar"] = True -def add_perm_strata(cross, perm_strata): - col_string = 'c("the_strata")' - perm_strata_string = "c(" - for item in perm_strata: - perm_strata_string += str(item) + "," + out_file = requests.post(GN3_RQTL_URL, data=post_data).json()['output_file'] - perm_strata_string = perm_strata_string[:-1] + ")" + return process_rqtl_results(out_file) - cross = add_phenotype(cross, perm_strata_string, "the_strata") - strata_ob = pull_var("perm_strata", cross, col_string) +def process_rqtl_results(out_file: str) -> List: + """Given the output filename, read in results and + return as a list of dictionaries representing each + marker - return cross, strata_ob + """ - -def sanitize_rqtl_phenotype(vals): - pheno_as_string = "c(" - for i, val in enumerate(vals): - if val == "x": - if i < (len(vals) - 1): - pheno_as_string += "NA," + marker_obs = [] + # Later I should probably redo this using csv.read to avoid the + # awkwardness with removing quotes with [1:-1] + with open(GN3_TMP_PATH + "/output/" + out_file, "r") as the_file: + for line in the_file: + line_items = line.split(",") + if line_items[1][1:-1] == "chr" or not line_items: + continue else: - pheno_as_string += "NA" + # Convert chr to int if possible + try: + the_chr = int(line_items[1][1:-1]) + except: + the_chr = line_items[1][1:-1] + this_marker = { + "name": line_items[0][1:-1], + "chr": the_chr, + "cM": float(line_items[2]), + "Mb": float(line_items[2]), + "lod_score": float(line_items[3]) + } + marker_obs.append(this_marker) + + return marker_obs + +def get_hash_of_textio(the_file: TextIO) -> str: + """Given a StringIO, return the hash of its contents""" + + the_file.seek(0) + hash_of_file = hashlib.md5(the_file.read().encode()).hexdigest() + + return hash_of_file + + +def write_phenotype_file(trait_name: str, + samples: List[str], + vals: List, + dataset_ob, + cofactors: Optional[str] = None) -> TextIO: + """Given trait name, sample list, value list, dataset ob, and optional string + representing cofactors, return the file's full path/name + + """ + + cofactor_data = cofactors_to_dict(cofactors, dataset_ob, samples) + + pheno_file = io.StringIO() + writer = csv.writer(pheno_file, delimiter="\t", quoting=csv.QUOTE_NONE) + + header_row = ["Samples", trait_name] + header_row += [cofactor for cofactor in cofactor_data] + + writer.writerow(header_row) + for i, sample in enumerate(samples): + this_row = [sample] + if vals[i] != "x": + this_row.append(vals[i]) else: - if i < (len(vals) - 1): - pheno_as_string += str(val) + "," - else: - pheno_as_string += str(val) - pheno_as_string += ")" - - return pheno_as_string - - -def sanitize_rqtl_names(vals): - pheno_as_string = "c(" - for i, val in enumerate(vals): - if val == "x": - if i < (len(vals) - 1): - pheno_as_string += "NA," - else: - pheno_as_string += "NA" - else: - if i < (len(vals) - 1): - pheno_as_string += "'" + str(val) + "'," - else: - pheno_as_string += "'" + str(val) + "'" - pheno_as_string += ")" - - return pheno_as_string - - -def add_phenotype(cross, pheno_as_string, col_name): - ro.globalenv["the_cross"] = cross - ro.r('pheno <- data.frame(pull.pheno(the_cross))') - ro.r('the_cross$pheno <- cbind(pheno, ' + col_name + \ - ' = as.numeric(' + pheno_as_string + '))') - return ro.r["the_cross"] - - -def add_categorical_covar(cross, covar_as_string, i): - ro.globalenv["the_cross"] = cross - logger.info("cross set") - ro.r('covar <- as.factor(' + covar_as_string + ')') - logger.info("covar set") - ro.r('newcovar <- model.matrix(~covar-1)') - logger.info("model.matrix finished") - ro.r('cat("new covar columns", ncol(newcovar), "\n")') - nCol = ro.r('ncol(newcovar)') - logger.info("ncol covar done: " + str(nCol[0])) - ro.r('pheno <- data.frame(pull.pheno(the_cross))') - logger.info("pheno pulled from cross") - nCol = int(nCol[0]) - logger.info("nCol python int:" + str(nCol)) - col_names = [] - # logger.info("loop") - for x in range(1, (nCol + 1)): - #logger.info("loop" + str(x)); - col_name = "covar_" + str(i) + "_" + str(x) - #logger.info("col_name" + col_name); - ro.r('the_cross$pheno <- cbind(pheno, ' + \ - col_name + ' = newcovar[,' + str(x) + '])') - col_names.append(col_name) - #logger.info("loop" + str(x) + "done"); - - logger.info("returning from add_categorical_covar") - return ro.r["the_cross"], col_names - - -def add_names(cross, names_as_string, col_name): - ro.globalenv["the_cross"] = cross - ro.r('pheno <- data.frame(pull.pheno(the_cross))') - ro.r('the_cross$pheno <- cbind(pheno, ' + \ - col_name + ' = ' + names_as_string + ')') - return ro.r["the_cross"] - - -def pull_var(var_name, cross, var_string): - ro.globalenv["the_cross"] = cross - ro.r(var_name + ' <- pull.pheno(the_cross, ' + var_string + ')') - - return ro.r[var_name] - - -def add_cofactors(cross, this_dataset, covariates, samples): - ro.numpy2ri.activate() - - covariate_list = covariates.split(",") - covar_name_string = "c(" - for i, covariate in enumerate(covariate_list): - logger.info("Covariate: " + covariate) - this_covar_data = [] - covar_as_string = "c(" - trait_name = covariate.split(":")[0] - dataset_ob = create_dataset(covariate.split(":")[1]) - trait_ob = create_trait(dataset=dataset_ob, - name=trait_name, - cellid=None) - - this_dataset.group.get_samplelist() - trait_samples = this_dataset.group.samplelist - trait_sample_data = trait_ob.data - for index, sample in enumerate(samples): - if sample in trait_samples: - if sample in trait_sample_data: - sample_value = trait_sample_data[sample].value - this_covar_data.append(sample_value) - else: - this_covar_data.append("NA") - - for j, item in enumerate(this_covar_data): - if j < (len(this_covar_data) - 1): - covar_as_string += str(item) + "," - else: - covar_as_string += str(item) - - covar_as_string += ")" - - datatype = get_trait_data_type(covariate) - logger.info("Covariate: " + covariate + " is of type: " + datatype) - if(datatype == "categorical"): # Cat variable - logger.info("call of add_categorical_covar") - cross, col_names = add_categorical_covar( - cross, covar_as_string, i) # Expand and add it to the cross - logger.info("add_categorical_covar returned") - # Go through the additional covar names - for z, col_name in enumerate(col_names): - if i < (len(covariate_list) - 1): - covar_name_string += '"' + col_name + '", ' - else: - if(z < (len(col_names) - 1)): - covar_name_string += '"' + col_name + '", ' + this_row.append("NA") + for cofactor in cofactor_data: + this_row.append(cofactor_data[cofactor][i]) + writer.writerow(this_row) + + hash_of_file = get_hash_of_textio(pheno_file) + file_path = TMPDIR + hash_of_file + ".csv" + + with open(file_path, "w") as fd: + pheno_file.seek(0) + shutil.copyfileobj(pheno_file, fd) + + return file_path + + +def cofactors_to_dict(cofactors: str, dataset_ob, samples) -> Dict: + """Given a string of cofactors, the trait being mapped's dataset ob, + and list of samples, return cofactor data as a Dict + + """ + cofactors = {} + if cofactors: + dataset_ob.group.get_samplelist() + sample_list = dataset_ob.group.samplelist + for cofactor in cofactors.split(","): + cofactor_name, cofactor_dataset = cofactor.split(":") + if cofactor_dataset == this_dataset.name: + cofactors[cofactor_name] = [] + trait_ob = create_trait(dataset=dataset_ob, + name=cofactor_name) + sample_data = trait_ob.data + for index, sample in enumerate(samples): + #if (sample in sample_list) and (sample in sample_data): + if sample in sample_data: + sample_value = sample_data[sample].value + cofactors[cofactor_name].append(sample_value) else: - covar_name_string += '"' + col_name + '"' - else: - col_name = "covar_" + str(i) - cross = add_phenotype(cross, covar_as_string, col_name) - if i < (len(covariate_list) - 1): - covar_name_string += '"' + col_name + '", ' - else: - covar_name_string += '"' + col_name + '"' - - covar_name_string += ")" - covars_ob = pull_var("trait_covars", cross, covar_name_string) - return cross, covars_ob - - -def create_marker_covariates(control_marker, cross): - ro.globalenv["the_cross"] = cross - # Get the genotype matrix - ro.r('genotypes <- pull.geno(the_cross)') - # TODO: sanitize user input, Never Ever trust a user - userinput_sanitized = control_marker.replace(" ", "").split(",") - logger.debug(userinput_sanitized) - if len(userinput_sanitized) > 0: - covariate_names = ', '.join('"{0}"'.format(w) - for w in userinput_sanitized) - ro.r('covnames <- c(' + covariate_names + ')') - else: - ro.r('covnames <- c()') - ro.r('covInGeno <- which(covnames %in% colnames(genotypes))') - ro.r('covnames <- covnames[covInGeno]') - ro.r("cat('covnames (purged): ', covnames,'\n')") - # Get the covariate matrix by using the marker name as index to the genotype file - ro.r('marker_covars <- genotypes[,covnames]') - # TODO: Create a design matrix from the marker covars for the markers in case of an F2, 4way, etc - return ro.r["marker_covars"] - - -def process_pair_scan_results(result): - pair_scan_results = [] - - result = result[1] - output = [tuple([result[j][i] for j in range(result.ncol)]) - for i in range(result.nrow)] - - for i, line in enumerate(result.iter_row()): - marker = {} - marker['name'] = result.rownames[i] - marker['chr1'] = output[i][0] - marker['Mb'] = output[i][1] - marker['chr2'] = int(output[i][2]) - pair_scan_results.append(marker) - - return pair_scan_results - - -def process_rqtl_perm_results(num_perm, results): - perm_vals = [item[0] for item in results] - - perm_output = perm_vals - suggestive = np.percentile(np.array(perm_vals), 67) - significant = np.percentile(np.array(perm_vals), 95) - - return perm_output, suggestive, significant - - -def process_rqtl_results(marker_names, results, species_name): # TODO: how to make this a one liner and not copy the stuff in a loop - qtl_results = [] - - for i, line in enumerate(results): - marker = {} - marker['name'] = marker_names[i] - if species_name == "mouse" and line[0] == 20: - marker['chr'] = "X" - else: - try: - marker['chr'] = int(line[0]) - except: - marker['chr'] = line[0] - marker['cM'] = marker['Mb'] = line[1] - marker['lod_score'] = line[2] - qtl_results.append(marker) - - return qtl_results + cofactors[cofactor_name].append("NA") + return cofactors \ No newline at end of file diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index a3b579ec..be1186c0 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -242,10 +242,10 @@ class RunMapping: # if start_vars['pair_scan'] == "true": # self.pair_scan = True if self.permCheck and self.num_perm > 0: - self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno( - self.vals, self.samples, self.dataset, self.mapping_scale, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates) + self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl( + self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates) else: - results = rqtl_mapping.run_rqtl_geno(self.vals, self.samples, self.dataset, self.mapping_scale, self.method, self.model, self.permCheck, + results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates) elif self.mapping_method == "reaper": if "startMb" in start_vars: # ZS: Check if first time page loaded, so it can default to ON -- cgit v1.2.3 From 0a52d87dc4f5dab40f8ec59a048324b73ce2a8cd Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 21 May 2021 15:14:23 +0300 Subject: locustfile: Add new load test for a simple test-case Load test fetching a single trait using Locust. --- locustfile.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 locustfile.py diff --git a/locustfile.py b/locustfile.py new file mode 100644 index 00000000..4abf3f1c --- /dev/null +++ b/locustfile.py @@ -0,0 +1,13 @@ +"""Load test a single trait page""" +from locust import HttpUser, task, between + + +class LoadTest(HttpUser): + wait_time = between(1, 2.5) + + @task + def fetch_trait(self): + """Fetch a single trait""" + # /api/v_pre1/gen_dropdown + self.client.get("/show_trait?trait_id=" + "1457545_at&dataset=HC_M2_0606_P") -- cgit v1.2.3 From efbe47b58f6e0b9a4b509a28ec8788a1c6dae343 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 21 May 2021 19:43:23 +0000 Subject: run_rqtl now reads in permutation output --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 34 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 3f4899b0..c3f047fc 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -8,6 +8,8 @@ from typing import List from typing import Optional from typing import TextIO +import numpy as np + from base.webqtlConfig import TMPDIR from base.trait import create_trait from utility.tools import locate @@ -46,7 +48,11 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, method, model, p out_file = requests.post(GN3_RQTL_URL, data=post_data).json()['output_file'] - return process_rqtl_results(out_file) + if num_perm > 0: + perm_results, suggestive, significant = process_perm_results(out_file) + return perm_results, suggestive, significant, process_rqtl_results(out_file) + else: + return process_rqtl_results(out_file) def process_rqtl_results(out_file: str) -> List: @@ -63,6 +69,7 @@ def process_rqtl_results(out_file: str) -> List: for line in the_file: line_items = line.split(",") if line_items[1][1:-1] == "chr" or not line_items: + # Skip header line continue else: # Convert chr to int if possible @@ -81,6 +88,30 @@ def process_rqtl_results(out_file: str) -> List: return marker_obs + +def process_perm_results(out_file: str): + """Given base filename, read in R/qtl permutation output and calculate + suggestive and significant thresholds + + """ + perm_results = [] + with open(GN3_TMP_PATH + "/output/PERM_" + out_file, "r") as the_file: + for i, line in enumerate(the_file): + if i == 0: + # Skip header line + continue + else: + line_items = line.split(",") + perm_results.append(float(line_items[1])) + + logger.debug("PERM RESULTS:", perm_results) + + suggestive = np.percentile(np.array(perm_results), 67) + significant = np.percentile(np.array(perm_results), 95) + + return perm_results, suggestive, significant + + def get_hash_of_textio(the_file: TextIO) -> str: """Given a StringIO, return the hash of its contents""" @@ -146,7 +177,6 @@ def cofactors_to_dict(cofactors: str, dataset_ob, samples) -> Dict: name=cofactor_name) sample_data = trait_ob.data for index, sample in enumerate(samples): - #if (sample in sample_list) and (sample in sample_data): if sample in sample_data: sample_value = sample_data[sample].value cofactors[cofactor_name].append(sample_value) -- cgit v1.2.3 From d2edff1a05a81cd2ca703de940ea2bec5bd1dd10 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 21 May 2021 20:32:15 +0000 Subject: Removed a couple unused parameters for run_rqtl --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 2 +- wqflask/wqflask/marker_regression/run_mapping.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index c3f047fc..d18f6a7b 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -20,7 +20,7 @@ logger = utility.logger.getLogger(__name__) GN3_RQTL_URL = "http://localhost:8086/api/rqtl/compute" GN3_TMP_PATH = "/export/local/home/zas1024/genenetwork3/tmp" -def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, method, model, permCheck, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, pair_scan, cofactors): +def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" pheno_file = write_phenotype_file(trait_name, samples, vals, cofactors) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index be1186c0..d727e3ff 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -243,10 +243,10 @@ class RunMapping: # self.pair_scan = True if self.permCheck and self.num_perm > 0: self.perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl( - self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.method, self.model, self.permCheck, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates) + self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) else: - results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.method, self.model, self.permCheck, - self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan, self.covariates) + results = rqtl_mapping.run_rqtl(self.this_trait.name, self.vals, self.samples, self.dataset, self.mapping_scale, self.model, self.method, + self.num_perm, perm_strata, self.do_control, self.control_marker, self.manhattan_plot, self.covariates) elif self.mapping_method == "reaper": if "startMb" in start_vars: # ZS: Check if first time page loaded, so it can default to ON if "additiveCheck" in start_vars: -- cgit v1.2.3 From 579ec56e488db19e7cd489b906ae6da7cd453239 Mon Sep 17 00:00:00 2001 From: zsloan Date: Sat, 22 May 2021 22:31:51 +0000 Subject: Rewrote test_rqtl_mapping.py, though haven't done tests for all functions yet --- .../wqflask/marker_regression/test_rqtl_mapping.py | 76 ++++++++++++---------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 91d2c587..5c679c05 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -1,42 +1,46 @@ import unittest from unittest import mock -from wqflask import app -from wqflask.marker_regression.rqtl_mapping import get_trait_data_type -from wqflask.marker_regression.rqtl_mapping import sanitize_rqtl_phenotype -from wqflask.marker_regression.rqtl_mapping import sanitize_rqtl_names +from wqflask.marker_regression.rqtl_mapping import run_rqtl +class AttributeSetter: + def __init__(self, obj): + for key, val in obj.items(): + setattr(self, key, val) + +class MockGroup(AttributeSetter): + def get_samplelist(self): + return None class TestRqtlMapping(unittest.TestCase): - def setUp(self): - self.app_context = app.app_context() - self.app_context.push() - - def tearDown(self): - self.app_context.pop() - - @mock.patch("wqflask.marker_regression.rqtl_mapping.g") - @mock.patch("wqflask.marker_regression.rqtl_mapping.logger") - def test_get_trait_data(self, mock_logger, mock_db): - """test for getting trait data_type return True""" - query_value = """SELECT value FROM TraitMetadata WHERE type='trait_data_type'""" - mock_db.db.execute.return_value.fetchone.return_value = [ - """{"type":"trait_data_type","name":"T1","traid_id":"fer434f"}"""] - results = get_trait_data_type("traid_id") - mock_db.db.execute.assert_called_with(query_value) - self.assertEqual(results, "fer434f") - - def test_sanitize_rqtl_phenotype(self): - """test for sanitizing rqtl phenotype""" - vals = ['f', "x", "r", "x", "x"] - results = sanitize_rqtl_phenotype(vals) - expected_phenotype_string = 'c(f,NA,r,NA,NA)' - - self.assertEqual(results, expected_phenotype_string) - - def test_sanitize_rqtl_names(self): - """test for sanitzing rqtl names""" - vals = ['f', "x", "r", "x", "x"] - expected_sanitized_name = "c('f',NA,'r',NA,NA)" - results = sanitize_rqtl_names(vals) - self.assertEqual(expected_sanitized_name, results) + @mock.patch("wqflask.marker_regression.rqtl_mapping.process_rqtl_results") + @mock.patch("wqflask.marker_regression.rqtl_mapping.process_perm_results") + @mock.patch("wqflask.marker_regression.rqtl_mapping.requests.post") + @mock.patch("wqflask.marker_regression.rqtl_mapping.locate") + @mock.patch("wqflask.marker_regression.gemma_mapping.write_phenotype_file") + def test_run_rqtl_with_perm(self, mock_write_pheno_file, mock_locate, mock_post, mock_process_perm, mock_process_rqtl): + """Test for run_rqtl with permutations > 0""" + dataset_group = MockGroup( + {"name": "GP1", "genofile": "file_geno"}) + + dataset = AttributeSetter({"group": dataset_group}) + + mock_write_pheno_file.return_value = "pheno_filename" + mock_locate.return_value = "geno_filename" + + mock_post.return_value = "output_filename" + + mock_process_perm.return_value = [[], 3, 4] + mock_process_rqtl.return_value = [] + + results = run_rqtl(trait_name="the_trait", vals=[], samples=[], + dataset=dataset, mapping_scale="cM", model="normal", method="hk", + num_perm=5, perm_strata_list=[], do_control="false", control_marker="", + manhattan_plot=True, cofactors="") + + mock_write_pheno_file.assert_called_once() + mock_locate.assert_called_once() + mock_post.assert_called_once() + mock_process_perm.assert_called_once() + mock_process_rqtl.assert_called_once() + self.assertEqual(results, ([], 3, 4, [])) \ No newline at end of file -- cgit v1.2.3 From 1ec1219eae75f1cf9a1b20ea479ea6e2934eaa68 Mon Sep 17 00:00:00 2001 From: zsloan Date: Sat, 22 May 2021 22:43:58 +0000 Subject: Fixed a couple broken tests --- wqflask/tests/unit/wqflask/api/test_mapping.py | 2 +- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/tests/unit/wqflask/api/test_mapping.py b/wqflask/tests/unit/wqflask/api/test_mapping.py index b094294a..159c982b 100644 --- a/wqflask/tests/unit/wqflask/api/test_mapping.py +++ b/wqflask/tests/unit/wqflask/api/test_mapping.py @@ -58,7 +58,7 @@ class TestMapping(unittest.TestCase): self.assertEqual(results_2, expected_results) - @mock.patch("wqflask.api.mapping.rqtl_mapping.run_rqtl_geno") + @mock.patch("wqflask.api.mapping.rqtl_mapping.run_rqtl") @mock.patch("wqflask.api.mapping.gemma_mapping.run_gemma") @mock.patch("wqflask.api.mapping.initialize_parameters") @mock.patch("wqflask.api.mapping.retrieve_sample_data") diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 5c679c05..626869b8 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -17,7 +17,7 @@ class TestRqtlMapping(unittest.TestCase): @mock.patch("wqflask.marker_regression.rqtl_mapping.process_perm_results") @mock.patch("wqflask.marker_regression.rqtl_mapping.requests.post") @mock.patch("wqflask.marker_regression.rqtl_mapping.locate") - @mock.patch("wqflask.marker_regression.gemma_mapping.write_phenotype_file") + @mock.patch("wqflask.marker_regression.rqtl_mapping.write_phenotype_file") def test_run_rqtl_with_perm(self, mock_write_pheno_file, mock_locate, mock_post, mock_process_perm, mock_process_rqtl): """Test for run_rqtl with permutations > 0""" dataset_group = MockGroup( -- cgit v1.2.3 From cf9f0bef4454f97270bcacd16080f9d7f6bbb5e6 Mon Sep 17 00:00:00 2001 From: zsloan Date: Sat, 22 May 2021 22:51:19 +0000 Subject: Fixed the mocked return_value for requests.post to fix broken test --- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 626869b8..6853c021 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -28,7 +28,8 @@ class TestRqtlMapping(unittest.TestCase): mock_write_pheno_file.return_value = "pheno_filename" mock_locate.return_value = "geno_filename" - mock_post.return_value = "output_filename" + mock_post.return_value = {"output_file": "output_filename", + "rqtl_cmd": "the_command"} mock_process_perm.return_value = [[], 3, 4] mock_process_rqtl.return_value = [] -- cgit v1.2.3 From 4d15c32e7d86885cffce1720db0df394bd6a02ff Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 25 May 2021 20:19:37 +0000 Subject: Removed code that processes output files, since this functionality is moved to GN3 --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 67 ++--------------------- 1 file changed, 4 insertions(+), 63 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index d18f6a7b..495678e9 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -46,70 +46,11 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n if cofactors: post_data["addcovar"] = True - out_file = requests.post(GN3_RQTL_URL, data=post_data).json()['output_file'] - + rqtl_output = requests.post(GN3_RQTL_URL, data=post_data).json() if num_perm > 0: - perm_results, suggestive, significant = process_perm_results(out_file) - return perm_results, suggestive, significant, process_rqtl_results(out_file) + return rqtl_output['perm_results'], rqtl_output['suggestive'], rqtl_output['significant'], rqtl_output['results'] else: - return process_rqtl_results(out_file) - - -def process_rqtl_results(out_file: str) -> List: - """Given the output filename, read in results and - return as a list of dictionaries representing each - marker - - """ - - marker_obs = [] - # Later I should probably redo this using csv.read to avoid the - # awkwardness with removing quotes with [1:-1] - with open(GN3_TMP_PATH + "/output/" + out_file, "r") as the_file: - for line in the_file: - line_items = line.split(",") - if line_items[1][1:-1] == "chr" or not line_items: - # Skip header line - continue - else: - # Convert chr to int if possible - try: - the_chr = int(line_items[1][1:-1]) - except: - the_chr = line_items[1][1:-1] - this_marker = { - "name": line_items[0][1:-1], - "chr": the_chr, - "cM": float(line_items[2]), - "Mb": float(line_items[2]), - "lod_score": float(line_items[3]) - } - marker_obs.append(this_marker) - - return marker_obs - - -def process_perm_results(out_file: str): - """Given base filename, read in R/qtl permutation output and calculate - suggestive and significant thresholds - - """ - perm_results = [] - with open(GN3_TMP_PATH + "/output/PERM_" + out_file, "r") as the_file: - for i, line in enumerate(the_file): - if i == 0: - # Skip header line - continue - else: - line_items = line.split(",") - perm_results.append(float(line_items[1])) - - logger.debug("PERM RESULTS:", perm_results) - - suggestive = np.percentile(np.array(perm_results), 67) - significant = np.percentile(np.array(perm_results), 95) - - return perm_results, suggestive, significant + return rqtl_output['results'] def get_hash_of_textio(the_file: TextIO) -> str: @@ -182,4 +123,4 @@ def cofactors_to_dict(cofactors: str, dataset_ob, samples) -> Dict: cofactors[cofactor_name].append(sample_value) else: cofactors[cofactor_name].append("NA") - return cofactors \ No newline at end of file + return cofactors -- cgit v1.2.3 From 666494b49bef48a46145cf415262d532530aba46 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 25 May 2021 20:49:48 +0000 Subject: Fixed/shortened get_categorical_variables function and correctly pass them to show_trait template --- wqflask/wqflask/show_trait/show_trait.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 837c7a54..a02da872 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -279,6 +279,8 @@ class ShowTrait: hddn['suggestive'] = 0 hddn['num_perm'] = 0 hddn['categorical_vars'] = "" + if categorical_var_list: + hddn['categorical_vars'] = ",".join(categorical_var_list) hddn['manhattan_plot'] = "" hddn['control_marker'] = "" if not self.temp_trait: @@ -684,23 +686,13 @@ def get_ncbi_summary(this_trait): return None -def get_categorical_variables(this_trait, sample_list): +def get_categorical_variables(this_trait, sample_list) -> list: categorical_var_list = [] if len(sample_list.attributes) > 0: for attribute in sample_list.attributes: - attribute_vals = [] - for sample_name in list(this_trait.data.keys()): - if sample_list.attributes[attribute].name in this_trait.data[sample_name].extra_attributes: - attribute_vals.append( - this_trait.data[sample_name].extra_attributes[sample_list.attributes[attribute].name]) - else: - attribute_vals.append("N/A") - num_distinct = len(set(attribute_vals)) - - if num_distinct < 10: - categorical_var_list.append( - sample_list.attributes[attribute].name) + if len(sample_list.attributes[attribute].distinct_values) < 10: + categorical_var_list.append(sample_list.attributes[attribute].name) return categorical_var_list -- cgit v1.2.3 From 9c583587b64d26dd9c554e092318435facc510f3 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 4 May 2021 21:15:52 +0300 Subject: templates: edit_trait: Add form for editing a dataset (WIP) --- wqflask/wqflask/templates/edit_trait.html | 165 ++++++++++++++++-------------- 1 file changed, 90 insertions(+), 75 deletions(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index b9a31fc6..ae6bc0d5 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -2,85 +2,100 @@ {% block title %}Trait Submission{% endblock %} {% block content %} -
      -
      +Edit Trait for Published Database +Submit Trait | Reset - {{ flash_me() }} - -
      -
      -
      -
      -

      Introduction

      -
      -

      The trait values that you enter are statistically compared with verified genotypes collected at a set of microsatellite markers in each RI set. The markers are drawn from a set of over 750, but for each set redundant markers have been removed, preferentially retaining those that are most informative.

      -

      These error-checked RI mapping data match theoretical expectations for RI strain sets. The cumulative adjusted length of the RI maps are approximately 1400 cM, a value that matches those of both MIT maps and Chromosome Committee Report maps. See our full description of the genetic data collected as part of the WebQTL project.

      -
      -
      -
      -
      -
      -

      About Your Data

      -
      -

      You can open a separate window giving the number of strains for each data set and sample data.

      -

      None of your submitted data is copied or stored by this system except during the actual processing of your submission. By the time the reply page displays in your browser, your submission has been cleared from this system.

      -
      -
      -
      -
      -
      -
      -

      Edit Trait Form

      -
      -
      -

      Choose Dataset to Edit:

      -
      -
      - -
      - -
      - -
      -
      - -
      - -
      - -
      -
      - -
      - -
      - -
      -
      - -
      - -
      - -
      -
      -
      -
      - - -
      -
      -
      -
      + +

      Trait Information:

      +
      + + +
      + +
      +
      +
      + +
      + +
      + +
      +
      + +
      + +
      + +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      +
      - +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      + + {%endblock%} - {% block js %} - - +{% block js %} + + {% endblock %} -- cgit v1.2.3 From b5779f798ca5ef30a000af726921db43b33291a6 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 6 May 2021 09:17:13 +0300 Subject: wqflask: database: Replace one-line function with it's body * wqflask/wqflask/database.py (init_db): Remove it. --- wqflask/wqflask/database.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/database.py b/wqflask/wqflask/database.py index 38c37d2c..11f8d287 100644 --- a/wqflask/wqflask/database.py +++ b/wqflask/wqflask/database.py @@ -14,9 +14,5 @@ db_session = scoped_session(sessionmaker(autocommit=False, Base = declarative_base() Base.query = db_session.query_property() - -def init_db(): - Base.metadata.create_all(bind=engine) - - -init_db() +# Initialise the db +Base.metadata.create_all(bind=engine) -- cgit v1.2.3 From d63a66c78121772e51c0a931e4740b62e18c62d1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 6 May 2021 11:50:45 +0300 Subject: db: webqtlDatabaseFunction: Remove logger info --- wqflask/db/webqtlDatabaseFunction.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index 29112949..fa747782 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -21,10 +21,6 @@ # This module is used by GeneNetwork project (www.genenetwork.org) from db.call import fetch1 -from utility.tools import USE_GN_SERVER - -from utility.logger import getLogger -logger = getLogger(__name__) ########################################################################### # output: cursor instance @@ -38,13 +34,10 @@ def retrieve_species(group): """ result = fetch1("select Species.Name from Species, InbredSet where InbredSet.Name = '%s' and InbredSet.SpeciesId = Species.Id" % ( group), "/cross/" + group + ".json", lambda r: (r["species"],))[0] - logger.debug("retrieve_species result:", result) return result def retrieve_species_id(group): - result = fetch1("select SpeciesId from InbredSet where Name = '%s'" % ( group), "/cross/" + group + ".json", lambda r: (r["species_id"],))[0] - logger.debug("retrieve_species_id result:", result) return result -- cgit v1.2.3 From 2e99ad2c1893a2a09a282f3ea0a9beb39f688f0b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 6 May 2021 11:51:11 +0300 Subject: db: webqtlDatabaseFunction: Remove comment --- wqflask/db/webqtlDatabaseFunction.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/wqflask/db/webqtlDatabaseFunction.py b/wqflask/db/webqtlDatabaseFunction.py index fa747782..9ec650a4 100644 --- a/wqflask/db/webqtlDatabaseFunction.py +++ b/wqflask/db/webqtlDatabaseFunction.py @@ -22,11 +22,6 @@ from db.call import fetch1 -########################################################################### -# output: cursor instance -# function: connect to database and return cursor instance -########################################################################### - def retrieve_species(group): """Get the species of a group (e.g. returns string "mouse" on "BXD" -- cgit v1.2.3 From 4b3e16df73502c1b4802b6de60f743093351ebc5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Mon, 10 May 2021 21:01:45 +0300 Subject: api: gen_menu: Replace get_species with get_all_species from gn3 * wqflask/wqflask/api/gen_menu.py (gen_dropdown_json): Use new `get_all_species` function. (get_species): Remove it. --- wqflask/wqflask/api/gen_menu.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/api/gen_menu.py b/wqflask/wqflask/api/gen_menu.py index 0c1120ab..a699a484 100644 --- a/wqflask/wqflask/api/gen_menu.py +++ b/wqflask/wqflask/api/gen_menu.py @@ -1,29 +1,17 @@ +from gn3.db.species import get_all_species def gen_dropdown_json(conn): """Generates and outputs (as json file) the data for the main dropdown menus on the home page """ - - species = get_species(conn) + species = get_all_species(conn) groups = get_groups(species, conn) types = get_types(groups, conn) datasets = get_datasets(types, conn) - - data = dict(species=species, + return dict(species=species, groups=groups, types=types, datasets=datasets) - return data - - -def get_species(conn): - """Build species list""" - with conn.cursor() as cursor: - cursor.execute("SELECT Name, MenuName FROM Species " - "ORDER BY OrderId") - results = cursor.fetchall() - return [[name, menu_name] for name, menu_name in results] - def get_groups(species, conn): """Build groups list""" -- cgit v1.2.3 From 919a2ee38412270cd5baf1e490b2f733f81cacec Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 13 May 2021 12:49:31 +0300 Subject: tests: test_gen_menu: Remove get_species --- wqflask/tests/unit/wqflask/api/test_gen_menu.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/wqflask/tests/unit/wqflask/api/test_gen_menu.py b/wqflask/tests/unit/wqflask/api/test_gen_menu.py index 743b3bde..7e477da2 100644 --- a/wqflask/tests/unit/wqflask/api/test_gen_menu.py +++ b/wqflask/tests/unit/wqflask/api/test_gen_menu.py @@ -3,7 +3,6 @@ import unittest from unittest import mock from wqflask.api.gen_menu import gen_dropdown_json -from wqflask.api.gen_menu import get_species from wqflask.api.gen_menu import get_groups from wqflask.api.gen_menu import get_types from wqflask.api.gen_menu import get_datasets @@ -67,20 +66,6 @@ class TestGenMenu(unittest.TestCase): } } - def test_get_species(self): - """Test that assertion is raised when dataset and dataset_name - are defined""" - db_mock = mock.MagicMock() - with db_mock.cursor() as cursor: - cursor.fetchall.return_value = ( - ('human', 'Human'), - ('mouse', 'Mouse')) - self.assertEqual(get_species(db_mock), - [['human', 'Human'], ['mouse', 'Mouse']]) - cursor.execute.assert_called_once_with( - "SELECT Name, MenuName FROM Species ORDER BY OrderId" - ) - def test_get_groups(self): """Test that species groups are grouped correctly""" db_mock = mock.MagicMock() @@ -411,7 +396,7 @@ class TestGenMenu(unittest.TestCase): @mock.patch('wqflask.api.gen_menu.get_datasets') @mock.patch('wqflask.api.gen_menu.get_types') @mock.patch('wqflask.api.gen_menu.get_groups') - @mock.patch('wqflask.api.gen_menu.get_species') + @mock.patch('wqflask.api.gen_menu.get_all_species') def test_gen_dropdown_json(self, species_mock, groups_mock, -- cgit v1.2.3 From 41408d44d23dc93afc85f8428fa2f6944fc902bb Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 18 May 2021 11:21:57 +0300 Subject: templates: show_trait_details: Add an "[edit]" button --- wqflask/wqflask/templates/show_trait_details.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html index d2999eef..83f7b0ac 100644 --- a/wqflask/wqflask/templates/show_trait_details.html +++ b/wqflask/wqflask/templates/show_trait_details.html @@ -235,7 +235,7 @@ {% endif %} {% if admin_status == "owner" or admin_status == "edit-admins" or admin_status == "edit-access" %} - + {% endif %}
      -- cgit v1.2.3 From 2de0bdc7c3350dacf14218bb1a1e4721ac4d1237 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 May 2021 23:22:49 +0300 Subject: wqflask: views: Add extra endpoint to display individual trait edit * wqflask/wqflask/views.py (show_edit_trait_page): New function/ end-point. --- wqflask/wqflask/views.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4834ee63..51266ada 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -21,6 +21,7 @@ import base64 import array import sqlalchemy from wqflask import app +from flask import current_app from flask import g from flask import Response from flask import request @@ -414,13 +415,35 @@ def submit_trait_form(): version=GN_VERSION) -@app.route("/edit_trait_form") -def edit_trait_page(): - species_and_groups = get_species_groups() +@app.route("/trait//edit/") +def show_edit_trait_page(name, inbred_set_id): + from gn3.db.phenotypes import (Phenotype, + PublishXRef, + Publication, + fetchone) + import MySQLdb + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + publish_xref = fetchone( + conn=conn, + table="PublishXRef", + where=PublishXRef(id_=name, + inbred_set_id=inbred_set_id)) + phenotype_ = fetchone( + conn=conn, + table="Phenotype", + where=Phenotype(id_=publish_xref.phenotype_id)) + publication_ = fetchone( + conn=conn, + table="Publication", + where=Publication(id_=publish_xref.publication_id)) return render_template( "edit_trait.html", - species_and_groups=species_and_groups, - gn_server_url=GN_SERVER_URL, + publish_xref=publish_xref, + phenotype=phenotype_, + publication=publication_, version=GN_VERSION) -- cgit v1.2.3 From 4dea9436f3678c9f5640001b0e5b66c582048498 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 20 May 2021 23:23:50 +0300 Subject: templates: edit_trait: Update edit btn to point to editing endpoint --- wqflask/wqflask/templates/edit_trait.html | 171 +++++++++++++++++++++++------- 1 file changed, 135 insertions(+), 36 deletions(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index ae6bc0d5..e945e730 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -5,96 +5,195 @@ Edit Trait for Published Database Submit Trait | Reset +{% if publish_xref.comments %} +

      Update History:

      + +{{ publish_xref.comments }} + +{% endif %} +

      Trait Information:

      - + -
      - +
      +
      -
      - +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      +
      + +
      + +
      +
      -
      - +
      +
      -
      - -
      - + +
      +
      - -
      - + +
      + +
      +
      +
      + +
      +
      -
      - +
      +
      -
      - +
      +
      -
      - +
      +
      -
      - +
      +
      - -
      - + +
      +
      - -
      - -
      + +
      +
      - +
      +
      + + +
      {%endblock%} {% block js %} - -- cgit v1.2.3 From cbfc07ac1c06fc3c17e8159a4afe661d2e21bee5 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 21 May 2021 15:59:36 +0300 Subject: wqflask: views: Add edit_trait page Should add some some authentication logic to this later since it's not supposed to be accessible to everyone. --- wqflask/wqflask/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 51266ada..87686d8f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -416,7 +416,7 @@ def submit_trait_form(): @app.route("/trait//edit/") -def show_edit_trait_page(name, inbred_set_id): +def edit_trait(name, inbred_set_id): from gn3.db.phenotypes import (Phenotype, PublishXRef, Publication, -- cgit v1.2.3 From 3efc5eab756d82e998354e144fe61976dd8110f1 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 21 May 2021 16:28:09 +0300 Subject: templates: edit_trait: Make form functional(add ability to submit) --- wqflask/wqflask/templates/edit_trait.html | 94 +++++++++---------------------- 1 file changed, 28 insertions(+), 66 deletions(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index e945e730..46778f63 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -12,27 +12,23 @@ Submit Trait | Reset {% endif %} -
      +

      Trait Information:

      - +
      - +
      - +
      - +
      - +
      - +
      - +
      - +
      - +
      -
      -
      @@ -88,27 +74,21 @@ Submit Trait | Reset Lab Code:
      - +
      -
      - +
      @@ -116,78 +96,60 @@ Submit Trait | Reset Authorized Users:
      - +
      - +
      - +
      - +
      - +
      - +
      - +
      - +
      - +
      - +
      -- cgit v1.2.3 From e97408ad909030a2e8cbbfcb0311c90b7fc9b706 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 21 May 2021 16:30:33 +0300 Subject: wqflask: views: Make imports top-level --- wqflask/wqflask/views.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 87686d8f..4b305cf2 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -9,6 +9,7 @@ import csv import simplejson as json import xlsxwriter import io # Todo: Use cStringIO? +import MySQLdb from zipfile import ZipFile, ZIP_DEFLATED @@ -21,6 +22,12 @@ import base64 import array import sqlalchemy from wqflask import app + +from gn3.db.phenotypes import Phenotype +from gn3.db.phenotypes import PublishXRef +from gn3.db.phenotypes import Publication +from gn3.db.phenotypes import fetchone + from flask import current_app from flask import g from flask import Response @@ -417,11 +424,6 @@ def submit_trait_form(): @app.route("/trait//edit/") def edit_trait(name, inbred_set_id): - from gn3.db.phenotypes import (Phenotype, - PublishXRef, - Publication, - fetchone) - import MySQLdb conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), passwd=current_app.config.get("DB_PASS"), -- cgit v1.2.3 From 8edc79a45219a0514433ddedf588550eecf3b904 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Fri, 21 May 2021 16:30:49 +0300 Subject: wqflask: views: Remove unused imports --- wqflask/wqflask/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 4b305cf2..9936f65c 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -39,10 +39,8 @@ from flask import redirect from flask import url_for from flask import send_file -from wqflask import collect from wqflask import search_results from wqflask import server_side -from wqflask.submit_bnw import get_bnw_input from base.data_set import create_dataset # Used by YAML in marker_regression from wqflask.show_trait import show_trait from wqflask.show_trait import export_trait_data -- cgit v1.2.3 From 01d742d4dd5ee9d193e1c0ccb3d32ec9592c099f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 25 May 2021 13:15:36 +0300 Subject: templates: edit_trait: Submit publish_xref data --- wqflask/wqflask/templates/edit_trait.html | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 46778f63..2cd8a6fd 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -148,6 +148,10 @@ Submit Trait | Reset
      + + + +
      -- cgit v1.2.3 From 7d3edb7ce668df5d072ed87a755f61782bb30a82 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 25 May 2021 13:19:14 +0300 Subject: wqflask: views: Add method that updates the trait * wqflask/wqflask/views.py (update_trait): New function. --- wqflask/wqflask/views.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 9936f65c..cd77f64f 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -27,6 +27,7 @@ from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import Publication from gn3.db.phenotypes import fetchone +from gn3.db.phenotypes import update from flask import current_app from flask import g @@ -447,6 +448,49 @@ def edit_trait(name, inbred_set_id): version=GN_VERSION) +@app.route("/trait/update", methods=["POST"]) +def update_trait(): + conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + # Filter out empty values + data_ = {k: v for k, v in request.form.items() if v is not ''} + + # Run updates: + update(conn, "Phenotype", + data=Phenotype( + pre_pub_description=data_.get("pre-pub-desc"), + post_pub_description=data_.get("post-pub-desc"), + original_description=data_.get("orig-desc"), + units=data_.get("units"), + pre_pub_abbreviation=data_.get("pre-pub-abbrev"), + post_pub_abbreviation=data_.get("post-pub-abbrev"), + lab_code=data_.get("labcode"), + submitter=data_.get("submitter"), + owner=data_.get("owner"), + ), + where=Phenotype(id_=data_.get("phenotype-id"))) + update(conn, "Publication", + data=Publication( + abstract=data_.get("abstract"), + authors=data_.get("authors"), + title=data_.get("title"), + journal=data_.get("journal"), + volume=data_.get("volume"), + pages=data_.get("pages"), + month=data_.get("month"), + year=data_.get("year")), + where=Publication(id_=data_.get("pubmed-id"))) + update(conn, "PublishXRef", + data=PublishXRef(comments="", + publication_id=data_.get("pubmed-id")), + where=PublishXRef( + id_=data_.get("dataset-name"), + inbred_set_id=data_.get("inbred-set-id"))) + return redirect("/trait/10007/edit/1") + + @app.route("/create_temp_trait", methods=('POST',)) def create_temp_trait(): logger.info(request.url) -- cgit v1.2.3 From 9496c645af96c31ee20c7cf15721396c7d16248f Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 25 May 2021 14:04:32 +0300 Subject: Use @admin_login_required to only enable Rob to do edits to traits * wqflask/wqflask/decorators.py (admin_login_required): New decorator. * wqflask/wqflask/views.py (edit_trait): Use admin_login_required decorator. --- wqflask/wqflask/decorators.py | 14 ++++++++++++++ wqflask/wqflask/views.py | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 wqflask/wqflask/decorators.py diff --git a/wqflask/wqflask/decorators.py b/wqflask/wqflask/decorators.py new file mode 100644 index 00000000..f0978fd3 --- /dev/null +++ b/wqflask/wqflask/decorators.py @@ -0,0 +1,14 @@ +"""This module contains gn2 decorators""" +from flask import g +from functools import wraps + + +def admin_login_required(f): + """Use this for endpoints where admins are required""" + @wraps(f) + def wrap(*args, **kwargs): + if g.user_session.record.get(b"user_email_address") not in [ + b"labwilliams@gmail.com"]: + return "You need to be admin", 401 + return f(*args, **kwargs) + return wrap diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index cd77f64f..807e162e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -65,6 +65,7 @@ from wqflask.export_traits import export_search_results_csv from wqflask.gsearch import GSearch from wqflask.update_search_results import GSearch as UpdateGSearch from wqflask.docs import Docs, update_text +from wqflask.decorators import admin_login_required from wqflask.db_info import InfoPage from utility import temp_data @@ -422,6 +423,7 @@ def submit_trait_form(): @app.route("/trait//edit/") +@admin_login_required def edit_trait(name, inbred_set_id): conn = MySQLdb.Connect(db=current_app.config.get("DB_NAME"), user=current_app.config.get("DB_USER"), -- cgit v1.2.3 From 2be6758dfb7ea55ab4366c3a8c5117e9d40f31bf Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 May 2021 10:50:37 +0300 Subject: wqflask: views: Import "fetchone" & "update" from "gn3.db" --- wqflask/wqflask/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 807e162e..96cadab8 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -23,11 +23,12 @@ import array import sqlalchemy from wqflask import app +from gn3.db import fetchone +from gn3.db import update from gn3.db.phenotypes import Phenotype from gn3.db.phenotypes import PublishXRef from gn3.db.phenotypes import Publication -from gn3.db.phenotypes import fetchone -from gn3.db.phenotypes import update + from flask import current_app from flask import g -- cgit v1.2.3 From c95eb2f9dedb746e237ac16ef1796e78de5ccdf7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 May 2021 13:10:41 +0300 Subject: wqflask: views: Update comment when data is edited For now the exact column that's edited isn't tracked. --- wqflask/wqflask/views.py | 71 ++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 96cadab8..ee69900e 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -461,36 +461,47 @@ def update_trait(): data_ = {k: v for k, v in request.form.items() if v is not ''} # Run updates: - update(conn, "Phenotype", - data=Phenotype( - pre_pub_description=data_.get("pre-pub-desc"), - post_pub_description=data_.get("post-pub-desc"), - original_description=data_.get("orig-desc"), - units=data_.get("units"), - pre_pub_abbreviation=data_.get("pre-pub-abbrev"), - post_pub_abbreviation=data_.get("post-pub-abbrev"), - lab_code=data_.get("labcode"), - submitter=data_.get("submitter"), - owner=data_.get("owner"), - ), - where=Phenotype(id_=data_.get("phenotype-id"))) - update(conn, "Publication", - data=Publication( - abstract=data_.get("abstract"), - authors=data_.get("authors"), - title=data_.get("title"), - journal=data_.get("journal"), - volume=data_.get("volume"), - pages=data_.get("pages"), - month=data_.get("month"), - year=data_.get("year")), - where=Publication(id_=data_.get("pubmed-id"))) - update(conn, "PublishXRef", - data=PublishXRef(comments="", - publication_id=data_.get("pubmed-id")), - where=PublishXRef( - id_=data_.get("dataset-name"), - inbred_set_id=data_.get("inbred-set-id"))) + updated_phenotypes = update( + conn, "Phenotype", + data=Phenotype( + pre_pub_description=data_.get("pre-pub-desc"), + post_pub_description=data_.get("post-pub-desc"), + original_description=data_.get("orig-desc"), + units=data_.get("units"), + pre_pub_abbreviation=data_.get("pre-pub-abbrev"), + post_pub_abbreviation=data_.get("post-pub-abbrev"), + lab_code=data_.get("labcode"), + submitter=data_.get("submitter"), + owner=data_.get("owner"), + ), + where=Phenotype(id_=data_.get("phenotype-id"))) + updated_publications = update( + conn, "Publication", + data=Publication( + abstract=data_.get("abstract"), + authors=data_.get("authors"), + title=data_.get("title"), + journal=data_.get("journal"), + volume=data_.get("volume"), + pages=data_.get("pages"), + month=data_.get("month"), + year=data_.get("year")), + where=Publication(id_=data_.get("pubmed-id"))) + if updated_phenotypes or updated_publications: + comments = data_.get("comments") + if comments: + comments = (f"{comments}\r\n" + f"{g.user_session.record.get(b'user_name')}") + update(conn, "PublishXRef", + data=PublishXRef( + comments=(f"{data_.get('comments')}\r\n" + "Modified by: " + f"{g.user_session.record.get(b'user_name').decode('utf-8')} " + f"on {str(datetime.datetime.now())}"), + publication_id=data_.get("pubmed-id")), + where=PublishXRef( + id_=data_.get("dataset-name"), + inbred_set_id=data_.get("inbred-set-id"))) return redirect("/trait/10007/edit/1") -- cgit v1.2.3 From 43d4e9b42f9ebcb9012bc791aa4ac34a15f290a7 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 May 2021 18:31:40 +0300 Subject: workflows: main: Use "bonfacekilz/genenetwork2:latest" docker image --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c6b56a81..f279a7e5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,7 +11,7 @@ on: jobs: unittest: runs-on: ubuntu-latest - container: bonfacekilz/python3-genenetwork2:00ba1f8 + container: bonfacekilz/genenetwork2:latest steps: # First start with mariadb set then checkout. The checkout gives -- cgit v1.2.3 From c22fdbd6a128ab9c9ddfc14299115a40cec37f07 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 26 May 2021 18:33:27 +0300 Subject: doc: docker-container: Rename python3-genenetwork2 to genenetwork2 See: https://is.gd/gHJvhe --- doc/docker-container.org | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/docker-container.org b/doc/docker-container.org index c894c4b5..ef0d71fc 100644 --- a/doc/docker-container.org +++ b/doc/docker-container.org @@ -32,13 +32,13 @@ First create the gn2 tar archive by running: env GUIX_PACKAGE_PATH="/home/bonface/projects/guix-bioinformatics:/home/bonface/projects/guix-past/modules" \ ./pre-inst-env guix pack --no-grafts\ -S /gn2-profile=/ \ - screen genenetwork2 + screen python2-genenetwork2 # For the Python 3 version: env GUIX_PACKAGE_PATH="/home/bonface/projects/guix-bioinformatics:/home/bonface/projects/guix-past/modules" \ ./pre-inst-env guix pack --no-grafts\ -S /gn2-profile=/ \ - screen python3-genenetwork2 + screen genenetwork2 #+end_src The output will look something similar to: @@ -59,16 +59,16 @@ RUN tar -xzf /tmp/gn2.tar.gz -C / && rm -f /tmp/gn2.tar.gz && \ Build the image(Note the fullstop at the end): -: sudo docker build -t python3-genenetwork2:latest -f Dockerfile . +: sudo docker build -t genenetwork2:latest -f Dockerfile . To load the image interactively you've just created: -: docker run -ti "python3-genenetwork2:latest" bash +: docker run -ti "genenetwork2:latest" bash Assuming you have a docker instance running, you could always run commands in it e.g: -: docker run "python2-genenetwork2:latest" python --version +: docker run "genenetwork2:latest" python --version * Pushing to DockerHub @@ -78,7 +78,7 @@ CI environment using Github Actions. To push to dockerhub, first get the image name by running =docker images=. Push to dockerhub using a command similar to: -: docker push bonfacekilz/python3-genenetwork2:latest +: docker push bonfacekilz/genenetwork2:latest Right now, we have 2 images on DockerHub: -- cgit v1.2.3 From 92f84ab7c544ac9d7ffdbf302cf35b8b67ef7819 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 19:59:24 +0000 Subject: Add code writing perm_strata to phenotype input file in rqtl_mapping.py + fix issue in cofactors_to_dict where two variables were both called 'cofactors' --- wqflask/wqflask/marker_regression/rqtl_mapping.py | 25 +++++++++++++++-------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py index 495678e9..09afb8d1 100644 --- a/wqflask/wqflask/marker_regression/rqtl_mapping.py +++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py @@ -23,7 +23,7 @@ GN3_TMP_PATH = "/export/local/home/zas1024/genenetwork3/tmp" def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, num_perm, perm_strata_list, do_control, control_marker, manhattan_plot, cofactors): """Run R/qtl by making a request to the GN3 endpoint and reading in the output file(s)""" - pheno_file = write_phenotype_file(trait_name, samples, vals, cofactors) + pheno_file = write_phenotype_file(trait_name, samples, vals, dataset, cofactors, perm_strata_list) if dataset.group.genofile: geno_file = locate(dataset.group.genofile, "genotype") else: @@ -46,6 +46,9 @@ def run_rqtl(trait_name, vals, samples, dataset, mapping_scale, model, method, n if cofactors: post_data["addcovar"] = True + if perm_strata_list: + post_data["pstrata"] = True + rqtl_output = requests.post(GN3_RQTL_URL, data=post_data).json() if num_perm > 0: return rqtl_output['perm_results'], rqtl_output['suggestive'], rqtl_output['significant'], rqtl_output['results'] @@ -66,12 +69,12 @@ def write_phenotype_file(trait_name: str, samples: List[str], vals: List, dataset_ob, - cofactors: Optional[str] = None) -> TextIO: + cofactors: Optional[str] = None, + perm_strata_list: Optional[List] = None) -> TextIO: """Given trait name, sample list, value list, dataset ob, and optional string representing cofactors, return the file's full path/name """ - cofactor_data = cofactors_to_dict(cofactors, dataset_ob, samples) pheno_file = io.StringIO() @@ -79,6 +82,8 @@ def write_phenotype_file(trait_name: str, header_row = ["Samples", trait_name] header_row += [cofactor for cofactor in cofactor_data] + if perm_strata_list: + header_row.append("Strata") writer.writerow(header_row) for i, sample in enumerate(samples): @@ -89,6 +94,8 @@ def write_phenotype_file(trait_name: str, this_row.append("NA") for cofactor in cofactor_data: this_row.append(cofactor_data[cofactor][i]) + if perm_strata_list: + this_row.append(perm_strata_list[i]) writer.writerow(this_row) hash_of_file = get_hash_of_textio(pheno_file) @@ -106,21 +113,21 @@ def cofactors_to_dict(cofactors: str, dataset_ob, samples) -> Dict: and list of samples, return cofactor data as a Dict """ - cofactors = {} + cofactor_dict = {} if cofactors: dataset_ob.group.get_samplelist() sample_list = dataset_ob.group.samplelist for cofactor in cofactors.split(","): cofactor_name, cofactor_dataset = cofactor.split(":") - if cofactor_dataset == this_dataset.name: - cofactors[cofactor_name] = [] + if cofactor_dataset == dataset_ob.name: + cofactor_dict[cofactor_name] = [] trait_ob = create_trait(dataset=dataset_ob, name=cofactor_name) sample_data = trait_ob.data for index, sample in enumerate(samples): if sample in sample_data: sample_value = sample_data[sample].value - cofactors[cofactor_name].append(sample_value) + cofactor_dict[cofactor_name].append(sample_value) else: - cofactors[cofactor_name].append("NA") - return cofactors + cofactor_dict[cofactor_name].append("NA") + return cofactor_dict -- cgit v1.2.3 From 6a54fa84542458fe8cfd288bf4c507214a1ddd04 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:08:14 +0000 Subject: Make all attribute names lowercase to deal with situations where sometimes they're lower-case and other times not --- wqflask/wqflask/marker_regression/run_mapping.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py index d727e3ff..c5b980a7 100644 --- a/wqflask/wqflask/marker_regression/run_mapping.py +++ b/wqflask/wqflask/marker_regression/run_mapping.py @@ -765,9 +765,9 @@ def get_perm_strata(this_trait, sample_list, categorical_vars, used_samples): if sample in list(sample_list.sample_attribute_values.keys()): combined_string = "" for var in categorical_vars: - if var in list(sample_list.sample_attribute_values[sample].keys()): + if var.lower() in sample_list.sample_attribute_values[sample]: combined_string += str( - sample_list.sample_attribute_values[sample][var]) + sample_list.sample_attribute_values[sample][var.lower()]) else: combined_string += "NA" else: -- cgit v1.2.3 From 2c2e1dc3d4be635dbf73aab380e22acca6a2014b Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:18:36 +0000 Subject: Improved test_rqtl_mapping.py with Bonface's recommendation of uses dataclasses --- .../wqflask/marker_regression/test_rqtl_mapping.py | 23 ++++++++++------------ 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 6853c021..bd97b2d2 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -2,17 +2,17 @@ import unittest from unittest import mock from wqflask.marker_regression.rqtl_mapping import run_rqtl -class AttributeSetter: - def __init__(self, obj): - for key, val in obj.items(): - setattr(self, key, val) +@dataclass +class MockGroup: + name: str, + genofile: str -class MockGroup(AttributeSetter): - def get_samplelist(self): - return None +@dataclass +class MockDataset: + group: MockGroup class TestRqtlMapping(unittest.TestCase): - + """Tests for functions in rqtl_mapping.py""" @mock.patch("wqflask.marker_regression.rqtl_mapping.process_rqtl_results") @mock.patch("wqflask.marker_regression.rqtl_mapping.process_perm_results") @mock.patch("wqflask.marker_regression.rqtl_mapping.requests.post") @@ -22,15 +22,12 @@ class TestRqtlMapping(unittest.TestCase): """Test for run_rqtl with permutations > 0""" dataset_group = MockGroup( {"name": "GP1", "genofile": "file_geno"}) - - dataset = AttributeSetter({"group": dataset_group}) + dataset = MockDataset(dataset_group) mock_write_pheno_file.return_value = "pheno_filename" mock_locate.return_value = "geno_filename" - mock_post.return_value = {"output_file": "output_filename", "rqtl_cmd": "the_command"} - mock_process_perm.return_value = [[], 3, 4] mock_process_rqtl.return_value = [] @@ -44,4 +41,4 @@ class TestRqtlMapping(unittest.TestCase): mock_post.assert_called_once() mock_process_perm.assert_called_once() mock_process_rqtl.assert_called_once() - self.assertEqual(results, ([], 3, 4, [])) \ No newline at end of file + self.assertEqual(results, ([], 3, 4, [])) -- cgit v1.2.3 From 37941d2724aaf22c1aeca18a77e4c17248c5b7bc Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:27:26 +0000 Subject: Updated test_run_mapping.py to account for attribute keys being checked as lowercase, though not sure if this will fully fix the test --- .../unit/wqflask/marker_regression/test_run_mapping.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py index 78cd3be9..c220a072 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_run_mapping.py @@ -229,20 +229,20 @@ class TestRunMapping(unittest.TestCase): used_samples = ["S1", "S2"] sample_list = AttributeSetter({"sample_attribute_values": { "S1": { - "C1": "c1_value", - "C2": "c2_value", - "W1": "w1_value" + "c1": "c1_value", + "c2": "c2_value", + "w1": "w1_value" }, "S2": { - "W1": "w2_value", - "W2": "w2_value" + "w1": "w2_value", + "w2": "w2_value" }, "S3": { - "C1": "c1_value", - "C2": "c2_value" + "c1": "c1_value", + "c2": "c2_value" }, -- cgit v1.2.3 From 09768ed9c255cdd10561eeba28bf3752747bf378 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:29:57 +0000 Subject: Removed parts of test_rqtl_mapping.py referring to process_perm_results and process_rqtl_results since that functionality was moved to GN3 --- .../tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index bd97b2d2..00a05d2d 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -13,12 +13,10 @@ class MockDataset: class TestRqtlMapping(unittest.TestCase): """Tests for functions in rqtl_mapping.py""" - @mock.patch("wqflask.marker_regression.rqtl_mapping.process_rqtl_results") - @mock.patch("wqflask.marker_regression.rqtl_mapping.process_perm_results") @mock.patch("wqflask.marker_regression.rqtl_mapping.requests.post") @mock.patch("wqflask.marker_regression.rqtl_mapping.locate") @mock.patch("wqflask.marker_regression.rqtl_mapping.write_phenotype_file") - def test_run_rqtl_with_perm(self, mock_write_pheno_file, mock_locate, mock_post, mock_process_perm, mock_process_rqtl): + def test_run_rqtl_with_perm(self, mock_write_pheno_file, mock_locate, mock_post): """Test for run_rqtl with permutations > 0""" dataset_group = MockGroup( {"name": "GP1", "genofile": "file_geno"}) @@ -28,8 +26,6 @@ class TestRqtlMapping(unittest.TestCase): mock_locate.return_value = "geno_filename" mock_post.return_value = {"output_file": "output_filename", "rqtl_cmd": "the_command"} - mock_process_perm.return_value = [[], 3, 4] - mock_process_rqtl.return_value = [] results = run_rqtl(trait_name="the_trait", vals=[], samples=[], dataset=dataset, mapping_scale="cM", model="normal", method="hk", @@ -39,6 +35,5 @@ class TestRqtlMapping(unittest.TestCase): mock_write_pheno_file.assert_called_once() mock_locate.assert_called_once() mock_post.assert_called_once() - mock_process_perm.assert_called_once() - mock_process_rqtl.assert_called_once() + self.assertEqual(results, ([], 3, 4, [])) -- cgit v1.2.3 From 0cc4376462b7f136e59d99dba1975d6023326126 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:32:58 +0000 Subject: Fixed syntax mistake when creating dataclass in test_rqtl_mapping.py --- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 00a05d2d..31a2c9e4 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -4,7 +4,7 @@ from wqflask.marker_regression.rqtl_mapping import run_rqtl @dataclass class MockGroup: - name: str, + name: str genofile: str @dataclass -- cgit v1.2.3 From 97df46292e8322f01466d36b1e1e3ecdad64e5c0 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:50:03 +0000 Subject: Forgot to import dataclass --- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 31a2c9e4..ff451230 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -1,5 +1,7 @@ import unittest from unittest import mock +from dataclasses import dataclass + from wqflask.marker_regression.rqtl_mapping import run_rqtl @dataclass -- cgit v1.2.3 From 767ff7e97a751f164da610b2f9c536b6660ec420 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 20:53:22 +0000 Subject: Fixed way MockGroup was initialized --- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index ff451230..a7f708e1 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -20,8 +20,7 @@ class TestRqtlMapping(unittest.TestCase): @mock.patch("wqflask.marker_regression.rqtl_mapping.write_phenotype_file") def test_run_rqtl_with_perm(self, mock_write_pheno_file, mock_locate, mock_post): """Test for run_rqtl with permutations > 0""" - dataset_group = MockGroup( - {"name": "GP1", "genofile": "file_geno"}) + dataset_group = MockGroup("GP1", "file_geno") dataset = MockDataset(dataset_group) mock_write_pheno_file.return_value = "pheno_filename" -- cgit v1.2.3 From f69eb61c5f58b008233f7bee0e551fe9151c7724 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 21:09:18 +0000 Subject: Change test_rqtl_mapping.py to account for full results being returned from the GN3 request --- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index a7f708e1..9a5162ae 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -25,8 +25,11 @@ class TestRqtlMapping(unittest.TestCase): mock_write_pheno_file.return_value = "pheno_filename" mock_locate.return_value = "geno_filename" - mock_post.return_value = {"output_file": "output_filename", - "rqtl_cmd": "the_command"} + mock_post.return_value = Mock(ok=True) + mock_post.return_value.json.return_value = {"perm_results": [], + "suggestive": 3, + "significant": 4, + "results" : []} results = run_rqtl(trait_name="the_trait", vals=[], samples=[], dataset=dataset, mapping_scale="cM", model="normal", method="hk", -- cgit v1.2.3 From e78c84aa10849465e0272daa6d94bfbd3419b072 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 27 May 2021 21:11:16 +0000 Subject: Fix the way Mock is initialized --- wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py index 9a5162ae..9d13e943 100644 --- a/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py +++ b/wqflask/tests/unit/wqflask/marker_regression/test_rqtl_mapping.py @@ -25,7 +25,7 @@ class TestRqtlMapping(unittest.TestCase): mock_write_pheno_file.return_value = "pheno_filename" mock_locate.return_value = "geno_filename" - mock_post.return_value = Mock(ok=True) + mock_post.return_value = mock.Mock(ok=True) mock_post.return_value.json.return_value = {"perm_results": [], "suggestive": 3, "significant": 4, -- cgit v1.2.3 From afc7a742da58a318c2b9c8b1c93209fc3820009a Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 28 May 2021 18:24:50 +0000 Subject: Re-added collect import in views.py since it was removed before (I'm guessing because it appeared to not be used, but it actually is used because it includes the collection endpoints --- wqflask/wqflask/views.py | 1 + 1 file changed, 1 insertion(+) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index ee69900e..86219faa 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -41,6 +41,7 @@ from flask import redirect from flask import url_for from flask import send_file +from wqflask import collect from wqflask import search_results from wqflask import server_side from base.data_set import create_dataset # Used by YAML in marker_regression -- cgit v1.2.3 From d0b78700c788281b8be0372784088cc021f24b00 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 28 May 2021 18:41:54 +0000 Subject: Added comment clarifying the situation with views.py imports --- wqflask/wqflask/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 86219faa..2c5b6c6b 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -41,6 +41,8 @@ from flask import redirect from flask import url_for from flask import send_file +# Some of these (like collect) might contain endpoints, so they're still used. +# Blueprints should probably be used instead. from wqflask import collect from wqflask import search_results from wqflask import server_side -- cgit v1.2.3 From 8de5c32c4846b0382ded05ddb633ab67e5e062c8 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 1 Jun 2021 16:34:08 +0300 Subject: templates: edit_trait: Only submit form elements that have changed We only submit form elements that have changed, and use that to update the relevant tables. See: https://stackoverflow.com/a/5221915 --- wqflask/wqflask/templates/edit_trait.html | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 2cd8a6fd..05bdebd1 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -162,5 +162,13 @@ Submit Trait | Reset {% block js %} {% endblock %} -- cgit v1.2.3 From 23c9d5eef66f159c422b136be9a837b6700208a4 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 1 Jun 2021 16:40:16 +0300 Subject: templates: edit_trait: Add "changed" class to hidden input values When a form element changes, it's marked as changed. Only input elements with "changed" get submitted. Since we need to submit the "hidden" input fields, we add "changed" to them anyways. --- wqflask/wqflask/templates/edit_trait.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 05bdebd1..393b8c39 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -148,12 +148,12 @@ Submit Trait | Reset
      - - - - - - + + + + + +
      -- cgit v1.2.3 From 271be90f028bf7027f7398251e989ab77f7a9499 Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Tue, 1 Jun 2021 17:09:31 +0300 Subject: templates: edit_trait: Use correct textarea "name" for post_pub_desc --- wqflask/wqflask/templates/edit_trait.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index 393b8c39..f01c497c 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -38,7 +38,7 @@ Submit Trait | Reset
      - +
      -- cgit v1.2.3 From c196bced9203625b4fa35d8d2752b70893ad3488 Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 1 Jun 2021 20:59:33 +0000 Subject: Make the container div for the trait table have a minimum width scaling off of the trait table width, to avoid the border overlapping with the table --- wqflask/wqflask/show_trait/show_trait.py | 2 -- wqflask/wqflask/templates/show_trait.html | 2 +- wqflask/wqflask/templates/show_trait_edit_data.html | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index a02da872..9ee6a16d 100644 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -624,8 +624,6 @@ def get_table_widths(sample_groups, sample_column_width, has_num_cases=False): trait_table_width += 80 trait_table_width += len(sample_groups[0].attributes) * 88 - trait_table_width = str(trait_table_width) + "px" - return stats_table_width, trait_table_width diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html index fc14822c..3dbf5f57 100644 --- a/wqflask/wqflask/templates/show_trait.html +++ b/wqflask/wqflask/templates/show_trait.html @@ -102,7 +102,7 @@
      -
      +
      1100) %}style="min-width: {{ trait_table_width|int + 30 }}px;"{% endif %}>

      Review and Edit Data diff --git a/wqflask/wqflask/templates/show_trait_edit_data.html b/wqflask/wqflask/templates/show_trait_edit_data.html index 0d34bebc..5939c953 100644 --- a/wqflask/wqflask/templates/show_trait_edit_data.html +++ b/wqflask/wqflask/templates/show_trait_edit_data.html @@ -53,7 +53,7 @@

      {% set outer_loop = loop %} -
      +

      {{ sample_type.header }}

      -- cgit v1.2.3 From a73e34707d88e849a6d62afc58fe4aa1d88c5dec Mon Sep 17 00:00:00 2001 From: zsloan Date: Tue, 1 Jun 2021 21:04:01 +0000 Subject: Made Genotype drop-down a little wider, since the genotype file names can be longer --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index c3575454..3dd44c85 100755 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -39,7 +39,7 @@ {% if genofiles and genofiles|length>0 %}
      -
      +
      {% for item in genofiles %} @@ -218,7 +218,7 @@
      -
      +
      ' } @@ -166,45 +167,53 @@ { 'title': "Index", 'type': "natural", + 'width': "30px", 'data': "index" }, { 'title': "Record", 'type': "natural", - 'data': null, 'orderDataType': "dom-inner-text", + 'width': "60px", + 'data': null, 'render': function(data, type, row, meta) { return '' + data.name + '' } }, { 'title': "Species", - 'type': "natural", + 'type': "natural", + 'width': "60px", 'data': "species" }, { 'title': "Group", 'type': "natural", + 'width': "150px", 'data': "group" }, { 'title': "Tissue", 'type': "natural", + 'width': "150px", 'data': "tissue" }, { 'title': "Dataset", 'type': "natural", + 'width': "300px", 'data': "dataset_fullname" }, { 'title': "Symbol", 'type': "natural", + 'width': "60px", 'data': "symbol" }, { 'title': "Description", 'type': "natural", + 'width': "300px", 'data': null, 'render': function(data, type, row, meta) { try { @@ -217,25 +226,27 @@ { 'title': "Location", 'type': "natural-minus-na", - 'width': "100px", + 'width': "125px", 'data': "location_repr" }, { 'title': "Mean", 'type': "natural-minus-na", - 'data': "mean", - 'orderSequence': [ "desc", "asc"] + 'orderSequence': [ "desc", "asc"], + 'width': "30px", + 'data': "mean" }, { 'title': "Max
      LRS?", 'type': "natural-minus-na", + 'width': "60px", 'data': "LRS_score_repr", 'orderSequence': [ "desc", "asc"] }, { - 'title': "Max LRS
      Location", + 'title': "Max LRS Location", 'type': "natural-minus-na", - 'width': "100px", + 'width': "125px", 'data': "max_lrs_text" }, { -- cgit v1.2.3 From 34c7ad6ebd62dc2baa5ab7e36e2bff9639925748 Mon Sep 17 00:00:00 2001 From: jgart Date: Fri, 4 Jun 2021 22:12:35 +0000 Subject: Update class docstring Author: jgart Date: Fri Jun 4 22:12:35 2021 +0000 --- wqflask/wqflask/server_side.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/wqflask/wqflask/server_side.py b/wqflask/wqflask/server_side.py index 7f68efad..e661c407 100644 --- a/wqflask/wqflask/server_side.py +++ b/wqflask/wqflask/server_side.py @@ -9,13 +9,11 @@ class ServerSideTable: the client-side and reduces the size of data interchanged. Usage: - ServerSideTable(table_data, request_values) + ServerSideTable(rows_count, table_rows, header_data_names, request_values) where, - `table_data` must have data members `rows_count` as number of rows in the table, `table_rows` as data rows of the table, `header_data_names` as headers names of the table. - `request_values` must have request arguments values including the DataTables server-side processing arguments. -- cgit v1.2.3 From 1415a0c29b1f06a40f2f73af85b3d71f0ce5bf5b Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Wed, 2 Jun 2021 08:33:51 +0300 Subject: templates: edit_trait: Add hidden inputs that contain old values --- wqflask/wqflask/templates/edit_trait.html | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/wqflask/wqflask/templates/edit_trait.html b/wqflask/wqflask/templates/edit_trait.html index f01c497c..e0f2f1d4 100644 --- a/wqflask/wqflask/templates/edit_trait.html +++ b/wqflask/wqflask/templates/edit_trait.html @@ -23,12 +23,14 @@ Submit Trait | Reset character in this field. -->
      +
      +