diff options
Diffstat (limited to 'gn2/wqflask/show_trait')
| -rw-r--r-- | gn2/wqflask/show_trait/SampleList.py | 223 | ||||
| -rw-r--r-- | gn2/wqflask/show_trait/__init__.py | 0 | ||||
| -rw-r--r-- | gn2/wqflask/show_trait/export_trait_data.py | 113 | ||||
| -rw-r--r-- | gn2/wqflask/show_trait/show_trait.py | 859 |
4 files changed, 1195 insertions, 0 deletions
diff --git a/gn2/wqflask/show_trait/SampleList.py b/gn2/wqflask/show_trait/SampleList.py new file mode 100644 index 00000000..64fc8fe6 --- /dev/null +++ b/gn2/wqflask/show_trait/SampleList.py @@ -0,0 +1,223 @@ +import re +import itertools + +from gn2.wqflask.database import database_connection +from gn2.base import webqtlCaseData, webqtlConfig +from pprint import pformat as pf + +from gn2.utility import Plot +from gn2.utility import Bunch +from gn2.utility.tools import get_setting + +class SampleList: + def __init__(self, + dataset, + sample_names, + this_trait, + sample_group_type="primary", + header="Samples"): + + self.dataset = dataset + self.this_trait = this_trait + self.sample_group_type = sample_group_type # primary or other + self.header = header + + self.sample_list = [] # The actual list + self.sample_attribute_values = {} + + self.get_attributes() + + if self.this_trait and self.dataset: + self.get_extra_attribute_values() + + for counter, sample_name in enumerate(sample_names, 1): + sample_name = sample_name.replace("_2nd_", "") + + # self.this_trait will be a list if it is a Temp trait + if isinstance(self.this_trait, list): + sample = webqtlCaseData.webqtlCaseData(name=sample_name) + if counter <= len(self.this_trait): + if isinstance(self.this_trait[counter - 1], (bytes, bytearray)): + if (self.this_trait[counter - 1].decode("utf-8").lower() != 'x'): + sample = webqtlCaseData.webqtlCaseData( + name=sample_name, + value=float(self.this_trait[counter - 1])) + else: + if (self.this_trait[counter - 1].lower() != 'x'): + sample = webqtlCaseData.webqtlCaseData( + name=sample_name, + value=float(self.this_trait[counter - 1])) + else: + # If there's no value for the sample/strain, + # create the sample object (so samples with no value + # are still displayed in the table) + try: + sample = self.this_trait.data[sample_name] + except KeyError: + sample = webqtlCaseData.webqtlCaseData(name=sample_name) + + sample.extra_info = {} + if (self.dataset.group.name == 'AXBXA' + and sample_name in ('AXB18/19/20', 'AXB13/14', 'BXA8/17')): + sample.extra_info['url'] = "/mouseCross.html#AXB/BXA" + sample.extra_info['css_class'] = "fs12" + + sample.this_id = str(counter) + + # For extra attribute columns; currently only used by + # several datasets + if self.sample_attribute_values: + sample.extra_attributes = self.sample_attribute_values.get( + sample_name, {}) + + # Add a url so RRID case attributes can be displayed as links + if '36' in sample.extra_attributes: + rrid_string = str(sample.extra_attributes['36']) + if self.dataset.group.species == "mouse": + if len(rrid_string.split(":")) > 1: + the_rrid = rrid_string.split(":")[1] + sample.extra_attributes['36'] = [ + rrid_string] + sample.extra_attributes['36'].append( + webqtlConfig.RRID_MOUSE_URL % the_rrid) + elif self.dataset.group.species == "rat": + if len(rrid_string): + the_rrid = rrid_string.split("_")[1] + sample.extra_attributes['36'] = [ + rrid_string] + sample.extra_attributes['36'].append( + webqtlConfig.RRID_RAT_URL % the_rrid) + + self.sample_list.append(sample) + + self.se_exists = any(sample.variance for sample in self.sample_list) + self.num_cases_exists = False + if (any(sample.num_cases for sample in self.sample_list) and + any((sample.num_cases and sample.num_cases != "1") for sample in self.sample_list)): + self.num_cases_exists = True + + first_attr_col = self.get_first_attr_col() + for sample in self.sample_list: + sample.first_attr_col = first_attr_col + + self.do_outliers() + + def __repr__(self): + return "<SampleList> --> %s" % (pf(self.__dict__)) + + def do_outliers(self): + values = [sample.value for sample in self.sample_list + if sample.value is not None] + upper_bound, lower_bound = Plot.find_outliers(values) + + for sample in self.sample_list: + if sample.value: + if upper_bound and sample.value > upper_bound: + sample.outlier = True + elif lower_bound and sample.value < lower_bound: + sample.outlier = True + else: + sample.outlier = False + + def get_attributes(self): + """Finds which extra attributes apply to this dataset""" + + # Get attribute names and distinct values for each attribute + with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT DISTINCT CaseAttribute.CaseAttributeId, " + "CaseAttribute.Name, CaseAttribute.Description, " + "CaseAttributeXRefNew.Value FROM " + "CaseAttribute, CaseAttributeXRefNew WHERE " + "CaseAttributeXRefNew.CaseAttributeId = CaseAttribute.CaseAttributeId " + "AND CaseAttributeXRefNew.InbredSetId = %s " + "ORDER BY CaseAttribute.CaseAttributeId", (str(self.dataset.group.id),) + ) + + self.attributes = {} + for attr, values in itertools.groupby( + cursor.fetchall(), lambda row: (row[0], row[1], row[2]) + ): + key, name, description = attr + self.attributes[key] = Bunch() + self.attributes[key].id = key + self.attributes[key].name = name + self.attributes[key].description = description + self.attributes[key].distinct_values = [ + item[3] for item in values] + self.attributes[key].distinct_values = natural_sort( + self.attributes[key].distinct_values) + all_numbers = True + for value in self.attributes[key].distinct_values: + try: + val_as_float = float(value) + except: + all_numbers = False + break + + if all_numbers: + self.attributes[key].alignment = "right" + else: + self.attributes[key].alignment = "left" + + def get_extra_attribute_values(self): + if self.attributes: + with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT Strain.Name AS SampleName, " + "CaseAttributeId AS Id, " + "CaseAttributeXRefNew.Value FROM Strain, " + "StrainXRef, InbredSet, CaseAttributeXRefNew " + "WHERE StrainXRef.StrainId = Strain.Id " + "AND InbredSet.Id = StrainXRef.InbredSetId " + "AND CaseAttributeXRefNew.StrainId = Strain.Id " + "AND InbredSet.Id = CaseAttributeXRefNew.InbredSetId " + "AND CaseAttributeXRefNew.InbredSetId = %s " + "ORDER BY SampleName", (self.dataset.group.id,) + ) + + for sample_name, items in itertools.groupby( + cursor.fetchall(), lambda row: row[0] + ): + attribute_values = {} + # Make a list of attr IDs without values (that have values for other samples) + valueless_attr_ids = [self.attributes[key].id for key in self.attributes.keys()] + for item in items: + sample_name, _id, value = item + valueless_attr_ids.remove(_id) + attribute_value = value + + # If it's an int, turn it into one for sorting + # (for example, 101 would be lower than 80 if + # they're strings instead of ints) + try: + attribute_value = int(attribute_value) + except ValueError: + pass + + attribute_values[str(_id)] = attribute_value + for attr_id in valueless_attr_ids: + attribute_values[str(attr_id)] = "" + + self.sample_attribute_values[sample_name] = attribute_values + + def get_first_attr_col(self): + first_attr_col = 4 + if self.se_exists: + first_attr_col += 2 + if self.num_cases_exists: + first_attr_col += 1 + + return first_attr_col + + +def natural_sort(a_list, key=lambda s: s): + """ + Sort the list into natural alphanumeric order. + """ + def get_alphanum_key_func(key): + def convert(text): return int(text) if text.isdigit() else text + return lambda s: [convert(c) for c in re.split('([0-9]+)', key(s))] + sort_key = get_alphanum_key_func(key) + sorted_list = sorted(a_list, key=sort_key) + return sorted_list diff --git a/gn2/wqflask/show_trait/__init__.py b/gn2/wqflask/show_trait/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/gn2/wqflask/show_trait/__init__.py diff --git a/gn2/wqflask/show_trait/export_trait_data.py b/gn2/wqflask/show_trait/export_trait_data.py new file mode 100644 index 00000000..06c1c502 --- /dev/null +++ b/gn2/wqflask/show_trait/export_trait_data.py @@ -0,0 +1,113 @@ +import datetime +import simplejson as json + +from pprint import pformat as pf +from functools import cmp_to_key +from gn2.base.trait import create_trait +from gn2.base import data_set + + +def export_sample_table(targs): + + sample_data = json.loads(targs['export_data']) + trait_name = targs['trait_display_name'] + + meta_data = get_export_metadata(targs) + + final_sample_data = meta_data + + column_headers = ["Index", "Name", "Value"] + attr_pos = 2 + if any(sample["se"] for sample in sample_data['primary_samples']): + column_headers.append("SE") + attr_pos = 3 + if any(sample["num_cases"] for sample in sample_data['primary_samples']): + column_headers.append("N") + attr_pos = 4 + + for key in sample_data["primary_samples"][0].keys(): + if key not in ["name", "value", "se", "num_cases"]: + column_headers.append(key) + + final_sample_data.append(column_headers) + for sample_group in ['primary_samples', 'other_samples']: + for i, row in enumerate(sample_data[sample_group]): + sorted_row = [i + 1] + dict_to_sorted_list(row)[:attr_pos] + for attr in sample_data['attributes']: + sorted_row.append(row[attr]) + final_sample_data.append(sorted_row) + + return trait_name, final_sample_data + + +def get_export_metadata(trait_metadata): + + trait_id, display_name, dataset_name, group_name = trait_metadata['trait_id'], trait_metadata['trait_display_name'], trait_metadata['dataset'], trait_metadata['group'] + + dataset = data_set.create_dataset(dataset_name, group_name=group_name) + this_trait = create_trait(dataset=dataset, + name=trait_id, + cellid=None, + get_qtl_info=False) + + metadata = [] + if dataset.type == "Publish": + metadata.append(["Phenotype ID:", display_name]) + metadata.append(["Phenotype URL: ", "http://genenetwork.org/show_trait?trait_id=" + \ + trait_id + "&dataset=" + dataset_name]) + metadata.append(["Group: ", dataset.group.name]) + metadata.append( + ["Phenotype: ", this_trait.description_display.replace(",", "\",\"")]) + metadata.append( + ["Authors: ", (this_trait.authors if this_trait.authors else "N/A")]) + metadata.append( + ["Title: ", (this_trait.title if this_trait.title else "N/A")]) + metadata.append( + ["Journal: ", (this_trait.journal if this_trait.journal else "N/A")]) + + metadata.append( + ["Dataset Link: ", "http://gn1.genenetwork.org/webqtl/main.py?FormID=sharinginfo&InfoPageName=" + dataset.name]) + else: + metadata.append(["Record ID: ", trait_id]) + metadata.append(["Trait URL: ", "http://genenetwork.org/show_trait?trait_id=" + \ + trait_id + "&dataset=" + dataset_name]) + if this_trait.symbol: + metadata.append(["Symbol: ", this_trait.symbol]) + metadata.append(["Dataset: ", dataset.name]) + metadata.append(["Group: ", dataset.group.name]) + metadata.append( + ["Export Date: ", datetime.datetime.now().strftime("%B %d, %Y")]) + metadata.append( + ["Export Time: ", datetime.datetime.now().strftime("%H:%M GMT")]) + + + return metadata + + +def dict_to_sorted_list(dictionary): + sorted_list = [item for item in list(dictionary.items())] + sorted_list = sorted(sorted_list, key=cmp_to_key(cmp_samples)) + sorted_values = [item[1] for item in sorted_list] + return sorted_values + + +def cmp_samples(a, b): + if b[0] == 'name': + return 1 + elif b[0] == 'value': + if a[0] == 'name': + return -1 + else: + return 1 + elif b[0] == 'se': + if a[0] == 'name' or a[0] == 'value': + return -1 + else: + return 1 + elif b[0] == 'num_cases': + if a[0] == 'name' or a[0] == 'value' or a[0] == 'se': + return -1 + else: + return 1 + else: + return -1 diff --git a/gn2/wqflask/show_trait/show_trait.py b/gn2/wqflask/show_trait/show_trait.py new file mode 100644 index 00000000..64d9be3b --- /dev/null +++ b/gn2/wqflask/show_trait/show_trait.py @@ -0,0 +1,859 @@ +from typing import Dict + +import string +import datetime +import uuid +import requests +import json as json + +from collections import OrderedDict + +import numpy as np +import scipy.stats as ss + +from gn2.wqflask.database import database_connection + +from gn2.base import webqtlConfig +from gn2.wqflask.show_trait.SampleList import SampleList +from gn2.base.trait import create_trait +from gn2.base import data_set +from gn2.utility import helper_functions +from gn2.utility.tools import get_setting, locate_ignore_error +from gn2.utility.tools import GN_PROXY_URL +from gn2.utility.redis_tools import get_redis_conn, get_resource_id + +from gn3.authentication import get_highest_user_access_role + +Redis = get_redis_conn() +ONE_YEAR = 60 * 60 * 24 * 365 + + +############################################### +# +# Todo: Put in security to ensure that user has permission to access +# confidential data sets And add i.p.limiting as necessary +# +############################################## + + +class ShowTrait: + def __init__(self, db_cursor, user_id, kw): + if 'trait_id' in kw and kw['dataset'] != "Temp": + self.temp_trait = False + self.trait_id = kw['trait_id'] + helper_functions.get_species_dataset_trait(self, kw) + self.resource_id = get_resource_id(self.dataset, + self.trait_id) + elif 'group' in kw: + self.temp_trait = True + self.trait_id = "Temp_" + kw['species'] + "_" + kw['group'] + \ + "_" + datetime.datetime.now().strftime("%m%d%H%M%S") + self.temp_species = kw['species'] + self.temp_group = kw['group'] + self.dataset = data_set.create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=self.temp_group) + + # Put values in Redis so they can be looked up later if + # added to a collection + Redis.set(self.trait_id, kw['trait_paste'], ex=ONE_YEAR) + self.trait_vals = kw['trait_paste'].split() + self.this_trait = create_trait(dataset=self.dataset, + name=self.trait_id, + cellid=None) + else: + self.temp_trait = True + self.trait_id = kw['trait_id'] + self.temp_species = self.trait_id.split("_")[1] + self.temp_group = self.trait_id.split("_")[2] + self.dataset = data_set.create_dataset( + dataset_name="Temp", dataset_type="Temp", group_name=self.temp_group) + self.this_trait = create_trait(dataset=self.dataset, + name=self.trait_id, + cellid=None) + self.trait_vals = Redis.get(self.trait_id).split() + + # Get verify/rna-seq link URLs + try: + blatsequence = self.this_trait.blatseq if self.dataset.type == "ProbeSet" else self.this_trait.sequence + if not blatsequence: + # XZ, 06/03/2009: ProbeSet name is not unique among platforms. We should use ProbeSet Id instead. + seqs = () + if self.dataset.type == "ProbeSet": + db_cursor.execute( + "SELECT Probe.Sequence, Probe.Name " + "FROM Probe, ProbeSet, ProbeSetFreeze, " + "ProbeSetXRef WHERE " + "ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id " + "AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND " + "ProbeSetFreeze.Name = %s AND " + "ProbeSet.Name = %s AND " + "Probe.ProbeSetId = ProbeSet.Id ORDER " + "BY Probe.SerialOrder", + (self.this_trait.dataset.name, self.this_trait.name,) + ) + else: + db_cursor.execute( + "SELECT Geno.Sequence " + "FROM Geno, GenoXRef, GenoFreeze " + "WHERE Geno.Name = %s AND " + "Geno.Id = GenoXRef.GenoId AND " + "GenoXRef.GenoFreezeId = GenoFreeze.Id AND " + "GenoFreeze.Name = %s", + (self.this_trait.name, self.this_trait.dataset.name) + ) + seqs = db_cursor.fetchall() + if not seqs: + raise ValueError + else: + blatsequence = '' + for seqt in seqs: + if int(seqt[1][-1]) % 2 == 1: + blatsequence += string.strip(seqt[0]) + + # --------Hongqiang add this part in order to not only blat ProbeSet, but also blat Probe + blatsequence = '%3E' + self.this_trait.name + '%0A' + blatsequence + '%0A' + + # XZ, 06/03/2009: ProbeSet name is not unique among platforms. We should use ProbeSet Id instead. + seqs = () + db_cursor.execute( + "SELECT Probe.Sequence, Probe.Name " + "FROM Probe, ProbeSet, ProbeSetFreeze, " + "ProbeSetXRef WHERE " + "ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id " + "AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND " + "ProbeSetFreeze.Name = %s AND ProbeSet.Name = %s " + "AND Probe.ProbeSetId = ProbeSet.Id " + "ORDER BY Probe.SerialOrder", + (self.this_trait.dataset.name, self.this_trait.name,) + ) + seqs = db_cursor.fetchall() + for seqt in seqs: + if int(seqt[1][-1]) % 2 == 1: + blatsequence += '%3EProbe_' + \ + seqt[1].strip() + '%0A' + seqt[0].strip() + '%0A' + + if self.dataset.group.species == "rat": + self.UCSC_BLAT_URL = webqtlConfig.UCSC_BLAT % ( + 'rat', 'rn7', blatsequence) + self.UTHSC_BLAT_URL = "" + elif self.dataset.group.species == "mouse": + self.UCSC_BLAT_URL = webqtlConfig.UCSC_BLAT % ( + 'mouse', 'mm10', blatsequence) + self.UTHSC_BLAT_URL = webqtlConfig.UTHSC_BLAT % ( + 'mouse', 'mm10', blatsequence) + elif self.dataset.group.species == "human": + self.UCSC_BLAT_URL = webqtlConfig.UCSC_BLAT % ( + 'human', 'hg38', blatsequence) + self.UTHSC_BLAT_URL = "" + else: + self.UCSC_BLAT_URL = "" + self.UTHSC_BLAT_URL = "" + except: + self.UCSC_BLAT_URL = "" + self.UTHSC_BLAT_URL = "" + + if self.dataset.type == "ProbeSet": + self.show_probes = "True" + + trait_units = get_trait_units(self.this_trait) + self.get_external_links() + self.build_correlation_tools() + + self.ncbi_summary = get_ncbi_summary(self.this_trait) + + # Get nearest marker for composite mapping + if not self.temp_trait: + if check_if_attr_exists(self.this_trait, 'locus_chr') and self.dataset.type != "Geno" and self.dataset.type != "Publish": + self.nearest_marker = get_nearest_marker( + self.this_trait, self.dataset) + else: + self.nearest_marker = "" + + self.make_sample_lists() + + trait_vals_by_group = [] + for sample_type in self.sample_groups: + trait_vals_by_group.append(get_trait_vals(sample_type.sample_list)) + + self.max_digits_by_group, self.no_decimal_place = get_max_digits(trait_vals_by_group) + + self.qnorm_vals = quantile_normalize_vals(self.sample_groups, trait_vals_by_group) + self.z_scores = get_z_scores(self.sample_groups, trait_vals_by_group) + + self.temp_uuid = uuid.uuid4() + + self.sample_group_types = OrderedDict() + if len(self.sample_groups) > 1: + self.sample_group_types['samples_primary'] = self.dataset.group.name + self.sample_group_types['samples_other'] = "Other" + self.sample_group_types['samples_all'] = "All" + else: + self.sample_group_types['samples_primary'] = self.dataset.group.name + sample_lists = [group.sample_list for group in self.sample_groups] + + self.categorical_var_list = [] + self.numerical_var_list = [] + if not self.temp_trait: + # ZS: Only using first samplelist, since I think mapping only uses those samples + self.categorical_var_list = get_categorical_variables( + self.this_trait, self.sample_groups[0]) + self.numerical_var_list = get_numerical_variables( + self.this_trait, self.sample_groups[0]) + + # ZS: Get list of chromosomes to select for mapping + self.chr_list = [["All", -1]] + for i, this_chr in enumerate(self.dataset.species.chromosomes.chromosomes(db_cursor)): + self.chr_list.append( + [self.dataset.species.chromosomes.chromosomes(db_cursor)[this_chr].name, i]) + + self.genofiles = self.dataset.group.get_genofiles() + study_samplelist_json = self.dataset.group.get_study_samplelists() + self.study_samplelists = [study["title"] for study in study_samplelist_json] + + # ZS: No need to grab scales from .geno file unless it's using + # a mapping method that reads .geno files + if "QTLReaper" or "R/qtl" in dataset.group.mapping_names: + if self.genofiles: + self.scales_in_geno = get_genotype_scales(self.genofiles) + else: + self.scales_in_geno = get_genotype_scales( + self.dataset.group.name + ".geno") + else: + self.scales_in_geno = {} + + self.has_num_cases = has_num_cases(self.this_trait) + + # ZS: Needed to know whether to display bar chart + get max + # sample name length in order to set table column width + self.num_values = 0 + # ZS: So it knows whether to display the Binary R/qtl mapping + # method, which doesn't work unless all values are 0 or 1 + self.binary = "true" + # ZS: Since we don't want to show log2 transform option for + # situations where it doesn't make sense + self.negative_vals_exist = "false" + max_samplename_width = 1 + for group in self.sample_groups: + for sample in group.sample_list: + if len(sample.name) > max_samplename_width: + max_samplename_width = len(sample.name) + if sample.display_value != "x": + self.num_values += 1 + if sample.display_value != 0 or sample.display_value != 1: + self.binary = "false" + if sample.value < 0: + self.negative_vals_exist = "true" + + # ZS: Check whether any attributes have few enough distinct + # values to show the "Block samples by group" option + self.categorical_attr_exists = "false" + for attribute in self.sample_groups[0].attributes: + if len(self.sample_groups[0].attributes[attribute].distinct_values) <= 10: + self.categorical_attr_exists = "true" + break + + sample_column_width = max_samplename_width * 8 + + self.stats_table_width, self.trait_table_width = get_table_widths( + self.sample_groups, sample_column_width, self.has_num_cases) + + if self.num_values >= 5000: + self.maf = 0.01 + else: + self.maf = 0.05 + + trait_symbol = None + short_description = None + if not self.temp_trait: + if self.this_trait.symbol: + trait_symbol = self.this_trait.symbol + short_description = trait_symbol + + elif hasattr(self.this_trait, 'post_publication_abbreviation'): + short_description = self.this_trait.post_publication_abbreviation + + elif hasattr(self.this_trait, 'pre_publication_abbreviation'): + short_description = self.this_trait.pre_publication_abbreviation + + # Todo: Add back in the ones we actually need from below, as we discover we need them + hddn = OrderedDict() + + if self.dataset.group.allsamples: + hddn['allsamples'] = ','.join(self.dataset.group.allsamples) + hddn['primary_samples'] = ','.join(self.primary_sample_names) + hddn['trait_id'] = self.trait_id + hddn['trait_display_name'] = self.this_trait.display_name + hddn['dataset'] = self.dataset.name + hddn['temp_trait'] = False + if self.temp_trait: + hddn['temp_trait'] = True + hddn['group'] = self.temp_group + hddn['species'] = self.temp_species + else: + hddn['group'] = self.dataset.group.name + hddn['species'] = self.dataset.group.species + hddn['use_outliers'] = False + hddn['method'] = "gemma" + hddn['mapmethod_rqtl'] = "hk" + hddn['mapmodel_rqtl'] = "normal" + hddn['pair_scan'] = "" + hddn['selected_chr'] = -1 + hddn['mapping_display_all'] = True + hddn['suggestive'] = 0 + hddn['study_samplelists'] = json.dumps(study_samplelist_json) + hddn['num_perm'] = 0 + hddn['categorical_vars'] = "" + if self.categorical_var_list: + hddn['categorical_vars'] = ",".join(self.categorical_var_list) + hddn['manhattan_plot'] = "" + hddn['control_marker'] = "" + if not self.temp_trait: + if hasattr(self.this_trait, 'locus_chr') and self.this_trait.locus_chr != "" and self.dataset.type != "Geno" and self.dataset.type != "Publish": + hddn['control_marker'] = self.nearest_marker + hddn['do_control'] = False + hddn['maf'] = 0.05 + hddn['mapping_scale'] = "physic" + hddn['compare_traits'] = [] + hddn['export_data'] = "" + hddn['export_format'] = "excel" + if len(self.scales_in_geno) < 2 and bool(self.scales_in_geno): + hddn['mapping_scale'] = self.scales_in_geno[list( + self.scales_in_geno.keys())[0]][0][0] + + # We'll need access to this_trait and hddn in the Jinja2 + # Template, so we put it inside self + self.hddn = hddn + + js_data = dict(trait_id=self.trait_id, + trait_symbol=trait_symbol, + max_digits = self.max_digits_by_group, + no_decimal_place = self.no_decimal_place, + short_description=short_description, + unit_type=trait_units, + dataset_type=self.dataset.type, + species=self.dataset.group.species, + scales_in_geno=self.scales_in_geno, + data_scale=self.dataset.data_scale, + sample_group_types=self.sample_group_types, + sample_lists=sample_lists, + se_exists=self.sample_groups[0].se_exists, + has_num_cases=self.has_num_cases, + attributes=self.sample_groups[0].attributes, + categorical_attr_exists=self.categorical_attr_exists, + categorical_vars=",".join(self.categorical_var_list), + num_values=self.num_values, + qnorm_values=self.qnorm_vals, + zscore_values=self.z_scores, + sample_column_width=sample_column_width, + temp_uuid=self.temp_uuid) + self.js_data = js_data + + def get_external_links(self): + # ZS: There's some weirdness here because some fields don't + # exist while others are empty strings + self.pubmed_link = webqtlConfig.PUBMEDLINK_URL % self.this_trait.pubmed_id if check_if_attr_exists( + self.this_trait, 'pubmed_id') else None + self.ncbi_gene_link = webqtlConfig.NCBI_LOCUSID % self.this_trait.geneid if check_if_attr_exists( + self.this_trait, 'geneid') else None + self.omim_link = webqtlConfig.OMIM_ID % self.this_trait.omim if check_if_attr_exists( + self.this_trait, 'omim') else None + self.homologene_link = webqtlConfig.HOMOLOGENE_ID % self.this_trait.homologeneid if check_if_attr_exists( + self.this_trait, 'homologeneid') else None + + self.genbank_link = None + if check_if_attr_exists(self.this_trait, 'genbankid'): + genbank_id = '|'.join(self.this_trait.genbankid.split('|')[0:10]) + if genbank_id[-1] == '|': + genbank_id = genbank_id[0:-1] + self.genbank_link = webqtlConfig.GENBANK_ID % genbank_id + + self.uniprot_link = None + if check_if_attr_exists(self.this_trait, 'uniprotid'): + self.uniprot_link = webqtlConfig.UNIPROT_URL % self.this_trait.uniprotid + + self.genotation_link = self.rgd_link = self.phenogen_link = self.gtex_link = self.genebridge_link = self.ucsc_blat_link = self.biogps_link = self.protein_atlas_link = None + self.string_link = self.panther_link = self.aba_link = self.ebi_gwas_link = self.wiki_pi_link = self.genemania_link = self.ensembl_link = None + if self.this_trait.symbol: + self.genotation_link = webqtlConfig.GENOTATION_URL % self.this_trait.symbol + self.gtex_link = webqtlConfig.GTEX_URL % self.this_trait.symbol + self.string_link = webqtlConfig.STRING_URL % self.this_trait.symbol + self.panther_link = webqtlConfig.PANTHER_URL % self.this_trait.symbol + self.ebi_gwas_link = webqtlConfig.EBIGWAS_URL % self.this_trait.symbol + self.protein_atlas_link = webqtlConfig.PROTEIN_ATLAS_URL % self.this_trait.symbol + + if self.dataset.group.species == "mouse" or self.dataset.group.species == "human": + self.rgd_link = webqtlConfig.RGD_URL % ( + self.this_trait.symbol, self.dataset.group.species.capitalize()) + if self.dataset.group.species == "mouse": + self.genemania_link = webqtlConfig.GENEMANIA_URL % ( + "mus-musculus", self.this_trait.symbol) + else: + self.genemania_link = webqtlConfig.GENEMANIA_URL % ( + "homo-sapiens", self.this_trait.symbol) + + if self.dataset.group.species == "mouse": + self.aba_link = webqtlConfig.ABA_URL % self.this_trait.symbol + results = () + with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT chromosome, txStart, txEnd FROM " + "GeneList WHERE geneSymbol = %s", + (self.this_trait.symbol,) + ) + results = cursor.fetchone() + if results: + chr, transcript_start, transcript_end = results + else: + chr = transcript_start = transcript_end = None + + if chr and transcript_start and transcript_end and self.this_trait.refseq_transcriptid: + transcript_start = int(transcript_start * 1000000) + transcript_end = int(transcript_end * 1000000) + self.ucsc_blat_link = webqtlConfig.UCSC_REFSEQ % ( + 'mm10', self.this_trait.refseq_transcriptid, chr, transcript_start, transcript_end) + + if self.dataset.group.species == "rat": + self.rgd_link = webqtlConfig.RGD_URL % ( + self.this_trait.symbol, self.dataset.group.species.capitalize()) + self.phenogen_link = webqtlConfig.PHENOGEN_URL % ( + self.this_trait.symbol) + self.genemania_link = webqtlConfig.GENEMANIA_URL % ( + "rattus-norvegicus", self.this_trait.symbol) + + results = () + with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT kgID, chromosome, txStart, txEnd " + "FROM GeneList_rn33 WHERE geneSymbol = %s", + (self.this_trait.symbol,) + ) + if results := cursor.fetchone(): + kgId, chr, transcript_start, transcript_end = results + else: + kgId = chr = transcript_start = transcript_end = None + + if chr and transcript_start and transcript_end and kgId: + # Convert to bases from megabases + transcript_start = int(transcript_start * 1000000) + transcript_end = int(transcript_end * 1000000) + self.ucsc_blat_link = webqtlConfig.UCSC_REFSEQ % ( + 'rn7', kgId, chr, transcript_start, transcript_end) + + if self.this_trait.geneid and (self.dataset.group.species == "mouse" or self.dataset.group.species == "rat" or self.dataset.group.species == "human"): + self.biogps_link = webqtlConfig.BIOGPS_URL % ( + self.dataset.group.species, self.this_trait.geneid) + self.gemma_link = webqtlConfig.GEMMA_URL % self.this_trait.geneid + + if self.dataset.group.species == "human": + self.aba_link = webqtlConfig.ABA_URL % self.this_trait.geneid + + def build_correlation_tools(self): + if self.temp_trait == True: + this_group = self.temp_group + else: + this_group = self.dataset.group.name + + # We're checking a string here! + assert isinstance(this_group, str), "We need a string type thing here" + if this_group[:3] == 'BXD' and this_group not in webqtlConfig.BXD_GROUP_EXCEPTIONS: + this_group = 'BXD' + + if this_group: + if self.temp_trait == True: + dataset_menu = data_set.datasets(this_group) + else: + dataset_menu = data_set.datasets( + this_group, self.dataset.group) + dataset_menu_selected = None + if len(dataset_menu): + if self.dataset: + dataset_menu_selected = self.dataset.name + + return_results_menu = (100, 200, 500, 1000, + 2000, 5000, 10000, 15000, 20000) + return_results_menu_selected = 500 + + self.corr_tools = dict(dataset_menu=dataset_menu, + dataset_menu_selected=dataset_menu_selected, + return_results_menu=return_results_menu, + return_results_menu_selected=return_results_menu_selected,) + + def make_sample_lists(self): + + all_samples_ordered = self.dataset.group.all_samples_ordered() + + parent_f1_samples = [] + if self.dataset.group.parlist and self.dataset.group.f1list: + parent_f1_samples = self.dataset.group.parlist + self.dataset.group.f1list + + primary_sample_names = list(all_samples_ordered) + + if not self.temp_trait: + other_sample_names = [] + + for sample in list(self.this_trait.data.keys()): + if (self.this_trait.data[sample].name2 != self.this_trait.data[sample].name): + if ((self.this_trait.data[sample].name2 in primary_sample_names) + and (self.this_trait.data[sample].name not in primary_sample_names)): + primary_sample_names.append( + self.this_trait.data[sample].name) + primary_sample_names.remove( + self.this_trait.data[sample].name2) + + all_samples_set = set(all_samples_ordered) + if sample not in all_samples_set: + all_samples_ordered.append(sample) + other_sample_names.append(sample) + + # CFW is here because the .geno file doesn't properly + # contain its full list of samples. This should probably + # be fixed. + if self.dataset.group.species == "human" or (set(primary_sample_names) == set(parent_f1_samples)) or self.dataset.group.name == "CFW": + primary_sample_names += other_sample_names + other_sample_names = [] + + if other_sample_names: + primary_header = "%s Only" % (self.dataset.group.name) + else: + primary_header = "Samples" + + primary_samples = SampleList(dataset=self.dataset, + sample_names=primary_sample_names, + this_trait=self.this_trait, + sample_group_type='primary', + header=primary_header) + + # if other_sample_names and self.dataset.group.species != + # "human" and self.dataset.group.name != "CFW": + if len(other_sample_names) > 0: + other_sample_names.sort() # Sort other samples + other_samples = SampleList(dataset=self.dataset, + sample_names=other_sample_names, + this_trait=self.this_trait, + sample_group_type='other', + header="Other") + + self.sample_groups = (primary_samples, other_samples) + else: + self.sample_groups = (primary_samples,) + else: + primary_samples = SampleList(dataset=self.dataset, + sample_names=primary_sample_names, + this_trait=self.trait_vals, + sample_group_type='primary', + header="%s Only" % (self.dataset.group.name)) + self.sample_groups = (primary_samples,) + self.primary_sample_names = primary_sample_names + self.dataset.group.allsamples = all_samples_ordered + + +def get_trait_vals(sample_list): + trait_vals = [] + for sample in sample_list: + try: + trait_vals.append(float(sample.value)) + except: + continue + + return trait_vals + +def get_max_digits(trait_vals): + def __max_digits__(these_vals): + if not bool(these_vals): + return None + return len(str(max(these_vals))) - 1 + + return [__max_digits__(val) for val in trait_vals], all([val.is_integer() for val in sum(trait_vals, [])]) + +def normf(trait_vals): + ranked_vals = ss.rankdata(trait_vals) + p_list = [] + for i, val in enumerate(trait_vals): + p_list.append(((i + 1) - 0.5) / len(trait_vals)) + + z = ss.norm.ppf(p_list) + + normed_vals = [] + for rank in ranked_vals: + normed_vals.append("%0.3f" % z[int(rank) - 1]) + + return normed_vals + +def quantile_normalize_vals(sample_groups, trait_vals): + qnorm_by_group = [] + for i, sample_type in enumerate(sample_groups): + qnorm_vals = normf(trait_vals[i]) + qnorm_vals_with_x = [] + counter = 0 + for sample in sample_type.sample_list: + if sample.display_value == "x": + qnorm_vals_with_x.append("x") + else: + qnorm_vals_with_x.append(qnorm_vals[counter]) + counter += 1 + + qnorm_by_group.append(qnorm_vals_with_x) + + return qnorm_by_group + + +def get_z_scores(sample_groups, trait_vals): + zscore_by_group = [] + for i, sample_type in enumerate(sample_groups): + zscores = ss.mstats.zscore(np.array(trait_vals[i])).tolist() + zscores_with_x = [] + counter = 0 + for sample in sample_type.sample_list: + if sample.display_value == "x": + zscores_with_x.append("x") + else: + zscores_with_x.append("%0.3f" % zscores[counter]) + counter += 1 + + zscore_by_group.append(zscores_with_x) + + return zscore_by_group + + +def get_nearest_marker(this_trait, this_db): + this_chr = this_trait.locus_chr + this_mb = this_trait.locus_mb + # One option is to take flanking markers, another is to take the + # two (or one) closest + with database_connection(get_setting("SQL_URI")) as conn, conn.cursor() as cursor: + cursor.execute( + "SELECT Geno.Name FROM Geno, GenoXRef, " + "GenoFreeze WHERE Geno.Chr = %s AND " + "GenoXRef.GenoId = Geno.Id AND " + "GenoFreeze.Id = GenoXRef.GenoFreezeId " + "AND GenoFreeze.Name = %s ORDER BY " + "ABS( Geno.Mb - %s) LIMIT 1", + (this_chr, f"{this_db.group.name}Geno", this_mb,) + ) + if result := cursor.fetchall(): + return result[0][0] + return "" + + +def get_table_widths(sample_groups, sample_column_width, has_num_cases=False): + stats_table_width = 250 + if len(sample_groups) > 1: + stats_table_width = 450 + + trait_table_width = 300 + sample_column_width + if sample_groups[0].se_exists: + trait_table_width += 80 + if has_num_cases: + trait_table_width += 80 + trait_table_width += len(sample_groups[0].attributes) * 88 + + return stats_table_width, trait_table_width + + +def has_num_cases(this_trait): + has_n = False + if this_trait.dataset.type != "ProbeSet" and this_trait.dataset.type != "Geno": + for name, sample in list(this_trait.data.items()): + if sample.num_cases and sample.num_cases != "1": + has_n = True + break + + return has_n + + +def get_trait_units(this_trait): + unit_type = "" + inside_brackets = False + if this_trait.description_fmt: + if ("[" in this_trait.description_fmt) and ("]" in this_trait.description_fmt): + for i in this_trait.description_fmt: + if inside_brackets: + if i != "]": + unit_type += i + else: + inside_brackets = False + if i == "[": + inside_brackets = True + + if unit_type == "": + unit_type = "value" + + return unit_type + + +def check_if_attr_exists(the_trait, id_type): + if hasattr(the_trait, id_type): + if getattr(the_trait, id_type) == None or getattr(the_trait, id_type) == "": + return False + else: + return True + else: + return False + + +def get_ncbi_summary(this_trait): + if check_if_attr_exists(this_trait, 'geneid'): + # ZS: Need to switch this try/except to something that checks + # the output later + try: + response = requests.get( + "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=gene&id=%s&retmode=json" % this_trait.geneid) + summary = json.loads(response.content)[ + 'result'][this_trait.geneid]['summary'] + return summary + except: + return None + else: + return None + + +def get_categorical_variables(this_trait, sample_list) -> list: + categorical_var_list = [] + + if len(sample_list.attributes) > 0: + for attribute in sample_list.attributes: + if len(sample_list.attributes[attribute].distinct_values) < 10: + categorical_var_list.append(str(sample_list.attributes[attribute].id)) + + return categorical_var_list + +def get_numerical_variables(this_trait, sample_list) -> list: + numerical_var_list = [] + + if len(sample_list.attributes) > 0: + for attribute in sample_list.attributes: + all_numeric = True + all_none = True + for attr_val in sample_list.attributes[attribute].distinct_values: + if not attr_val: + continue + try: + val_as_float = float(attr_val) + all_none = False + except: + all_numeric = False + break + if all_numeric and not all_none: + numerical_var_list.append(sample_list.attributes[attribute].name) + + return numerical_var_list + +def get_genotype_scales(genofiles): + geno_scales = {} + if isinstance(genofiles, list): + for the_file in genofiles: + file_location = the_file['location'] + geno_scales[file_location] = get_scales_from_genofile( + file_location) + else: + geno_scales[genofiles] = get_scales_from_genofile(genofiles) + + return geno_scales + + +def get_scales_from_genofile(file_location): + geno_path = locate_ignore_error(file_location, 'genotype') + # ZS: This is just to allow the code to run when + if not geno_path: + return [["physic", "Mb"]] + cm_and_mb_cols_exist = True + cm_column = None + mb_column = None + with open(geno_path, "r") as geno_fh: + for i, line in enumerate(geno_fh): + if line[0] == "#" or line[0] == "@": + # ZS: If the scale is made explicit in the metadata, + # use that + if "@scale" in line: + scale = line.split(":")[1].strip() + if scale == "morgan": + return [["morgan", "cM"]] + else: + return [["physic", "Mb"]] + else: + continue + if line[:3] == "Chr": + first_marker_line = i + 1 + if line.split("\t")[2].strip() == "cM": + cm_column = 2 + elif line.split("\t")[3].strip() == "cM": + cm_column = 3 + if line.split("\t")[2].strip() == "Mb": + mb_column = 2 + elif line.split("\t")[3].strip() == "Mb": + mb_column = 3 + break + + # ZS: This attempts to check whether the cM and Mb columns are + # 'real', since some .geno files have one column be a copy of + # the other column, or have one column that is all 0s + cm_all_zero = True + mb_all_zero = True + cm_mb_all_equal = True + for i, line in enumerate(geno_fh): + # ZS: I'm assuming there won't be more than 10 markers + # where the position is listed as 0 + if first_marker_line <= i < first_marker_line + 10: + if cm_column: + cm_val = line.split("\t")[cm_column].strip() + if cm_val != "0": + cm_all_zero = False + if mb_column: + mb_val = line.split("\t")[mb_column].strip() + if mb_val != "0": + mb_all_zero = False + if cm_column and mb_column: + if cm_val != mb_val: + cm_mb_all_equal = False + else: + if i > first_marker_line + 10: + break + + # ZS: This assumes that both won't be all zero, since if that's + # the case mapping shouldn't be an option to begin with + if mb_all_zero: + return [["morgan", "cM"]] + elif cm_mb_all_equal: + return [["physic", "Mb"]] + elif cm_and_mb_cols_exist: + return [["physic", "Mb"], ["morgan", "cM"]] + else: + return [["physic", "Mb"]] + + + +def get_diff_of_vals(new_vals: Dict, trait_id: str) -> Dict: + """ Get the diff between current sample values and the values in the DB + + Given a dict of the changed values and the trait/dataset ID, return a Dict + with keys corresponding to each sample with a changed value and a value + that is a dict with keys for the old_value and new_value + + """ + + trait_name = trait_id.split(":")[0] + dataset_name = trait_id.split(":")[1] + trait_ob = create_trait(name=trait_name, dataset_name=dataset_name) + + old_vals = {sample : trait_ob.data[sample].value for sample in trait_ob.data} + + shared_samples = set.union(set(new_vals.keys()), set(old_vals.keys())) + + diff_dict = {} + for sample in shared_samples: + try: + new_val = round(float(new_vals[sample]), 3) + except: + new_val = "x" + try: + old_val = round(float(old_vals[sample]), 3) + except: + old_val = "x" + + if new_val != old_val: + diff_dict[sample] = { + "new_val": new_val, + "old_val": old_val + } + + return diff_dict |