diff options
Diffstat (limited to 'wqflask/base/trait.py')
| -rw-r--r-- | wqflask/base/trait.py | 356 |
1 files changed, 113 insertions, 243 deletions
diff --git a/wqflask/base/trait.py b/wqflask/base/trait.py index bf87e879..5525472e 100644 --- a/wqflask/base/trait.py +++ b/wqflask/base/trait.py @@ -3,14 +3,17 @@ from __future__ import absolute_import, division, print_function import string import resource import codecs +import requests -from htmlgen import HTMLgen2 as HT +import redis +Redis = redis.StrictRedis() from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset from db import webqtlDatabaseFunction from utility import webqtlUtil +from utility import hmac from wqflask import app @@ -18,17 +21,11 @@ import simplejson as json from MySQLdb import escape_string as escape from pprint import pformat as pf -from flask import Flask, g, request +from flask import Flask, g, request, url_for from utility.logger import getLogger logger = getLogger(__name__ ) -from wqflask import user_manager - -def print_mem(stage=""): - mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - print("{}: {}".format(stage, mem/1024)) - class GeneralTrait(object): """ Trait class defines a trait in webqtl, can be either Microarray, @@ -39,13 +36,15 @@ class GeneralTrait(object): def __init__(self, get_qtl_info=False, get_sample_info=True, **kw): # xor assertion assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; + self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. if kw.get('dataset_name'): - self.dataset = create_dataset(kw.get('dataset_name')) - #print(" in GeneralTrait created dataset:", self.dataset) + if kw.get('dataset_name') == "Temp": + temp_group = self.name.split("_")[2] + self.dataset = create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = temp_group) + else: + self.dataset = create_dataset(kw.get('dataset_name')) else: self.dataset = kw.get('dataset') - self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. - #print("THE NAME IS:", self.name) self.cellid = kw.get('cellid') self.identification = kw.get('identification', 'un-named trait') self.haveinfo = kw.get('haveinfo', False) @@ -61,6 +60,10 @@ class GeneralTrait(object): self.num_overlap = None self.strand_probe = None self.symbol = None + self.display_name = self.name + + self.LRS_score_repr = "N/A" + self.LRS_location_repr = "N/A" if kw.get('fullname'): name2 = value.split("::") @@ -72,94 +75,11 @@ class GeneralTrait(object): # Todo: These two lines are necessary most of the time, but perhaps not all of the time # So we could add a simple if statement to short-circuit this if necessary - self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info) + if self.dataset.type != "Temp": + self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info) if get_sample_info != False: self = retrieve_sample_data(self, self.dataset) - - def get_name(self): - stringy = "" - if self.dataset and self.name: - stringy = "%s::%s" % (self.dataset, self.name) - if self.cellid: - stringy += "::" + self.cellid - else: - stringy = self.description - return stringy - - - def get_given_name(self): - """ - when user enter a trait or GN generate a trait, user want show the name - not the name that generated by GN randomly, the two follow function are - used to give the real name and the database. displayName() will show the - database also, getGivenName() just show the name. - For other trait, displayName() as same as getName(), getGivenName() as - same as self.name - - Hongqiang 11/29/07 - - """ - stringy = self.name - if self.dataset and self.name: - desc = self.dataset.get_desc() - if desc: - #desc = self.handle_pca(desc) - stringy = desc - return stringy - - - def display_name(self): - stringy = "" - if self.dataset and self.name: - desc = self.dataset.get_desc() - #desc = self.handle_pca(desc) - if desc: - #desc = self.handle_pca(desc) - #stringy = desc - #if desc.__contains__('PCA'): - # desc = desc[desc.rindex(':')+1:].strip() - #else: - # desc = desc[:desc.index('entered')].strip() - #desc = self.handle_pca(desc) - stringy = "%s::%s" % (self.dataset, desc) - else: - stringy = "%s::%s" % (self.dataset, self.name) - if self.cellid: - stringy += "::" + self.cellid - else: - stringy = self.description - - return stringy - - - #def __str__(self): - # #return "%s %s" % (self.getName(), self.group) - # return self.getName() - #__str__ = getName - #__repr__ = __str__ - - def export_data(self, samplelist, the_type="val"): - """ - export data according to samplelist - mostly used in calculating correlation - - """ - result = [] - for sample in samplelist: - if self.data.has_key(sample): - if the_type=='val': - result.append(self.data[sample].val) - elif the_type=='var': - result.append(self.data[sample].var) - elif the_type=='N': - result.append(self.data[sample].N) - else: - raise KeyError, `the_type`+' the_type is incorrect.' - else: - result.append(None) - return result - def export_informative(self, include_variance=0): """ export informative sample @@ -179,19 +99,6 @@ class GeneralTrait(object): sample_aliases.append(sample_data.name2) return samples, vals, the_vars, sample_aliases - - @property - def name_header_fmt(self): - '''Return a human-readable name for use in page header''' - if self.dataset.type == 'ProbeSet': - return self.symbol - elif self.dataset.type == 'Geno': - return self.name - elif self.dataset.type == 'Publish': - return self.post_publication_abbreviation - else: - return "unnamed" - @property def description_fmt(self): '''Return a text formated description''' @@ -200,6 +107,8 @@ class GeneralTrait(object): formatted = self.description if self.probe_target_description: formatted += "; " + self.probe_target_description + else: + formatted = "Not available" elif self.dataset.type == 'Publish': if self.confidential: formatted = self.pre_publication_description @@ -212,11 +121,36 @@ class GeneralTrait(object): @property def alias_fmt(self): '''Return a text formatted alias''' + + alias = 'Not available' if self.alias: alias = string.replace(self.alias, ";", " ") alias = string.join(string.split(alias), ", ") - else: - alias = 'Not available' + + return alias + + @property + def wikidata_alias_fmt(self): + '''Return a text formatted alias''' + + alias = 'Not available' + if self.symbol: + human_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.upper()) + mouse_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.capitalize()) + other_response = requests.get("http://gn2.genenetwork.org/gn3/gene/aliases/" + self.symbol.lower()) + + if human_response and mouse_response and other_response: + alias_list = json.loads(human_response.content) + json.loads(mouse_response.content) + json.loads(other_response.content) + + filtered_aliases = [] + seen = set() + for item in alias_list: + if item in seen: + continue + else: + filtered_aliases.append(item) + seen.add(item) + alias = "; ".join(filtered_aliases) return alias @@ -244,60 +178,33 @@ class GeneralTrait(object): fmt += (' on the minus strand ') return fmt - -# In ProbeSet, there are maybe several annotations match one sequence -# so we need use sequence(BlatSeq) as the identification, when we update -# one annotation, we update the others who match the sequence also. -# -# Hongqiang Li, 3/3/2008 -def getSequence(trait, dataset_name): - dataset = create_dataset(dataset_name) - - if dataset.type == 'ProbeSet': - results = g.db.execute(''' - SELECT - ProbeSet.BlatSeq - FROM - ProbeSet, ProbeSetFreeze, ProbeSetXRef - WHERE - ProbeSet.Id=ProbeSetXRef.ProbeSetId and - ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and - ProbeSet.Name = %s - ProbeSetFreeze.Name = %s - ''', trait.name, dataset.name).fetchone() - - return results[0] def retrieve_sample_data(trait, dataset, samplelist=None): if samplelist == None: samplelist = [] - results = dataset.retrieve_sample_data(trait.name) + if dataset.type == "Temp": + results = Redis.get(trait.name).split() + else: + results = dataset.retrieve_sample_data(trait.name) # Todo: is this necessary? If not remove trait.data.clear() - all_samples_ordered = dataset.group.all_samples_ordered() - if results: - for item in results: - name, value, variance, num_cases, name2 = item - if not samplelist or (samplelist and name in samplelist): - trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) - - return trait - -def convert_location_to_value(chromosome, mb): - try: - location_value = int(chromosome)*1000 + float(mb) - except ValueError: - if chromosome.upper() == 'X': - location_value = 20*1000 + float(mb) + if dataset.type == "Temp": + all_samples_ordered = dataset.group.all_samples_ordered() + for i, item in enumerate(results): + try: + trait.data[all_samples_ordered[i]] = webqtlCaseData(all_samples_ordered[i], float(item)) + except: + pass else: - location_value = (ord(str(chromosome).upper()[0])*1000 + - float(mb)) - - return location_value + for item in results: + name, value, variance, num_cases, name2 = item + if not samplelist or (samplelist and name in samplelist): + trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) + return trait @app.route("/trait/get_sample_data") def get_sample_data(): @@ -307,25 +214,37 @@ def get_sample_data(): trait_ob = GeneralTrait(name=trait, dataset_name=dataset) - return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }]) - - #jsonable_sample_data = {} - #for sample in trait_ob.data.iteritems(): - # jsonable_sample_data[sample] = trait_ob.data[sample].value - # - #return jsonable_sample_data + trait_dict = {} + trait_dict['name'] = trait + trait_dict['db'] = dataset + trait_dict['type'] = trait_ob.dataset.type + trait_dict['group'] = trait_ob.dataset.group.name + trait_dict['tissue'] = trait_ob.dataset.tissue + trait_dict['species'] = trait_ob.dataset.group.species + trait_dict['url'] = url_for('show_trait_page', trait_id = trait, dataset = dataset) + trait_dict['description'] = trait_ob.description_display + if trait_ob.dataset.type == "ProbeSet": + trait_dict['symbol'] = trait_ob.symbol + trait_dict['location'] = trait_ob.location_repr + elif trait_ob.dataset.type == "Publish": + if trait_ob.pubmed_id: + trait_dict['pubmed_link'] = trait_ob.pubmed_link + trait_dict['pubmed_text'] = trait_ob.pubmed_text + + return json.dumps([trait_dict, {key: value.value for key, value in trait_ob.data.iteritems() }]) -def jsonable(trait, dataset_name): +def jsonable(trait): """Return a dict suitable for using as json Actual turning into json doesn't happen here though""" - dataset = create_dataset(dataset_name) + dataset = create_dataset(dataset_name = trait.dataset.name, dataset_type = trait.dataset.type, group_name = trait.dataset.group.name) if dataset.type == "ProbeSet": return dict(name=trait.name, symbol=trait.symbol, dataset=dataset.name, + dataset_name = dataset.shortname, description=trait.description_display, mean=trait.mean, location=trait.location_repr, @@ -337,7 +256,9 @@ def jsonable(trait, dataset_name): if trait.pubmed_id: return dict(name=trait.name, dataset=dataset.name, + dataset_name = dataset.shortname, description=trait.description_display, + abbreviation=trait.abbreviation, authors=trait.authors, pubmed_text=trait.pubmed_text, pubmed_link=trait.pubmed_link, @@ -348,7 +269,9 @@ def jsonable(trait, dataset_name): else: return dict(name=trait.name, dataset=dataset.name, + dataset_name = dataset.shortname, description=trait.description_display, + abbreviation=trait.abbreviation, authors=trait.authors, pubmed_text=trait.pubmed_text, lrs_score=trait.LRS_score_repr, @@ -358,6 +281,7 @@ def jsonable(trait, dataset_name): elif dataset.type == "Geno": return dict(name=trait.name, dataset=dataset.name, + dataset_name = dataset.shortname, location=trait.location_repr ) else: @@ -379,7 +303,7 @@ def jsonable_table_row(trait, dataset_name, index): additive = "N/A" else: additive = "%.3f" % round(float(trait.additive), 2) - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.symbol, @@ -395,7 +319,7 @@ def jsonable_table_row(trait, dataset_name, index): else: additive = "%.2f" % round(float(trait.additive), 2) if trait.pubmed_id: - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.description_display, @@ -405,7 +329,7 @@ def jsonable_table_row(trait, dataset_name, index): trait.LRS_location_repr, additive] else: - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.description_display, @@ -415,7 +339,7 @@ def jsonable_table_row(trait, dataset_name, index): trait.LRS_location_repr, additive] elif dataset.type == "Geno": - return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + user_manager.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', + return ['<input type="checkbox" name="searchResult" class="checkbox trait_checkbox" value="' + hmac.data_hmac('{}:{}'.format(str(trait.name), dataset.name)) + '">', index, '<a href="/show_trait?trait_id='+str(trait.name)+'&dataset='+dataset.name+'">'+str(trait.name)+'</a>', trait.location_repr] @@ -428,21 +352,22 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if dataset.type == 'Publish': query = """ SELECT - PublishXRef.Id, Publication.PubMed_ID, + PublishXRef.Id, InbredSet.InbredSetCode, Publication.PubMed_ID, Phenotype.Pre_publication_description, Phenotype.Post_publication_description, Phenotype.Original_description, - Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, + Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, PublishXRef.mean, Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users, Publication.Authors, Publication.Title, Publication.Abstract, Publication.Journal, Publication.Volume, Publication.Pages, Publication.Month, Publication.Year, PublishXRef.Sequence, Phenotype.Units, PublishXRef.comments FROM - PublishXRef, Publication, Phenotype, PublishFreeze + PublishXRef, Publication, Phenotype, PublishFreeze, InbredSet WHERE PublishXRef.Id = %s AND Phenotype.Id = PublishXRef.PhenotypeId AND Publication.Id = PublishXRef.PublicationId AND PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND + PublishXRef.InbredSetId = InbredSet.Id AND PublishFreeze.Id = %s """ % (trait.name, dataset.id) @@ -492,17 +417,25 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait_info = g.db.execute(query, (string.join(dataset.display_fields,','), dataset.type, trait.name)).fetchone() + if trait_info: trait.haveinfo = True #XZ: assign SQL query result to trait attributes. for i, field in enumerate(dataset.display_fields): holder = trait_info[i] + # if isinstance(trait_info[i], basestring): + # logger.debug("HOLDER:", holder) + # logger.debug("HOLDER2:", holder.decode(encoding='latin1')) + # holder = unicode(trait_info[i], "utf-8", "ignore") if isinstance(trait_info[i], basestring): - holder = unicode(trait_info[i], "utf-8", "ignore") + holder = holder.encode('latin1') setattr(trait, field, holder) if dataset.type == 'Publish': + if trait.group_code: + trait.display_name = trait.group_code + "_" + str(trait.name) + trait.confidential = 0 if trait.pre_publication_description and not trait.pubmed_id: trait.confidential = 1 @@ -513,6 +446,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): #phenotype traits, then display the pre-publication description instead #of the post-publication description if trait.confidential: + trait.abbreviation = trait.pre_publication_abbreviation trait.description_display = trait.pre_publication_description #if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( @@ -522,11 +456,16 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): # # description = self.pre_publication_description else: + trait.abbreviation = trait.post_publication_abbreviation if description: trait.description_display = description.strip() else: trait.description_display = "" + trait.abbreviation = unicode(str(trait.abbreviation).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.description_display = unicode(str(trait.description_display).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + trait.authors = unicode(str(trait.authors).strip(codecs.BOM_UTF8), 'utf-8', errors="replace") + if not trait.year.isdigit(): trait.pubmed_text = "N/A" else: @@ -535,38 +474,7 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): if trait.pubmed_id: trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id - - trait.homologeneid = None if dataset.type == 'ProbeSet' and dataset.group: - if trait.geneid: - #XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number. - #XZ: So I have to test if geneid is number before execute the query. - #XZ: The geneid values in database should be cleaned up. - #try: - # float(self.geneid) - # geneidIsNumber = True - #except ValueError: - # geneidIsNumber = False - #if geneidIsNumber: - query = """ - SELECT - HomologeneId - FROM - Homologene, Species, InbredSet - WHERE - Homologene.GeneId ='%s' AND - InbredSet.Name = '%s' AND - InbredSet.SpeciesId = Species.Id AND - Species.TaxonomyId = Homologene.TaxonomyId - """ % (escape(str(trait.geneid)), escape(dataset.group.name)) - logger.sql(query) - result = g.db.execute(query).fetchone() - #else: - # result = None - - if result: - trait.homologeneid = result[0] - description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') @@ -582,47 +490,22 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): # Save it for the jinja2 template trait.description_display = description_display - #XZ: trait_location_value is used for sorting trait.location_repr = 'N/A' - trait.location_value = 1000000 - if trait.chr and trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = convert_location_to_value(trait.chr, trait.mb) - #try: - # trait_location_value = int(self.chr)*1000 + self.mb - #except ValueError: - # if self.chr.upper() == 'X': - # trait_location_value = 20*1000 + self.mb - # else: - # trait_location_value = (ord(str(self.chr).upper()[0])*1000 + - # self.mb) - - #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) - trait.location_value = trait_location_value elif dataset.type == "Geno": trait.location_repr = 'N/A' - trait.location_value = 1000000 - if trait.chr and trait.mb: - #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") - #This is so we can convert the location to a number used for sorting - trait_location_value = convert_location_to_value(trait.chr, trait.mb) - - #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) - trait.location_value = trait_location_value if get_qtl_info: #LRS and its location trait.LRS_score_repr = "N/A" - trait.LRS_score_value = 0 trait.LRS_location_repr = "N/A" - trait.LRS_location_value = 1000000 + trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.additive = "" if dataset.type == 'ProbeSet' and not trait.cellid: + trait.mean = "" query = """ SELECT ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive @@ -653,9 +536,6 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" else: trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" - else: - trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.mean = trait.additive = "" - if dataset.type == 'Publish': query = """ @@ -692,19 +572,9 @@ def retrieve_trait_info(trait, dataset, get_qtl_info=False): trait.locus = trait.lrs = trait.additive = "" if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "": - #XZ: LRS_location_value is used for sorting - try: - LRS_location_value = int(trait.locus_chr)*1000 + float(trait.locus_mb) - except: - if trait.locus_chr.upper() == 'X': - LRS_location_value = 20*1000 + float(trait.locus_mb) - else: - LRS_location_value = ord(str(trait.locus_chr).upper()[0])*1000 + float(trait.locus_mb) - trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb)) if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs - trait.LRS_score_value = LRS_score_value = trait.lrs else: raise KeyError, `trait.name`+' information is not found in the database.' |