from __future__ import absolute_import, division, print_function import string import resource import codecs from htmlgen import HTMLgen2 as HT from base import webqtlConfig from base.webqtlCaseData import webqtlCaseData from base.data_set import create_dataset from db import webqtlDatabaseFunction from utility import webqtlUtil from wqflask import app import simplejson as json from MySQLdb import escape_string as escape from pprint import pformat as pf from flask import Flask, g, request from utility.logger import getLogger logger = getLogger(__name__ ) from wqflask import user_manager def print_mem(stage=""): mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss print("{}: {}".format(stage, mem/1024)) class GeneralTrait(object): """ Trait class defines a trait in webqtl, can be either Microarray, Published phenotype, genotype, or user input trait """ def __init__(self, get_qtl_info=False, get_sample_info=True, **kw): # xor assertion assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name"; if kw.get('dataset_name'): self.dataset = create_dataset(kw.get('dataset_name')) #print(" in GeneralTrait created dataset:", self.dataset) else: self.dataset = kw.get('dataset') self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc. #print("THE NAME IS:", self.name) self.cellid = kw.get('cellid') self.identification = kw.get('identification', 'un-named trait') self.haveinfo = kw.get('haveinfo', False) self.sequence = kw.get('sequence') # Blat sequence, available for ProbeSet self.data = kw.get('data', {}) # Sets defaults self.locus = None self.lrs = None self.pvalue = None self.mean = None self.additive = None self.num_overlap = None self.strand_probe = None self.symbol = None if kw.get('fullname'): name2 = value.split("::") if len(name2) == 2: self.dataset, self.name = name2 # self.cellid is set to None above elif len(name2) == 3: self.dataset, self.name, self.cellid = name2 # Todo: These two lines are necessary most of the time, but perhaps not all of the time # So we could add a simple if statement to short-circuit this if necessary self = retrieve_trait_info(self, self.dataset, get_qtl_info=get_qtl_info) if get_sample_info != False: self = retrieve_sample_data(self, self.dataset) def get_name(self): stringy = "" if self.dataset and self.name: stringy = "%s::%s" % (self.dataset, self.name) if self.cellid: stringy += "::" + self.cellid else: stringy = self.description return stringy def get_given_name(self): """ when user enter a trait or GN generate a trait, user want show the name not the name that generated by GN randomly, the two follow function are used to give the real name and the database. displayName() will show the database also, getGivenName() just show the name. For other trait, displayName() as same as getName(), getGivenName() as same as self.name Hongqiang 11/29/07 """ stringy = self.name if self.dataset and self.name: desc = self.dataset.get_desc() if desc: #desc = self.handle_pca(desc) stringy = desc return stringy def display_name(self): stringy = "" if self.dataset and self.name: desc = self.dataset.get_desc() #desc = self.handle_pca(desc) if desc: #desc = self.handle_pca(desc) #stringy = desc #if desc.__contains__('PCA'): # desc = desc[desc.rindex(':')+1:].strip() #else: # desc = desc[:desc.index('entered')].strip() #desc = self.handle_pca(desc) stringy = "%s::%s" % (self.dataset, desc) else: stringy = "%s::%s" % (self.dataset, self.name) if self.cellid: stringy += "::" + self.cellid else: stringy = self.description return stringy #def __str__(self): # #return "%s %s" % (self.getName(), self.group) # return self.getName() #__str__ = getName #__repr__ = __str__ def export_data(self, samplelist, the_type="val"): """ export data according to samplelist mostly used in calculating correlation """ result = [] for sample in samplelist: if self.data.has_key(sample): if the_type=='val': result.append(self.data[sample].val) elif the_type=='var': result.append(self.data[sample].var) elif the_type=='N': result.append(self.data[sample].N) else: raise KeyError, `the_type`+' the_type is incorrect.' else: result.append(None) return result def export_informative(self, include_variance=0): """ export informative sample mostly used in qtl regression """ samples = [] vals = [] the_vars = [] sample_aliases = [] for sample_name, sample_data in self.data.items(): if sample_data.value != None: if not include_variance or sample_data.variance != None: samples.append(sample_name) vals.append(sample_data.value) the_vars.append(sample_data.variance) sample_aliases.append(sample_data.name2) return samples, vals, the_vars, sample_aliases @property def name_header_fmt(self): '''Return a human-readable name for use in page header''' if self.dataset.type == 'ProbeSet': return self.symbol elif self.dataset.type == 'Geno': return self.name elif self.dataset.type == 'Publish': return self.post_publication_abbreviation else: return "unnamed" @property def description_fmt(self): '''Return a text formated description''' if self.dataset.type == 'ProbeSet': if self.description: formatted = self.description if self.probe_target_description: formatted += "; " + self.probe_target_description elif self.dataset.type == 'Publish': if self.confidential: formatted = self.pre_publication_description else: formatted = self.post_publication_description else: formatted = "Not available" return formatted.capitalize() @property def alias_fmt(self): '''Return a text formatted alias''' if self.alias: alias = string.replace(self.alias, ";", " ") alias = string.join(string.split(alias), ", ") else: alias = 'Not available' return alias @property def location_fmt(self): '''Return a text formatted location While we're at it we set self.location in case we need it later (do we?) ''' if self.chr and self.mb: self.location = 'Chr %s @ %s Mb' % (self.chr,self.mb) elif self.chr: self.location = 'Chr %s @ Unknown position' % (self.chr) else: self.location = 'Not available' fmt = self.location ##XZ: deal with direction if self.strand_probe == '+': fmt += (' on the plus strand ') elif self.strand_probe == '-': fmt += (' on the minus strand ') return fmt # In ProbeSet, there are maybe several annotations match one sequence # so we need use sequence(BlatSeq) as the identification, when we update # one annotation, we update the others who match the sequence also. # # Hongqiang Li, 3/3/2008 def getSequence(trait, dataset_name): dataset = create_dataset(dataset_name) if dataset.type == 'ProbeSet': results = g.db.execute(''' SELECT ProbeSet.BlatSeq FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef WHERE ProbeSet.Id=ProbeSetXRef.ProbeSetId and ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and ProbeSet.Name = %s ProbeSetFreeze.Name = %s ''', trait.name, dataset.name).fetchone() return results[0] def retrieve_sample_data(trait, dataset, samplelist=None): if samplelist == None: samplelist = [] results = dataset.retrieve_sample_data(trait.name) # Todo: is this necessary? If not remove trait.data.clear() all_samples_ordered = dataset.group.all_samples_ordered() if results: for item in results: name, value, variance, num_cases, name2 = item if not samplelist or (samplelist and name in samplelist): trait.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases) return trait def convert_location_to_value(chromosome, mb): try: location_value = int(chromosome)*1000 + float(mb) except ValueError: if chromosome.upper() == 'X': location_value = 20*1000 + float(mb) else: location_value = (ord(str(chromosome).upper()[0])*1000 + float(mb)) return location_value @app.route("/trait/get_sample_data") def get_sample_data(): params = request.args trait = params['trait'] dataset = params['dataset'] trait_ob = GeneralTrait(name=trait, dataset_name=dataset) return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }]) #jsonable_sample_data = {} #for sample in trait_ob.data.iteritems(): # jsonable_sample_data[sample] = trait_ob.data[sample].value # #return jsonable_sample_data def jsonable(trait, dataset_name): """Return a dict suitable for using as json Actual turning into json doesn't happen here though""" dataset = create_dataset(dataset_name) if dataset.type == "ProbeSet": return dict(name=trait.name, symbol=trait.symbol, dataset=dataset.name, description=trait.description_display, mean=trait.mean, location=trait.location_repr, lrs_score=trait.LRS_score_repr, lrs_location=trait.LRS_location_repr, additive=trait.additive ) elif dataset.type == "Publish": if trait.pubmed_id: return dict(name=trait.name, dataset=dataset.name, description=trait.description_display, authors=trait.authors, pubmed_text=trait.pubmed_text, pubmed_link=trait.pubmed_link, lrs_score=trait.LRS_score_repr, lrs_location=trait.LRS_location_repr, additive=trait.additive ) else: return dict(name=trait.name, dataset=dataset.name, description=trait.description_display, authors=trait.authors, pubmed_text=trait.pubmed_text, lrs_score=trait.LRS_score_repr, lrs_location=trait.LRS_location_repr, additive=trait.additive ) elif dataset.type == "Geno": return dict(name=trait.name, dataset=dataset.name, location=trait.location_repr ) else: return dict() def jsonable_table_row(trait, dataset_name, index): """Return a list suitable for json and intended to be displayed in a table Actual turning into json doesn't happen here though""" dataset = create_dataset(dataset_name) if dataset.type == "ProbeSet": if trait.mean == "": mean = "N/A" else: mean = "%.3f" % round(float(trait.mean), 2) if trait.additive == "": additive = "N/A" else: additive = "%.3f" % round(float(trait.additive), 2) return ['', index, ''+str(trait.name)+'', trait.symbol, trait.description_display, trait.location_repr, mean, trait.LRS_score_repr, trait.LRS_location_repr, additive] elif dataset.type == "Publish": if trait.additive == "": additive = "N/A" else: additive = "%.2f" % round(float(trait.additive), 2) if trait.pubmed_id: return ['', index, ''+str(trait.name)+'', trait.description_display, trait.authors, '' + trait.pubmed_text + '', trait.LRS_score_repr, trait.LRS_location_repr, additive] else: return ['', index, ''+str(trait.name)+'', trait.description_display, trait.authors, trait.pubmed_text, trait.LRS_score_repr, trait.LRS_location_repr, additive] elif dataset.type == "Geno": return ['', index, ''+str(trait.name)+'', trait.location_repr] else: return dict() def retrieve_trait_info(trait, dataset, get_qtl_info=False): assert dataset, "Dataset doesn't exist" if dataset.type == 'Publish': query = """ SELECT PublishXRef.Id, Publication.PubMed_ID, Phenotype.Pre_publication_description, Phenotype.Post_publication_description, Phenotype.Original_description, Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation, Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users, Publication.Authors, Publication.Title, Publication.Abstract, Publication.Journal, Publication.Volume, Publication.Pages, Publication.Month, Publication.Year, PublishXRef.Sequence, Phenotype.Units, PublishXRef.comments FROM PublishXRef, Publication, Phenotype, PublishFreeze WHERE PublishXRef.Id = %s AND Phenotype.Id = PublishXRef.PhenotypeId AND Publication.Id = PublishXRef.PublicationId AND PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND PublishFreeze.Id = %s """ % (trait.name, dataset.id) logger.sql(query) trait_info = g.db.execute(query).fetchone() #XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name #XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms. elif dataset.type == 'ProbeSet': display_fields_string = ', ProbeSet.'.join(dataset.display_fields) display_fields_string = 'ProbeSet.' + display_fields_string query = """ SELECT %s FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef WHERE ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND ProbeSetXRef.ProbeSetId = ProbeSet.Id AND ProbeSetFreeze.Name = '%s' AND ProbeSet.Name = '%s' """ % (escape(display_fields_string), escape(dataset.name), escape(str(trait.name))) logger.sql(query) trait_info = g.db.execute(query).fetchone() #XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name # to avoid the problem of same marker name from different species. elif dataset.type == 'Geno': display_fields_string = string.join(dataset.display_fields,',Geno.') display_fields_string = 'Geno.' + display_fields_string query = """ SELECT %s FROM Geno, GenoFreeze, GenoXRef WHERE GenoXRef.GenoFreezeId = GenoFreeze.Id AND GenoXRef.GenoId = Geno.Id AND GenoFreeze.Name = '%s' AND Geno.Name = '%s' """ % (escape(display_fields_string), escape(dataset.name), escape(trait.name)) logger.sql(query) trait_info = g.db.execute(query).fetchone() else: #Temp type query = """SELECT %s FROM %s WHERE Name = %s""" logger.sql(query) trait_info = g.db.execute(query, (string.join(dataset.display_fields,','), dataset.type, trait.name)).fetchone() if trait_info: trait.haveinfo = True #XZ: assign SQL query result to trait attributes. for i, field in enumerate(dataset.display_fields): holder = trait_info[i] if isinstance(trait_info[i], basestring): holder = unicode(trait_info[i], "utf8", "ignore") setattr(trait, field, holder) if dataset.type == 'Publish': trait.confidential = 0 if trait.pre_publication_description and not trait.pubmed_id: trait.confidential = 1 description = trait.post_publication_description #If the dataset is confidential and the user has access to confidential #phenotype traits, then display the pre-publication description instead #of the post-publication description if trait.confidential: trait.description_display = trait.pre_publication_description #if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait( # privilege=self.dataset.privilege, # userName=self.dataset.userName, # authorized_users=self.authorized_users): # # description = self.pre_publication_description else: if description: trait.description_display = description.strip() else: trait.description_display = "" if not trait.year.isdigit(): trait.pubmed_text = "N/A" else: trait.pubmed_text = trait.year if trait.pubmed_id: trait.pubmed_link = webqtlConfig.PUBMEDLINK_URL % trait.pubmed_id trait.homologeneid = None if dataset.type == 'ProbeSet' and dataset.group: if trait.geneid: #XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number. #XZ: So I have to test if geneid is number before execute the query. #XZ: The geneid values in database should be cleaned up. #try: # float(self.geneid) # geneidIsNumber = True #except ValueError: # geneidIsNumber = False #if geneidIsNumber: query = """ SELECT HomologeneId FROM Homologene, Species, InbredSet WHERE Homologene.GeneId ='%s' AND InbredSet.Name = '%s' AND InbredSet.SpeciesId = Species.Id AND Species.TaxonomyId = Homologene.TaxonomyId """ % (escape(str(trait.geneid)), escape(dataset.group.name)) logger.sql(query) result = g.db.execute(query).fetchone() #else: # result = None if result: trait.homologeneid = result[0] description_string = unicode(str(trait.description).strip(codecs.BOM_UTF8), 'utf-8') target_string = unicode(str(trait.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8') if len(description_string) > 1 and description_string != 'None': description_display = description_string else: description_display = trait.symbol if (len(description_display) > 1 and description_display != 'N/A' and len(target_string) > 1 and target_string != 'None'): description_display = description_display + '; ' + target_string.strip() # Save it for the jinja2 template trait.description_display = description_display #XZ: trait_location_value is used for sorting trait.location_repr = 'N/A' trait.location_value = 1000000 if trait.chr and trait.mb: #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") #This is so we can convert the location to a number used for sorting trait_location_value = convert_location_to_value(trait.chr, trait.mb) #try: # trait_location_value = int(self.chr)*1000 + self.mb #except ValueError: # if self.chr.upper() == 'X': # trait_location_value = 20*1000 + self.mb # else: # trait_location_value = (ord(str(self.chr).upper()[0])*1000 + # self.mb) #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) trait.location_value = trait_location_value elif dataset.type == "Geno": trait.location_repr = 'N/A' trait.location_value = 1000000 if trait.chr and trait.mb: #Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y") #This is so we can convert the location to a number used for sorting trait_location_value = convert_location_to_value(trait.chr, trait.mb) #ZS: Put this in function currently called "convert_location_to_value" trait.location_repr = 'Chr%s: %.6f' % (trait.chr, float(trait.mb)) trait.location_value = trait_location_value if get_qtl_info: #LRS and its location trait.LRS_score_repr = "N/A" trait.LRS_score_value = 0 trait.LRS_location_repr = "N/A" trait.LRS_location_value = 1000000 if dataset.type == 'ProbeSet' and not trait.cellid: query = """ SELECT ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive FROM ProbeSetXRef, ProbeSet WHERE ProbeSetXRef.ProbeSetId = ProbeSet.Id AND ProbeSet.Name = "{}" AND ProbeSetXRef.ProbeSetFreezeId ={} """.format(trait.name, dataset.id) logger.sql(query) trait_qtl = g.db.execute(query).fetchone() if trait_qtl: trait.locus, trait.lrs, trait.pvalue, trait.mean, trait.additive = trait_qtl if trait.locus: query = """ select Geno.Chr, Geno.Mb from Geno, Species where Species.Name = '{}' and Geno.Name = '{}' and Geno.SpeciesId = Species.Id """.format(dataset.group.species, trait.locus) logger.sql(query) result = g.db.execute(query).fetchone() if result: trait.locus_chr = result[0] trait.locus_mb = result[1] else: trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" else: trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" else: trait.locus = trait.locus_chr = trait.locus_mb = trait.lrs = trait.pvalue = trait.mean = trait.additive = "" if dataset.type == 'Publish': query = """ SELECT PublishXRef.Locus, PublishXRef.LRS, PublishXRef.additive FROM PublishXRef, PublishFreeze WHERE PublishXRef.Id = %s AND PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND PublishFreeze.Id =%s """ % (trait.name, dataset.id) logger.sql(query) trait_qtl = g.db.execute(query).fetchone() if trait_qtl: trait.locus, trait.lrs, trait.additive = trait_qtl if trait.locus: query = """ select Geno.Chr, Geno.Mb from Geno, Species where Species.Name = '{}' and Geno.Name = '{}' and Geno.SpeciesId = Species.Id """.format(dataset.group.species, trait.locus) logger.sql(query) result = g.db.execute(query).fetchone() if result: trait.locus_chr = result[0] trait.locus_mb = result[1] else: trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" else: trait.locus = trait.locus_chr = trait.locus_mb = trait.additive = "" else: trait.locus = trait.lrs = trait.additive = "" if (dataset.type == 'Publish' or dataset.type == "ProbeSet") and trait.locus_chr != "" and trait.locus_mb != "": #XZ: LRS_location_value is used for sorting try: LRS_location_value = int(trait.locus_chr)*1000 + float(trait.locus_mb) except: if trait.locus_chr.upper() == 'X': LRS_location_value = 20*1000 + float(trait.locus_mb) else: LRS_location_value = ord(str(trait.locus_chr).upper()[0])*1000 + float(trait.locus_mb) trait.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (trait.locus_chr, float(trait.locus_mb)) if trait.lrs != "": trait.LRS_score_repr = LRS_score_repr = '%3.1f' % trait.lrs trait.LRS_score_value = LRS_score_value = trait.lrs else: raise KeyError, `trait.name`+' information is not found in the database.' return trait