from __future__ import absolute_import, division, print_function
import string
import resource
import codecs
from htmlgen import HTMLgen2 as HT
from base import webqtlConfig
from base.webqtlCaseData import webqtlCaseData
from base.data_set import create_dataset
from dbFunction import webqtlDatabaseFunction
from utility import webqtlUtil
from wqflask import app
import simplejson as json
from MySQLdb import escape_string as escape
from pprint import pformat as pf
from flask import Flask, g, request
def print_mem(stage=""):
mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
print("{}: {}".format(stage, mem/1024))
class GeneralTrait(object):
"""
Trait class defines a trait in webqtl, can be either Microarray,
Published phenotype, genotype, or user input trait
"""
def __init__(self, get_qtl_info=False, get_sample_info=True, **kw):
# xor assertion
assert bool(kw.get('dataset')) != bool(kw.get('dataset_name')), "Needs dataset ob. or name";
if kw.get('dataset_name'):
self.dataset = create_dataset(kw.get('dataset_name'))
#print(" in GeneralTrait created dataset:", self.dataset)
else:
self.dataset = kw.get('dataset')
self.name = kw.get('name') # Trait ID, ProbeSet ID, Published ID, etc.
#print("THE NAME IS:", self.name)
self.cellid = kw.get('cellid')
self.identification = kw.get('identification', 'un-named trait')
self.haveinfo = kw.get('haveinfo', False)
self.sequence = kw.get('sequence') # Blat sequence, available for ProbeSet
self.data = kw.get('data', {})
# Sets defaults
self.locus = None
self.lrs = None
self.pvalue = None
self.mean = None
self.num_overlap = None
self.strand_probe = None
self.symbol = None
if kw.get('fullname'):
name2 = value.split("::")
if len(name2) == 2:
self.dataset, self.name = name2
# self.cellid is set to None above
elif len(name2) == 3:
self.dataset, self.name, self.cellid = name2
# Todo: These two lines are necessary most of the time, but perhaps not all of the time
# So we could add a simple if statement to short-circuit this if necessary
self.retrieve_info(get_qtl_info=get_qtl_info)
if get_sample_info != False:
self.retrieve_sample_data()
def jsonable(self):
"""Return a dict suitable for using as json
Actual turning into json doesn't happen here though"""
return dict(name=self.name,
dataset=self.dataset.name,
description=self.description_display,
mean=self.mean)
def get_info(self):
"""For lots of traits just use get_trait_info in dataset instead...that will be way
more efficient"""
self.dataset.get_trait_info([self],
webqtlDatabaseFunction.retrieve_species(self.dataset.group.name))
def get_name(self):
stringy = ""
if self.dataset and self.name:
stringy = "%s::%s" % (self.dataset, self.name)
if self.cellid:
stringy += "::" + self.cellid
else:
stringy = self.description
return stringy
def get_given_name(self):
"""
when user enter a trait or GN generate a trait, user want show the name
not the name that generated by GN randomly, the two follow function are
used to give the real name and the database. displayName() will show the
database also, getGivenName() just show the name.
For other trait, displayName() as same as getName(), getGivenName() as
same as self.name
Hongqiang 11/29/07
"""
stringy = self.name
if self.dataset and self.name:
desc = self.dataset.get_desc()
if desc:
#desc = self.handle_pca(desc)
stringy = desc
return stringy
def display_name(self):
stringy = ""
if self.dataset and self.name:
desc = self.dataset.get_desc()
#desc = self.handle_pca(desc)
if desc:
#desc = self.handle_pca(desc)
#stringy = desc
#if desc.__contains__('PCA'):
# desc = desc[desc.rindex(':')+1:].strip()
#else:
# desc = desc[:desc.index('entered')].strip()
#desc = self.handle_pca(desc)
stringy = "%s::%s" % (self.dataset, desc)
else:
stringy = "%s::%s" % (self.dataset, self.name)
if self.cellid:
stringy += "::" + self.cellid
else:
stringy = self.description
return stringy
#def __str__(self):
# #return "%s %s" % (self.getName(), self.group)
# return self.getName()
#__str__ = getName
#__repr__ = __str__
def export_data(self, samplelist, the_type="val"):
"""
export data according to samplelist
mostly used in calculating correlation
"""
result = []
for sample in samplelist:
if self.data.has_key(sample):
if the_type=='val':
result.append(self.data[sample].val)
elif the_type=='var':
result.append(self.data[sample].var)
elif the_type=='N':
result.append(self.data[sample].N)
else:
raise KeyError, `the_type`+' the_type is incorrect.'
else:
result.append(None)
return result
def export_informative(self, include_variance=0):
"""
export informative sample
mostly used in qtl regression
"""
samples = []
vals = []
the_vars = []
for sample_name, sample_data in self.data.items():
if sample_data.value != None:
if not include_variance or sample_data.variance != None:
samples.append(sample_name)
vals.append(sample_data.value)
the_vars.append(sample_data.variance)
return samples, vals, the_vars
#
# In ProbeSet, there are maybe several annotations match one sequence
# so we need use sequence(BlatSeq) as the identification, when we update
# one annotation, we update the others who match the sequence also.
#
# Hongqiang Li, 3/3/2008
#
#def getSequence(self):
# assert self.cursor
# if self.dataset.type == 'ProbeSet':
# self.cursor.execute('''
# SELECT
# ProbeSet.BlatSeq
# FROM
# ProbeSet, ProbeSetFreeze, ProbeSetXRef
# WHERE
# ProbeSet.Id=ProbeSetXRef.ProbeSetId and
# ProbeSetFreeze.Id = ProbeSetXRef.ProbSetFreezeId and
# ProbeSet.Name = %s
# ProbeSetFreeze.Name = %s
# ''', self.name, self.dataset.name)
# #self.cursor.execute(query)
# results = self.fetchone()
#
# return results[0]
def retrieve_sample_data(self, samplelist=None):
if samplelist == None:
samplelist = []
results = self.dataset.retrieve_sample_data(self.name)
# Todo: is this necessary? If not remove
self.data.clear()
all_samples_ordered = self.dataset.group.all_samples_ordered()
if results:
for item in results:
name, value, variance, num_cases = item
if not samplelist or (samplelist and name in samplelist):
self.data[name] = webqtlCaseData(*item) #name, value, variance, num_cases)
def retrieve_info(self, get_qtl_info=False):
assert self.dataset, "Dataset doesn't exist"
if self.dataset.type == 'Publish':
query = """
SELECT
PublishXRef.Id, Publication.PubMed_ID,
Phenotype.Pre_publication_description, Phenotype.Post_publication_description, Phenotype.Original_description,
Phenotype.Pre_publication_abbreviation, Phenotype.Post_publication_abbreviation,
Phenotype.Lab_code, Phenotype.Submitter, Phenotype.Owner, Phenotype.Authorized_Users,
Publication.Authors, Publication.Title, Publication.Abstract,
Publication.Journal, Publication.Volume, Publication.Pages,
Publication.Month, Publication.Year, PublishXRef.Sequence,
Phenotype.Units, PublishXRef.comments
FROM
PublishXRef, Publication, Phenotype, PublishFreeze
WHERE
PublishXRef.Id = %s AND
Phenotype.Id = PublishXRef.PhenotypeId AND
Publication.Id = PublishXRef.PublicationId AND
PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND
PublishFreeze.Id = %s
""" % (self.name, self.dataset.id)
trait_info = g.db.execute(query).fetchone()
#XZ, 05/08/2009: Xiaodong add this block to use ProbeSet.Id to find the probeset instead of just using ProbeSet.Name
#XZ, 05/08/2009: to avoid the problem of same probeset name from different platforms.
elif self.dataset.type == 'ProbeSet':
display_fields_string = ', ProbeSet.'.join(self.dataset.display_fields)
display_fields_string = 'ProbeSet.' + display_fields_string
query = """
SELECT %s
FROM ProbeSet, ProbeSetFreeze, ProbeSetXRef
WHERE
ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id AND
ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
ProbeSetFreeze.Name = '%s' AND
ProbeSet.Name = '%s'
""" % (escape(display_fields_string),
escape(self.dataset.name),
escape(str(self.name)))
trait_info = g.db.execute(query).fetchone()
#XZ, 05/08/2009: We also should use Geno.Id to find marker instead of just using Geno.Name
# to avoid the problem of same marker name from different species.
elif self.dataset.type == 'Geno':
display_fields_string = string.join(self.dataset.display_fields,',Geno.')
display_fields_string = 'Geno.' + display_fields_string
query = """
SELECT %s
FROM Geno, GenoFreeze, GenoXRef
WHERE
GenoXRef.GenoFreezeId = GenoFreeze.Id AND
GenoXRef.GenoId = Geno.Id AND
GenoFreeze.Name = '%s' AND
Geno.Name = '%s'
""" % (escape(display_fields_string),
escape(self.dataset.name),
escape(self.name))
trait_info = g.db.execute(query).fetchone()
else: #Temp type
query = """SELECT %s FROM %s WHERE Name = %s"""
trait_info = g.db.execute(query,
(string.join(self.dataset.display_fields,','),
self.dataset.type, self.name)).fetchone()
if trait_info:
self.haveinfo = True
#XZ: assign SQL query result to trait attributes.
for i, field in enumerate(self.dataset.display_fields):
holder = trait_info[i]
if isinstance(trait_info[i], basestring):
holder = unicode(trait_info[i], "utf8", "ignore")
setattr(self, field, holder)
if self.dataset.type == 'Publish':
self.confidential = 0
if self.pre_publication_description and not self.pubmed_id:
self.confidential = 1
description = self.post_publication_description
#If the dataset is confidential and the user has access to confidential
#phenotype traits, then display the pre-publication description instead
#of the post-publication description
if self.confidential:
self.description_display = ""
#if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(
# privilege=self.dataset.privilege,
# userName=self.dataset.userName,
# authorized_users=self.authorized_users):
#
# description = self.pre_publication_description
if len(description) > 0:
self.description_display = description.strip()
else:
self.description_display = ""
if not self.year.isdigit():
self.pubmed_text = "N/A"
else:
self.pubmed_text = self.year
if self.pubmed_id:
self.pubmed_link = webqtlConfig.PUBMEDLINK_URL % self.pubmed_id
self.homologeneid = None
if self.dataset.type == 'ProbeSet' and self.dataset.group and self.geneid:
#XZ, 05/26/2010: From time to time, this query get error message because some geneid values in database are not number.
#XZ: So I have to test if geneid is number before execute the query.
#XZ: The geneid values in database should be cleaned up.
#try:
# float(self.geneid)
# geneidIsNumber = True
#except ValueError:
# geneidIsNumber = False
#if geneidIsNumber:
query = """
SELECT
HomologeneId
FROM
Homologene, Species, InbredSet
WHERE
Homologene.GeneId =%s AND
InbredSet.Name = '%s' AND
InbredSet.SpeciesId = Species.Id AND
Species.TaxonomyId = Homologene.TaxonomyId
""" % (escape(str(self.geneid)), escape(self.dataset.group.name))
result = g.db.execute(query).fetchone()
#else:
# result = None
if result:
self.homologeneid = result[0]
description_string = unicode(str(self.description).strip(codecs.BOM_UTF8), 'utf-8')
target_string = unicode(str(self.probe_target_description).strip(codecs.BOM_UTF8), 'utf-8')
if len(description_string) > 1 and description_string != 'None':
description_display = description_string
else:
description_display = self.symbol
if (len(description_display) > 1 and description_display != 'N/A' and
len(target_string) > 1 and target_string != 'None'):
description_display = description_display + '; ' + target_string.strip()
# Save it for the jinja2 template
self.description_display = description_display
#XZ: trait_location_value is used for sorting
trait_location_repr = 'N/A'
trait_location_value = 1000000
if self.chr and self.mb:
#Checks if the chromosome number can be cast to an int (i.e. isn't "X" or "Y")
#This is so we can convert the location to a number used for sorting
trait_location_value = convert_location_to_value(self.chr, self.mb)
#try:
# trait_location_value = int(self.chr)*1000 + self.mb
#except ValueError:
# if self.chr.upper() == 'X':
# trait_location_value = 20*1000 + self.mb
# else:
# trait_location_value = (ord(str(self.chr).upper()[0])*1000 +
# self.mb)
#ZS: Put this in function currently called "convert_location_to_value"
self.location_repr = 'Chr%s: %.6f' % (self.chr, float(self.mb))
self.location_value = trait_location_value
if get_qtl_info:
#LRS and its location
self.LRS_score_repr = "N/A"
self.LRS_score_value = 0
self.LRS_location_repr = "N/A"
self.LRS_location_value = 1000000
if self.dataset.type == 'ProbeSet' and not self.cellid:
query = """
SELECT
ProbeSetXRef.Locus, ProbeSetXRef.LRS, ProbeSetXRef.pValue, ProbeSetXRef.mean, ProbeSetXRef.additive
FROM
ProbeSetXRef, ProbeSet
WHERE
ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
ProbeSet.Name = "{}" AND
ProbeSetXRef.ProbeSetFreezeId ={}
""".format(self.name, self.dataset.id)
trait_qtl = g.db.execute(query).fetchone()
if trait_qtl:
self.locus, self.lrs, self.pvalue, self.mean, self.additive= trait_qtl
if self.locus:
query = """
select Geno.Chr, Geno.Mb from Geno, Species
where Species.Name = '{}' and
Geno.Name = '{}' and
Geno.SpeciesId = Species.Id
""".format(self.dataset.group.species, self.locus)
result = g.db.execute(query).fetchone()
if result:
self.locus_chr = result[0]
self.locus_mb = result[1]
else:
self.locus = self.locus_chr = self.locus_mb = ""
else:
self.locus = self.locus_chr = self.locus_mb = ""
else:
self.locus = self.locus_chr = self.locus_mb = self.lrs = self.pvalue = self.mean = self.additive = ""
if self.dataset.type == 'Publish':
trait_qtl = g.db.execute("""
SELECT
PublishXRef.Locus, PublishXRef.LRS, PublishXRef.additive
FROM
PublishXRef, PublishFreeze
WHERE
PublishXRef.Id = %s AND
PublishXRef.InbredSetId = PublishFreeze.InbredSetId AND
PublishFreeze.Id =%s
""", (self.name, self.dataset.id)).fetchone()
if trait_qtl:
self.locus, self.lrs, self.additive = trait_qtl
if self.locus:
query = """
select Geno.Chr, Geno.Mb from Geno, Species
where Species.Name = '{}' and
Geno.Name = '{}' and
Geno.SpeciesId = Species.Id
""".format(self.dataset.group.species, self.locus)
result = g.db.execute(query).fetchone()
if result:
self.locus_chr = result[0]
self.locus_mb = result[1]
else:
self.locus = self.locus_chr = self.locus_mb = ""
else:
self.locus = self.locus_chr = self.locus_mb = ""
else:
self.locus = self.lrs = self.additive = ""
if self.locus_chr != "" and self.locus_mb != "":
#XZ: LRS_location_value is used for sorting
try:
LRS_location_value = int(self.locus_chr)*1000 + float(self.locus_mb)
except:
if self.locus_chr.upper() == 'X':
LRS_location_value = 20*1000 + float(self.locus_mb)
else:
LRS_location_value = ord(str(self.locus_chr).upper()[0])*1000 + float(self.locus_mb)
self.LRS_location_repr = LRS_location_repr = 'Chr%s: %.6f' % (self.locus_chr, float(self.locus_mb))
self.LRS_score_repr = LRS_score_repr = '%3.1f' % self.lrs
self.LRS_score_value = LRS_score_value = self.lrs
else:
raise KeyError, `self.name`+' information is not found in the database.'
def genHTML(self, formName = "", dispFromDatabase=0, privilege="guest", userName="Guest", authorized_users=""):
if not self.haveinfo:
self.retrieveInfo()
if self.dataset.type == 'Publish':
PubMedLink = ""
if self.pubmed_id:
PubMedLink = HT.Href(text="PubMed %d : " % self.pubmed_id,
target = "_blank", url = webqtlConfig.PUBMEDLINK_URL % self.pubmed_id)
else:
PubMedLink = HT.Span("Unpublished : ", Class="fs15")
if formName:
setDescription2 = HT.Href(url="javascript:showDatabase3('%s','%s','%s','')" %
(formName, self.dataset.name, self.name), Class = "fs14")
else:
setDescription2 = HT.Href(url="javascript:showDatabase2('%s','%s','')" %
(self.dataset.name,self.name), Class = "fs14")
if self.confidential and not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=privilege, userName=userName, authorized_users=authorized_users):
setDescription2.append('RecordID/%s - %s' % (self.name, self.pre_publication_description))
else:
setDescription2.append('RecordID/%s - %s' % (self.name, self.post_publication_description))
#XZ 03/26/2011: Xiaodong comment out the following two lins as Rob asked. Need to check with Rob why in PublishXRef table, there are few row whose Sequence > 1.
#if self.sequence > 1:
# setDescription2.append(' btach %d' % self.sequence)
if self.authors:
a1 = string.split(self.authors,',')[0]
while a1[0] == '"' or a1[0] == "'" :
a1 = a1[1:]
setDescription2.append(' by ')
setDescription2.append(HT.Italic('%s, and colleagues' % a1))
setDescription = HT.Span(PubMedLink, setDescription2)
elif self.dataset.type == 'Temp':
setDescription = HT.Href(text="%s" % (self.description),url="javascript:showDatabase2\
('%s','%s','')" % (self.dataset.name,self.name), Class = "fs14")
setDescription = HT.Span(setDescription)
elif self.dataset.type == 'Geno': # Genome DB only available for single search
if formName:
setDescription = HT.Href(text="Locus %s [Chr %s @ %s Mb]" % (self.name,self.chr,\
'%2.3f' % self.mb),url="javascript:showDatabase3('%s','%s','%s','')" % \
(formName, self.dataset.name, self.name), Class = "fs14")
else:
setDescription = HT.Href(text="Locus %s [Chr %s @ %s Mb]" % (self.name,self.chr,\
'%2.3f' % self.mb),url="javascript:showDatabase2('%s','%s','')" % \
(self.dataset.name,self.name), Class = "fs14")
setDescription = HT.Span(setDescription)
else:
if self.cellid:
if formName:
setDescription = HT.Href(text="ProbeSet/%s/%s" % (self.name, self.cellid),url=\
"javascript:showDatabase3('%s','%s','%s','%s')" % (formName, self.dataset.name,self.name,self.cellid), \
Class = "fs14")
else:
setDescription = HT.Href(text="ProbeSet/%s/%s" % (self.name,self.cellid),url=\
"javascript:showDatabase2('%s','%s','%s')" % (self.dataset.name,self.name,self.cellid), \
Class = "fs14")
else:
if formName:
setDescription = HT.Href(text="ProbeSet/%s" % self.name, url=\
"javascript:showDatabase3('%s','%s','%s','')" % (formName, self.dataset.name,self.name), \
Class = "fs14")
else:
setDescription = HT.Href(text="ProbeSet/%s" % self.name, url=\
"javascript:showDatabase2('%s','%s','')" % (self.dataset.name,self.name), \
Class = "fs14")
if self.symbol and self.chr and self.mb:
setDescription.append(' [')
setDescription.append(HT.Italic('%s' % self.symbol,Class="cdg fwb"))
setDescription.append(' on Chr %s @ %s Mb]' % (self.chr,self.mb))
if self.description:
setDescription.append(': %s' % self.description)
if self.probe_target_description:
setDescription.append('; %s' % self.probe_target_description)
setDescription = HT.Span(setDescription)
if self.dataset.type != 'Temp' and dispFromDatabase:
setDescription.append( ' --- FROM : ')
setDescription.append(self.dataset.genHTML(Class='cori'))
return setDescription
@property
def name_header_fmt(self):
'''Return a human-readable name for use in page header'''
if self.dataset.type == 'ProbeSet':
return self.symbol
elif self.dataset.type == 'Geno':
return self.name
elif self.dataset.type == 'Publish':
return self.post_publication_abbreviation
else:
return "unnamed"
@property
def description_fmt(self):
'''Return a text formated description'''
if self.dataset.type == 'ProbeSet':
if self.description:
formatted = self.description
if self.probe_target_description:
formatted += "; " + self.probe_target_description
elif self.dataset.type == 'Publish':
formatted = self.post_publication_description
else:
formatted = "Not available"
return formatted.capitalize()
@property
def alias_fmt(self):
'''Return a text formatted alias'''
if self.alias:
alias = string.replace(self.alias, ";", " ")
alias = string.join(string.split(alias), ", ")
else:
alias = 'Not available'
return alias
@property
def location_fmt(self):
'''Return a text formatted location
While we're at it we set self.location in case we need it later (do we?)
'''
if self.chr and self.mb:
self.location = 'Chr %s @ %s Mb' % (self.chr,self.mb)
elif self.chr:
self.location = 'Chr %s @ Unknown position' % (self.chr)
else:
self.location = 'Not available'
fmt = self.location
##XZ: deal with direction
if self.strand_probe == '+':
fmt += (' on the plus strand ')
elif self.strand_probe == '-':
fmt += (' on the minus strand ')
return fmt
def get_database(self):
"""
Returns the database, and the url referring to the database if it exists
We're going to to return two values here, and we don't want to have to call this twice from
the template. So it's not a property called from the template, but instead is called from the view
"""
if self.cellid:
self.cursor.execute("""
select ProbeFreeze.Name from ProbeFreeze, ProbeSetFreeze
where
ProbeFreeze.Id = ProbeSetFreeze.ProbeFreezeId AND
ProbeSetFreeze.Id = %d""" % thisTrait.dataset.id)
probeDBName = self.cursor.fetchone()[0]
return dict(name = probeDBName,
url = None)
else:
return dict(name = self.dataset.fullname,
url = webqtlConfig.INFOPAGEHREF % self.dataset.name)
def calculate_correlation(self, values, method):
"""Calculate the correlation value and p value according to the method specified"""
#ZS: This takes the list of values of the trait our selected trait is being correlated against and removes the values of the samples our trait has no value for
#There's probably a better way of dealing with this, but I'll have to ask Christian
updated_raw_values = []
updated_values = []
for i in range(len(values)):
if values[i] != "None":
updated_raw_values.append(self.raw_values[i])
updated_values.append(values[i])
self.raw_values = updated_raw_values
values = updated_values
if method == METHOD_SAMPLE_PEARSON or method == METHOD_LIT or method == METHOD_TISSUE_PEARSON:
corr, nOverlap = webqtlUtil.calCorrelation(self.raw_values, values, len(values))
else:
corr, nOverlap = webqtlUtil.calCorrelationRank(self.raw_values, values, len(values))
self.correlation = corr
self.overlap = nOverlap
if self.overlap < 3:
self.p_value = 1.0
else:
#ZS - This is probably the wrong way to deal with this. Correlation values of 1.0 definitely exist (the trait correlated against itself), so zero division needs to br prevented.
if abs(self.correlation) >= 1.0:
self.p_value = 0.0
else:
ZValue = 0.5*log((1.0+self.correlation)/(1.0-self.correlation))
ZValue = ZValue*sqrt(self.overlap-3)
self.p_value = 2.0*(1.0 - reaper.normp(abs(ZValue)))
def convert_location_to_value(chromosome, mb):
try:
location_value = int(chromosome)*1000 + float(mb)
except ValueError:
if chromosome.upper() == 'X':
location_value = 20*1000 + float(mb)
else:
location_value = (ord(str(chromosome).upper()[0])*1000 +
float(mb))
return location_value
@app.route("/trait/get_sample_data")
def get_sample_data():
params = request.args
trait = params['trait']
dataset = params['dataset']
trait_ob = GeneralTrait(name=trait, dataset_name=dataset)
return json.dumps([trait, {key: value.value for key, value in trait_ob.data.iteritems() }])
#jsonable_sample_data = {}
#for sample in trait_ob.data.iteritems():
# jsonable_sample_data[sample] = trait_ob.data[sample].value
#
#return jsonable_sample_data
