diff options
Diffstat (limited to 'web/webqtl/correlation')
| -rwxr-xr-x | web/webqtl/correlation/CorrelationPage.py | 1958 | ||||
| -rwxr-xr-x | web/webqtl/correlation/PartialCorrDBPage.py | 1359 | ||||
| -rwxr-xr-x | web/webqtl/correlation/PartialCorrInputPage.py | 484 | ||||
| -rwxr-xr-x | web/webqtl/correlation/PartialCorrTraitPage.py | 310 | ||||
| -rwxr-xr-x | web/webqtl/correlation/PlotCorrelationPage.py | 683 | ||||
| -rwxr-xr-x | web/webqtl/correlation/__init__.py | 0 | ||||
| -rwxr-xr-x | web/webqtl/correlation/correlationFunction.py | 923 |
7 files changed, 5717 insertions, 0 deletions
diff --git a/web/webqtl/correlation/CorrelationPage.py b/web/webqtl/correlation/CorrelationPage.py new file mode 100755 index 00000000..8ce669cb --- /dev/null +++ b/web/webqtl/correlation/CorrelationPage.py @@ -0,0 +1,1958 @@ +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by NL 2011/02/11 + +import string +from math import * +import cPickle +import os +import time +import pyXLWriter as xl +import pp +import math + +from htmlgen import HTMLgen2 as HT +import reaper + +from base import webqtlConfig +from utility.THCell import THCell +from utility.TDCell import TDCell +from base.webqtlTrait import webqtlTrait +from base.webqtlDataset import webqtlDataset +from base.templatePage import templatePage +from utility import webqtlUtil +from dbFunction import webqtlDatabaseFunction +import utility.webqtlUtil #this is for parallel computing only. +from correlation import correlationFunction + + +class CorrelationPage(templatePage): + + corrMinInformative = 4 + + def __init__(self, fd): + + #XZ, 01/14/2009: This method is for parallel computing only. + #XZ: It is supposed to be called when "Genetic Correlation, Pearson's r" (method 1) + #XZ: or "Genetic Correlation, Spearman's rho" (method 2) is selected + def compute_corr( input_nnCorr, input_trait, input_list, computing_method): + + allcorrelations = [] + + for line in input_list: + tokens = line.split('","') + tokens[-1] = tokens[-1][:-2] #remove the last " + tokens[0] = tokens[0][1:] #remove the first " + + traitdataName = tokens[0] + database_trait = tokens[1:] + + if computing_method == "1": #XZ: Pearson's r + corr,nOverlap = utility.webqtlUtil.calCorrelationText(input_trait, database_trait, input_nnCorr) + else: #XZ: Spearman's rho + corr,nOverlap = utility.webqtlUtil.calCorrelationRankText(input_trait, database_trait, input_nnCorr) + traitinfo = [traitdataName,corr,nOverlap] + allcorrelations.append(traitinfo) + + return allcorrelations + + + templatePage.__init__(self, fd) + + if not self.openMysql(): + return + + if not fd.genotype: + fd.readGenotype() + + #XZ, 09/18/2008: get the information such as value, variance of the input strain names from the form. + if fd.allstrainlist: + mdpchoice = fd.formdata.getvalue('MDPChoice') + #XZ, in HTML source code, it is "BXD Only" or "BXH only", and so on + if mdpchoice == "1": + strainlist = fd.f1list + fd.strainlist + #XZ, in HTML source code, it is "MDP Only" + elif mdpchoice == "2": + strainlist = [] + strainlist2 = fd.f1list + fd.strainlist + for strain in fd.allstrainlist: + if strain not in strainlist2: + strainlist.append(strain) + #So called MDP Panel + if strainlist: + strainlist = fd.f1list+fd.parlist+strainlist + #XZ, in HTML source code, it is "All Cases" + else: + strainlist = fd.allstrainlist + #XZ, 09/18/2008: put the trait data into dictionary fd.allTraitData + fd.readData(fd.allstrainlist) + else: + mdpchoice = None + strainlist = fd.strainlist + #XZ, 09/18/2008: put the trait data into dictionary fd.allTraitData + fd.readData() + + #XZ, 3/16/2010: variable RISet must be pass by the form + RISet = fd.RISet + #XZ, 12/12/2008: get species infomation + species = webqtlDatabaseFunction.retrieveSpecies(cursor=self.cursor, RISet=RISet) + + #XZ, 09/18/2008: get all information about the user selected database. + self.target_db_name = fd.formdata.getvalue('database') + + try: + self.db = webqtlDataset(self.target_db_name, self.cursor) + except: + heading = "Correlation Table" + detail = ["The database you just requested has not been established yet."] + self.error(heading=heading,detail=detail) + return + + #XZ, 09/18/2008: check if user has the authority to get access to the database. + if self.db.type == 'ProbeSet': + self.cursor.execute('SELECT Id, Name, FullName, confidentiality, AuthorisedUsers FROM ProbeSetFreeze WHERE Name = "%s"' % self.target_db_name) + indId, indName, indFullName, confidential, AuthorisedUsers = self.cursor.fetchall()[0] + + if confidential == 1: + access_to_confidential_dataset = 0 + + #for the dataset that confidentiality is 1 + #1. 'admin' and 'root' can see all of the dataset + #2. 'user' can see the dataset that AuthorisedUsers contains his id(stored in the Id field of User table) + if webqtlConfig.USERDICT[self.privilege] > webqtlConfig.USERDICT['user']: + access_to_confidential_dataset = 1 + else: + AuthorisedUsersList=AuthorisedUsers.split(',') + if AuthorisedUsersList.__contains__(self.userName): + access_to_confidential_dataset = 1 + + if not access_to_confidential_dataset: + #Error, Confidential Database + heading = "Correlation Table" + detail = ["The %s database you selected is not open to the public at this time, please go back and select other database." % indFullName] + self.error(heading=heading,detail=detail,error="Confidential Database") + return + + #XZ, 09/18/2008: filter out the strains that have no value. + _strains, _vals, _vars, N = fd.informativeStrains(strainlist) + + N = len(_strains) + + if N < self.corrMinInformative: + heading = "Correlation Table" + detail = ['Fewer than %d strain data were entered for %s data set. No calculation of correlation has been attempted.' % (self.corrMinInformative, RISet)] + self.error(heading=heading,detail=detail) + return + + #XZ, 09/28/2008: if user select "1", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "2", then display 2, 3 and 5. + #XZ, 09/28/2008: if user select "3", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "4", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "5", then display 2, 3 and 5. + methodDict = {"1":"Genetic Correlation (Pearson's r)","2":"Genetic Correlation (Spearman's rho)","3":"SGO Literature Correlation","4":"Tissue Correlation (Pearson's r)", "5":"Tissue Correlation (Spearman's rho)"} + self.method = fd.formdata.getvalue('method') + if self.method not in ("1","2","3","4","5"): + self.method = "1" + + self.returnNumber = int(fd.formdata.getvalue('criteria')) + + myTrait = fd.formdata.getvalue('fullname') + if myTrait: + myTrait = webqtlTrait(fullname=myTrait, cursor=self.cursor) + myTrait.retrieveInfo() + + # We will not get Literature Correlations if there is no GeneId because there is nothing to look against + try: + input_trait_GeneId = int(fd.formdata.getvalue('GeneId')) + except: + input_trait_GeneId = None + + # We will not get Tissue Correlations if there is no gene symbol because there is nothing to look against + try: + input_trait_symbol = myTrait.symbol + except: + input_trait_symbol = None + + + #XZ, 12/12/2008: if the species is rat or human, translate the geneid to mouse geneid + input_trait_mouse_geneid = self.translateToMouseGeneID(species, input_trait_GeneId) + + + #XZ: As of Nov/13/2010, this dataset is 'UTHSC Illumina V6.2 RankInv B6 D2 average CNS GI average (May 08)' + TissueProbeSetFreezeId = 1 + + #XZ, 09/22/2008: If we need search by GeneId, + #XZ, 09/22/2008: we have to check if this GeneId is in the literature or tissue correlation table. + #XZ, 10/15/2008: We also to check if the selected database is probeset type. + if self.method == "3" or self.method == "4" or self.method == "5": + if self.db.type != "ProbeSet": + self.error(heading="Wrong correlation type",detail="It is not possible to compute the %s between your trait and data in this %s database. Please try again after selecting another type of correlation." % (methodDict[self.method],self.db.name),error="Correlation Type Error") + return + + """ + if not input_trait_GeneId: + self.error(heading="No Associated GeneId",detail="This trait has no associated GeneId, so we are not able to show any literature or tissue related information.",error="No GeneId Error") + return + """ + + #XZ: We have checked geneid did exist + + if self.method == "3": + if not input_trait_GeneId or not self.checkForLitInfo(input_trait_mouse_geneid): + self.error(heading="No Literature Info",detail="This gene does not have any associated Literature Information.",error="Literature Correlation Error") + return + + if self.method == "4" or self.method == "5": + if not input_trait_symbol: + self.error(heading="No Tissue Correlation Information",detail="This gene does not have any associated Tissue Correlation Information.",error="Tissue Correlation Error") + return + + if not self.checkSymbolForTissueCorr(TissueProbeSetFreezeId, myTrait.symbol): + self.error(heading="No Tissue Correlation Information",detail="This gene does not have any associated Tissue Correlation Information.",error="Tissue Correlation Error") + return + +############################################################################################################################################ + + allcorrelations = [] + nnCorr = len(_vals) + + #XZ: Use the fast method only for probeset dataset, and this dataset must have been created. + #XZ: Otherwise, use original method + + useFastMethod = False + + if self.db.type == "ProbeSet": + + DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) + DirectoryList = os.listdir(webqtlConfig.TEXTDIR) ### List of existing text files. Used to check if a text file already exists + + if DatabaseFileName in DirectoryList: + useFastMethod = True + + if useFastMethod: + if 1: + #try: + useLit = False + if self.method == "3": + litCorrs = self.fetchLitCorrelations(species=species, GeneId=input_trait_GeneId, db=self.db, returnNumber=self.returnNumber) + useLit = True + + useTissueCorr = False + if self.method == "4" or self.method == "5": + tissueCorrs = self.fetchTissueCorrelations(db=self.db, primaryTraitSymbol=input_trait_symbol, TissueProbeSetFreezeId=TissueProbeSetFreezeId, method=self.method, returnNumber = self.returnNumber) + useTissueCorr = True + + datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r') + + #XZ, 01/08/2009: read the first line + line = datasetFile.readline() + dataset_strains = webqtlUtil.readLineCSV(line)[1:] + + #XZ, 01/08/2009: This step is critical. It is necessary for this new method. + #XZ: The original function fetchAllDatabaseData uses all strains stored in variable _strains to + #XZ: retrieve the values of each strain from database in real time. + #XZ: The new method uses all strains stored in variable dataset_strains to create a new variable + #XZ: _newvals. _newvals has the same length as dataset_strains. The items in _newvals is in + #XZ: the same order of items in dataset_strains. The value of each item in _newvals is either + #XZ: the value of correspinding strain in _vals or 'None'. + _newvals = [] + for item in dataset_strains: + if item in _strains: + _newvals.append(_vals[_strains.index(item)]) + else: + _newvals.append('None') + + nnCorr = len(_newvals) + + #XZ, 01/14/2009: If literature corr or tissue corr is selected, + #XZ: there is no need to use parallel computing. + if useLit or useTissueCorr: + for line in datasetFile: + traitdata=webqtlUtil.readLineCSV(line) + traitdataName = traitdata[0] + traitvals = traitdata[1:] + + if useLit: + if not litCorrs.has_key( traitdataName ): + continue + + if useTissueCorr: + if not tissueCorrs.has_key( traitdataName ): + continue + + if self.method == "3" or self.method == "4": + corr,nOverlap = webqtlUtil.calCorrelationText(traitvals,_newvals,nnCorr) + else: + corr,nOverlap = webqtlUtil.calCorrelationRankText(traitvals,_newvals,nnCorr) + + traitinfo = [traitdataName,corr,nOverlap] + + if useLit: + traitinfo.append(litCorrs[traitdataName]) + + if useTissueCorr: + tempCorr, tempPValue = tissueCorrs[traitdataName] + traitinfo.append(tempCorr) + traitinfo.append(tempPValue) + + allcorrelations.append(traitinfo) + #XZ, 01/14/2009: If genetic corr is selected, use parallel computing + else: + input_line_list = datasetFile.readlines() + all_line_number = len(input_line_list) + + step = 1000 + job_number = math.ceil( float(all_line_number)/step ) + + job_input_lists = [] + + for job_index in range( int(job_number) ): + starti = job_index*step + endi = min((job_index+1)*step, all_line_number) + + one_job_input_list = [] + + for i in range( starti, endi ): + one_job_input_list.append( input_line_list[i] ) + + job_input_lists.append( one_job_input_list ) + + ppservers = () + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + + jobs = [] + results = [] + + for one_job_input_list in job_input_lists: #pay attention to modules from outside + jobs.append( job_server.submit(func=compute_corr, args=(nnCorr, _newvals, one_job_input_list, self.method), depfuncs=(), modules=("utility.webqtlUtil",)) ) + + for one_job in jobs: + one_result = one_job() + results.append( one_result ) + + for one_result in results: + for one_traitinfo in one_result: + allcorrelations.append( one_traitinfo ) + + datasetFile.close() + totalTraits = len(allcorrelations) + #except: + # useFastMethod = False + # self.error(heading="No computation method",detail="Something is wrong within the try except block in CorrelationPage python module.",error="Computation Error") + # return + + #XZ, 01/08/2009: use the original method to retrieve from database and compute. + if not useFastMethod: + + traitdatabase, dataStartPos = self.fetchAllDatabaseData(species=species, GeneId=input_trait_GeneId, GeneSymbol=input_trait_symbol, strains=_strains, db=self.db, method=self.method, returnNumber=self.returnNumber, tissueProbeSetFreezeId=TissueProbeSetFreezeId) + + totalTraits = len(traitdatabase) #XZ, 09/18/2008: total trait number + + for traitdata in traitdatabase: + traitdataName = traitdata[0] + traitvals = traitdata[dataStartPos:] + if self.method == "1" or self.method == "3" or self.method == "4": + corr,nOverlap = webqtlUtil.calCorrelation(traitvals,_vals,nnCorr) + else: + corr,nOverlap = webqtlUtil.calCorrelationRank(traitvals,_vals,nnCorr) + + traitinfo = [traitdataName,corr,nOverlap] + + #XZ, 09/28/2008: if user select '3', then fetchAllDatabaseData would give us LitCorr in the [1] position + #XZ, 09/28/2008: if user select '4' or '5', then fetchAllDatabaseData would give us Tissue Corr in the [1] position + #XZ, 09/28/2008: and Tissue Corr P Value in the [2] position + if input_trait_GeneId and self.db.type == "ProbeSet": + if self.method == "3": + traitinfo.append( traitdata[1] ) + if self.method == "4" or self.method == "5": + traitinfo.append( traitdata[1] ) + traitinfo.append( traitdata[2] ) + + allcorrelations.append(traitinfo) + + +############################################################# + + if self.method == "3" and input_trait_GeneId: + allcorrelations.sort(webqtlUtil.cmpLitCorr) + #XZ, 3/31/2010: Theoretically, we should create one function 'comTissueCorr' + #to compare each trait by their tissue corr p values. + #But because the tissue corr p values are generated by permutation test, + #the top ones always have p value 0. So comparing p values actually does nothing. + #In addition, for the tissue data in our database, the N is always the same. + #So it's safe to compare with tissue corr statistic value. + #That's the same as literature corr. + elif self.method in ["4","5"] and input_trait_GeneId: + allcorrelations.sort(webqtlUtil.cmpLitCorr) + else: + allcorrelations.sort(webqtlUtil.cmpCorr) + + + #XZ, 09/20/2008: we only need the top ones. + self.returnNumber = min(self.returnNumber,len(allcorrelations)) + allcorrelations = allcorrelations[:self.returnNumber] + + addLiteratureCorr = False + addTissueCorr = False + + traitList = [] + for item in allcorrelations: + thisTrait = webqtlTrait(db=self.db, name=item[0], cursor=self.cursor) + thisTrait.retrieveInfo( QTL='Yes' ) + + nOverlap = item[2] + corr = item[1] + + #XZ: calculate corrPValue + if nOverlap < 3: + corrPValue = 1.0 + else: + if abs(corr) >= 1.0: + corrPValue = 0.0 + else: + ZValue = 0.5*log((1.0+corr)/(1.0-corr)) + ZValue = ZValue*sqrt(nOverlap-3) + corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue))) + + thisTrait.Name = item[0] + thisTrait.corr = corr + thisTrait.nOverlap = nOverlap + thisTrait.corrPValue = corrPValue + # NL, 07/19/2010 + # js function changed, add a new parameter rankOrder for js function 'showTissueCorrPlot' + rankOrder = 0; + if self.method in ["2","5"]: + rankOrder = 1; + thisTrait.rankOrder =rankOrder + + #XZ, 26/09/2008: Method is 4 or 5. Have fetched tissue corr, but no literature correlation yet. + if len(item) == 5: + thisTrait.tissueCorr = item[3] + thisTrait.tissuePValue = item[4] + addLiteratureCorr = True + + #XZ, 26/09/2008: Method is 3, Have fetched literature corr, but no tissue corr yet. + elif len(item) == 4: + thisTrait.LCorr = item[3] + thisTrait.mouse_geneid = self.translateToMouseGeneID(species, thisTrait.geneid) + addTissueCorr = True + + #XZ, 26/09/2008: Method is 1 or 2. Have NOT fetched literature corr and tissue corr yet. + # Phenotype data will not have geneid, and neither will some probes + # we need to handle this because we will get an attribute error + else: + if input_trait_mouse_geneid and self.db.type=="ProbeSet": + addLiteratureCorr = True + if input_trait_symbol and self.db.type=="ProbeSet": + addTissueCorr = True + + traitList.append(thisTrait) + + if addLiteratureCorr: + traitList = self.getLiteratureCorrelationByList(input_trait_mouse_geneid, species, traitList) + if addTissueCorr: + traitList = self.getTissueCorrelationByList( primaryTraitSymbol=input_trait_symbol, traitList=traitList,TissueProbeSetFreezeId =TissueProbeSetFreezeId, method=self.method) + +######################################################## + + TD_LR = HT.TD(height=200,width="100%",bgColor='#eeeeee') + + mainfmName = webqtlUtil.genRandStr("fm_") + form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name= mainfmName, submit=HT.Input(type='hidden')) + hddn = {'FormID':'showDatabase', 'ProbeSetID':'_','database':self.target_db_name, 'databaseFull':self.db.fullname, 'CellID':'_', 'RISet':RISet, 'identification':fd.identification} + + if myTrait: + hddn['fullname']=fd.formdata.getvalue('fullname') + if mdpchoice: + hddn['MDPChoice']=mdpchoice + + + #XZ, 09/18/2008: pass the trait data to next page by hidden parameters. + webqtlUtil.exportData(hddn, fd.allTraitData) + + if fd.incparentsf1: + hddn['incparentsf1']='ON' + + if fd.allstrainlist: + hddn['allstrainlist'] = string.join(fd.allstrainlist, ' ') + + + for key in hddn.keys(): + form.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + #XZ, 11/21/2008: add two parameters to form + form.append(HT.Input(name="X_geneSymbol", value="", type='hidden')) + form.append(HT.Input(name="Y_geneSymbol", value="", type='hidden')) + + #XZ, 3/11/2010: add one parameter to record if the method is rank order. + form.append(HT.Input(name="rankOrder", value="%s" % rankOrder, type='hidden')) + + form.append(HT.Input(name="TissueProbeSetFreezeId", value="%s" % TissueProbeSetFreezeId, type='hidden')) + + #################################### + # generate the info on top of page # + #################################### + + info = self.getTopInfo(myTrait=myTrait, method=self.method, db=self.db, target_db_name=self.target_db_name, returnNumber=self.returnNumber, methodDict=methodDict, totalTraits=totalTraits, identification=fd.identification ) + + ############## + # Excel file # + ############## + filename= webqtlUtil.genRandStr("Corr_") + xlsUrl = HT.Input(type='button', value = 'Download Table', onClick= "location.href='/tmp/%s.xls'" % filename, Class='button') + # Create a new Excel workbook + workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+filename)) + headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") + + #XZ, 3/18/2010: pay attention to the line number of header in this file. As of today, there are 7 lines. + worksheet = self.createExcelFileWithTitleAndFooter(workbook=workbook, identification=fd.identification, db=self.db, returnNumber=self.returnNumber) + + newrow = 7 + + +##################################################################### + + + + mintmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'showIntMap');" % mainfmName) + mintmap_img = HT.Image("/images/multiple_interval_mapping1_final.jpg", name='mintmap', alt="Multiple Interval Mapping", title="Multiple Interval Mapping", style="border:none;") + mintmap.append(mintmap_img) + mcorr = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'compCorr');" % mainfmName) + mcorr_img = HT.Image("/images/compare_correlates2_final.jpg", alt="Compare Correlates", title="Compare Correlates", style="border:none;") + mcorr.append(mcorr_img) + cormatrix = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'corMatrix');" % mainfmName) + cormatrix_img = HT.Image("/images/correlation_matrix1_final.jpg", alt="Correlation Matrix and PCA", title="Correlation Matrix and PCA", style="border:none;") + cormatrix.append(cormatrix_img) + networkGraph = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'networkGraph');" % mainfmName) + networkGraph_img = HT.Image("/images/network_graph1_final.jpg", name='mintmap', alt="Network Graphs", title="Network Graphs", style="border:none;") + networkGraph.append(networkGraph_img) + heatmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'heatmap');" % mainfmName) + heatmap_img = HT.Image("/images/heatmap2_final.jpg", name='mintmap', alt="QTL Heat Map and Clustering", title="QTL Heatmap and Clustering", style="border:none;") + heatmap.append(heatmap_img) + partialCorr = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'partialCorrInput');" % mainfmName) + partialCorr_img = HT.Image("/images/partial_correlation_final.jpg", name='partialCorr', alt="Partial Correlation", title="Partial Correlation", style="border:none;") + partialCorr.append(partialCorr_img) + addselect = HT.Href(url="#redirect", onClick="addRmvSelection('%s', document.getElementsByName('%s')[0], 'addToSelection');" % (RISet, mainfmName)) + addselect_img = HT.Image("/images/add_collection1_final.jpg", name="addselect", alt="Add To Collection", title="Add To Collection", style="border:none;") + addselect.append(addselect_img) + selectall = HT.Href(url="#redirect", onClick="checkAll(document.getElementsByName('%s')[0]);" % mainfmName) + selectall_img = HT.Image("/images/select_all2_final.jpg", name="selectall", alt="Select All", title="Select All", style="border:none;") + selectall.append(selectall_img) + selectinvert = HT.Href(url="#redirect", onClick = "checkInvert(document.getElementsByName('%s')[0]);" % mainfmName) + selectinvert_img = HT.Image("/images/invert_selection2_final.jpg", name="selectinvert", alt="Invert Selection", title="Invert Selection", style="border:none;") + selectinvert.append(selectinvert_img) + reset = HT.Href(url="#redirect", onClick="checkNone(document.getElementsByName('%s')[0]); return false;" % mainfmName) + reset_img = HT.Image("/images/select_none2_final.jpg", alt="Select None", title="Select None", style="border:none;") + reset.append(reset_img) + selecttraits = HT.Input(type='button' ,name='selecttraits',value='Select Traits', onClick="checkTraits(this.form);",Class="button") + selectgt = HT.Input(type='text' ,name='selectgt',value='-1.0', size=6,maxlength=10,onChange="checkNumeric(this,1.0,'-1.0','gthan','greater than filed')") + selectlt = HT.Input(type='text' ,name='selectlt',value='1.0', size=6,maxlength=10,onChange="checkNumeric(this,-1.0,'1.0','lthan','less than field')") + selectandor = HT.Select(name='selectandor') + selectandor.append(('AND','and')) + selectandor.append(('OR','or')) + selectandor.selected.append('AND') + + pageTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="100%", border=0, align="Left") + + containerTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="90%",border=0, align="Left") + + optionsTable = HT.TableLite(cellSpacing=2, cellPadding=0,width="320", height="80", border=0, align="Left") + optionsTable.append(HT.TR(HT.TD(selectall), HT.TD(reset), HT.TD(selectinvert), HT.TD(addselect), align="left")) + optionsTable.append(HT.TR(HT.TD(" "*1,"Select"), HT.TD("Deselect"), HT.TD(" "*1,"Invert"), HT.TD(" "*3,"Add"))) + containerTable.append(HT.TR(HT.TD(optionsTable))) + + functionTable = HT.TableLite(cellSpacing=2,cellPadding=0,width="480",height="80", border=0, align="Left") + functionRow = HT.TR(HT.TD(networkGraph, width="16.7%"), HT.TD(cormatrix, width="16.7%"), HT.TD(partialCorr, width="16.7%"), HT.TD(mcorr, width="16.7%"), HT.TD(mintmap, width="16.7%"), HT.TD(heatmap), align="left") + labelRow = HT.TR(HT.TD(" "*1,HT.Text("Graph")), HT.TD(" "*1,HT.Text("Matrix")), HT.TD(" "*1,HT.Text("Partial")), HT.TD(HT.Text("Compare")), HT.TD(HT.Text("QTL Map")), HT.TD(HT.Text(text="Heat Map"))) + functionTable.append(functionRow, labelRow) + containerTable.append(HT.TR(HT.TD(functionTable), HT.BR())) + + #more_options = HT.Image("/images/more_options1_final.jpg", name='more_options', alt="Expand Options", title="Expand Options", style="border:none;", Class="toggleShowHide") + + #containerTable.append(HT.TR(HT.TD(more_options, HT.BR(), HT.BR()))) + + moreOptions = HT.Input(type='button',name='options',value='More Options', onClick="",Class="toggle") + fewerOptions = HT.Input(type='button',name='options',value='Fewer Options', onClick="",Class="toggle") + + if (fd.formdata.getvalue('showHideOptions') == 'less'): + containerTable.append(HT.TR(HT.TD(" "), height="10"), HT.TR(HT.TD(HT.Div(fewerOptions, Class="toggleShowHide")))) + containerTable.append(HT.TR(HT.TD(" "))) + else: + containerTable.append(HT.TR(HT.TD(" "), height="10"), HT.TR(HT.TD(HT.Div(moreOptions, Class="toggleShowHide")))) + containerTable.append(HT.TR(HT.TD(" "))) + + containerTable.append(HT.TR(HT.TD(HT.Span(selecttraits,' with r > ',selectgt, ' ',selectandor, ' r < ',selectlt,Class="bd1 cbddf fs11")), style="display:none;", Class="extra_options")) + + chrMenu = HT.Input(type='hidden',name='chromosomes',value='all') + + corrHeading = HT.Paragraph('Correlation Table', Class="title") + + + tblobj = {} + + if self.db.type=="Geno": + + containerTable.append(HT.TR(HT.TD(xlsUrl, height=40))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + tblobj['header'], worksheet = self.getTableHeaderForGeno( method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForGeno(traitList=traitList, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript) + + workbook.close() + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + + pageTable.append(HT.TR(HT.TD(div))) + + form.append(HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + HT.P(), HT.P(), pageTable) + TD_LR.append(corrHeading, info, form, HT.P()) + + self.dict['body'] = str(TD_LR) + self.dict['js1'] = '' + self.dict['title'] = 'Correlation' + + elif self.db.type=="Publish": + + containerTable.append(HT.TR(HT.TD(xlsUrl, height=40))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + tblobj['header'], worksheet = self.getTableHeaderForPublish(method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForPublish(traitList=traitList, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript, species=species) + + workbook.close() + + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + + pageTable.append(HT.TR(HT.TD(div))) + + form.append( + HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + HT.P(), pageTable) + TD_LR.append(corrHeading, info, form, HT.P()) + + self.dict['body'] = str(TD_LR) + self.dict['js1'] = '' + self.dict['title'] = 'Correlation' + + + elif self.db.type=="ProbeSet": + + tblobj['header'], worksheet = self.getTableHeaderForProbeSet(method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForProbeSet(traitList=traitList, primaryTrait=myTrait, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript, species=species) + + workbook.close() + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + + #XZ: here is the table of traits + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1", hiddenColumns=["Gene ID","Homologene ID"]), corrScript, Id="sortable") + + + #XZ, 01/12/2009: create database menu for 'Add Correlation' + self.cursor.execute(""" + select + ProbeSetFreeze.FullName, ProbeSetFreeze.Id, Tissue.name + from + ProbeSetFreeze, ProbeFreeze, ProbeSetFreeze as ps2, ProbeFreeze as p2, Tissue + where + ps2.Id = %d + and ps2.ProbeFreezeId = p2.Id + and ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id + and (ProbeFreeze.InbredSetId = p2.InbredSetId or (ProbeFreeze.InbredSetId in (1, 3) and p2.InbredSetId in (1, 3))) + and p2.ChipId = ProbeFreeze.ChipId + and ps2.Id != ProbeSetFreeze.Id + and ProbeFreeze.TissueId = Tissue.Id + and ProbeSetFreeze.public > %d + order by + ProbeFreeze.TissueId, ProbeSetFreeze.CreateTime desc + """ % (self.db.id, webqtlConfig.PUBLICTHRESH)) + + results = self.cursor.fetchall() + dbCustomizer = HT.Select(results, name = "customizer") + databaseMenuSub = preTissue = "" + for item in results: + TName, TId, TTissue = item + if TTissue != preTissue: + if databaseMenuSub: + dbCustomizer.append(databaseMenuSub) + databaseMenuSub = HT.Optgroup(label = '%s mRNA ------' % TTissue) + preTissue = TTissue + + databaseMenuSub.append(item[:2]) + if databaseMenuSub: + dbCustomizer.append(databaseMenuSub) + + #updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + #variables: filename, strainIds and vals are required by getquerystring function + strainIds=self.getStrainIds(species=species, strains=_strains) + var1 = HT.Input(name="filename", value=filename, type='hidden') + var2 = HT.Input(name="strainIds", value=strainIds, type='hidden') + var3 = HT.Input(name="vals", value=_vals, type='hidden') + customizerButton = HT.Input(type="button", Class="button", value="Add Correlation", onClick = "xmlhttpPost('%smain.py?FormID=AJAX_table', 'sortable', (getquerystring(this.form)))" % webqtlConfig.CGIDIR) + + containerTable.append(HT.TR(HT.TD(HT.Span(var1,var2,var3,customizerButton, "with", dbCustomizer, Class="bd1 cbddf fs11"), HT.BR(), HT.BR()), style="display:none;", Class="extra_options")) + + #outside analysis part + GCATButton = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'GCAT');" % mainfmName) + GCATButton_img = HT.Image("/images/GCAT_logo_final.jpg", name="GCAT", alt="GCAT", title="GCAT", style="border:none") + GCATButton.append(GCATButton_img) + + ODE = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'ODE');" % mainfmName) + ODE_img = HT.Image("/images/ODE_logo_final.jpg", name="ode", alt="ODE", title="ODE", style="border:none") + ODE.append(ODE_img) + + ''' + #XZ, 07/07/2010: I comment out this block of code. + WebGestaltScript = HT.Script(language="Javascript") + WebGestaltScript.append(""" +setTimeout('openWebGestalt()', 2000); +function openWebGestalt(){ + var thisForm = document['WebGestalt']; + makeWebGestaltTree(thisForm, '%s', %d, 'edag_only.php'); +} + """ % (mainfmName, len(traitList))) + ''' + + self.cursor.execute('SELECT GeneChip.GO_tree_value FROM GeneChip, ProbeFreeze, ProbeSetFreeze WHERE GeneChip.Id = ProbeFreeze.ChipId and ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id and ProbeSetFreeze.Name = "%s"' % self.db.name) + result = self.cursor.fetchone() + + if result: + GO_tree_value = result[0] + + if GO_tree_value: + + WebGestalt = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'GOTree');" % mainfmName) + WebGestalt_img = HT.Image("/images/webgestalt_icon_final.jpg", name="webgestalt", alt="Gene Set Analysis Toolkit", title="Gene Set Analysis Toolkit", style="border:none") + WebGestalt.append(WebGestalt_img) + + hddnWebGestalt = { + 'id_list':'', + 'correlation':'', + 'id_value':'', + 'llid_list':'', + 'id_type':GO_tree_value, + 'idtype':'', + 'species':'', + 'list':'', + 'client':''} + + hddnWebGestalt['ref_type'] = hddnWebGestalt['id_type'] + hddnWebGestalt['cat_type'] = 'GO' + hddnWebGestalt['significancelevel'] = 'Top10' + + if species == 'rat': + hddnWebGestalt['org'] = 'Rattus norvegicus' + elif species == 'human': + hddnWebGestalt['org'] = 'Homo sapiens' + elif species == 'mouse': + hddnWebGestalt['org'] = 'Mus musculus' + else: + hddnWebGestalt['org'] = '' + + for key in hddnWebGestalt.keys(): + form.append(HT.Input(name=key, value=hddnWebGestalt[key], type='hidden')) + + + LinkOutTable = HT.TableLite(cellSpacing=2,cellPadding=0,width="320",height="80", border=0, align="Left") + + if not GO_tree_value: + LinkOutRow = HT.TR(HT.TD(GCATButton, width="50%"), HT.TD(ODE, width="50%"), align="left") + LinkOutLabels = HT.TR(HT.TD(" ", HT.Text("GCAT"), width="50%"), HT.TD(" ",HT.Text("ODE"), width="50%"), align="left") + else: + LinkOutRow = HT.TR(HT.TD(WebGestalt, width="25%"), HT.TD(GCATButton, width="25%"), HT.TD(ODE, width="25%"), align="left") + LinkOutLabels = HT.TR(HT.TD(HT.Text("Gene Set")), HT.TD(" "*2, HT.Text("GCAT")), HT.TD(" "*3, HT.Text("ODE")), style="display:none;", Class="extra_options") + + LinkOutTable.append(LinkOutRow,LinkOutLabels) + + containerTable.append(HT.TR(HT.TD(LinkOutTable), Class="extra_options", style="display:none;")) + + containerTable.append(HT.TR(HT.TD(xlsUrl, HT.BR(), HT.BR()))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + pageTable.append(HT.TR(HT.TD(div))) + + if species == 'human': + heatmap = "" + + form.append(HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + info, HT.BR(), pageTable, HT.BR()) + + TD_LR.append(corrHeading, form, HT.P()) + + + self.dict['body'] = str(TD_LR) + self.dict['title'] = 'Correlation' + # updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + self.dict['js1'] = '' + self.dict['js2'] = 'onLoad="pageOffset()"' + self.dict['layer'] = self.generateWarningLayer() + else: + self.dict['body'] = "" + + +############################# +# # +# CorrelationPage Functions # +# # +############################# + + + def getSortByValue(self, calculationMethod): + + if calculationMethod == "1": + sortby = ("Sample p(r)", "up") + elif calculationMethod == "2": + sortby = ("Sample p(rho)", "up") + elif calculationMethod == "3": #XZ: literature correlation + sortby = ("Lit Corr","down") + elif calculationMethod == "4": #XZ: tissue correlation + sortby = ("Tissue r", "down") + elif calculationMethod == "5": + sortby = ("Tissue rho", "down") + + return sortby + + + + def generateWarningLayer(self): + + layerString = """ +<!-- BEGIN FLOATING LAYER CODE //--> +<div id="warningLayer" style="padding:3px; border: 1px solid #222; + background-color: #fff; position:absolute;width:250px;left:100;top:100;visibility:hidden"> +<table border="0" width="250" class="cbrb" cellspacing="0" cellpadding="5"> +<tr> +<td width="100%"> + <table border="0" width="100%" cellspacing="0" cellpadding="0" height="36"> + <tr> + <td class="cbrb cw ff15 fwb" align="Center" width="100%" style="padding:4px"> + Sort Table + </td> + </tr> + <tr> + <td width="100%" bgcolor="#eeeeee" align="Center" style="padding:4px"> +<!-- PLACE YOUR CONTENT HERE //--> +Resorting this table <br> +<!-- END OF CONTENT AREA //--> + </td> + </tr> + </table> +</td> +</tr> +</table> +</div> +<!-- END FLOATING LAYER CODE //--> + + """ + + return layerString + + + #XZ, 01/07/2009: In HTML code, the variable 'database' corresponds to the column 'Name' in database table. + def getFileName(self, target_db_name): ### dcrowell August 2008 + """Returns the name of the reference database file with which correlations are calculated. + Takes argument cursor which is a cursor object of any instance of a subclass of templatePage + Used by correlationPage""" + + query = 'SELECT Id, FullName FROM ProbeSetFreeze WHERE Name = "%s"' % target_db_name + self.cursor.execute(query) + result = self.cursor.fetchone() + Id = result[0] + FullName = result[1] + FullName = FullName.replace(' ','_') + FullName = FullName.replace('/','_') + + FileName = 'ProbeSetFreezeId_' + str(Id) + '_FullName_' + FullName + '.txt' + + return FileName + + + #XZ, 01/29/2009: I modified this function. + #XZ: Note that the type of StrainIds must be number, not string. + def getStrainIds(self, species=None, strains=[]): + StrainIds = [] + for item in strains: + self.cursor.execute('''SELECT Strain.Id FROM Strain, Species WHERE + Strain.Name="%s" and Strain.SpeciesId=Species.Id and Species.name = "%s" ''' % (item, species)) + Id = self.cursor.fetchone()[0] + StrainIds.append(Id) + + return StrainIds + + + #XZ, 12/12/2008: if the species is rat or human, translate the geneid to mouse geneid + #XZ, 12/12/2008: if the input geneid is 'None', return 0 + #XZ, 12/12/2008: if the input geneid has no corresponding mouse geneid, return 0 + def translateToMouseGeneID (self, species, geneid): + mouse_geneid = 0; + + #if input geneid is None, return 0. + if not geneid: + return mouse_geneid + + if species == 'mouse': + mouse_geneid = geneid + elif species == 'rat': + self.cursor.execute( "SELECT mouse FROM GeneIDXRef WHERE rat=%d" % int(geneid) ) + record = self.cursor.fetchone() + if record: + mouse_geneid = record[0] + elif species == 'human': + self.cursor.execute( "SELECT mouse FROM GeneIDXRef WHERE human=%d" % int(geneid) ) + record = self.cursor.fetchone() + if record: + mouse_geneid = record[0] + + return mouse_geneid + + + #XZ, 12/16/2008: the input geneid is of mouse type + def checkForLitInfo(self,geneId): + q = 'SELECT 1 FROM LCorrRamin3 WHERE GeneId1=%s LIMIT 1' % geneId + self.cursor.execute(q) + try: + x = self.cursor.fetchone() + if x: return True + else: raise + except: return False + + + #XZ, 12/16/2008: the input geneid is of mouse type + def checkSymbolForTissueCorr(self, tissueProbeSetFreezeId=0, symbol=""): + q = "SELECT 1 FROM TissueProbeSetXRef WHERE TissueProbeSetFreezeId=%s and Symbol='%s' LIMIT 1" % (tissueProbeSetFreezeId,symbol) + self.cursor.execute(q) + try: + x = self.cursor.fetchone() + if x: return True + else: raise + except: return False + + + + def fetchAllDatabaseData(self, species, GeneId, GeneSymbol, strains, db, method, returnNumber, tissueProbeSetFreezeId): + + StrainIds = [] + for item in strains: + self.cursor.execute('''SELECT Strain.Id FROM Strain, Species WHERE Strain.Name="%s" and Strain.SpeciesId=Species.Id and Species.name = "%s" ''' % (item, species)) + Id = self.cursor.fetchone()[0] + StrainIds.append('%d' % Id) + + # break it into smaller chunks so we don't overload the MySql server + nnn = len(StrainIds) / 25 + if len(StrainIds) % 25: + nnn += 1 + oridata = [] + + #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + tempTable = None + if GeneId and db.type == "ProbeSet": + if method == "3": + tempTable = self.getTempLiteratureTable(species=species, input_species_geneid=GeneId, returnNumber=returnNumber) + + if method == "4" or method == "5": + tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, TissueProbeSetFreezeId=tissueProbeSetFreezeId, method=method, returnNumber=returnNumber) + + for step in range(nnn): + temp = [] + StrainIdstep = StrainIds[step*25:min(len(StrainIds), (step+1)*25)] + for item in StrainIdstep: temp.append('T%s.value' % item) + + if db.type == "Publish": + query = "SELECT PublishXRef.Id, " + dataStartPos = 1 + query += string.join(temp,', ') + query += ' FROM (PublishXRef, PublishFreeze)' + #XZ, 03/04/2009: Xiaodong changed Data to PublishData + for item in StrainIdstep: + query += 'left join PublishData as T%s on T%s.Id = PublishXRef.DataId and T%s.StrainId=%s\n' %(item,item,item,item) + query += "WHERE PublishXRef.InbredSetId = PublishFreeze.InbredSetId and PublishFreeze.Name = '%s'" % (db.name, ) + #XZ, 09/20/2008: extract literature correlation value together with gene expression values. + #XZ, 09/20/2008: notice the difference between the code in next block. + elif tempTable: + # we can get a little performance out of selecting our LitCorr here + # but also we need to do this because we are unconcerned with probes that have no geneId associated with them + # as we would not have litCorr data. + + if method == "3": + query = "SELECT %s.Name, %s.value," % (db.type,tempTable) + dataStartPos = 2 + if method == "4" or method == "5": + query = "SELECT %s.Name, %s.Correlation, %s.PValue," % (db.type,tempTable, tempTable) + dataStartPos = 3 + + query += string.join(temp,', ') + query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) + if method == "3": + query += ' LEFT JOIN %s ON %s.GeneId2=ProbeSet.GeneId ' % (tempTable,tempTable) + if method == "4" or method == "5": + query += ' LEFT JOIN %s ON %s.Symbol=ProbeSet.Symbol ' % (tempTable,tempTable) + #XZ, 03/04/2009: Xiaodong changed Data to %sData and changed parameters from %(item,item, db.type,item,item) to %(db.type, item,item, db.type,item,item) + for item in StrainIdstep: + query += 'left join %sData as T%s on T%s.Id = %sXRef.DataId and T%s.StrainId=%s\n' %(db.type, item,item, db.type,item,item) + + if method == "3": + query += "WHERE ProbeSet.GeneId IS NOT NULL AND %s.value IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable,db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + if method == "4" or method == "5": + query += "WHERE ProbeSet.Symbol IS NOT NULL AND %s.Correlation IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable,db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + else: + query = "SELECT %s.Name," % db.type + dataStartPos = 1 + query += string.join(temp,', ') + query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) + #XZ, 03/04/2009: Xiaodong changed Data to %sData and changed parameters from %(item,item, db.type,item,item) to %(db.type, item,item, db.type,item,item) + for item in StrainIdstep: + query += 'left join %sData as T%s on T%s.Id = %sXRef.DataId and T%s.StrainId=%s\n' %(db.type, item,item, db.type,item,item) + query += "WHERE %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + + self.cursor.execute(query) + results = self.cursor.fetchall() + oridata.append(results) + + datasize = len(oridata[0]) + traitdatabase = [] + # put all of the seperate data together into a huge list of lists + for j in range(datasize): + traitdata = list(oridata[0][j]) + for i in range(1,nnn): + traitdata += list(oridata[i][j][dataStartPos:]) + traitdatabase.append(traitdata) + + if tempTable: + self.cursor.execute( 'DROP TEMPORARY TABLE %s' % tempTable ) + + return traitdatabase, dataStartPos + + + # XZ, 09/20/2008: This function creates TEMPORARY TABLE tmpTableName_2 and return its name. + # XZ, 09/20/2008: It stores top literature correlation values associated with the input geneId. + # XZ, 09/20/2008: Attention: In each row, the input geneId is always in column GeneId1. + #XZ, 12/16/2008: the input geneid can be of mouse, rat or human type + def getTempLiteratureTable(self, species, input_species_geneid, returnNumber): + # according to mysql the TEMPORARY TABLE name should not have to be unique because + # it is only available to the current connection. This program will be invoked via command line, but if it + # were to be invoked over mod_python this could cuase problems. mod_python will keep the connection alive + # in its executing threads ( i think) so there is a potential for the table not being dropped between users. + #XZ, 01/29/2009: To prevent the potential risk, I generate random table names and drop the tables after use them. + + + # the 'input_species_geneid' could be rat or human geneid, need to translate it to mouse geneid + translated_mouse_geneid = self.translateToMouseGeneID (species, input_species_geneid) + + tmpTableName_1 = webqtlUtil.genRandStr(prefix="LITERATURE") + + q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName_1 + q2 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId1,GeneId2,value FROM LCorrRamin3 WHERE GeneId1=%s' % (tmpTableName_1, translated_mouse_geneid) + q3 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId2,GeneId1,value FROM LCorrRamin3 WHERE GeneId2=%s AND GeneId1!=%s' % (tmpTableName_1, translated_mouse_geneid,translated_mouse_geneid) + for x in [q1,q2,q3]: self.cursor.execute(x) + + #XZ, 09/23/2008: Just use the top records insteard of using all records + tmpTableName_2 = webqtlUtil.genRandStr(prefix="TOPLITERATURE") + + q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName_2 + self.cursor.execute(q1) + q2 = 'SELECT GeneId1, GeneId2, value FROM %s ORDER BY value DESC' % tmpTableName_1 + self.cursor.execute(q2) + result = self.cursor.fetchall() + + counter = 0 #this is to count how many records being inserted into table + for one_row in result: + mouse_geneid1, mouse_geneid2, lit_corr_alue = one_row + + #mouse_geneid1 has been tested before, now should test if mouse_geneid2 has corresponding geneid in other species + translated_species_geneid = 0 + if species == 'mouse': + translated_species_geneid = mouse_geneid2 + elif species == 'rat': + self.cursor.execute( "SELECT rat FROM GeneIDXRef WHERE mouse=%d" % int(mouse_geneid2) ) + record = self.cursor.fetchone() + if record: + translated_species_geneid = record[0] + elif species == 'human': + self.cursor.execute( "SELECT human FROM GeneIDXRef WHERE mouse=%d" % int(mouse_geneid2) ) + record = self.cursor.fetchone() + if record: + translated_species_geneid = record[0] + + if translated_species_geneid: + self.cursor.execute( 'INSERT INTO %s (GeneId1, GeneId2, value) VALUES (%d,%d,%f)' % (tmpTableName_2, int(input_species_geneid),int(translated_species_geneid), float(lit_corr_alue)) ) + counter = counter + 1 + + #pay attention to the number + if (counter > 2*returnNumber): + break + + self.cursor.execute('DROP TEMPORARY TABLE %s' % tmpTableName_1) + + return tmpTableName_2 + + + + #XZ, 09/23/2008: In tissue correlation tables, there is no record of GeneId1 == GeneId2 + #XZ, 09/24/2008: Note that the correlation value can be negative. + def getTempTissueCorrTable(self, primaryTraitSymbol="", TissueProbeSetFreezeId=0, method="", returnNumber=0): + + def cmpTissCorrAbsoluteValue(A, B): + try: + if abs(A[1]) < abs(B[1]): return 1 + elif abs(A[1]) == abs(B[1]): + return 0 + else: return -1 + except: + return 0 + + symbolCorrDict, symbolPvalueDict = self.calculateCorrOfAllTissueTrait(primaryTraitSymbol=primaryTraitSymbol, TissueProbeSetFreezeId=TissueProbeSetFreezeId, method=method) + + symbolCorrList = symbolCorrDict.items() + + symbolCorrList.sort(cmpTissCorrAbsoluteValue) + symbolCorrList = symbolCorrList[0 : 2*returnNumber] + + tmpTableName = webqtlUtil.genRandStr(prefix="TOPTISSUE") + + q1 = 'CREATE TEMPORARY TABLE %s (Symbol varchar(100) PRIMARY KEY, Correlation float, PValue float)' % tmpTableName + self.cursor.execute(q1) + + for one_pair in symbolCorrList: + one_symbol = one_pair[0] + one_corr = one_pair[1] + one_p_value = symbolPvalueDict[one_symbol] + + self.cursor.execute( "INSERT INTO %s (Symbol, Correlation, PValue) VALUES ('%s',%f,%f)" % (tmpTableName, one_symbol, float(one_corr), float(one_p_value)) ) + + return tmpTableName + + + #XZ, 01/09/2009: This function was created by David Crowell. Xiaodong cleaned up and modified it. + def fetchLitCorrelations(self, species, GeneId, db, returnNumber): ### Used to generate Lit Correlations when calculations are done from text file. dcrowell August 2008 + """Uses getTempLiteratureTable to generate table of literatire correlations. This function then gathers that data and + pairs it with the TraitID string. Takes as its arguments a formdata instance, and a database instance. + Returns a dictionary of 'TraitID':'LitCorr' for the requested correlation""" + + tempTable = self.getTempLiteratureTable(species=species, input_species_geneid=GeneId, returnNumber=returnNumber) + + query = "SELECT %s.Name, %s.value" % (db.type,tempTable) + query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) + query += ' LEFT JOIN %s ON %s.GeneId2=ProbeSet.GeneId ' % (tempTable,tempTable) + query += "WHERE ProbeSet.GeneId IS NOT NULL AND %s.value IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable, db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + + self.cursor.execute(query) + results = self.cursor.fetchall() + + litCorrDict = {} + + for entry in results: + traitName,litcorr = entry + litCorrDict[traitName] = litcorr + + self.cursor.execute('DROP TEMPORARY TABLE %s' % tempTable) + + return litCorrDict + + + + #XZ, 01/09/2009: Xiaodong created this function. + def fetchTissueCorrelations(self, db, primaryTraitSymbol="", TissueProbeSetFreezeId=0, method="", returnNumber = 0): + """Uses getTempTissueCorrTable to generate table of tissue correlations. This function then gathers that data and + pairs it with the TraitID string. Takes as its arguments a formdata instance, and a database instance. + Returns a dictionary of 'TraitID':(tissueCorr, tissuePValue) for the requested correlation""" + + + tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=primaryTraitSymbol, TissueProbeSetFreezeId=TissueProbeSetFreezeId, method=method, returnNumber=returnNumber) + + query = "SELECT ProbeSet.Name, %s.Correlation, %s.PValue" % (tempTable, tempTable) + query += ' FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze)' + query += ' LEFT JOIN %s ON %s.Symbol=ProbeSet.Symbol ' % (tempTable,tempTable) + query += "WHERE ProbeSetFreeze.Name = '%s' and ProbeSetFreeze.Id=ProbeSetXRef.ProbeSetFreezeId and ProbeSet.Id = ProbeSetXRef.ProbeSetId and ProbeSet.Symbol IS NOT NULL AND %s.Correlation IS NOT NULL" % (db.name, tempTable) + + self.cursor.execute(query) + results = self.cursor.fetchall() + + tissueCorrDict = {} + + for entry in results: + traitName, tissueCorr, tissuePValue = entry + tissueCorrDict[traitName] = (tissueCorr, tissuePValue) + + self.cursor.execute('DROP TEMPORARY TABLE %s' % tempTable) + + return tissueCorrDict + + + + #XZ, 01/13/2008 + def getLiteratureCorrelationByList(self, input_trait_mouse_geneid=None, species=None, traitList=None): + + tmpTableName = webqtlUtil.genRandStr(prefix="LITERATURE") + + q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName + q2 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId1,GeneId2,value FROM LCorrRamin3 WHERE GeneId1=%s' % (tmpTableName, input_trait_mouse_geneid) + q3 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId2,GeneId1,value FROM LCorrRamin3 WHERE GeneId2=%s AND GeneId1!=%s' % (tmpTableName, input_trait_mouse_geneid, input_trait_mouse_geneid) + + for x in [q1,q2,q3]: + self.cursor.execute(x) + + for thisTrait in traitList: + try: + if thisTrait.geneid: + thisTrait.mouse_geneid = self.translateToMouseGeneID(species, thisTrait.geneid) + else: + thisTrait.mouse_geneid = 0 + except: + thisTrait.mouse_geneid = 0 + + if thisTrait.mouse_geneid and str(thisTrait.mouse_geneid).find(";") == -1: + try: + self.cursor.execute("SELECT value FROM %s WHERE GeneId2 = %s" % (tmpTableName, thisTrait.mouse_geneid)) + result = self.cursor.fetchone() + if result: + thisTrait.LCorr = result[0] + else: + thisTrait.LCorr = None + except: + thisTrait.LCorr = None + else: + thisTrait.LCorr = None + + self.cursor.execute("DROP TEMPORARY TABLE %s" % tmpTableName) + + return traitList + + + + def calculateCorrOfAllTissueTrait(self, primaryTraitSymbol=None, TissueProbeSetFreezeId=None, method=None): + + symbolCorrDict = {} + symbolPvalueDict = {} + + primaryTraitSymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=[primaryTraitSymbol], TissueProbeSetFreezeId=TissueProbeSetFreezeId) + primaryTraitValue = primaryTraitSymbolValueDict.values()[0] + + SymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=[], TissueProbeSetFreezeId=TissueProbeSetFreezeId) + + if method in ["2","5"]: + symbolCorrDict, symbolPvalueDict = correlationFunction.batchCalTissueCorr(primaryTraitValue,SymbolValueDict,method='spearman') + else: + symbolCorrDict, symbolPvalueDict = correlationFunction.batchCalTissueCorr(primaryTraitValue,SymbolValueDict) + + + return (symbolCorrDict, symbolPvalueDict) + + + + #XZ, 10/13/2010 + def getTissueCorrelationByList(self, primaryTraitSymbol=None, traitList=None, TissueProbeSetFreezeId=None, method=None): + + primaryTraitSymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=[primaryTraitSymbol], TissueProbeSetFreezeId=TissueProbeSetFreezeId) + + if primaryTraitSymbol.lower() in primaryTraitSymbolValueDict: + primaryTraitValue = primaryTraitSymbolValueDict[primaryTraitSymbol.lower()] + + geneSymbolList = [] + + for thisTrait in traitList: + if hasattr(thisTrait, 'symbol'): + geneSymbolList.append(thisTrait.symbol) + + SymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=geneSymbolList, TissueProbeSetFreezeId=TissueProbeSetFreezeId) + + for thisTrait in traitList: + if hasattr(thisTrait, 'symbol') and thisTrait.symbol and thisTrait.symbol.lower() in SymbolValueDict: + oneTraitValue = SymbolValueDict[thisTrait.symbol.lower()] + if method in ["2","5"]: + result = correlationFunction.calZeroOrderCorrForTiss( primaryTraitValue, oneTraitValue, method='spearman' ) + else: + result = correlationFunction.calZeroOrderCorrForTiss( primaryTraitValue, oneTraitValue) + thisTrait.tissueCorr = result[0] + thisTrait.tissuePValue = result[2] + else: + thisTrait.tissueCorr = None + thisTrait.tissuePValue = None + else: + for thisTrait in traitList: + thisTrait.tissueCorr = None + thisTrait.tissuePValue = None + + return traitList + + + def getTopInfo(self, myTrait=None, method=None, db=None, target_db_name=None, returnNumber=None, methodDict=None, totalTraits=None, identification=None ): + + if myTrait: + if method in ["1","2"]: #genetic correlation + info = HT.Paragraph("Values of Record %s in the " % myTrait.getGivenName(), HT.Href(text=myTrait.db.fullname,url=webqtlConfig.INFOPAGEHREF % myTrait.db.name,target="_blank", Class="fwn"), + " database were compared to all %d records in the " % totalTraits, HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank", Class="fwn"), + ' database. The top %d correlations ranked by the %s are displayed.' % (returnNumber,methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.') + else: + #myTrait.retrieveInfo()#need to know geneid and symbol + if method == "3":#literature correlation + searchDBName = "Literature Correlation" + searchDBLink = "/correlationAnnotation.html#literatureCorr" + else: #tissue correlation + searchDBName = "Tissue Correlation" + searchDBLink = "/correlationAnnotation.html#tissueCorr" + info = HT.Paragraph("Your input record %s in the " % myTrait.getGivenName(), HT.Href(text=myTrait.db.fullname,url=webqtlConfig.INFOPAGEHREF % myTrait.db.name,target="_blank", Class="fwn"), + " database corresponds to ", + HT.Href(text='gene Id %s, and gene symbol %s' % (myTrait.geneid, myTrait.symbol), target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" % myTrait.geneid, Class="fs12 fwn"), + '. GN ranked all genes in the ', HT.Href(text=searchDBName,url=searchDBLink,target="_blank", Class="fwn"),' database by the %s.' % methodDict[method], + ' The top %d probes or probesets in the ' % returnNumber, HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank", Class="fwn"), + ' database corresponding to the top genes ranked by the %s are displayed.' %( methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.' ) + + elif identification: + info = HT.Paragraph('Values of %s were compared to all %d traits in ' % (identification, totalTraits), + HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank",Class="fwn"), + ' database. The TOP %d correlations ranked by the %s are displayed.' % (returnNumber,methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.') + + else: + info = HT.Paragraph('Trait values were compared to all values in ', + HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank",Class="fwn"), + ' database. The TOP %d correlations ranked by the %s are displayed.' % (returnNumber,methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.') + + if db.type=="Geno": + info.append(HT.BR(),HT.BR(),'Clicking on the Locus will open the genotypes data for that locus. Click on the correlation to see a scatter plot of the trait data.') + elif db.type=="Publish": + info.append(HT.BR(),HT.BR(),'Clicking on the record ID will open the published phenotype data for that publication. Click on the correlation to see a scatter plot of the trait data. ') + elif db.type=="ProbeSet": + info.append(HT.BR(),'Click the correlation values to generate scatter plots. Select the Record ID to open the Trait Data and Analysis form. Select the symbol to open NCBI Entrez.') + else: + pass + + + return info + + + def createExcelFileWithTitleAndFooter(self, workbook=None, identification=None, db=None, returnNumber=None): + + worksheet = workbook.add_worksheet() + + titleStyle = workbook.add_format(align = 'left', bold = 0, size=14, border = 1, border_color="gray") + + ##Write title Info + # Modified by Hongqiang Li + worksheet.write([1, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle) + worksheet.write([1, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle) + worksheet.write([2, 0], "Trait : %s" % identification, titleStyle) + worksheet.write([3, 0], "Database : %s" % db.fullname, titleStyle) + worksheet.write([4, 0], "Date : %s" % time.strftime("%B %d, %Y", time.gmtime()), titleStyle) + worksheet.write([5, 0], "Time : %s GMT" % time.strftime("%H:%M ", time.gmtime()), titleStyle) + worksheet.write([6, 0], "Status of data ownership: Possibly unpublished data; please see %s/statusandContact.html for details on sources, ownership, and usage of these data." % webqtlConfig.PORTADDR, titleStyle) + #Write footer info + worksheet.write([9 + returnNumber, 0], "Funding for The GeneNetwork: NIAAA (U01AA13499, U24AA13513), NIDA, NIMH, and NIAAA (P20-DA21131), NCI MMHCC (U01CA105417), and NCRR (U01NR 105417)", titleStyle) + worksheet.write([10 + returnNumber, 0], "PLEASE RETAIN DATA SOURCE INFORMATION WHENEVER POSSIBLE", titleStyle) + + return worksheet + + + def getTableHeaderForGeno(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Record', HT.BR(), 'ID', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Record ID', idx=1), + THCell(HT.TD('Location', HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Location (Chr and Mb)', idx=2), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample r", idx=3), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=4), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(r)", idx=5)]] + + for ncol, item in enumerate(['Record ID', 'Location (Chr, Mb)', 'Sample r', 'N Cases', 'Sample p(r)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Record', HT.BR(), 'ID', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Record ID', idx=1), + THCell(HT.TD('Location', HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Location (Chr and Mb)', idx=2), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample rho", idx=3), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=4), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(rho)", idx=5)]] + + for ncol, item in enumerate(['Record ID', 'Location (Chr, Mb)', 'Sample rho', 'N Cases', 'Sample p(rho)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + + return tblobj_header, worksheet + + + def getTableBodyForGeno(self, traitList, formName=None, worksheet=None, newrow=None, corrScript=None): + + tblobj_body = [] + + for thisTrait in traitList: + tr = [] + + trId = str(thisTrait) + + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.corr)) + + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"),align="left", Class="fs12 fwn ffl b1 c222"), text=thisTrait.name, val=thisTrait.name.upper())) + + #XZ: trait_location_value is used for sorting + trait_location_repr = '--' + trait_location_value = 1000000 + + if thisTrait.chr and thisTrait.mb: + try: + trait_location_value = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if thisTrait.chr.upper() == 'X': + trait_location_value = 20*1000 + thisTrait.mb + else: + trait_location_value = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + trait_location_repr = 'Chr%s: %.6f' % (thisTrait.chr, float(thisTrait.mb) ) + + tr.append(TDCell(HT.TD(trait_location_repr, Class="fs12 fwn b1 c222", nowrap="on"), trait_location_repr, trait_location_value)) + + + repr='%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlot('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"), Class="fs12 fwn ffl b1 c222", nowrap='ON', align='right'),repr,abs(thisTrait.corr))) + + repr = '%d' % thisTrait.nOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222",align='right'),repr,thisTrait.nOverlap)) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, trait_location_repr, thisTrait.corr, thisTrait.nOverlap, thisTrait.corrPValue]): + worksheet.write([newrow, ncol], item) + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getTableHeaderForPublish(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Record ID", idx=1), + THCell(HT.TD('Phenotype', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Phenotype", idx=2), + THCell(HT.TD('Authors', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Authors", idx=3), + THCell(HT.TD('Year', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Year", idx=4), + THCell(HT.TD('Max',HT.BR(), 'LRS', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS", idx=5), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS Location", idx=6), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample r", idx=7), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=8), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(r)", idx=9)]] + + for ncol, item in enumerate(["Record", "Phenotype", "Authors", "Year", "Pubmed Id", "Max LRS", "Max LRS Location (Chr: Mb)", "Sample r", "N Cases", "Sample p(r)"]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Record ID", idx=1), + THCell(HT.TD('Phenotype', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Phenotype", idx=2), + THCell(HT.TD('Authors', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Authors", idx=3), + THCell(HT.TD('Year', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Year", idx=4), + THCell(HT.TD('Max',HT.BR(), 'LRS', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS", idx=5), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS Location", idx=6), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample rho", idx=7), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=8), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(rho)", idx=9)]] + + for ncol, item in enumerate(["Record", "Phenotype", "Authors", "Year", "Pubmed Id", "Max LRS", "Max LRS Location (Chr: Mb)", "Sample rho", "N Cases", "Sample p(rho)"]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + + return tblobj_header, worksheet + + + def getTableBodyForPublish(self, traitList, formName=None, worksheet=None, newrow=None, corrScript=None, species=''): + + tblobj_body = [] + + for thisTrait in traitList: + tr = [] + + trId = str(thisTrait) + + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.corr)) + + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn"), nowrap="yes",align="center", Class="fs12 fwn b1 c222"),str(thisTrait.name), thisTrait.name)) + + PhenotypeString = thisTrait.post_publication_description + if thisTrait.confidential: + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users): + PhenotypeString = thisTrait.pre_publication_description + + tr.append(TDCell(HT.TD(PhenotypeString, Class="fs12 fwn b1 c222"), PhenotypeString, PhenotypeString.upper())) + + tr.append(TDCell(HT.TD(thisTrait.authors, Class="fs12 fwn b1 c222 fsI"),thisTrait.authors, thisTrait.authors.strip().upper())) + + try: + PubMedLinkText = myear = repr = int(thisTrait.year) + except: + PubMedLinkText = repr = "--" + myear = 0 + if thisTrait.pubmed_id: + PubMedLink = HT.Href(text= repr,url= webqtlConfig.PUBMEDLINK_URL % thisTrait.pubmed_id,target='_blank', Class="fs12 fwn") + else: + PubMedLink = repr + + tr.append(TDCell(HT.TD(PubMedLink, Class="fs12 fwn b1 c222", align='center'), repr, myear)) + + #LRS and its location + LRS_score_repr = '--' + LRS_score_value = 0 + LRS_location_repr = '--' + LRS_location_value = 1000000 + LRS_flag = 1 + + #Max LRS and its Locus location + if thisTrait.lrs and thisTrait.locus: + self.cursor.execute(""" + select Geno.Chr, Geno.Mb from Geno, Species + where Species.Name = '%s' and + Geno.Name = '%s' and + Geno.SpeciesId = Species.Id + """ % (species, thisTrait.locus)) + result = self.cursor.fetchone() + + if result: + if result[0] and result[1]: + LRS_Chr = result[0] + LRS_Mb = result[1] + + #XZ: LRS_location_value is used for sorting + try: + LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) + except: + if LRS_Chr.upper() == 'X': + LRS_location_value = 20*1000 + float(LRS_Mb) + else: + LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) + + + LRS_score_repr = '%3.1f' % thisTrait.lrs + LRS_score_value = thisTrait.lrs + LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb) ) + LRS_flag = 0 + + #tr.append(TDCell(HT.TD(HT.Href(text=LRS_score_repr,url="javascript:showIntervalMapping('%s', '%s : %s')" % (formName, thisTrait.db.shortname, thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn ffl b1 c222", align='right', nowrap="on"),LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222", align='right', nowrap="on"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222"), LRS_location_repr, LRS_location_value)) + + if LRS_flag: + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222"), LRS_location_repr, LRS_location_value)) + + repr = '%3.4f' % thisTrait.corr + tr.append(TDCell(HT.TD(HT.Href(text=repr,url="javascript:showCorrPlot('%s', '%s')" % (formName,thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn b1 c222", align='right',nowrap="on"), repr, abs(thisTrait.corr))) + + repr = '%d' % thisTrait.nOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.nOverlap)) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, PhenotypeString, thisTrait.authors, thisTrait.year, thisTrait.pubmed_id, LRS_score_repr, LRS_location_repr, thisTrait.corr, thisTrait.nOverlap, thisTrait.corrPValue]): + worksheet.write([newrow, ncol], item) + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getTableHeaderForProbeSet(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Record ID", idx=1), + THCell(HT.TD('Gene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Gene ID", idx=2), + THCell(HT.TD('Homologene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Homologene ID", idx=3), + THCell(HT.TD('Symbol',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Symbol", idx=4), + THCell(HT.TD('Description',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Description", idx=5), + THCell(HT.TD('Location',HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Location (Chr: Mb)", idx=6), + THCell(HT.TD('Mean',HT.BR(),'Expr',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Mean Expr", idx=7), + THCell(HT.TD('Max',HT.BR(),'LRS',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS", idx=8), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS Location (Chr: Mb)", idx=9), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample r", idx=10), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=11), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(r)", idx=12), + THCell(HT.TD(HT.Href( + text = HT.Span('Lit',HT.BR(), 'Corr', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#literatureCorr"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Lit Corr", idx=13), + #XZ, 09/22/2008: tissue correlation + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue r", idx=14), + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue p(r)", idx=15)]] + + for ncol, item in enumerate(['Record', 'Gene ID', 'Homologene ID', 'Symbol', 'Description', 'Location (Chr: Mb)', 'Mean Expr', 'Max LRS', 'Max LRS Location (Chr: Mb)', 'Sample r', 'N Cases', 'Sample p(r)', 'Lit Corr', 'Tissue r', 'Tissue p(r)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Record ID", idx=1), + THCell(HT.TD('Gene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Gene ID", idx=2), + THCell(HT.TD('Homologene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Homologene ID", idx=3), + THCell(HT.TD('Symbol',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Symbol", idx=4), + THCell(HT.TD('Description',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Description", idx=5), + THCell(HT.TD('Location',HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Location (Chr: Mb)", idx=6), + THCell(HT.TD('Mean',HT.BR(),'Expr',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Mean Expr", idx=7), + THCell(HT.TD('Max',HT.BR(),'LRS',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS", idx=8), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS Location (Chr: Mb)", idx=9), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample rho", idx=10), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=11), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(rho)", idx=12), + THCell(HT.TD(HT.Href( + text = HT.Span('Lit',HT.BR(), 'Corr', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#literatureCorr"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Lit Corr", idx=13), + #XZ, 09/22/2008: tissue correlation + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue rho", idx=14), + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue p(rho)", idx=15)]] + + for ncol, item in enumerate(['Record ID', 'Gene ID', 'Homologene ID', 'Symbol', 'Description', 'Location (Chr: Mb)', 'Mean Expr', 'Max LRS', 'Max LRS Location (Chr: Mb)', 'Sample rho', 'N Cases', 'Sample p(rho)', 'Lit Corr', 'Tissue rho', 'Tissue p(rho)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + return tblobj_header, worksheet + + + def getTableBodyForProbeSet(self, traitList=[], primaryTrait=None, formName=None, worksheet=None, newrow=None, corrScript=None, species=''): + + tblobj_body = [] + + for thisTrait in traitList: + + if thisTrait.symbol: + pass + else: + thisTrait.symbol = "--" + + if thisTrait.geneid: + symbolurl = HT.Href(text=thisTrait.symbol,target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" % thisTrait.geneid, Class="fs12 fwn") + else: + symbolurl = HT.Href(text=thisTrait.symbol,target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?CMD=search&DB=gene&term=%s" % thisTrait.symbol, Class="fs12 fwn") + + tr = [] + + trId = str(thisTrait) + + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.corr)) + + #XZ, 12/08/2008: checkbox + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + #XZ, 12/08/2008: probeset name + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName,thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn b1 c222"), thisTrait.name, thisTrait.name.upper())) + + #XZ, 12/08/2008: gene id + if thisTrait.geneid: + tr.append(TDCell(None, thisTrait.geneid, val=999)) + else: + tr.append(TDCell(None, thisTrait.geneid, val=999)) + + #XZ, 12/08/2008: homologene id + if thisTrait.homologeneid: + tr.append(TDCell("", thisTrait.homologeneid, val=999)) + else: + tr.append(TDCell("", thisTrait.homologeneid, val=999)) + + #XZ, 12/08/2008: gene symbol + tr.append(TDCell(HT.TD(symbolurl, Class="fs12 fwn b1 c222 fsI"),thisTrait.symbol, thisTrait.symbol.upper())) + + #XZ, 12/08/2008: description + #XZ, 06/05/2009: Rob asked to add probe target description + description_string = str(thisTrait.description).strip() + target_string = str(thisTrait.probe_target_description).strip() + + description_display = '' + + if len(description_string) > 1 and description_string != 'None': + description_display = description_string + else: + description_display = thisTrait.symbol + + if len(description_display) > 1 and description_display != 'N/A' and len(target_string) > 1 and target_string != 'None': + description_display = description_display + '; ' + target_string.strip() + + tr.append(TDCell(HT.TD(description_display, Class="fs12 fwn b1 c222"), description_display, description_display)) + + #XZ: trait_location_value is used for sorting + trait_location_repr = '--' + trait_location_value = 1000000 + + if thisTrait.chr and thisTrait.mb: + try: + trait_location_value = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if thisTrait.chr.upper() == 'X': + trait_location_value = 20*1000 + thisTrait.mb + else: + trait_location_value = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + trait_location_repr = 'Chr%s: %.6f' % (thisTrait.chr, float(thisTrait.mb) ) + + tr.append(TDCell(HT.TD(trait_location_repr, Class="fs12 fwn b1 c222", nowrap="on"), trait_location_repr, trait_location_value)) + + """ + #XZ, 12/08/2008: chromosome number + #XZ, 12/10/2008: use Mbvalue to sort chromosome + tr.append(TDCell( HT.TD(thisTrait.chr, Class="fs12 fwn b1 c222", align='right'), thisTrait.chr, Mbvalue) ) + + #XZ, 12/08/2008: Rob wants 6 digit precision, and we have to deal with that the mb could be None + if not thisTrait.mb: + tr.append(TDCell(HT.TD(thisTrait.mb, Class="fs12 fwn b1 c222",align='right'), thisTrait.mb, Mbvalue)) + else: + tr.append(TDCell(HT.TD('%.6f' % thisTrait.mb, Class="fs12 fwn b1 c222", align='right'), thisTrait.mb, Mbvalue)) + """ + + + + #XZ, 01/12/08: This SQL query is much faster. + self.cursor.execute(""" + select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet + where ProbeSetXRef.ProbeSetFreezeId = %d and + ProbeSet.Id = ProbeSetXRef.ProbeSetId and + ProbeSet.Name = '%s' + """ % (thisTrait.db.id, thisTrait.name)) + result = self.cursor.fetchone() + if result: + if result[0]: + mean = result[0] + else: + mean=0 + else: + mean = 0 + + #XZ, 06/05/2009: It is neccessary to turn on nowrap + repr = "%2.3f" % mean + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right', nowrap='ON'),repr, mean)) + + #LRS and its location + LRS_score_repr = '--' + LRS_score_value = 0 + LRS_location_repr = '--' + LRS_location_value = 1000000 + LRS_flag = 1 + + #Max LRS and its Locus location + if thisTrait.lrs and thisTrait.locus: + self.cursor.execute(""" + select Geno.Chr, Geno.Mb from Geno, Species + where Species.Name = '%s' and + Geno.Name = '%s' and + Geno.SpeciesId = Species.Id + """ % (species, thisTrait.locus)) + result = self.cursor.fetchone() + + if result: + if result[0] and result[1]: + LRS_Chr = result[0] + LRS_Mb = result[1] + + #XZ: LRS_location_value is used for sorting + try: + LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) + except: + if LRS_Chr.upper() == 'X': + LRS_location_value = 20*1000 + float(LRS_Mb) + else: + LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) + + + LRS_score_repr = '%3.1f' % thisTrait.lrs + LRS_score_value = thisTrait.lrs + LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb) ) + LRS_flag = 0 + + #tr.append(TDCell(HT.TD(HT.Href(text=LRS_score_repr,url="javascript:showIntervalMapping('%s', '%s : %s')" % (formName, thisTrait.db.shortname, thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn ffl b1 c222", align='right', nowrap="on"),LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222", align='right', nowrap="on"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222", nowrap="on"), LRS_location_repr, LRS_location_value)) + + if LRS_flag: + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222"), LRS_location_repr, LRS_location_value)) + + + #XZ, 12/08/2008: generic correlation + repr='%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlot('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"), Class="fs12 fwn ffl b1 c222", align='right'),repr,abs(thisTrait.corr))) + + #XZ, 12/08/2008: number of overlaped cases + repr = '%d' % thisTrait.nOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.nOverlap)) + + #XZ, 12/08/2008: p value of genetic correlation + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + #XZ, 12/08/2008: literature correlation + LCorr = 0.0 + LCorrStr = "--" + if hasattr(thisTrait, 'LCorr') and thisTrait.LCorr: + LCorr = thisTrait.LCorr + LCorrStr = "%2.3f" % thisTrait.LCorr + tr.append(TDCell(HT.TD(LCorrStr, Class="fs12 fwn b1 c222", align='right'), LCorrStr, abs(LCorr))) + + #XZ, 09/22/2008: tissue correlation. + TCorr = 0.0 + TCorrStr = "--" + #XZ, 11/20/2008: need to pass two geneids: input_trait_mouse_geneid and thisTrait.mouse_geneid + if hasattr(thisTrait, 'tissueCorr') and thisTrait.tissueCorr: + TCorr = thisTrait.tissueCorr + TCorrStr = "%2.3f" % thisTrait.tissueCorr + # NL, 07/19/2010: add a new parameter rankOrder for js function 'showTissueCorrPlot' + rankOrder = thisTrait.rankOrder + TCorrPlotURL = "javascript:showTissueCorrPlot('%s','%s','%s',%d)" %(formName, primaryTrait.symbol, thisTrait.symbol,rankOrder) + tr.append(TDCell(HT.TD(HT.Href(text=TCorrStr, url=TCorrPlotURL, Class="fs12 fwn ff1"), Class="fs12 fwn ff1 b1 c222", align='right'), TCorrStr, abs(TCorr))) + else: + tr.append(TDCell(HT.TD(TCorrStr, Class="fs12 fwn b1 c222", align='right'), TCorrStr, abs(TCorr))) + + #XZ, 12/08/2008: p value of tissue correlation + TPValue = 1.0 + TPValueStr = "--" + if hasattr(thisTrait, 'tissueCorr') and thisTrait.tissuePValue: #XZ, 09/22/2008: thisTrait.tissuePValue can't be used here because it could be 0 + TPValue = thisTrait.tissuePValue + TPValueStr = "%2.3f" % thisTrait.tissuePValue + tr.append(TDCell(HT.TD(TPValueStr, Class="fs12 fwn b1 c222", align='right'), TPValueStr, TPValue)) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, thisTrait.geneid, thisTrait.homologeneid, thisTrait.symbol, thisTrait.description, trait_location_repr, mean, LRS_score_repr, LRS_location_repr, thisTrait.corr, thisTrait.nOverlap, thisTrait.corrPValue, LCorr, TCorr, TPValue]): + worksheet.write([newrow, ncol], item) + + newrow += 1 + + return tblobj_body, worksheet, corrScript diff --git a/web/webqtl/correlation/PartialCorrDBPage.py b/web/webqtl/correlation/PartialCorrDBPage.py new file mode 100755 index 00000000..ecd1e623 --- /dev/null +++ b/web/webqtl/correlation/PartialCorrDBPage.py @@ -0,0 +1,1359 @@ +import string +import cPickle +import os +import pyXLWriter as xl + +from htmlgen import HTMLgen2 as HT + +from base import webqtlConfig +#import webqtlData +from utility.THCell import THCell +from utility.TDCell import TDCell +from base.webqtlTrait import webqtlTrait +from base.webqtlDataset import webqtlDataset +from base.templatePage import templatePage +from utility import webqtlUtil +from CorrelationPage import CorrelationPage +import correlationFunction +from dbFunction import webqtlDatabaseFunction + + +######################################### +# Partial Correlation Dataset Page +######################################### + + +class PartialCorrDBPage(CorrelationPage): + + corrMinInformative = 4 + + def __init__(self, fd): + + + templatePage.__init__(self, fd) + + if not self.openMysql(): + return + + + primaryTraitString = fd.formdata.getvalue('primaryTrait') + primaryTrait = (webqtlTrait(fullname=primaryTraitString, cursor=self.cursor)) + + controlTraitsString = fd.formdata.getvalue('controlTraits') + controlTraitsList = list(string.split(controlTraitsString,',')) + controlTraits = [] + for item in controlTraitsList: + controlTraits.append(webqtlTrait(fullname=item, cursor=self.cursor)) + + #XZ, 3/16/2010: variable RISet must be pass by the form + RISet = fd.RISet + #XZ, 12/12/2008: get species infomation + species = webqtlDatabaseFunction.retrieveSpecies(cursor=self.cursor, RISet=RISet) + + #XZ, 09/18/2008: get all information about the user selected database. + self.target_db_name = fd.formdata.getvalue('database2') + + try: + self.db = webqtlDataset(self.target_db_name, self.cursor) + except: + heading = "Partial Correlation Table" + detail = ["The database you just requested has not been established yet."] + self.error(heading=heading,detail=detail) + return + + #XZ, 09/18/2008: check if user has the authority to get access to the database. + if self.db.type == 'ProbeSet': + self.cursor.execute('SELECT Id, Name, FullName, confidentiality, AuthorisedUsers FROM ProbeSetFreeze WHERE Name = "%s"' % self.target_db_name) + indId, indName, indFullName, confidential, AuthorisedUsers = self.cursor.fetchall()[0] + + if confidential == 1: + access_to_confidential_dataset = 0 + + #for the dataset that confidentiality is 1 + #1. 'admin' and 'root' can see all of the dataset + #2. 'user' can see the dataset that AuthorisedUsers contains his id(stored in the Id field of User table) + if webqtlConfig.USERDICT[self.privilege] > webqtlConfig.USERDICT['user']: + access_to_confidential_dataset = 1 + else: + AuthorisedUsersList=AuthorisedUsers.split(',') + if AuthorisedUsersList.__contains__(self.userName): + access_to_confidential_dataset = 1 + + if not access_to_confidential_dataset: + #Error, Confidential Database + heading = "Partial Correlation Table" + detail = ["The %s database you selected is not open to the public at this time, please go back and select another database." % indFullName] + self.error(heading=heading,detail=detail,error="Confidential Database") + return + + + primaryTrait.retrieveData() + _primarystrains, _primaryvals, _primaryvars = primaryTrait.exportInformative() + + controlTraitNames = fd.formdata.getvalue('controlTraits') + _controlstrains,_controlvals,_controlvars,_controlNs = correlationFunction.controlStrains(controlTraitNames,_primarystrains) + + ## If the strains for which each of the control traits and the primary trait have values are not identical, + ## we must remove from the calculation all vlaues for strains that are not present in each. Without doing this, + ## undesirable biases would be introduced. + + common_primary_control_strains = _primarystrains #keep _primarystrains + fixed_primary_vals = _primaryvals #keep _primaryvals + fixed_control_vals = _controlvals + + allsame = True + ##allsame is boolean for whether or not primary and control trait have values for the same strains + for i in _controlstrains: + if _primarystrains != i: + allsame=False + break + + if not allsame: + common_primary_control_strains, fixed_primary_vals, fixed_control_vals, _vars, _controlvars = correlationFunction.fixStrains(_primarystrains,_controlstrains,_primaryvals,_controlvals,_primaryvars,_controlvars) + + N = len(common_primary_control_strains) + if N < self.corrMinInformative: + heading = "Partial Correlation Table" + detail = ['Fewer than %d strain data were entered for %s data set. No calculation of correlation has been attempted.' % (self.corrMinInformative, RISet)] + self.error(heading=heading,detail=detail) + return + + #XZ: We should check the value of control trait and primary trait here. + nameOfIdenticalTraits = correlationFunction.findIdenticalTraits ( fixed_primary_vals, primaryTraitString, fixed_control_vals, controlTraitsList ) + if nameOfIdenticalTraits: + heading = "Partial Correlation Table" + detail = ['%s and %s have same values for the %s strains that will be used to calculate partial correlation (common for all primary and control traits). In such case, partial correlation can NOT be calculated. Please re-select your traits.' % (nameOfIdenticalTraits[0], nameOfIdenticalTraits[1], len(fixed_primary_vals))] + self.error(heading=heading,detail=detail) + return + + + #XZ, 09/28/2008: if user select "1", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "2", then display 2, 3 and 5. + #XZ, 09/28/2008: if user select "3", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "4", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "5", then display 2, 3 and 5. + methodDict = {"1":"Genetic Correlation (Pearson's r)","2":"Genetic Correlation (Spearman's rho)","3":"SGO Literature Correlation","4":"Tissue Correlation (Pearson's r)", "5":"Tissue Correlation (Spearman's rho)"} + self.method = fd.formdata.getvalue('method') + if self.method not in ("1","2","3","4","5"): + self.method = "1" + + self.returnNumber = int(fd.formdata.getvalue('criteria')) + + myTrait = primaryTrait + myTrait.retrieveInfo() + + # We will not get Literature Correlations if there is no GeneId because there is nothing to look against + try: + input_trait_GeneId = myTrait.geneid + except: + input_trait_GeneId = None + + # We will not get Tissue Correlations if there is no gene symbol because there is nothing to look against + try: + input_trait_symbol = myTrait.symbol + except: + input_trait_symbol = None + + + #XZ, 12/12/2008: if the species is rat or human, translate the geneid to mouse geneid + input_trait_mouse_geneid = self.translateToMouseGeneID(species, input_trait_GeneId) + + #XZ: As of Nov/13/2010, this dataset is 'UTHSC Illumina V6.2 RankInv B6 D2 average CNS GI average (May 08)' + TissueProbeSetFreezeId = 1 + + + #XZ, 09/22/2008: If we need search by GeneId, + #XZ, 09/22/2008: we have to check if this GeneId is in the literature or tissue correlation table. + #XZ, 10/15/2008: We also to check if the selected database is probeset type. + if self.method == "3" or self.method == "4" or self.method == "5": + if self.db.type != "ProbeSet": + self.error(heading="Wrong correlation type",detail="It is not possible to compute the %s between your trait and data in this %s database. Please try again after selecting another type of correlation." % (methodDict[self.method],self.db.name),error="Correlation Type Error") + return + + """ + if not input_trait_GeneId: + self.error(heading="No Associated GeneId",detail="This trait has no associated GeneId, so we are not able to show any literature or tissue related information.",error="No GeneId Error") + return + """ + + #XZ: We have checked geneid did exist + + if self.method == "3": + if not input_trait_GeneId or not self.checkForLitInfo(input_trait_mouse_geneid): + self.error(heading="No Literature Info",detail="This gene does not have any associated Literature Information.",error="Literature Correlation Error") + return + + if self.method == "4" or self.method == "5": + if not input_trait_symbol: + self.error(heading="No Tissue Correlation Information",detail="This gene does not have any associated Tissue Correlation Information.",error="Tissue Correlation Error") + return + + if not self.checkSymbolForTissueCorr(TissueProbeSetFreezeId, myTrait.symbol): + self.error(heading="No Tissue Correlation Information",detail="This gene does not have any associated Tissue Correlation Information.",error="Tissue Correlation Error") + return + +####################################################################################################################################### + + nnCorr = len(fixed_primary_vals) + + #XZ: Use the fast method only for probeset dataset, and this dataset must have been created. + #XZ: Otherwise, use original method + + useFastMethod = False + + if self.db.type == "ProbeSet": + DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) + DirectoryList = os.listdir(webqtlConfig.TEXTDIR) # List of existing text files. Used to check if a text file already exists + if DatabaseFileName in DirectoryList: + useFastMethod = True + + if useFastMethod: + totalTraits, allcorrelations = self.getPartialCorrelationsFast(common_primary_control_strains , fixed_primary_vals, fixed_control_vals, nnCorr, DatabaseFileName, species, input_trait_GeneId, input_trait_symbol, TissueProbeSetFreezeId) + + if totalTraits == 0: + useFastMethod = False + + #XZ, 01/08/2009: use the original method to retrieve from database and compute. + if not useFastMethod: + totalTraits, allcorrelations = self.getPartialCorrelationsNormal(common_primary_control_strains, fixed_primary_vals, fixed_control_vals, nnCorr, species, input_trait_GeneId, input_trait_symbol,TissueProbeSetFreezeId) + +############################################################# + + if self.method == "3" and input_trait_GeneId: + allcorrelations.sort(self.cmpLitCorr) + elif self.method in ["4","5"] and input_trait_GeneId: + allcorrelations.sort(self.cmpLitCorr) + else: + allcorrelations.sort(self.cmpPartialCorrPValue) + + #XZ, 09/20/2008: we only need the top ones. + self.returnNumber = min(self.returnNumber,len(allcorrelations)) + allcorrelations = allcorrelations[:self.returnNumber] + + addLiteratureCorr = False + addTissueCorr = False + + traitList = [] + for item in allcorrelations: + thisTrait = webqtlTrait(db=self.db, name=item[0], cursor=self.cursor) + thisTrait.retrieveInfo() + + thisTrait.Name = item[0] + thisTrait.NOverlap = item[1] + + thisTrait.partial_corr = item[2] + thisTrait.partial_corrPValue = item[3] + + thisTrait.corr = item[4] + thisTrait.corrPValue = item[5] + # NL, 07/19/2010 + # js function changed, add a new parameter rankOrder for js function 'showTissueCorrPlot' + rankOrder = 0; + if self.method in ["2","5"]: + rankOrder = 1; + thisTrait.rankOrder = rankOrder + + #XZ, 26/09/2008: Method is 4 or 5. Have fetched tissue corr, but no literature correlation yet. + if len(item) == 8: + thisTrait.tissueCorr = item[6] + thisTrait.tissuePValue = item[7] + addLiteratureCorr = True + + #XZ, 26/09/2008: Method is 3, Have fetched literature corr, but no tissue corr yet. + elif len(item) == 7: + thisTrait.LCorr = item[6] + thisTrait.mouse_geneid = self.translateToMouseGeneID(species, thisTrait.geneid) + addTissueCorr = True + + #XZ, 26/09/2008: Method is 1 or 2. Have NOT fetched literature corr and tissue corr yet. + # Phenotype data will not have geneid, and neither will some probes + # we need to handle this because we will get an attribute error + else: + if input_trait_mouse_geneid and self.db.type=="ProbeSet": + addLiteratureCorr = True + if input_trait_symbol and self.db.type=="ProbeSet": + addTissueCorr = True + + traitList.append(thisTrait) + + if addLiteratureCorr: + traitList = self.getLiteratureCorrelationByList(input_trait_mouse_geneid, species, traitList) + if addTissueCorr: + traitList = self.getTissueCorrelationByList(primaryTraitSymbol=input_trait_symbol, traitList=traitList,TissueProbeSetFreezeId=TissueProbeSetFreezeId, method=self.method) + +######################################################## + + TD_LR = HT.TD(height=200,width="100%",bgColor='#eeeeee') + + mainfmName = webqtlUtil.genRandStr("fm_") + form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name= mainfmName, submit=HT.Input(type='hidden')) + hddn = {'FormID':'showDatabase', 'ProbeSetID':'_','database':self.target_db_name, 'CellID':'_', 'RISet':RISet, 'identification':fd.identification} + + if myTrait: + hddn['fullname']=str(myTrait) + + + for key in hddn.keys(): + form.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + #XZ, 11/21/2008: add two parameters to form + form.append(HT.Input(name="X_geneSymbol", value="", type='hidden')) + form.append(HT.Input(name="Y_geneSymbol", value="", type='hidden')) + + #XZ, 3/11/2010: add one parameter to record if the method is rank order. + + form.append(HT.Input(name="rankOrder", value="%s" % rankOrder, type='hidden')) + + form.append(HT.Input(name="TissueProbeSetFreezeId", value="%s" % TissueProbeSetFreezeId, type='hidden')) + + + #################################### + # generate the info on top of page # + #################################### + + info_form = self.getFormForPrimaryAndControlTraits (primaryTrait, controlTraits) + info = self.getTopInfo(myTrait=myTrait, method=self.method, db=self.db, target_db_name=self.target_db_name, returnNumber=self.returnNumber, methodDict=methodDict, totalTraits=totalTraits, identification=fd.identification ) + + ############## + # Excel file # + ############## + filename= webqtlUtil.genRandStr("Corr_") + xlsUrl = HT.Input(type='button', value = 'Download Table', onClick= "location.href='/tmp/%s.xls'" % filename, Class='button') + # Create a new Excel workbook + workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+filename)) + headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") + + #XZ, 3/18/2010: pay attention to the line number of header in this file. As of today, there are 7 lines. + worksheet = self.createExcelFileWithTitleAndFooter(workbook=workbook, identification=fd.identification, db=self.db, returnNumber=self.returnNumber) + + newrow = 7 + + + +##################################################################### + + mintmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'showIntMap');" % mainfmName) + mintmap_img = HT.Image("/images/multiple_interval_mapping1_final.jpg", name='mintmap', alt="Multiple Interval Mapping", title="Multiple Interval Mapping", style="border:none;") + mintmap.append(mintmap_img) + mcorr = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'compCorr');" % mainfmName) + mcorr_img = HT.Image("/images/compare_correlates2_final.jpg", alt="Compare Correlates", title="Compare Correlates", style="border:none;") + mcorr.append(mcorr_img) + cormatrix = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'corMatrix');" % mainfmName) + cormatrix_img = HT.Image("/images/correlation_matrix1_final.jpg", alt="Correlation Matrix and PCA", title="Correlation Matrix and PCA", style="border:none;") + cormatrix.append(cormatrix_img) + networkGraph = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'networkGraph');" % mainfmName) + networkGraph_img = HT.Image("/images/network_graph1_final.jpg", name='mintmap', alt="Network Graphs", title="Network Graphs", style="border:none;") + networkGraph.append(networkGraph_img) + heatmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'heatmap');" % mainfmName) + heatmap_img = HT.Image("/images/heatmap2_final.jpg", name='mintmap', alt="QTL Heat Map and Clustering", title="QTL Heatmap and Clustering", style="border:none;") + heatmap.append(heatmap_img) + partialCorr = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'partialCorrInput');" % mainfmName) + partialCorr_img = HT.Image("/images/partial_correlation_final.jpg", name='partialCorr', alt="Partial Correlation", title="Partial Correlation", style="border:none;") + partialCorr.append(partialCorr_img) + addselect = HT.Href(url="#redirect", onClick="addRmvSelection('%s', document.getElementsByName('%s')[0], 'addToSelection');" % (RISet, mainfmName)) + addselect_img = HT.Image("/images/add_collection1_final.jpg", name="addselect", alt="Add To Collection", title="Add To Collection", style="border:none;") + addselect.append(addselect_img) + selectall = HT.Href(url="#redirect", onClick="checkAll(document.getElementsByName('%s')[0]);" % mainfmName) + selectall_img = HT.Image("/images/select_all2_final.jpg", name="selectall", alt="Select All", title="Select All", style="border:none;") + selectall.append(selectall_img) + selectinvert = HT.Href(url="#redirect", onClick = "checkInvert(document.getElementsByName('%s')[0]);" % mainfmName) + selectinvert_img = HT.Image("/images/invert_selection2_final.jpg", name="selectinvert", alt="Invert Selection", title="Invert Selection", style="border:none;") + selectinvert.append(selectinvert_img) + reset = HT.Href(url="#redirect", onClick="checkNone(document.getElementsByName('%s')[0]); return false;" % mainfmName) + reset_img = HT.Image("/images/select_none2_final.jpg", alt="Select None", title="Select None", style="border:none;") + reset.append(reset_img) + selecttraits = HT.Input(type='button' ,name='selecttraits',value='Select Traits', onClick="checkTraits(this.form);",Class="button") + selectgt = HT.Input(type='text' ,name='selectgt',value='-1.0', size=6,maxlength=10,onChange="checkNumeric(this,1.0,'-1.0','gthan','greater than filed')") + selectlt = HT.Input(type='text' ,name='selectlt',value='1.0', size=6,maxlength=10,onChange="checkNumeric(this,-1.0,'1.0','lthan','less than field')") + selectandor = HT.Select(name='selectandor') + selectandor.append(('AND','and')) + selectandor.append(('OR','or')) + selectandor.selected.append('AND') + + chrMenu = HT.Input(type='hidden',name='chromosomes',value='all') + + corrHeading = HT.Paragraph('Partial Correlation Table', Class="title") + + + pageTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="100%", border=0, align="Left") + containerTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="90%",border=0, align="Left") + + optionsTable = HT.TableLite(cellSpacing=2, cellPadding=0,width="320", height="80", border=0, align="Left") + optionsTable.append(HT.TR(HT.TD(selectall), HT.TD(reset), HT.TD(selectinvert), HT.TD(addselect), align="left")) + optionsTable.append(HT.TR(HT.TD(" "*1,"Select"), HT.TD("Deselect"), HT.TD(" "*1,"Invert"), HT.TD(" "*3,"Add"))) + containerTable.append(HT.TR(HT.TD(optionsTable))) + + functionTable = HT.TableLite(cellSpacing=2,cellPadding=0,width="480",height="80", border=0, align="Left") + functionRow = HT.TR(HT.TD(networkGraph, width="16.7%"), HT.TD(cormatrix, width="16.7%"), HT.TD(partialCorr, width="16.7%"), HT.TD(mcorr, width="16.7%"), HT.TD(mintmap, width="16.7%"), HT.TD(heatmap), align="left") + labelRow = HT.TR(HT.TD(" "*1,HT.Text("Graph")), HT.TD(" "*1,HT.Text("Matrix")), HT.TD(" "*1,HT.Text("Partial")), HT.TD(HT.Text("Compare")), HT.TD(HT.Text("QTL Map")), HT.TD(HT.Text(text="Heat Map"))) + functionTable.append(functionRow, labelRow) + containerTable.append(HT.TR(HT.TD(functionTable), HT.BR())) + + moreOptions = HT.Input(type='button',name='options',value='More Options', onClick="",Class="toggle") + fewerOptions = HT.Input(type='button',name='options',value='Fewer Options', onClick="",Class="toggle") + + if (fd.formdata.getvalue('showHideOptions') == 'less'): + containerTable.append(HT.TR(HT.TD(" "), height="10"), HT.TR(HT.TD(HT.Div(fewerOptions, Class="toggleShowHide")))) + containerTable.append(HT.TR(HT.TD(" "))) + else: + containerTable.append(HT.TR(HT.TD(" "), height="10"), HT.TR(HT.TD(HT.Div(moreOptions, Class="toggleShowHide")))) + containerTable.append(HT.TR(HT.TD(" "))) + + containerTable.append(HT.TR(HT.TD(HT.Span(selecttraits,' with partial r > ',selectgt, ' ',selectandor, ' r < ',selectlt,Class="bd1 cbddf fs11")), style="display:none;", Class="extra_options")) + + + tblobj = {} + + + if self.db.type=="Geno": + + containerTable.append(HT.TR(HT.TD(xlsUrl, height=40))) + pageTable.append(HT.TR(HT.TD(containerTable))) + + tblobj['header'], worksheet = self.getTableHeaderForGeno( method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + sortby = self.getSortByValue( calculationMethod = self.method ) + + tblobj['body'], worksheet, corrScript = self.getTableBodyForGeno(traitList=traitList, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript) + + workbook.close() + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + pageTable.append(HT.TR(HT.TD(div))) + form.append(HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + HT.P(),pageTable) + + TD_LR.append(corrHeading, info_form, HT.P(), info, form, HT.P()) + + self.dict['body'] = str(TD_LR) + # updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + self.dict['js1'] = '' + self.dict['title'] = 'Partial Correlation Result' + + elif self.db.type=="Publish": + + containerTable.append(HT.TR(HT.TD(xlsUrl, height=40))) + pageTable.append(HT.TR(HT.TD(containerTable))) + + tblobj['header'], worksheet = self.getTableHeaderForPublish(method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForPublish(traitList=traitList, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript) + + workbook.close() + + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + pageTable.append(HT.TR(HT.TD(div))) + + form.append( + HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + HT.P(),pageTable) + + TD_LR.append(corrHeading, info_form, HT.P(), info, form, HT.P()) + + self.dict['body'] = str(TD_LR) + #updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + self.dict['js1'] = '' + self.dict['title'] = 'Partial Correlation Result' + + elif self.db.type=="ProbeSet": + + tblobj['header'], worksheet = self.getTableHeaderForProbeSet(method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForProbeSet(traitList=traitList, primaryTrait=myTrait, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript) + + workbook.close() + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + + ''' + #XZ, 07/07/2010: I comment out this block of code. + WebGestaltScript = HT.Script(language="Javascript") + WebGestaltScript.append(""" +setTimeout('openWebGestalt()', 2000); +function openWebGestalt(){ + var thisForm = document['WebGestalt']; + makeWebGestaltTree(thisForm, '%s', %d, 'edag_only.php'); +} + """ % (mainfmName, len(traitList))) + ''' + + #XZ: here is the table of traits + # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + + self.cursor.execute('SELECT GeneChip.GO_tree_value FROM GeneChip, ProbeFreeze, ProbeSetFreeze WHERE GeneChip.Id = ProbeFreeze.ChipId and ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id and ProbeSetFreeze.Name = "%s"' % self.db.name) + result = self.cursor.fetchone() + + if result: + GO_tree_value = result[0] + + if GO_tree_value: + + hddnWebGestalt = { + 'id_list':'', + 'correlation':'', + 'id_value':'', + 'llid_list':'', + 'id_type':GO_tree_value, + 'idtype':'', + 'species':'', + 'list':'', + 'client':''} + + hddnWebGestalt['ref_type'] = hddnWebGestalt['id_type'] + hddnWebGestalt['cat_type'] = 'GO' + hddnWebGestalt['significancelevel'] = 'Top10' + + if species == 'rat': + hddnWebGestalt['org'] = 'Rattus norvegicus' + elif species == 'human': + hddnWebGestalt['org'] = 'Homo sapiens' + elif species == 'mouse': + hddnWebGestalt['org'] = 'Mus musculus' + else: + hddnWebGestalt['org'] = '' + + for key in hddnWebGestalt.keys(): + form.append(HT.Input(name=key, value=hddnWebGestalt[key], type='hidden')) + + #XZ, 01/12/2009: create database menu for 'Add Correlation' + self.cursor.execute(""" + select + ProbeSetFreeze.FullName, ProbeSetFreeze.Id, Tissue.name + from + ProbeSetFreeze, ProbeFreeze, ProbeSetFreeze as ps2, ProbeFreeze as p2, Tissue + where + ps2.Id = %d + and ps2.ProbeFreezeId = p2.Id + and ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id + and (ProbeFreeze.InbredSetId = p2.InbredSetId or (ProbeFreeze.InbredSetId in (1, 3) and p2.InbredSetId in (1, 3))) + and p2.ChipId = ProbeFreeze.ChipId + and ps2.Id != ProbeSetFreeze.Id + and ProbeFreeze.TissueId = Tissue.Id + and ProbeSetFreeze.public > %d + order by + ProbeFreeze.TissueId, ProbeSetFreeze.CreateTime desc + """ % (self.db.id, webqtlConfig.PUBLICTHRESH)) + + results = self.cursor.fetchall() + dbCustomizer = HT.Select(results, name = "customizer") + databaseMenuSub = preTissue = "" + for item in results: + TName, TId, TTissue = item + if TTissue != preTissue: + if databaseMenuSub: + dbCustomizer.append(databaseMenuSub) + databaseMenuSub = HT.Optgroup(label = '%s mRNA ------' % TTissue) + preTissue = TTissue + + databaseMenuSub.append(item[:2]) + if databaseMenuSub: + dbCustomizer.append(databaseMenuSub) + #updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + #variables: filename, strainIds and vals are required by getquerystring function + strainIds=self.getStrainIds(species=species, strains=_primarystrains) + var1 = HT.Input(name="filename", value=filename, type='hidden') + var2 = HT.Input(name="strainIds", value=strainIds, type='hidden') + var3 = HT.Input(name="vals", value=_primaryvals, type='hidden') + customizerButton = HT.Input(type="button", Class="button", value="Add Correlation", onClick = "xmlhttpPost('%smain.py?FormID=AJAX_table', 'sortable', (getquerystring(this.form)))" % webqtlConfig.CGIDIR) + + containerTable.append(HT.TR(HT.TD(HT.Span(var1,var2,var3,customizerButton, "with", dbCustomizer, Class="bd1 cbddf fs11"), HT.BR(), HT.BR()), style="display:none;", Class="extra_options")) + + #outside analysis part + GCATButton = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'GCAT');" % mainfmName) + GCATButton_img = HT.Image("/images/GCAT_logo_final.jpg", name="GCAT", alt="GCAT", title="GCAT", style="border:none") + GCATButton.append(GCATButton_img) + + ODE = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'ODE');" % mainfmName) + ODE_img = HT.Image("/images/ODE_logo_final.jpg", name="ode", alt="ODE", title="ODE", style="border:none") + ODE.append(ODE_img) + + WebGestalt = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'GOTree');" % mainfmName) + WebGestalt_img = HT.Image("/images/webgestalt_icon_final.jpg", name="webgestalt", alt="Gene Set Analysis Toolkit", title="Gene Set Analysis Toolkit", style="border:none") + WebGestalt.append(WebGestalt_img) + + LinkOutTable = HT.TableLite(cellSpacing=2,cellPadding=0,width="320",height="80", border=0, align="Left") + if not GO_tree_value: + LinkOutRow = HT.TR(HT.TD(GCATButton, width="50%"), HT.TD(ODE, width="50%"), align="left") + LinkOutLabels = HT.TR(HT.TD(" ", HT.Text("GCAT"), width="50%"), HT.TD(" ",HT.Text("ODE"), width="50%"), align="left") + else: + LinkOutRow = HT.TR(HT.TD(WebGestalt, width="25%"), HT.TD(GCATButton, width="25%"), HT.TD(ODE, width="25%"), align="left") + LinkOutLabels = HT.TR(HT.TD(HT.Text("Gene Set")), HT.TD(" "*2, HT.Text("GCAT")), HT.TD(" "*3, HT.Text("ODE")), style="display:none;", Class="extra_options") + LinkOutTable.append(LinkOutRow,LinkOutLabels) + + containerTable.append(HT.TR(HT.TD(LinkOutTable), Class="extra_options", style="display:none;")) + + containerTable.append(HT.TR(HT.TD(xlsUrl, HT.BR(), HT.BR(), height=40))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + pageTable.append(HT.TR(HT.TD(div))) + + if species == 'human': + heatmap = "" + + form.append(HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + info, HT.BR(), pageTable, HT.BR()) + + TD_LR.append(corrHeading, info_form, HT.P(), form, HT.P()) + + + self.dict['body'] = str(TD_LR) + self.dict['title'] = 'Partial Correlation Result' + # updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + self.dict['js1'] = '' + self.dict['js2'] = 'onLoad="pageOffset()"' + self.dict['layer'] = self.generateWarningLayer() + + else: + self.dict['body'] = "" + + + +#################################### +# # +#Partial CorrelationPage Functions # +# # +#################################### + + + def getSortByValue(self, calculationMethod): + + sortby = ("partial_pv", "up") + + if calculationMethod == "3": #XZ: literature correlation + sortby = ("lcorr","down") + elif calculationMethod == "4" or calculationMethod == "5": #XZ: tissue correlation + sortby = ("tissuecorr", "down") + + return sortby + + + #XZ, 3/31/2010: + #A[0] holds trait name. + #A[1] holds partial correlation coefficient number. + #A[2] holds N. + #A[3] holds p value of partial correlation. + def cmpPartialCorrPValue (self, A, B): + try: + if A[3] < B[3]: + return -1 + elif A[3] == B[3]: + return 0 + else: + return 1 + except: + return 0 + + + #XZ, 4/1/2010: + #A[0] holds trait name. + #A[1] holds N. + #A[2] holds partial correlation coefficient number. + #A[3] holds p value of partial correlation. + #A[6] holds literature corr or tissue corr value. + #Sort by literature corr or tissue corr first, then by partial corr p value. + def cmpLitCorr(self, A, B): + try: + if abs(A[6]) < abs(B[6]): + return 1 + elif abs(A[6]) == abs(B[6]): + if A[3] < B[3]: + return -1 + elif A[3] == B[3]: + return 0 + else: + return 1 + else: + return -1 + except: + return 0 + + + def getPartialCorrelationsFast(self, _strains, _vals, _controlvals, nnCorr, DatabaseFileName, species, input_trait_GeneId,gene_symbol,TissueProbeSetFreezeId ): + """Calculates and returns correlation coefficients using data from a csv text file.""" + + try: + allcorrelations = [] + + useLit = False + if self.method == "3": + litCorrs = self.fetchLitCorrelations(species=species, GeneId=input_trait_GeneId, db=self.db, returnNumber=self.returnNumber) + useLit = True + + useTissueCorr = False + if self.method == "4" or self.method == "5": + tissueCorrs = self.fetchTissueCorrelations(db=self.db,primaryTraitSymbol=gene_symbol, TissueProbeSetFreezeId=TissueProbeSetFreezeId, method=self.method, returnNumber=self.returnNumber) + useTissueCorr = True + + datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r') + + #XZ, 01/08/2009: read the first line + line = datasetFile.readline() + dataset_strains = webqtlUtil.readLineCSV(line)[1:] + + #XZ, 3/30/2010: This step is critical. + good_dataset_strains_index = [] + + for i in range(len(_strains)): + found_in_dataset_strains = 0 + for j, one_dataset_strain in enumerate(dataset_strains): + if one_dataset_strain == _strains[i]: + found_in_dataset_strains = 1 + good_dataset_strains_index.append(j) + break + + if not found_in_dataset_strains: + good_dataset_strains_index.append(-99999) + + allTargetTraitNames = [] + allTargetTraitValues = [] + + #XZ, 04/01/2009: If literature corr or tissue corr is selected, + #XZ: there is no need to compute partial correlation for all traits. + #XZ: If genetic corr is selected, compute partial correlation for all traits. + for line in datasetFile: + trait_line = webqtlUtil.readLineCSV(line) + trait_name = trait_line[0] + trait_data = trait_line[1:] + + if useLit: + if not litCorrs.has_key( trait_name ): + continue + + if useTissueCorr: + if not tissueCorrs.has_key( trait_name ): + continue + + #XZ, 04/01/2010: If useLit or useTissueCorr, and this trait should not be added, + #it will not go to the next step. + + good_dataset_vals = [] + for i in good_dataset_strains_index: + if i == -99999: + good_dataset_vals.append(None) + else: + good_dataset_vals.append( float(trait_data[i]) ) + + allTargetTraitNames.append(trait_name) + allTargetTraitValues.append(good_dataset_vals) + + datasetFile.close() + + if self.method in ["2", "5"]: #Spearman + allcorrelations = correlationFunction.determinePartialsByR(primaryVal=_vals, controlVals=_controlvals, targetVals=allTargetTraitValues, targetNames=allTargetTraitNames, method='s') + else: + allcorrelations = correlationFunction.determinePartialsByR(primaryVal=_vals, controlVals=_controlvals, targetVals=allTargetTraitValues, targetNames=allTargetTraitNames) + + totalTraits = len(allcorrelations) + + if useLit or useTissueCorr: + for i, item in enumerate(allcorrelations): + if useLit: + allcorrelations[i].append(litCorrs[ item[0] ]) + if useTissueCorr: + tempCorr, tempPValue = tissueCorrs[ item[0] ] + allcorrelations[i].append(tempCorr) + allcorrelations[i].append(tempPValue) + + return totalTraits, allcorrelations + except: + return 0, 0 + + + def getPartialCorrelationsNormal(self, _strains, _vals, _controlvals, nnCorr, species, input_trait_GeneId, input_trait_symbol,TissueProbeSetFreezeId): + """Calculates and returns correlation coefficients""" + + traitdatabase, dataStartPos = self.fetchAllDatabaseData(species=species, GeneId=input_trait_GeneId, GeneSymbol=input_trait_symbol, strains=_strains, db=self.db, method=self.method, returnNumber=self.returnNumber, tissueProbeSetFreezeId=TissueProbeSetFreezeId) + totalTraits = len(traitdatabase) #XZ, 09/18/2008: total trait number + + allcorrelations = [] + + allTargetTraitNames = [] + allTargetTraitValues = [] + + for traitdata in traitdatabase: + traitdataName = traitdata[0] + traitvals = traitdata[dataStartPos:] + allTargetTraitNames.append (traitdataName) + allTargetTraitValues.append (traitvals) + + if self.method in ["2", "5"]: #Spearman + allcorrelations = correlationFunction.determinePartialsByR(primaryVal=_vals, controlVals=_controlvals, targetVals=allTargetTraitValues, targetNames=allTargetTraitNames, method='s') + else: + allcorrelations = correlationFunction.determinePartialsByR(primaryVal=_vals, controlVals=_controlvals, targetVals=allTargetTraitValues, targetNames=allTargetTraitNames) + + #XZ, 09/28/2008: if user select '3', then fetchAllDatabaseData would give us LitCorr in the [1] position + #XZ, 09/28/2008: if user select '4' or '5', then fetchAllDatabaseData would give us Tissue Corr in the [1] position + #XZ, 09/28/2008: and Tissue Corr P Value in the [2] position + if input_trait_GeneId and self.db.type == "ProbeSet" and self.method in ["3", "4", "5"]: + for i, item in enumerate(allcorrelations): + if self.method == "3": + item.append( traitdatabase[1] ) + if self.method == "4" or self.method == "5": + item.append( traitdatabase[1] ) + item.append( traitdatabase[2] ) + + + return totalTraits, allcorrelations + + + def getTableHeaderForPublish(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1", "3", "4"]: + tblobj_header = [[THCell(HT.TD('', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), sort=0), + THCell(HT.TD('Record', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="id", idx=1), + THCell(HT.TD('Phenotype', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="pheno", idx=2), + THCell(HT.TD('Authors', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="auth", idx=3), + THCell(HT.TD('Year', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="year", idx=4), + THCell(HT.TD('N', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="nstr", idx=5), + THCell(HT.TD('Partial r ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="partial_corr", idx=6), + THCell(HT.TD('p(partial r)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="partial_pv", idx=7), + THCell(HT.TD('r ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="corr", idx=8), + THCell(HT.TD('p(r)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="pv", idx=9), + THCell(HT.TD('delta r', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="delta_corr", idx=10)]] + + for ncol, item in enumerate(["Record", "Phenotype", "Authors", "Year", "PubMedID", "N", "Partial r", "p(partial r)", "r ", "p(r)", "delta r"]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD('', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), sort=0), + THCell(HT.TD('Record', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="id", idx=1), + THCell(HT.TD('Phenotype', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="pheno", idx=2), + THCell(HT.TD('Authors', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="auth", idx=3), + THCell(HT.TD('Year', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="year", idx=4), + THCell(HT.TD('N', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="nstr", idx=5), + THCell(HT.TD('Partial rho ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="partial_corr", idx=6), + THCell(HT.TD('p(partial rho)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="partial_pv", idx=7), + THCell(HT.TD('rho ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="corr", idx=8), + THCell(HT.TD('p(rho)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="pv", idx=9), + THCell(HT.TD('delta rho', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="delta_corr", idx=10)]] + + for ncol, item in enumerate(["Record", "Phenotype", "Authors", "Year", "PubMedID", "N", "Partial rho", "p(partial rho)", "rho ", "p(rho)", "delta rho"]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + return tblobj_header, worksheet + + + def getTableBodyForPublish(self, traitList, formName=None, worksheet=None, newrow=None, corrScript=None): + + tblobj_body = [] + + for thisTrait in traitList: + tr = [] + + trId = str(thisTrait) + + #partial corr value could be string 'NA' + try: + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.partial_corr)) + except: + corrScript.append('corrArray["%s"] = {corr:"NA"};' % (trId)) + + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn"), nowrap="yes",align="center", Class="fs12 fwn b1 c222"),str(thisTrait.name), thisTrait.name)) + + PhenotypeString = thisTrait.post_publication_description + if thisTrait.confidential: + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users): + PhenotypeString = thisTrait.pre_publication_description + tr.append(TDCell(HT.TD(PhenotypeString, Class="fs12 fwn b1 c222"), PhenotypeString, PhenotypeString.upper())) + + tr.append(TDCell(HT.TD(thisTrait.authors, Class="fs12 fwn b1 c222 fsI"),thisTrait.authors, thisTrait.authors.strip().upper())) + + try: + PubMedLinkText = myear = repr = int(thisTrait.year) + except: + PubMedLinkText = repr = "N/A" + myear = 0 + if thisTrait.pubmed_id: + PubMedLink = HT.Href(text= repr,url= webqtlConfig.PUBMEDLINK_URL % thisTrait.pubmed_id,target='_blank', Class="fs12 fwn") + else: + PubMedLink = repr + + tr.append(TDCell(HT.TD(PubMedLink, Class="fs12 fwn b1 c222", align='center'), repr, myear)) + + repr = '%d' % thisTrait.NOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.NOverlap)) + + try: + repr = '%3.3f' % thisTrait.partial_corr + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn b1 c222", align='right', nowrap="on"), repr, abs(thisTrait.partial_corr))) + except: + repr = 'NA' + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='left'), text=repr, val=0 )) + + repr = webqtlUtil.SciFloat(thisTrait.partial_corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.partial_corrPValue)) + + repr = '%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn b1 c222", align='right', nowrap="on"), repr, abs(thisTrait.corr))) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + #delta + try: + delta = '%3.3f' % ( float(thisTrait.partial_corr) - float(thisTrait.corr) ) + tr.append(TDCell(HT.TD(delta, Class="fs12 fwn ffl b1 c222", align='right', nowrap="on"), text=delta, val=abs(float(delta)) )) + except: + delta = 'NA' + tr.append(TDCell(HT.TD(delta, Class="fs12 fwn ffl b1 c222", align='left'), text=delta, val=0 )) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, PhenotypeString, thisTrait.authors, thisTrait.year, thisTrait.pubmed_id, thisTrait.NOverlap, thisTrait.partial_corr, thisTrait.partial_corrPValue, thisTrait.corr, thisTrait.corrPValue, delta]): + worksheet.write([newrow, ncol], str(item) ) + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getTableHeaderForGeno(self, method=None, worksheet=None, newrow=None, headingStyle=None): + tblobj_header = [] + + if method in ["1", "3", "4"]: + tblobj_header = [[THCell(HT.TD('', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Locus', Class="fs13 fwb ffl b1 cw cbrb",align='center'), text='locus', idx=1), + THCell(HT.TD('Chr', Class="fs13 fwb ffl b1 cw cbrb"), text='chr', idx=2), + THCell(HT.TD('Megabase', Class="fs13 fwb ffl b1 cw cbrb"), text='Mb', idx=3), + THCell(HT.TD('N', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='nstr', idx=4), + THCell(HT.TD('Partial r ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='partial_corr', idx=5), + THCell(HT.TD('p(partial r)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='partial_pv', idx=6), + THCell(HT.TD('r ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='corr', idx=7), + THCell(HT.TD('p(r)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='pv', idx=8), + THCell(HT.TD('delta r', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='delta_corr', idx=9)]] + + for ncol, item in enumerate(['Locus', 'Chr', ' Mb ', ' N ', 'Partial r', 'p(partial r)', 'r ', 'p(r)', 'delta r' ]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD('', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Locus', Class="fs13 fwb ffl b1 cw cbrb",align='center'), text='locus', idx=1), + THCell(HT.TD('Chr', Class="fs13 fwb ffl b1 cw cbrb"), text='chr', idx=2), + THCell(HT.TD('Megabase', Class="fs13 fwb ffl b1 cw cbrb"), text='Mb', idx=3), + THCell(HT.TD('N', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='nstr', idx=4), + THCell(HT.TD('Partial rho', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='partial_corr', idx=5), + THCell(HT.TD('p(partial rho)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='partial_pv', idx=6), + THCell(HT.TD('rho ', Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text='corr', idx=7), + THCell(HT.TD('p(rho)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='pv', idx=8), + THCell(HT.TD('delta rho', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='delta_corr', idx=9)]] + + for ncol, item in enumerate(['Locus', 'Chr', ' Mb ', ' N ', 'Partial rho', 'p(partial rho)', 'rho ', 'p(rho)', 'delta rho' ]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + return tblobj_header, worksheet + + + + def getTableBodyForGeno(self, traitList, formName=None, worksheet=None, newrow=None, corrScript=None): + + tblobj_body = [] + + for thisTrait in traitList: + tr = [] + + trId = str(thisTrait) + + #partial corr value could be string 'NA' + try: + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.partial_corr)) + except: + corrScript.append('corrArray["%s"] = {corr:"NA"};' % (trId)) + + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"),align="center", Class="fs12 fwn ffl b1 c222"), text=thisTrait.name, val=thisTrait.name.upper())) + + #tr.append(TDCell(HT.TD(thisTrait.chr, Class="fs12 fwn ffl b1 c222", align='right'), text=str(thisTrait.chr))) + + try: + Mbvalue = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if not thisTrait.chr or not thisTrait.mb: + Mbvalue = 1000000 + elif thisTrait.chr.upper() == 'X': + Mbvalue = 20*1000 + thisTrait.mb + else: + Mbvalue = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + tr.append(TDCell( HT.TD(thisTrait.chr, Class="fs12 fwn b1 c222", align='right'), thisTrait.chr, Mbvalue) ) + tr.append(TDCell(HT.TD(thisTrait.mb, Class="fs12 fwn ffl b1 c222", align='right'), text=str(thisTrait.mb), val=Mbvalue)) + + repr = '%d' % thisTrait.NOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.NOverlap)) + + try: + repr='%3.3f' % thisTrait.partial_corr + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right',nowrap='ON'),repr,abs(thisTrait.partial_corr))) + except: + repr = 'NA' + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='left'), text=repr, val=0 )) + + repr = webqtlUtil.SciFloat(thisTrait.partial_corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.partial_corrPValue)) + + repr = '%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn b1 c222", align='right',nowrap='ON'), repr, abs(thisTrait.corr))) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + #delta + try: + delta = '%3.3f' % ( float(thisTrait.partial_corr) - float(thisTrait.corr) ) + tr.append(TDCell(HT.TD(delta, Class="fs12 fwn ffl b1 c222", align='right', nowrap='ON'), text=delta, val=abs(float(delta)) )) + except: + delta = 'NA' + tr.append(TDCell(HT.TD(delta, Class="fs12 fwn ffl b1 c222", align='left'), text=delta, val=0 )) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, thisTrait.chr, thisTrait.mb, thisTrait.NOverlap, thisTrait.partial_corr, thisTrait.partial_corrPValue, thisTrait.corr, thisTrait.corrPValue, delta]): + worksheet.write([newrow, ncol], item) + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getTableHeaderForProbeSet(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), sort=0), + THCell(HT.TD('Record',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="id", idx=1), + THCell(HT.TD('','Symbol',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="symbol", idx=2), + THCell(HT.TD('','Description',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="desc", idx=3), + #XZ, 12/09/2008: sort chr + THCell(HT.TD('','Chr',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="chr", idx=4), + THCell(HT.TD('','Mb',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Mb", idx=5), + THCell(HT.TD('Mean',HT.BR(),'Expr',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="mean", idx=6), + THCell(HT.TD('N',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="nstr", idx=7), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'Partial r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="partial_corr", idx=8), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(partial r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="partial_pv", idx=9), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="corr", idx=10), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="pv", idx=11), + THCell(HT.TD('delta',HT.BR(), 'r', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="delta_corr", idx=12), + THCell(HT.TD(HT.Href( + text = HT.Span('Pubmed',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#literatureCorr"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="lcorr", idx=13), + #XZ, 09/22/2008: tissue correlation + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="tissuecorr", idx=14), + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="tissuepvalue", idx=15)]] + + for ncol, item in enumerate(['Record', 'Gene ID', 'Symbol', 'Description', 'Chr', 'Megabase', 'Mean Expr', 'N ', 'Sample Partial r', 'Sample p(partial r)', 'Sample r', 'Sample p(r)', 'delta r', 'Lit Corr', 'Tissue r', 'Tissue p(r)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), sort=0), + THCell(HT.TD('Record',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="id", idx=1), + THCell(HT.TD('','Symbol',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="symbol", idx=2), + THCell(HT.TD('','Description',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="desc", idx=3), + THCell(HT.TD('','Chr',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="chr", idx=4), + THCell(HT.TD('','Mb',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Mb", idx=5), + THCell(HT.TD('Mean',HT.BR(),'Expr',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="mean", idx=6), + THCell(HT.TD('N',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="nstr", idx=7), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'Partial rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="partial_corr", idx=8), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(partial rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="partial_pv", idx=9), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="corr", idx=10), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="pv", idx=11), + THCell(HT.TD('delta',HT.BR(),'rho', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text="delta_corr", idx=12), + THCell(HT.TD(HT.Href( + text = HT.Span('Pubmed',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#literatureCorr"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="lcorr", idx=13), + #XZ, 09/22/2008: tissue correlation + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="tissuecorr", idx=14), + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="tissuepvalue", idx=15)]] + + for ncol, item in enumerate(['Record', 'Gene ID', 'Symbol', 'Description', 'Chr', 'Megabase', 'Mean Expr', 'N ', 'Sample Partial rho', 'Sample p(partial rho)', 'Sample rho', 'Sample p(rho)', 'delta rho', 'Pubmed r', 'Tissue rho', 'Tissue p(rho)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + return tblobj_header, worksheet + + + def getTableBodyForProbeSet(self, traitList=[], primaryTrait=None, formName=None, worksheet=None, newrow=None, corrScript=None): + + tblobj_body = [] + + for thisTrait in traitList: + + if thisTrait.symbol: + pass + else: + thisTrait.symbol = "N/A" + + if thisTrait.geneid: + symbolurl = HT.Href(text=thisTrait.symbol,target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" % thisTrait.geneid, Class="fs12 fwn") + else: + symbolurl = HT.Href(text=thisTrait.symbol,target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?CMD=search&DB=gene&term=%s" % thisTrait.symbol, Class="fs12 fwn") + + tr = [] + + trId = str(thisTrait) + + #partial corr value could be string 'NA' + try: + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.partial_corr)) + except: + corrScript.append('corrArray["%s"] = {corr:"NA"};' % (trId)) + + #XZ, 12/08/2008: checkbox + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + #XZ, 12/08/2008: probeset name + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName,thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn b1 c222"), thisTrait.name, thisTrait.name.upper())) + + #XZ, 12/08/2008: gene symbol + tr.append(TDCell(HT.TD(symbolurl, Class="fs12 fwn b1 c222 fsI"),thisTrait.symbol, thisTrait.symbol.upper())) + + #XZ, 12/08/2008: description + #XZ, 06/05/2009: Rob asked to add probe target description + description_string = str(thisTrait.description).strip() + target_string = str(thisTrait.probe_target_description).strip() + + description_display = '' + + if len(description_string) > 1 and description_string != 'None': + description_display = description_string + else: + description_display = thisTrait.symbol + + if len(description_display) > 1 and description_display != 'N/A' and len(target_string) > 1 and target_string != 'None': + description_display = description_display + '; ' + target_string.strip() + + tr.append(TDCell(HT.TD(description_display, Class="fs12 fwn b1 c222"), description_display, description_display)) + + #XZ, 12/08/2008: Mbvalue is used for sorting + try: + Mbvalue = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if not thisTrait.chr or not thisTrait.mb: + Mbvalue = 1000000 + elif thisTrait.chr.upper() == 'X': + Mbvalue = 20*1000 + thisTrait.mb + else: + Mbvalue = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + #XZ, 12/08/2008: chromosome number + #XZ, 12/10/2008: use Mbvalue to sort chromosome + tr.append(TDCell( HT.TD(thisTrait.chr, Class="fs12 fwn b1 c222", align='right'), thisTrait.chr, Mbvalue) ) + + #XZ, 12/08/2008: Rob wants 6 digit precision, and we have to deal with that the mb could be None + if not thisTrait.mb: + tr.append(TDCell(HT.TD(thisTrait.mb, Class="fs12 fwn b1 c222",align='right'), thisTrait.mb, Mbvalue)) + else: + tr.append(TDCell(HT.TD('%.6f' % thisTrait.mb, Class="fs12 fwn b1 c222", align='right'), thisTrait.mb, Mbvalue)) + + #XZ, 01/12/08: This SQL query is much faster. + self.cursor.execute(""" + select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet + where ProbeSetXRef.ProbeSetFreezeId = %d and + ProbeSet.Id = ProbeSetXRef.ProbeSetId and + ProbeSet.Name = '%s' + """ % (thisTrait.db.id, thisTrait.name)) + result = self.cursor.fetchone() + if result: + if result[0]: + mean = result[0] + else: + mean=0 + else: + mean = 0 + + #XZ, 06/05/2009: It is neccessary to turn on nowrap + repr = "%2.3f" % mean + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right', nowrap='ON'),repr, mean)) + + #XZ: number of overlaped cases for partial corr + repr = '%d' % thisTrait.NOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.NOverlap)) + + #XZ: sample partial correlation + try: + repr='%3.3f' % thisTrait.partial_corr + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right', nowrap='ON'),repr,abs(thisTrait.partial_corr))) + except: + repr = 'NA' + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='left'), text=repr, val=0 )) + + #XZ: p value of genetic partial correlation + repr = webqtlUtil.SciFloat(thisTrait.partial_corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.partial_corrPValue)) + + repr = '%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn b1 c222", align='right',nowrap='ON'), repr, abs(thisTrait.corr))) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + #delta + try: + delta = '%3.3f' % ( float(thisTrait.partial_corr) - float(thisTrait.corr) ) + tr.append(TDCell(HT.TD(delta, Class="fs12 fwn ffl b1 c222", align='right', nowrap='ON'), text=delta, val=abs(float(delta)) )) + except: + delta = 'NA' + tr.append(TDCell(HT.TD(delta, Class="fs12 fwn ffl b1 c222", align='left'), text=delta, val=0 )) + + #XZ, 12/08/2008: literature correlation + LCorr = 0.0 + LCorrStr = "N/A" + if hasattr(thisTrait, 'LCorr') and thisTrait.LCorr: + LCorr = thisTrait.LCorr + LCorrStr = "%2.3f" % thisTrait.LCorr + tr.append(TDCell(HT.TD(LCorrStr, Class="fs12 fwn b1 c222", align='right'), LCorrStr, abs(LCorr))) + + #XZ, 09/22/2008: tissue correlation. + TCorr = 0.0 + TCorrStr = "N/A" + #XZ, 11/18/2010: need to pass two gene symbols + if hasattr(thisTrait, 'tissueCorr') and thisTrait.tissueCorr: + TCorr = thisTrait.tissueCorr + TCorrStr = "%2.3f" % thisTrait.tissueCorr + #NL, 07/19/2010: add a new parameter rankOrder for js function 'showTissueCorrPlot' + rankOrder =thisTrait.rankOrder + TCorrPlotURL = "javascript:showTissueCorrPlot('%s','%s','%s',%d)" %(formName, primaryTrait.symbol, thisTrait.symbol,rankOrder) + tr.append(TDCell(HT.TD(HT.Href(text=TCorrStr, url=TCorrPlotURL, Class="fs12 fwn ff1"), Class="fs12 fwn ff1 b1 c222", align='right'), TCorrStr, abs(TCorr) )) + else: + tr.append(TDCell(HT.TD(TCorrStr, Class="fs12 fwn b1 c222", align='right'), TCorrStr, abs(TCorr))) + + #XZ, 12/08/2008: p value of tissue correlation + TPValue = 1.0 + TPValueStr = "N/A" + if hasattr(thisTrait, 'tissueCorr') and thisTrait.tissueCorr: #XZ, 09/22/2008: thisTrait.tissuePValue can't be used here because it could be 0 + TPValue = thisTrait.tissuePValue + TPValueStr = "%2.3f" % thisTrait.tissuePValue + tr.append(TDCell(HT.TD(TPValueStr, Class="fs12 fwn b1 c222", align='right'), TPValueStr, abs(TPValue) )) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, thisTrait.geneid, thisTrait.symbol, thisTrait.description, thisTrait.chr, thisTrait.mb, mean, thisTrait.NOverlap, thisTrait.partial_corr, thisTrait.partial_corrPValue, thisTrait.corr, thisTrait.corrPValue, delta, LCorrStr, TCorrStr, TPValueStr]): + worksheet.write([newrow, ncol], item) + + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getFormForPrimaryAndControlTraits (self, primaryTrait, controlTraits): + + info_form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase', submit=HT.Input(type='hidden')) + + hddn = {'FormID':'showDatabase', 'database':'_', 'ProbeSetID':'_', 'CellID':'_' }#XZ: These four parameters are required by javascript function showDatabase2. + + for key in hddn.keys(): + info_form.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + info_form.append(HT.Paragraph("Primary Trait", Class="subtitle"), '\n') + + primaryTraitTable = HT.TableLite(cellSpacing=4,cellPadding=0,width="90%",border=0) + descriptionString = primaryTrait.genHTML(dispFromDatabase=1) + if primaryTrait.db.type == 'Publish' and primaryTrait.confidential: + descriptionString = primaryTrait.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=primaryTrait.authorized_users) + primaryTraitTable.append(HT.TR(HT.TD(HT.Href(text='%s' % descriptionString, url="javascript:showDatabase2('%s','%s','%s')" % (primaryTrait.db.name,primaryTrait.name,primaryTrait.cellid), Class="fs12 fwn") ))) + + info_form.append(primaryTraitTable) + + info_form.append(HT.Paragraph("Control Traits", Class="subtitle"), '\n') + + controlTraitsTable = HT.TableLite(cellSpacing=4,cellPadding=0,width="90%",border=0) + + seq = 1 + + ## Generate the listing table for control traits + for thisTrait in controlTraits: + descriptionString = thisTrait.genHTML(dispFromDatabase=1) + if thisTrait.db.type == 'Publish' and thisTrait.confidential: + descriptionString = thisTrait.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users) + controlTraitsTable.append(HT.TR(HT.TD("%d."%seq,align="right",width=10), + HT.TD(HT.Href(text='%s' % descriptionString,url="javascript:showDatabase2('%s','%s','%s')" % (thisTrait.db.name,thisTrait.name,thisTrait.cellid), Class="fs12 fwn") ))) + seq += 1 + + info_form.append(controlTraitsTable) + + return info_form diff --git a/web/webqtl/correlation/PartialCorrInputPage.py b/web/webqtl/correlation/PartialCorrInputPage.py new file mode 100755 index 00000000..7d32da6d --- /dev/null +++ b/web/webqtl/correlation/PartialCorrInputPage.py @@ -0,0 +1,484 @@ +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by GeneNetwork Core Team 2010/10/20 + +import os +import string +import cPickle + +from htmlgen import HTMLgen2 as HT + +from base import webqtlConfig +from utility.THCell import THCell +from utility.TDCell import TDCell +from base.webqtlTrait import webqtlTrait +from base.templatePage import templatePage +from dbFunction import webqtlDatabaseFunction +from utility import webqtlUtil + + + +class PartialCorrInputPage(templatePage): + + def __init__(self,fd): + + templatePage.__init__(self, fd) + + if not self.openMysql(): + return + + searchResult = fd.formdata.getvalue('searchResult') + + if not searchResult: + heading = 'Partial Correlation' + detail = ['You need to select at least three traits in order to calculate partial correlation.'] + self.error(heading=heading,detail=detail) + return + + + ## Adds the Trait instance for each trait name from the collection + traits = [] + + for item in searchResult: + traits.append(webqtlTrait(fullname=item, cursor=self.cursor)) + + RISet = fd.formdata.getvalue('RISet') + species = webqtlDatabaseFunction.retrieveSpecies(cursor=self.cursor, RISet=RISet) + + #XZ: HTML part + TD_LR = HT.TD(colspan=2,height=200,width="100%",bgColor='#eeeeee') + TD_LR.append("Please select one primary trait, one to three control traits, and at least one target trait.", HT.P() ) + + mainFormName = 'showDatabase' + mainForm = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), name=mainFormName,submit=HT.Input(type='hidden')) + + #XZ: Add hidden form values + hddn = {'FormID':'calPartialCorrTrait', 'database':'', 'ProbeSetID':'', 'CellID':'', #XZ: These four parameters are required by javascript function showDatabase2. + 'controlTraits':'', + 'primaryTrait':'', + 'targetTraits':'', + 'pcMethod':'', + 'RISet':RISet + } + + + for key in hddn.keys(): + mainForm.append(HT.Input(type='hidden', name=key, value=hddn[key])) + + radioNames = [] + + for thisTrait in traits: + oneRadioName = thisTrait.getName() + radioNames.append(oneRadioName) + + radioNamesString = ','.join(radioNames) + + # Creates the image href that runs the javascript setting all traits as target or ignored + setAllTarget = HT.Href(url="#redirect", onClick="setAllAsTarget(document.getElementsByName('showDatabase')[0], '%s');" % radioNamesString) + setAllTargetImg = HT.Image("/images/select_all.gif", alt="Select All", title="Select All", style="border:none;") + setAllTarget.append(setAllTargetImg) + setAllIgnore = HT.Href(url="#redirect", onClick="setAllAsIgnore(document.getElementsByName('showDatabase')[0], '%s');" % radioNamesString) + setAllIgnoreImg = HT.Image("/images/select_all.gif", alt="Select All", title="Select All", style="border:none;") + setAllIgnore.append(setAllIgnoreImg) + + + tblobj = {} + tblobj['header'] = self.getCollectionTableHeader() + + sortby = self.getSortByValue() + + tblobj['body'] = self.getCollectionTableBody(traitList=traits, formName=mainFormName, species=species) + + filename= webqtlUtil.genRandStr("Search_") + + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), Id="sortable") + + mainForm.append(div) + + #XZ: Add button + radioNamesString = ','.join(radioNames) + jsCommand_1 = "validateTrait(this.form, \'" + radioNamesString + "\', 0, 1);" + jsCommand_2 = "validateTrait(this.form, \'" + radioNamesString + "\', 0, 2);" + partialCorrTraitButton_1 = HT.Input(type='button', name='submitPartialCorrTrait_1', value='Pearson\'s r', onClick='%s' % jsCommand_1, Class="button") + partialCorrTraitButton_2 = HT.Input(type='button', name='submitPartialCorrTrait_2', value='Spearman\'s rho', onClick='%s' % jsCommand_2, Class="button") + mainForm.append(HT.BR(), "Compute partial correlation for target selected above:", HT.BR(), partialCorrTraitButton_1, partialCorrTraitButton_2, HT.BR(), HT.BR(), HT.HR(color="gray",size=3) ) + + jsCommand = "validateTrait(this.form, \'" + radioNamesString + "\', 1);" + partialCorrDBButton = HT.Input(type='button', name='submitPartialCorrDB', value='Calculate', onClick='%s' % jsCommand,Class="button") + + methodText = HT.Span("Calculate:", Class="ffl fwb fs12") + + methodMenu = HT.Select(name='method') + methodMenu.append(('Genetic Correlation, Pearson\'s r','1')) + methodMenu.append(('Genetic Correlation, Spearman\'s rho','2')) + methodMenu.append(('SGO Literature Correlation','3')) + methodMenu.append(('Tissue Correlation, Pearson\'s r','4')) + methodMenu.append(('Tissue Correlation, Spearman\'s rho','5')) + + databaseText = HT.Span("Choose Database:", Class="ffl fwb fs12") + databaseMenu = HT.Select(name='database2') + + nmenu = 0 + + self.cursor.execute('SELECT PublishFreeze.FullName,PublishFreeze.Name FROM \ + PublishFreeze,InbredSet WHERE PublishFreeze.InbredSetId = InbredSet.Id \ + and InbredSet.Name = "%s" and PublishFreeze.public > %d' % \ + (RISet,webqtlConfig.PUBLICTHRESH)) + for item in self.cursor.fetchall(): + databaseMenu.append(item) + nmenu += 1 + + self.cursor.execute('SELECT GenoFreeze.FullName,GenoFreeze.Name FROM GenoFreeze,\ + InbredSet WHERE GenoFreeze.InbredSetId = InbredSet.Id and InbredSet.Name = \ + "%s" and GenoFreeze.public > %d' % (RISet,webqtlConfig.PUBLICTHRESH)) + for item in self.cursor.fetchall(): + databaseMenu.append(item) + nmenu += 1 + + #03/09/2009: Xiaodong changed the SQL query to order by Name as requested by Rob. + self.cursor.execute('SELECT Id, Name FROM Tissue order by Name') + for item in self.cursor.fetchall(): + TId, TName = item + databaseMenuSub = HT.Optgroup(label = '%s ------' % TName) + self.cursor.execute('SELECT ProbeSetFreeze.FullName,ProbeSetFreeze.Name FROM ProbeSetFreeze, ProbeFreeze, \ + InbredSet WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id and ProbeFreeze.TissueId = %d and \ + ProbeSetFreeze.public > %d and ProbeFreeze.InbredSetId = InbredSet.Id and InbredSet.Name like "%s%%" \ + order by ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId ' % (TId,webqtlConfig.PUBLICTHRESH, RISet)) + for item2 in self.cursor.fetchall(): + databaseMenuSub.append(item2) + nmenu += 1 + databaseMenu.append(databaseMenuSub) + + if nmenu: + criteriaText = HT.Span("Return:", Class="ffl fwb fs12") + criteriaMenu = HT.Select(name='criteria', selected='500') + criteriaMenu.append(('top 100','100')) + criteriaMenu.append(('top 200','200')) + criteriaMenu.append(('top 500','500')) + criteriaMenu.append(('top 1000','1000')) + criteriaMenu.append(('top 2000','2000')) + criteriaMenu.append(('top 5000','5000')) + criteriaMenu.append(('top 10000','10000')) + criteriaMenu.append(('top 15000','15000')) + criteriaMenu.append(('top 20000','20000')) + + self.MPDCell = HT.TD() + correlationMenus = HT.TableLite( + HT.TR( + HT.TD(databaseText,HT.BR(),databaseMenu, colspan=4) + ), + HT.TR( + HT.TD(methodText,HT.BR(),methodMenu), + self.MPDCell, + HT.TD(criteriaText,HT.BR(),criteriaMenu)), + border=0, cellspacing=4, cellpadding=0) + else: + correlationMenus = "" + + mainForm.append(HT.Font('or',color='red', size=4), HT.BR(), HT.BR(), "Compute partial correlation for each trait in the database selected below:", HT.BR() ) + mainForm.append( partialCorrDBButton, HT.BR(), HT.BR(), correlationMenus) + + TD_LR.append(mainForm) + + self.dict['body'] = str(TD_LR) + self.dict['js1'] ='' + self.dict['title'] = 'Partial Correlation Input' + + + def getCollectionTableHeader(self): + + tblobj_header = [] + + className = "fs13 fwb ffl b1 cw cbrb" + + tblobj_header = [[THCell(HT.TD('Index', Class=className, nowrap="on"), sort=0), + THCell(HT.TD("Primary (X)",align="center", Class="fs13 fwb ffl b1 cw cbrb", nowrap="ON"), text="primary", sort=0), + THCell(HT.TD("Control (Z)",align="center", Class="fs13 fwb ffl b1 cw cbrb", nowrap="ON"), text="control", sort=0), + THCell(HT.TD("Target (Y)",align="center", Class="fs13 fwb ffl b1 cw cbrb", nowrap="ON"), text="target", sort=0), + THCell(HT.TD("Ignored",align="center", Class="fs13 fwb ffl b1 cw cbrb", nowrap="ON"), text="target", sort=0), + THCell(HT.TD('Dataset', HT.BR(), HT.BR(), valign="top", Class=className, nowrap="on"), text="dataset", idx=1), + THCell(HT.TD('Trait', HT.BR(), 'ID', HT.BR(), valign="top", Class=className, nowrap="on"), text="name", idx=2), + THCell(HT.TD('Description', HT.BR(), HT.BR(), valign="top", Class=className, nowrap="on"), text="desc", idx=3), + THCell(HT.TD('Location', HT.BR(), HT.BR(), valign="top", Class=className, nowrap="on"), text="location", idx=4), + THCell(HT.TD('Mean', HT.BR(), HT.BR(), valign="top", Class=className, nowrap="on"), text="mean", idx=5), + THCell(HT.TD('N', HT.BR(), 'Cases', HT.BR(), valign="top", Class=className, nowrap="on"), text="samples", idx=6), + THCell(HT.TD('Max LRS', HT.BR(), HT.BR(), valign="top", Class=className, nowrap="on"), text="lrs", idx=7), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb', HT.BR(), valign="top", Class=className, nowrap="on"), text="lrs_location", idx=8)]] + + return tblobj_header + + + + def getCollectionTableBody(self, traitList=None, formName=None, species=''): + + tblobj_body = [] + + className = "fs12 fwn b1 c222" + + for thisTrait in traitList: + tr = [] + + if not thisTrait.haveinfo: + thisTrait.retrieveInfo(QTL=1) + + trId = str(thisTrait) + + oneRadioName = thisTrait.getName() + + tr.append(TDCell( HT.TD(' ',align="center",valign="center",Class=className) )) + tr.append(TDCell( HT.TD(HT.Input(type="radio", name=oneRadioName, value="primary"),align="center",valign="center",Class=className) )) + tr.append(TDCell( HT.TD(HT.Input(type="radio", name=oneRadioName, value="control"),align="center",valign="center",Class=className) )) + tr.append(TDCell( HT.TD(HT.Input(type="radio", name=oneRadioName, value="target", checked="true"),align="center",valign="center",Class=className) )) + tr.append(TDCell( HT.TD(HT.Input(type="radio", name=oneRadioName, value="ignored"),align="center",valign="center",Class=className) )) + + tr.append(TDCell(HT.TD(thisTrait.db.name, Class="fs12 fwn b1 c222"), thisTrait.db.name, thisTrait.db.name.upper())) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showDatabase3('%s','%s','%s','')" % (formName, thisTrait.db.name, thisTrait.name), Class="fs12 fwn"), nowrap="yes",align="left", Class=className +),str(thisTrait.name), thisTrait.name)) + + #description column + if (thisTrait.db.type == "Publish"): + PhenotypeString = thisTrait.post_publication_description + if thisTrait.confidential: + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users): + PhenotypeString = thisTrait.pre_publication_description + tr.append(TDCell(HT.TD(PhenotypeString, Class=className), PhenotypeString, PhenotypeString.upper())) + elif (thisTrait.db.type == "ProbeSet" or thisTrait.db.type == "Temp"): + description_string = str(thisTrait.description).strip() + if (thisTrait.db.type == "ProbeSet"): + target_string = str(thisTrait.probe_target_description).strip() + + description_display = '' + + if len(description_string) > 1 and description_string != 'None': + description_display = description_string + else: + description_display = thisTrait.symbol + + if len(description_display) > 1 and description_display != 'N/A' and len(target_string) > 1 and target_string != 'None': + description_display = description_display + '; ' + target_string.strip() + + description_string = description_display + + tr.append(TDCell(HT.TD(description_string, Class=className), description_string, description_string)) + else: + tr.append(TDCell(HT.TD("--", align="left", Class=className), "--", "Zz")) + + #location column + if (thisTrait.db.type == "Publish"): + tr.append(TDCell(HT.TD("--", align="left", Class=className), "--", "Zz")) + else: + #ZS: trait_location_value is used for sorting + trait_location_repr = "N/A" + trait_location_value = 1000000 + + if hasattr(thisTrait, 'chr') and hasattr(thisTrait, 'mb') and thisTrait.chr and thisTrait.mb: + try: + trait_location_value = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if thisTrait.chr.upper() == "X": + trait_location_value = 20*1000 + thisTrait.mb + else: + trait_location_value = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + trait_location_repr = "Chr%s: %.6f" % (thisTrait.chr, float(thisTrait.mb) ) + + tr.append( TDCell(HT.TD(trait_location_repr, nowrap="yes", Class=className), trait_location_repr, trait_location_value) ) + + if (thisTrait.db.type == "ProbeSet"): + self.cursor.execute(""" + select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet + where ProbeSetXRef.ProbeSetFreezeId = %d and + ProbeSet.Id = ProbeSetXRef.ProbeSetId and + ProbeSet.Name = '%s' + """ % (thisTrait.db.id, thisTrait.name)) + result = self.cursor.fetchone() + if result: + if result[0]: + mean = result[0] + else: + mean=0 + else: + mean = 0 + + #XZ, 06/05/2009: It is neccessary to turn on nowrap + repr = "%2.3f" % mean + tr.append(TDCell(HT.TD(repr, Class=className, align='right', nowrap='ON'),repr, mean)) + + elif (thisTrait.db.type == "Publish"): + self.cursor.execute(""" + select count(PublishData.value), sum(PublishData.value) from PublishData, PublishXRef, PublishFreeze + where PublishData.Id = PublishXRef.DataId and + PublishXRef.Id = %s and + PublishXRef.InbredSetId = PublishFreeze.InbredSetId and + PublishFreeze.Id = %d + """ % (thisTrait.name, thisTrait.db.id)) + result = self.cursor.fetchone() + + if result: + if result[0] and result[1]: + mean = result[1]/result[0] + else: + mean = 0 + else: + mean = 0 + + repr = "%2.3f" % mean + tr.append(TDCell(HT.TD(repr, Class=className, align='right', nowrap='ON'),repr, mean)) + else: + tr.append(TDCell(HT.TD("--", Class=className, align='left', nowrap='ON'),"--", 0)) + + #Number of cases + n_cases_value = 0 + n_cases_repr = "--" + if (thisTrait.db.type == "Publish"): + self.cursor.execute(""" + select count(PublishData.value) from PublishData, PublishXRef, PublishFreeze + where PublishData.Id = PublishXRef.DataId and + PublishXRef.Id = %s and + PublishXRef.InbredSetId = PublishFreeze.InbredSetId and + PublishFreeze.Id = %d + """ % (thisTrait.name, thisTrait.db.id)) + result = self.cursor.fetchone() + + if result: + if result[0]: + n_cases_value = result[0] + n_cases_repr = result[0] + + if (n_cases_value == "--"): + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='left', nowrap="on"), n_cases_repr, n_cases_value)) + else: + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='right', nowrap="on"), n_cases_repr, n_cases_value)) + + elif (thisTrait.db.type == "ProbeSet"): + self.cursor.execute(""" + select count(ProbeSetData.value) from ProbeSet, ProbeSetXRef, ProbeSetData, ProbeSetFreeze + where ProbeSet.Name='%s' and + ProbeSetXRef.ProbeSetId = ProbeSet.Id and + ProbeSetXRef.DataId = ProbeSetData.Id and + ProbeSetXRef.ProbeSetFreezeId = ProbeSetFreeze.Id and + ProbeSetFreeze.Name = '%s' + """ % (thisTrait.name, thisTrait.db.name)) + result = self.cursor.fetchone() + + if result: + if result[0]: + n_cases_value = result[0] + n_cases_repr = result[0] + if (n_cases_value == "--"): + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='left', nowrap="on"), n_cases_repr, n_cases_value)) + else: + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='right', nowrap="on"), n_cases_repr, n_cases_value)) + + elif (thisTrait.db.type == "Geno"): + self.cursor.execute(""" + select count(GenoData.value) from GenoData, GenoXRef, GenoFreeze, Geno, Strain + where Geno.SpeciesId = %s and Geno.Name='%s' and + GenoXRef.GenoId = Geno.Id and + GenoXRef.DataId = GenoData.Id and + GenoXRef.GenoFreezeId = GenoFreeze.Id and + GenoData.StrainId = Strain.Id and + GenoFreeze.Name = '%s' + """ % (webqtlDatabaseFunction.retrieveSpeciesId(self.cursor, thisTrait.db.riset), thisTrait.name, thisTrait.db.name)) + result = self.cursor.fetchone() + + if result: + if result[0]: + n_cases_value = result[0] + n_cases_repr = result[0] + if (n_cases_value == "--"): + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='left', nowrap="on"), n_cases_repr, n_cases_value)) + else: + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='right', nowrap="on"), n_cases_repr, n_cases_value)) + + else: + tr.append(TDCell(HT.TD(n_cases_repr, Class=className, align='left', nowrap="on"), n_cases_repr, n_cases_value)) + + + if (thisTrait.db.type != "Geno"): + #LRS and its location + LRS_score_repr = '--' + LRS_score_value = 0 + LRS_location_repr = '--' + LRS_location_value = 1000000 + LRS_flag = 1 + + #Max LRS and its Locus location + if hasattr(thisTrait, 'lrs') and hasattr(thisTrait, 'locus') and thisTrait.lrs and thisTrait.locus: + self.cursor.execute(""" + select Geno.Chr, Geno.Mb from Geno, Species + where Species.Name = '%s' and + Geno.Name = '%s' and + Geno.SpeciesId = Species.Id + """ % (species, thisTrait.locus)) + result = self.cursor.fetchone() + + if result: + if result[0] and result[1]: + LRS_Chr = result[0] + LRS_Mb = result[1] + + #XZ: LRS_location_value is used for sorting + try: + LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) + except: + if LRS_Chr.upper() == 'X': + LRS_location_value = 20*1000 + float(LRS_Mb) + else: + LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) + + + LRS_score_repr = '%3.1f' % thisTrait.lrs + LRS_score_value = thisTrait.lrs + LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb) ) + LRS_flag = 0 + + tr.append(TDCell(HT.TD(LRS_score_repr, Class=className, align='right', nowrap="on"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class=className), LRS_location_repr, LRS_location_value)) + + if LRS_flag: + tr.append(TDCell(HT.TD(LRS_score_repr, Class=className), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class=className), LRS_location_repr, LRS_location_value)) + else: + tr.append(TDCell(HT.TD("--", align="left", Class=className), "--", 0)) + tr.append(TDCell(HT.TD("--", align="left", Class=className), "--", 1000000)) + + tblobj_body.append(tr) + + return tblobj_body + + + + def getSortByValue(self): + + sortby = ("pv", "up") + + return sortby + diff --git a/web/webqtl/correlation/PartialCorrTraitPage.py b/web/webqtl/correlation/PartialCorrTraitPage.py new file mode 100755 index 00000000..1c79e250 --- /dev/null +++ b/web/webqtl/correlation/PartialCorrTraitPage.py @@ -0,0 +1,310 @@ +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by GeneNetwork Core Team 2010/10/20 + +import string +import cPickle +import os + +from htmlgen import HTMLgen2 as HT + +from base import webqtlConfig +from utility.THCell import THCell +from utility.TDCell import TDCell +from base.webqtlTrait import webqtlTrait +from base.templatePage import templatePage +from utility import webqtlUtil +from CorrelationPage import CorrelationPage +import correlationFunction + + + +class PartialCorrTraitPage(CorrelationPage): + + corrMinInformative = 4 + + + def __init__(self,fd): + + templatePage.__init__(self, fd) + + if not self.openMysql(): + return + + TD_LR = HT.TD(colspan=2,height=200,width="100%",bgColor='#eeeeee') + + TD_LR.append(HT.Paragraph("Partial Correlation Table", Class="title"), '\n') + + pc_method = fd.formdata.getvalue('pcMethod') + + primaryTraitString = fd.formdata.getvalue('primaryTrait') + primaryTrait = (webqtlTrait(fullname=primaryTraitString, cursor=self.cursor)) + + controlTraitsString = fd.formdata.getvalue('controlTraits') + controlTraitsList = list(string.split(controlTraitsString,',')) + controlTraits = [] + for item in controlTraitsList: + controlTraits.append(webqtlTrait(fullname=item, cursor=self.cursor)) + + targetTraitsString = fd.formdata.getvalue('targetTraits') + targetTraitsList = list(string.split(targetTraitsString,',')) + targetTraits = [] + _targetnames = [] + for item in targetTraitsList: + oneTargetTrait = webqtlTrait(fullname=item, cursor=self.cursor) + oneTargetTrait.retrieveInfo() + targetTraits.append( oneTargetTrait ) + _targetnames.append( oneTargetTrait.name ) + + #XZ: filter out the strains that have no value. + primaryTrait.retrieveData() + _strains, _vals, _vars = primaryTrait.exportInformative() + + #XZ: _controlstrains, _controlvals and _controlvars are list of list [ [], [], ...]. _controlNs is number + _controlstrains,_controlvals,_controlvars,_controlNs = correlationFunction.controlStrains(controlTraitsString,_strains) + + N = len(_strains) + + allsame = True + ##allsame is boolean for whether or not primary and control trait have values for the same strains + for i in _controlstrains: + if _strains != i: + allsame=False + break + + ## If the strains for which each of the control traits and the primary trait have values are not identical, + ## we must remove from the calculation all vlaues for strains that are not present in each. Without doing this, + ## undesirable biases would be introduced. + # XZ, 01/11/2010: After execution of function fixStrains, variables _vals,_controlvals,_vars,_controlvars have the same number and same order of strains as strains in variable _strains. The _controlstrains remains intact. + if not allsame: + _strains,_vals,_controlvals,_vars,_controlvars = correlationFunction.fixStrains(_strains,_controlstrains,_vals,_controlvals,_vars,_controlvars) + N = len(_strains) + + #XZ: We should check the value of control trait and primary trait here. + nameOfIdenticalTraits = correlationFunction.findIdenticalTraits ( _vals, primaryTraitString, _controlvals, controlTraitsList ) + if nameOfIdenticalTraits: + heading = "Partial Correlation Table" + detail = ['%s and %s have same values for the %s strains that will be used to calculate partial correlation (common for all primary and control traits). In such case, partial correlation can NOT be calculated. Please re-select your traits.' % (nameOfIdenticalTraits[0], nameOfIdenticalTraits[1], len(_vals))] + self.error(heading=heading,detail=detail) + return + + + if N < self.corrMinInformative: + heading = "Partial Correlation Table" + detail = ['Fewer than %d strain data were entered for %s data set. No calculation of correlation has been attempted.' % (self.corrMinInformative, fd.RISet)] + self.error(heading=heading,detail=detail) + return + + #XZ, 01/11/2010: Pay attention to the target trait strain number and order! + #XZ 03/29/2010: need to input target trait values to this function. + + _targetvals = [] + for oneTargetTrait in targetTraits: + oneTargetTrait.retrieveData() + oneTraitVals = oneTargetTrait.exportData( _strains ) + _targetvals.append(oneTraitVals) + + + if pc_method == 'spearman': + allcorrelations = correlationFunction.determinePartialsByR(primaryVal = _vals, controlVals = _controlvals, targetVals = _targetvals, targetNames = _targetnames, method='s') + else: + allcorrelations = correlationFunction.determinePartialsByR(primaryVal = _vals, controlVals = _controlvals, targetVals = _targetvals, targetNames = _targetnames) + + totalTraits = len(allcorrelations) + + + info_form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase', submit=HT.Input(type='hidden')) + + hddn = {'FormID':'showDatabase', 'database':'_', 'ProbeSetID':'_', 'CellID':'_' }#XZ: These four parameters are required by javascript function showDatabase2. + + for key in hddn.keys(): + info_form.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + info_form.append(HT.Paragraph("Primary Trait", Class="subtitle"), '\n') + + primaryTraitTable = HT.TableLite(cellSpacing=4,cellPadding=0,width="90%",border=0) + + descriptionString = primaryTrait.genHTML(dispFromDatabase=1) + if primaryTrait.db.type == 'Publish' and primaryTrait.confidential: + descriptionString = primaryTrait.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=primaryTrait.authorized_users) + primaryTraitTable.append(HT.TR(HT.TD(HT.Href(text='%s' % descriptionString, url="javascript:showDatabase2('%s','%s','%s')" % (primaryTrait.db.name,primaryTrait.name,primaryTrait.cellid), Class="fs12 fwn") ))) + + info_form.append(primaryTraitTable) + + info_form.append(HT.Paragraph("Control Traits", Class="subtitle"), '\n') + + controlTraitsTable = HT.TableLite(cellSpacing=4,cellPadding=0,width="90%",border=0) + + seq = 1 + + ## Generate the listing table for control traits + for thisTrait in controlTraits: + descriptionString = thisTrait.genHTML(dispFromDatabase=1) + if thisTrait.db.type == 'Publish' and thisTrait.confidential: + descriptionString = thisTrait.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users) + controlTraitsTable.append(HT.TR(HT.TD("%d."%seq,align="left", width=10), + HT.TD(HT.Href(text='%s' % descriptionString,url="javascript:showDatabase2('%s','%s','%s')" % (thisTrait.db.name,thisTrait.name,thisTrait.cellid), Class="fs12 fwn") ))) + seq += 1 + + info_form.append(controlTraitsTable) + + + TD_LR.append(info_form) + + + mainfmName = webqtlUtil.genRandStr("fm_") + form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name= mainfmName, submit=HT.Input(type='hidden')) + + hddn = {'FormID':'showDatabase', 'database':'_', 'ProbeSetID':'_', 'CellID':'_' }#XZ: These four parameters are required by javascript function showDatabase2. + + for key in hddn.keys(): + form.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + + filename= webqtlUtil.genRandStr("Corr_") + + tblobj = {} + + if pc_method == 'spearman': + tblobj['header'] = \ + [[THCell(HT.TD('', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Database', Class="fs13 fwb ffl b1 cw cbrb",align='center'), text='db', idx=1), + THCell(HT.TD('Record', Class="fs13 fwb ffl b1 cw cbrb",align='center'), text='id', idx=2), + THCell(HT.TD('Symbol', Class="fs13 fwb ffl b1 cw cbrb"), text='symbol', idx=3), + THCell(HT.TD('Description', Class="fs13 fwb ffl b1 cw cbrb", align='center'), text='desc', idx=4), + THCell(HT.TD('N ', nowrap="on", Class="fs13 fwb ffl b1 cw cbrb"), text='nstr', idx=5), + THCell(HT.TD('Partial rho ', nowrap="on", Class="fs13 fwb ffl b1 cw cbrb"), text='partial_corr', idx=6), + THCell(HT.TD('p(partial rho)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='partial_pv', idx=7), + THCell(HT.TD('rho ', nowrap="on", Class="fs13 fwb ffl b1 cw cbrb"), text='corr', idx=8), + THCell(HT.TD('p(rho)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='pv', idx=9), + THCell(HT.TD('delta rho', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='delta_rho', idx=10)]] + else: + tblobj['header'] = \ + [[THCell(HT.TD('', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Database', Class="fs13 fwb ffl b1 cw cbrb",align='center'), text='db', idx=1), + THCell(HT.TD('Record', Class="fs13 fwb ffl b1 cw cbrb",align='center'), text='id', idx=2), + THCell(HT.TD('Symbol', Class="fs13 fwb ffl b1 cw cbrb"), text='symbol', idx=3), + THCell(HT.TD('Description', Class="fs13 fwb ffl b1 cw cbrb", align='center'), text='desc', idx=4), + THCell(HT.TD('N ', nowrap="on", Class="fs13 fwb ffl b1 cw cbrb"), text='nstr', idx=5), + THCell(HT.TD('Partial r ', nowrap="on", Class="fs13 fwb ffl b1 cw cbrb"), text='partial_corr', idx=6), + THCell(HT.TD('p(partial r)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='partial_pv', idx=7), + THCell(HT.TD('r ', nowrap="on", Class="fs13 fwb ffl b1 cw cbrb"), text='corr', idx=8), + THCell(HT.TD('p(r)', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='pv', idx=9), + THCell(HT.TD('delta r', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), text='delta_r', idx=10)]] + + sortby = ("partial_pv", "up") + + tblobj['body'] = [] + for i, thisTrait in enumerate(targetTraits): + tr = [] + + trId = str(thisTrait) + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.db.fullname,url=webqtlConfig.INFOPAGEHREF % thisTrait.db.name,target="_blank", Class="fs12 fwn"), Class="fs12 fwn ffl b1 c222"), text=thisTrait.db.name, val=thisTrait +.db.name.upper())) + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showDatabase3('%s', '%s', '%s', '%s')" % (mainfmName,thisTrait.db.name,thisTrait.name,thisTrait.cellid), Class="fs12 fwn"), Class="fs12 fwn b1 c222"), text=thisTrait.name, val=thisTrait.name)) + + #XZ: Symbol column + if thisTrait.db.type =="ProbeSet": + if thisTrait.symbol: + tr.append(TDCell(HT.TD(thisTrait.symbol, Class="fs12 fwn ffl b1 c222"), text=thisTrait.symbol, val=thisTrait.symbol.upper())) + else: + tr.append(TDCell(HT.TD('NA', Class="fs12 fwn ffl b1 c222"), text='NA', val='NA')) + elif thisTrait.db.type =="Publish": + AbbreviationString = "--" + if (thisTrait.post_publication_abbreviation != None): + AbbreviationString = thisTrait.post_publication_abbreviation + + if thisTrait.confidential: + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users): + if thisTrait.pre_publication_abbreviation: + AbbreviationString = thisTrait.pre_publication_abbreviation + else: + AbbreviationString = "--" + + if AbbreviationString == "--": + tr.append(TDCell(HT.TD('NA', Class="fs12 fwn ffl b1 c222"), text='NA', val='NA')) + else: + tr.append(TDCell(HT.TD(AbbreviationString, Class="fs12 fwn ffl b1 c222"), text=AbbreviationString, val=AbbreviationString.upper())) + else: + tr.append(TDCell(HT.TD(thisTrait.name, Class="fs12 fwn ffl b1 c222"), text=thisTrait.name, val=thisTrait.name)) + + #XZ: Description column + if thisTrait.db.type =="ProbeSet" or thisTrait.db.type == "Temp": + tr.append(TDCell(HT.TD(thisTrait.description, Class="fs12 fwn ffl b1 c222"), text=thisTrait.description, val=thisTrait.description.upper())) + elif thisTrait.db.type =="Publish": + PhenotypeString = thisTrait.post_publication_description + if thisTrait.confidential: + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users): + PhenotypeString = thisTrait.pre_publication_description + tr.append(TDCell(HT.TD(PhenotypeString, Class="fs12 fwn ffl b1 c222"), text=PhenotypeString, val=PhenotypeString.upper())) + else: + tr.append(TDCell(HT.TD(thisTrait.name, Class="fs12 fwn ffl b1 c222"), text=thisTrait.name, val=thisTrait.name)) + + tr.append(TDCell(HT.TD(allcorrelations[i][1], Class="fs12 fwn ffl b1 c222", align='right'), text=allcorrelations[i][1], val=allcorrelations[i][1])) + + #partial correlation result + try: + repr = '%3.3f' % float(allcorrelations[i][2]) + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'), text=repr, val=abs(allcorrelations[i][2]))) + except: + repr = 'NA' + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='left'), text=repr, val=0 )) + + repr = webqtlUtil.SciFloat(allcorrelations[i][3]) + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", nowrap='ON', align='right'), text=repr, val=allcorrelations[i][3])) + + #zero order correlation result + repr = '%3.3f' % float(allcorrelations[i][4]) + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'), text=repr, val=abs(allcorrelations[i][4]))) + + repr = webqtlUtil.SciFloat(allcorrelations[i][5]) + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", nowrap='ON', align='right'), text=repr, val=allcorrelations[i][5])) + + #delta + try: + repr = '%3.3f' % ( float(allcorrelations[i][2]) - float(allcorrelations[i][4]) ) + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'), text=repr, val=repr )) + except: + repr = 'NA' + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='left'), text=repr, val=0 )) + + tblobj['body'].append(tr) + + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), Id="sortable") + form.append(div) + + + TD_LR.append(HT.Center(form),HT.P()) + + self.dict['body'] = str(TD_LR) + # updated by NL, moved js function xmlhttpPost() and updatepage() to dhtml.js + self.dict['js1'] = '' + self.dict['title'] = 'Partial Correlation Result' + diff --git a/web/webqtl/correlation/PlotCorrelationPage.py b/web/webqtl/correlation/PlotCorrelationPage.py new file mode 100755 index 00000000..23d2ccde --- /dev/null +++ b/web/webqtl/correlation/PlotCorrelationPage.py @@ -0,0 +1,683 @@ +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by Ning Liu 2011/01/11 + +import string +import piddle as pid +import os + +from htmlgen import HTMLgen2 as HT + +from utility import svg #Code using this module currently commented out +from utility import Plot +from base.webqtlTrait import webqtlTrait +from base.templatePage import templatePage +from utility import webqtlUtil +from base import webqtlConfig +from dbFunction import webqtlDatabaseFunction +from correlation import correlationFunction + +######################################### +# PlotCorrelationPage +######################################### +class PlotCorrelationPage(templatePage): + corrMinInformative = 4 + + def __init__(self, fd): + + templatePage.__init__(self, fd) + + self.initializeDisplayParameters(fd) + + if not fd.genotype: + fd.readGenotype() + + if fd.allstrainlist: + mdpchoice = fd.formdata.getvalue('MDPChoice') + if mdpchoice == "1": + strainlist = fd.f1list + fd.strainlist + elif mdpchoice == "2": + strainlist = [] + strainlist2 = fd.f1list + fd.strainlist + for strain in fd.allstrainlist: + if strain not in strainlist2: + strainlist.append(strain) + #So called MDP Panel + if strainlist: + strainlist = fd.f1list+fd.parlist+strainlist + else: + strainlist = fd.allstrainlist + fd.readData(fd.allstrainlist) + else: + mdpchoice = None + strainlist = fd.strainlist + fd.readData() + + #if fd.allstrainlist: + # fd.readData(fd.allstrainlist) + # strainlist = fd.allstrainlist + #else: + # fd.readData() + # strainlist = fd.strainlist + + + if not self.openMysql(): + return + + isSampleCorr = 0 #XZ: initial value is false + isTissueCorr = 0 #XZ: initial value is false + + #Javascript functions (showCorrelationPlot2, showTissueCorrPlot) have made sure the correlation type is either sample correlation or tissue correlation. + if (self.database and (self.ProbeSetID != 'none')): + isSampleCorr = 1 + elif (self.X_geneSymbol and self.Y_geneSymbol): + isTissueCorr = 1 + else: + heading = "Correlation Type Error" + detail = ["For the input parameters, GN can not recognize the correlation type is sample correlation or tissue correlation."] + self.error(heading=heading,detail=detail) + return + + + TD_LR = HT.TD(colspan=2,height=200,width="100%",bgColor='#eeeeee', align="left", wrap="off") + + + dataX=[] + dataY=[] + dataZ=[] # shortname + fullTissueName=[] + xlabel = '' + ylabel = '' + + if isTissueCorr: + dataX, dataY, xlabel, ylabel, dataZ, fullTissueName = self.getTissueLabelsValues(X_geneSymbol=self.X_geneSymbol, Y_geneSymbol=self.Y_geneSymbol, TissueProbeSetFreezeId=self.TissueProbeSetFreezeId) + plotHeading = HT.Paragraph('Tissue Correlation Scatterplot') + plotHeading.__setattr__("class","title") + + if isSampleCorr: + plotHeading = HT.Paragraph('Sample Correlation Scatterplot') + plotHeading.__setattr__("class","title") + + #XZ: retrieve trait 1 info, Y axis + trait1_data = [] #trait 1 data + trait1Url = '' + + try: + Trait1 = webqtlTrait(db=self.database, name=self.ProbeSetID, cellid=self.CellID, cursor=self.cursor) + Trait1.retrieveInfo() + Trait1.retrieveData() + except: + heading = "Retrieve Data" + detail = ["The database you just requested has not been established yet."] + self.error(heading=heading,detail=detail) + return + + trait1_data = Trait1.exportData(strainlist) + if Trait1.db.type == 'Publish' and Trait1.confidential: + trait1Url = Trait1.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=Trait1.authorized_users) + else: + trait1Url = Trait1.genHTML(dispFromDatabase=1) + ylabel = '%s : %s' % (Trait1.db.shortname, Trait1.name) + if Trait1.cellid: + ylabel += ' : ' + Trait1.cellid + + + #XZ, retrieve trait 2 info, X axis + traitdata2 = [] #trait 2 data + _vals = [] #trait 2 data + trait2Url = '' + + if ( self.database2 and (self.ProbeSetID2 != 'none') ): + try: + Trait2 = webqtlTrait(db=self.database2, name=self.ProbeSetID2, cellid=self.CellID2, cursor=self.cursor) + Trait2.retrieveInfo() + Trait2.retrieveData() + except: + heading = "Retrieve Data" + detail = ["The database you just requested has not been established yet."] + self.error(heading=heading,detail=detail) + return + + if Trait2.db.type == 'Publish' and Trait2.confidential: + trait2Url = Trait2.genHTML(dispFromDatabase=1, privilege=self.privilege, userName=self.userName, authorized_users=Trait2.authorized_users) + else: + trait2Url = Trait2.genHTML(dispFromDatabase=1) + traitdata2 = Trait2.exportData(strainlist) + _vals = traitdata2[:] + xlabel = '%s : %s' % (Trait2.db.shortname, Trait2.name) + if Trait2.cellid: + xlabel += ' : ' + Trait2.cellid + else: + for item in strainlist: + if fd.allTraitData.has_key(item): + _vals.append(fd.allTraitData[item].val) + else: + _vals.append(None) + + if fd.identification: + xlabel = fd.identification + else: + xlabel = "User Input Data" + + try: + Trait2 = webqtlTrait(fullname=fd.formdata.getvalue('fullname'), cursor=self.cursor) + trait2Url = Trait2.genHTML(dispFromDatabase=1) + except: + trait2Url = xlabel + + if (_vals and trait1_data): + if len(_vals) != len(trait1_data): + errors = HT.Blockquote(HT.Font('Error: ',color='red'),HT.Font('The number of traits are inconsistent, Program quit',color='black')) + errors.__setattr__("class","subtitle") + TD_LR.append(errors) + self.dict['body'] = str(TD_LR) + return + + for i in range(len(_vals)): + if _vals[i]!= None and trait1_data[i]!= None: + dataX.append(_vals[i]) + dataY.append(trait1_data[i]) + strainName = strainlist[i] + if self.showstrains: + dataZ.append(webqtlUtil.genShortStrainName(RISet=fd.RISet, input_strainName=strainName)) + else: + heading = "Correlation Plot" + detail = ['Empty Dataset for sample correlation, please check your data.'] + self.error(heading=heading,detail=detail) + return + + + #XZ: We have gotten all data for both traits. + if len(dataX) >= self.corrMinInformative: + + if self.rankOrder == 0: + rankPrimary = 0 + rankSecondary = 1 + else: + rankPrimary = 1 + rankSecondary = 0 + + lineColor = self.setLineColor(); + symbolColor = self.setSymbolColor(); + idColor = self.setIdColor(); + + c = pid.PILCanvas(size=(self.plotSize, self.plotSize*0.90)) + data_coordinate = Plot.plotXY(canvas=c, dataX=dataX, dataY=dataY, rank=rankPrimary, dataLabel = dataZ, labelColor=pid.black, lineSize=self.lineSize, lineColor=lineColor, idColor=idColor, idFont=self.idFont, idSize=self.idSize, symbolColor=symbolColor, symbolType=self.symbol, filled=self.filled, symbolSize=self.symbolSize, XLabel=xlabel, connectdot=0, YLabel=ylabel, title='', fitcurve=self.showline, displayR =1, offset= (90, self.plotSize/20, self.plotSize/10, 90), showLabel = self.showIdentifiers) + + if rankPrimary == 1: + dataXlabel, dataYlabel = webqtlUtil.calRank(xVals=dataX, yVals=dataY, N=len(dataX)) + else: + dataXlabel, dataYlabel = dataX, dataY + + gifmap1 = HT.Map(name='CorrelationPlotImageMap1') + + for i, item in enumerate(data_coordinate): + one_rect_coordinate = "%d, %d, %d, %d" % (item[0] - 5, item[1] - 5, item[0] + 5, item[1] + 5) + if isTissueCorr: + one_rect_title = "%s (%s, %s)" % (fullTissueName[i], dataXlabel[i], dataYlabel[i]) + else: + one_rect_title = "%s (%s, %s)" % (dataZ[i], dataXlabel[i], dataYlabel[i]) + gifmap1.areas.append(HT.Area(shape='rect',coords=one_rect_coordinate, title=one_rect_title) ) + + filename= webqtlUtil.genRandStr("XY_") + c.save(webqtlConfig.IMGDIR+filename, format='gif') + img1=HT.Image('/image/'+filename+'.gif',border=0, usemap='#CorrelationPlotImageMap1') + + mainForm_1 = HT.Form( cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase', submit=HT.Input(type='hidden')) + hddn = {'FormID':'showDatabase','ProbeSetID':'_','database':'_','CellID':'_','RISet':fd.RISet, 'ProbeSetID2':'_', 'database2':'_', 'CellID2':'_', 'allstrainlist':string.join(fd.strainlist, " "), 'traitList': fd.formdata.getvalue("traitList")} + if fd.incparentsf1: + hddn['incparentsf1'] = 'ON' + for key in hddn.keys(): + mainForm_1.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + if isSampleCorr: + mainForm_1.append(HT.P(), HT.Blockquote(HT.Strong('X axis:'),HT.Blockquote(trait2Url),HT.Strong('Y axis:'),HT.Blockquote(trait1Url), style='width: %spx;' % self.plotSize, wrap="hard")) + + graphForm = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), name='MDP_Form',submit=HT.Input(type='hidden')) + graph_hddn = self.setHiddenParameters(fd, rankPrimary) + webqtlUtil.exportData(graph_hddn, fd.allTraitData) #XZ: This is necessary to replot with different groups of strains + + for key in graph_hddn.keys(): + graphForm.append(HT.Input(name=key, value=graph_hddn[key], type='hidden')) + + options = self.createOptionsMenu(fd, mdpchoice) + + if (self.showOptions == '0'): + showOptionsButton = HT.Input(type='button' ,name='optionsButton',value='Hide Options', onClick="showHideOptions();", Class="button") + else: + showOptionsButton = HT.Input(type='button' ,name='optionsButton',value='Show Options', onClick="showHideOptions();", Class="button") + + # updated by NL: 12-07-2011 add variables for tissue abbreviation page + if isTissueCorr: + graphForm.append(HT.Input(name='shortTissueName', value='', type='hidden')) + graphForm.append(HT.Input(name='fullTissueName', value='', type='hidden')) + shortTissueNameStr=string.join(dataZ, ",") + fullTissueNameStr=string.join(fullTissueName, ",") + + tissueAbbrButton=HT.Input(type='button' ,name='tissueAbbrButton',value='Show Abbreviations', onClick="showTissueAbbr('MDP_Form','%s','%s')" % (shortTissueNameStr,fullTissueNameStr), Class="button") + graphForm.append(showOptionsButton,' ',tissueAbbrButton, HT.BR(), HT.BR()) + else: + graphForm.append(showOptionsButton, HT.BR(), HT.BR()) + + graphForm.append(options, HT.BR()) + graphForm.append(HT.HR(), HT.BR(), HT.P()) + + TD_LR.append(plotHeading, HT.BR(),graphForm, HT.BR(), gifmap1, HT.P(), img1, HT.P(), mainForm_1) + TD_LR.append(HT.BR(), HT.HR(color="grey", size=5, width="100%")) + + + + c = pid.PILCanvas(size=(self.plotSize, self.plotSize*0.90)) + data_coordinate = Plot.plotXY(canvas=c, dataX=dataX, dataY=dataY, rank=rankSecondary, dataLabel = dataZ, labelColor=pid.black,lineColor=lineColor, lineSize=self.lineSize, idColor=idColor, idFont=self.idFont, idSize=self.idSize, symbolColor=symbolColor, symbolType=self.symbol, filled=self.filled, symbolSize=self.symbolSize, XLabel=xlabel, connectdot=0, YLabel=ylabel,title='', fitcurve=self.showline, displayR =1, offset= (90, self.plotSize/20, self.plotSize/10, 90), showLabel = self.showIdentifiers) + + if rankSecondary == 1: + dataXlabel, dataYlabel = webqtlUtil.calRank(xVals=dataX, yVals=dataY, N=len(dataX)) + else: + dataXlabel, dataYlabel = dataX, dataY + + gifmap2 = HT.Map(name='CorrelationPlotImageMap2') + + for i, item in enumerate(data_coordinate): + one_rect_coordinate = "%d, %d, %d, %d" % (item[0] - 6, item[1] - 6, item[0] + 6, item[1] + 6) + if isTissueCorr: + one_rect_title = "%s (%s, %s)" % (fullTissueName[i], dataXlabel[i], dataYlabel[i]) + else: + one_rect_title = "%s (%s, %s)" % (dataZ[i], dataXlabel[i], dataYlabel[i]) + + gifmap2.areas.append(HT.Area(shape='rect',coords=one_rect_coordinate, title=one_rect_title) ) + + filename= webqtlUtil.genRandStr("XY_") + c.save(webqtlConfig.IMGDIR+filename, format='gif') + img2=HT.Image('/image/'+filename+'.gif',border=0, usemap='#CorrelationPlotImageMap2') + + mainForm_2 = HT.Form( cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name='showDatabase2', submit=HT.Input(type='hidden')) + hddn = {'FormID':'showDatabase2','ProbeSetID':'_','database':'_','CellID':'_','RISet':fd.RISet, 'ProbeSetID2':'_', 'database2':'_', 'CellID2':'_', 'allstrainlist':string.join(fd.strainlist, " "), 'traitList': fd.formdata.getvalue("traitList")} + if fd.incparentsf1: + hddn['incparentsf1'] = 'ON' + for key in hddn.keys(): + mainForm_2.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + if isSampleCorr: + mainForm_2.append(HT.P(), HT.Blockquote(HT.Strong('X axis:'),HT.Blockquote(trait2Url),HT.Strong('Y axis:'),HT.Blockquote(trait1Url), style='width:%spx;' % self.plotSize)) + + + TD_LR.append(HT.BR(), HT.P()) + TD_LR.append('\n', gifmap2, HT.P(), HT.P(), img2, HT.P(), mainForm_2) + + self.dict['body'] = str(TD_LR) + else: + heading = "Correlation Plot" + detail = ['Fewer than %d strain data were entered for %s data set. No statitical analysis has been attempted.' % (self.corrMinInformative, fd.RISet)] + self.error(heading=heading,detail=detail) + return + + + + def initializeDisplayParameters(self, fd): + """ + Initializes all of the PlotCorrelationPage class parameters, + acquiring most values from the formdata (fd) + """ + + rankOrderString = fd.formdata.getvalue('rankOrder') + if rankOrderString == "1": + self.rankOrder = 1 + else: + self.rankOrder = 0 + + self.dict['title'] = 'Correlation X-Y Scatterplot' + focusScript = "onLoad=\"document.getElementsByName('plotSize')[0].focus();\";" + self.dict['js2'] = focusScript + + self.showstrains = fd.formdata.getvalue('ShowStrains') + self.showline = fd.formdata.getvalue('ShowLine') + self.X_geneSymbol = fd.formdata.getvalue('X_geneSymbol','') + self.Y_geneSymbol = fd.formdata.getvalue('Y_geneSymbol','') + self.TissueProbeSetFreezeId = fd.formdata.getvalue('TissueProbeSetFreezeId', '1') + + self.symbolColor = fd.formdata.getvalue('symbolColor', 'black') + self.symbol = fd.formdata.getvalue('symbol', 'circle') + self.filled = fd.formdata.getvalue('filled', 'yes') + self.symbolSize = fd.formdata.getvalue('symbolSize', 'tiny') + self.idColor = fd.formdata.getvalue('idColor', 'blue') + self.idFont = fd.formdata.getvalue('idFont', 'arial') + self.idSize = fd.formdata.getvalue('idSize', '14') + self.lineColor = fd.formdata.getvalue('lineColor', 'grey') + self.lineSize = fd.formdata.getvalue('lineSize', 'medium') + self.showOptions = fd.formdata.getvalue('showOptions', '0') + + try: + self.plotSize = int(fd.formdata.getvalue('plotSize', 900)) + except: + self.plotSize = 900 + try: + self.showIdentifiers = int(fd.formdata.getvalue('showIdentifiers', 1)) + except: + self.showIdentifiers = 1 + + self.database = fd.formdata.getvalue('database') + self.ProbeSetID = fd.formdata.getvalue('ProbeSetID', 'none') + self.CellID = fd.formdata.getvalue('CellID') + + self.database2 = fd.formdata.getvalue('database2') + self.ProbeSetID2 = fd.formdata.getvalue('ProbeSetID2', 'none') + self.CellID2 = fd.formdata.getvalue('CellID2') + + def createOptionsMenu(self, fd, mdpchoice): + """ + Create all the HTML for the options menu; the first if/else statements + determine whether the Div container holding all the other html is visible + or not. + """ + + if (self.showOptions == '0'): + options = HT.Div(name="options", id="options", style="display: none") + self.showOptions = '1' + else: + options = HT.Div(name="options", id="options", style="display: ''") + self.showOptions = '0' + + if self.showIdentifiers: + containerTable = HT.TableLite(cellspacing=1, width=730, height=150, border=1) + else: + containerTable = HT.TableLite(cellspacing=1, width=730, height=130, border=1) + + if self.showIdentifiers: + containerTable = HT.TableLite(cellspacing=1, width=730, height=150, border=1) + else: + containerTable = HT.TableLite(cellspacing=1, width=730, height=130, border=1) + + containerRow = HT.TR() + containerCell = HT.TD(valign="middle", align="center") + + optionsTable = HT.TableLite(Class="collap", cellspacing=2, width=700) + + sizeOptions = HT.TR(align="right") + tagOptions = HT.TR(align="right") + markerOptions = HT.TR(align="right") + lineOptions = HT.TR(align="right") + replot_mdpOptions = HT.TR(align="right") + + sizeOptions.append(HT.TD(HT.Bold("Size: "), " "*1, HT.Input(type='text' ,name='plotSize', value=self.plotSize, style="background-color: #FFFFFF; width: 50px;", onChange="checkWidth();"), align="left")) + + idColorSel = HT.Select(name="idColorSel", onChange="changeIdColor(); submit();", selected=self.idColor) + idColorSel.append(("blue", "blue")) + idColorSel.append(("green", "green")) + idColorSel.append(("red", "red")) + idColorSel.append(("yellow", "yellow")) + idColorSel.append(("white", "white")) + idColorSel.append(("purple", "purple")) + idColorSel.append(("brown", "brown")) + idColorSel.append(("grey", "grey")) + idColorSel.append(("black","black")) + + idFontSel = HT.Select(name="idFontSel", onChange="changeIdFont(); submit();", selected=self.idFont) + idFontSel.append(("Arial", "arial")) + idFontSel.append(("Trebuchet", "trebuc")) + idFontSel.append(("Verdana", "verdana")) + idFontSel.append(("Georgia", "Georgia")) + idFontSel.append(("Courier", "cour")) + + idSizeSel = HT.Select(name="idSizeSel", onChange="changeIdSize(); submit();", selected=self.idSize) + idSizeSel.append(("10", "10")) + idSizeSel.append(("12", "12")) + idSizeSel.append(("14", "14")) + idSizeSel.append(("16", "16")) + idSizeSel.append(("18", "18")) + + if self.showIdentifiers: + tagButton = HT.TD(HT.Input(type='button' ,name='',value=' Hide Tags ',onClick="this.form.showIdentifiers.value=0;submit();", Class="button"), align="right") + else: + tagButton = HT.TD(HT.Input(type='button' ,name='',value=' Show Tags ',onClick="this.form.showIdentifiers.value=1;submit();", Class="button"), align="right") + + tagOptions.append(HT.TD(HT.Text(HT.Bold("Tag Settings: ")), align="left")) + tagOptions.append(HT.TD(HT.Text(text="Font: "), idFontSel)) + tagOptions.append(HT.TD(HT.Text(text="Color: "), idColorSel)) + tagOptions.append(HT.TD(HT.Text(text="Point: "), idSizeSel)) + tagOptions.append(tagButton) + optionsTable.append(sizeOptions, tagOptions) + + if fd.allstrainlist and mdpchoice: + allStrainList = HT.Input(name='allstrainlist', value=string.join(fd.allstrainlist, " "), type='hidden') + mdpChoice = HT.Input(name='MDPChoice', value=mdpchoice, type='hidden') + btn0 = HT.Input(type='button' ,name='',value='All Cases',onClick="this.form.MDPChoice.value=0;submit();", Class="button") + btn1 = HT.Input(type='button' ,name='',value='%s Only' % fd.RISet,onClick="this.form.MDPChoice.value=1;submit();", Class="button") + btn2 = HT.Input(type='button' ,name='',value='MDP Only', onClick="this.form.MDPChoice.value=2;submit();", Class="button") + + + colorSel = HT.Select(name="colorSel", onChange="changeSymbolColor(); submit();", selected=self.symbolColor) + colorSel.append(("red", "red")) + colorSel.append(("green", "green")) + colorSel.append(("blue", "blue")) + colorSel.append(("yellow", "yellow")) + colorSel.append(("purple", "purple")) + colorSel.append(("brown", "brown")) + colorSel.append(("grey", "grey")) + colorSel.append(("black","black")) + + symbolSel = HT.Select(name="symbolSel", onChange="changeSymbol(); submit();", selected=self.symbol) + symbolSel.append(("4-star","4-star")) + symbolSel.append(("3-star","3-star")) + symbolSel.append(("cross", "cross")) + symbolSel.append(("circle","circle")) + symbolSel.append(("diamond", "diamond")) + symbolSel.append(("square", "square")) + symbolSel.append(("vert rect", "vertRect")) + symbolSel.append(("hori rect", "horiRect")) + + sizeSel = HT.Select(name="sizeSel", onChange="changeSize(); submit();", selected=self.symbolSize) + sizeSel.append(("tiny","tiny")) + sizeSel.append(("small","small")) + sizeSel.append(("medium","medium")) + sizeSel.append(("large","large")) + + fillSel = HT.Select(name="fillSel", onChange="changeFilled(); submit();", selected=self.filled) + fillSel.append(("no","no")) + fillSel.append(("yes","yes")) + + lineColorSel = HT.Select(name="lineColorSel", onChange="changeLineColor(); submit();", selected=self.lineColor) + lineColorSel.append(("red", "red")) + lineColorSel.append(("green", "green")) + lineColorSel.append(("blue", "blue")) + lineColorSel.append(("yellow", "yellow")) + lineColorSel.append(("purple", "purple")) + lineColorSel.append(("brown", "brown")) + lineColorSel.append(("grey", "grey")) + lineColorSel.append(("black","black")) + + lineSizeSel = HT.Select(name="lineSizeSel", onChange="changeLineSize(); submit();", selected=self.lineSize) + lineSizeSel.append(("thin", "thin")) + lineSizeSel.append(("medium", "medium")) + lineSizeSel.append(("thick", "thick")) + + + markerOptions.append(HT.TD(HT.Text(HT.Bold("Marker Settings: ")), align="left")) + markerOptions.append(HT.TD(HT.Text(text="Marker: "), symbolSel)) + markerOptions.append(HT.TD(HT.Text(text="Color: "), colorSel)) + markerOptions.append(HT.TD(HT.Text(text="Fill: "), fillSel)) + markerOptions.append(HT.TD(HT.Text(text="Size: "), sizeSel)) + + lineOptions.append(HT.TD(HT.Text(HT.Bold("Line Settings: ")), align="left")) + lineOptions.append(HT.TD(HT.Text(text="Width: "), lineSizeSel)) + lineOptions.append(HT.TD(HT.Text(text="Color: "), lineColorSel)) + + replotButton = HT.Input(type='button', name='', value=' Replot ',onClick="checkWidth(); submit();", Class="button") + + if fd.allstrainlist and mdpchoice: + replot_mdpOptions.append(HT.TD(replotButton, align="left"), HT.TD(allStrainList, mdpChoice, btn0, btn1, btn2, align="center", colspan=3)) + optionsTable.append(markerOptions, lineOptions, HT.TR(HT.TD(HT.BR())), replot_mdpOptions ) + else: + replot_mdpOptions.append(HT.TD(replotButton, align="left")) + optionsTable.append(markerOptions, lineOptions, HT.TR(HT.TD(HT.BR())), replot_mdpOptions) + + containerCell.append(optionsTable) + containerRow.append(containerCell) + containerTable.append(containerRow) + + options.append(containerTable) + + return options + + def setHiddenParameters(self, fd, rankPrimary): + """ + Create the dictionary of hidden form parameters from PlotCorrelationPage's class parameters + """ + + graph_hddn = {'FormID':'showCorrelationPlot','RISet':fd.RISet, 'identification':fd.identification, "incparentsf1":1, "showIdentifiers":self.showIdentifiers} + + if self.database: graph_hddn['database']=self.database + if self.ProbeSetID: graph_hddn['ProbeSetID']=self.ProbeSetID + if self.CellID: graph_hddn['CellID']=self.CellID + if self.database2: graph_hddn['database2']=self.database2 + if self.ProbeSetID2: graph_hddn['ProbeSetID2']=self.ProbeSetID2 + if self.CellID2: graph_hddn['CellID2']=self.CellID2 + if self.showstrains: graph_hddn['ShowStrains']=self.showstrains + if self.showline: graph_hddn['ShowLine']=self.showline + if self.X_geneSymbol: graph_hddn['X_geneSymbol']=self.X_geneSymbol + if self.Y_geneSymbol: graph_hddn['Y_geneSymbol']=self.Y_geneSymbol + if self.TissueProbeSetFreezeId: graph_hddn['TissueProbeSetFreezeId']=self.TissueProbeSetFreezeId + if self.rankOrder: graph_hddn['rankOrder'] = rankPrimary + if fd.formdata.getvalue('fullname'): graph_hddn['fullname']=fd.formdata.getvalue('fullname') + if self.lineColor: graph_hddn['lineColor'] = self.lineColor + if self.lineSize: graph_hddn['lineSize'] = self.lineSize + if self.idColor: graph_hddn['idColor'] = self.idColor + if self.idFont: graph_hddn['idFont'] = self.idFont + if self.idSize: graph_hddn['idSize'] = self.idSize + if self.symbolColor: graph_hddn['symbolColor'] = self.symbolColor + if self.symbol: graph_hddn['symbol'] = self.symbol + if self.filled: graph_hddn['filled'] = self.filled + if self.symbolSize: graph_hddn['symbolSize'] = self.symbolSize + if self.showOptions: graph_hddn['showOptions'] = self.showOptions + + return graph_hddn + + def setIdColor(self): + """ + Set the plot tag/ID color based upon the value of the idColor class parameter + """ + + if self.idColor == 'black': + idColor = pid.black + elif self.idColor == 'white': + idColor = pid.white + elif self.idColor == 'yellow': + idColor = pid.yellow + elif self.idColor == 'grey': + idColor = pid.grey + elif self.idColor == 'blue': + idColor = pid.blue + elif self.idColor == 'purple': + idColor = pid.purple + elif self.idColor == 'brown': + idColor = pid.brown + elif self.idColor == 'green': + idColor = pid.green + else: + idColor = pid.red + + return idColor + + def setSymbolColor(self): + """ + Set the plot symbol color based upon the value of the symbolColor class parameter + """ + + if self.symbolColor == 'black': + symbolColor = pid.black + elif self.symbolColor == 'grey': + symbolColor = pid.grey + elif self.symbolColor == 'yellow': + symbolColor = pid.yellow + elif self.symbolColor == 'blue': + symbolColor = pid.blue + elif self.symbolColor == 'purple': + symbolColor = pid.purple + elif self.symbolColor == 'brown': + symbolColor = pid.brown + elif self.symbolColor== 'green': + symbolColor = pid.green + else: + symbolColor = pid.red + + return symbolColor + + def setLineColor(self): + """ + Set the plot line color based upon the lineColor class parameter + """ + + if self.lineColor == 'black': + lineColor = pid.black + elif self.lineColor == 'grey': + lineColor = pid.grey + elif self.lineColor == 'yellow': + lineColor = pid.yellow + elif self.lineColor == 'blue': + lineColor = pid.blue + elif self.lineColor == 'purple': + lineColor = pid.purple + elif self.lineColor == 'brown': + lineColor = pid.brown + elif self.lineColor== 'green': + lineColor = pid.green + else: + lineColor = pid.red + + return lineColor + + + def getTissueLabelsValues(self, X_geneSymbol=None, Y_geneSymbol=None, TissueProbeSetFreezeId=None ): + + dataX = [] + dataY = [] + data_fullLabel = [] + data_shortLabel = [] + # updated by NL, 2011-01-11 using new function getTissueProbeSetXRefInfo to get dataId value + X_symbolList,X_geneIdDict,X_dataIdDict,X_ChrDict,X_MbDict,X_descDict,X_pTargetDescDict = correlationFunction.getTissueProbeSetXRefInfo(cursor=self.cursor,GeneNameLst=[X_geneSymbol],TissueProbeSetFreezeId=TissueProbeSetFreezeId) + Y_symbolList,Y_geneIdDict,Y_dataIdDict,Y_ChrDict,Y_MbDict,Y_descDict,Y_pTargetDescDict = correlationFunction.getTissueProbeSetXRefInfo(cursor=self.cursor,GeneNameLst=[Y_geneSymbol],TissueProbeSetFreezeId=TissueProbeSetFreezeId) + # in dataIdDict, key is the lower cased geneSymbol + X_DataId = X_dataIdDict[X_geneSymbol.lower()] + Y_DataId = Y_dataIdDict[Y_geneSymbol.lower()] + + self.cursor.execute("SELECT TissueID,value FROM TissueProbeSetData WHERE Id = %d ORDER BY TissueID" % int(X_DataId) ) + results = self.cursor.fetchall() + for item in results: + TissueID, Value = item + dataX.append(Value) + self.cursor.execute("SELECT Tissue.Name, Tissue.Short_Name FROM Tissue WHERE Id = %d" % int(TissueID) ) + temp = self.cursor.fetchone() + data_fullLabel.append( temp[0] ) + data_shortLabel.append( temp[1] ) + + self.cursor.execute("SELECT TissueID,value FROM TissueProbeSetData WHERE Id = %d ORDER BY TissueID" % int(Y_DataId) ) + results = self.cursor.fetchall() + for item in results: + TissueID, Value = item + dataY.append(Value) + + X_label = "%s" % X_geneSymbol + Y_label = "%s" % Y_geneSymbol + + return dataX, dataY, X_label, Y_label, data_shortLabel, data_fullLabel diff --git a/web/webqtl/correlation/__init__.py b/web/webqtl/correlation/__init__.py new file mode 100755 index 00000000..e69de29b --- /dev/null +++ b/web/webqtl/correlation/__init__.py diff --git a/web/webqtl/correlation/correlationFunction.py b/web/webqtl/correlation/correlationFunction.py new file mode 100755 index 00000000..cc19f54e --- /dev/null +++ b/web/webqtl/correlation/correlationFunction.py @@ -0,0 +1,923 @@ +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by NL 2011/03/23 + + +import math +import rpy2.robjects +import pp +import string + +from utility import webqtlUtil +from base.webqtlTrait import webqtlTrait +from dbFunction import webqtlDatabaseFunction + + + +#XZ: The input 'controls' is String. It contains the full name of control traits. +#XZ: The input variable 'strainlst' is List. It contains the strain names of primary trait. +#XZ: The returned tcstrains is the list of list [[],[]...]. So are tcvals and tcvars. The last returned parameter is list of numbers. +#XZ, 03/29/2010: For each returned control trait, there is no None value in it. +def controlStrains(controls, strainlst): + + controls = controls.split(',') + + cvals = {} + for oneTraitName in controls: + oneTrait = webqtlTrait(fullname=oneTraitName, cursor=webqtlDatabaseFunction.getCursor() ) + oneTrait.retrieveData() + cvals[oneTraitName] = oneTrait.data + + tcstrains = [] + tcvals = [] + tcvars = [] + + for oneTraitName in controls: + strains = [] + vals = [] + vars = [] + + for _strain in strainlst: + if cvals[oneTraitName].has_key(_strain): + _val = cvals[oneTraitName][_strain].val + if _val != None: + strains.append(_strain) + vals.append(_val) + vars.append(None) + + tcstrains.append(strains) + tcvals.append(vals) + tcvars.append(vars) + + return tcstrains, tcvals, tcvars, [len(x) for x in tcstrains] + + + +#XZ, 03/29/2010: After execution of functon "controlStrains" and "fixStrains", primary trait and control traits have the same strains and in the same order. There is no 'None' value in them. +def fixStrains(_strains,_controlstrains,_vals,_controlvals,_vars,_controlvars): + """Corrects strains, vals, and vars so that all contrain only those strains common + to the reference trait and all control traits.""" + + def dictify(strains,vals,vars): + subdict = {} + for i in xrange(len(strains)): + subdict[strains[i]] = (vals[i],vars[i]) + return subdict + + #XZ: The 'dicts' is a list of dictionary. The first element is the dictionary of reference trait. The rest elements are for control traits. + dicts = [] + dicts.append(dictify(_strains,_vals,_vars)) + + nCstrains = len(_controlstrains) + for i in xrange(nCstrains): + dicts.append(dictify(_controlstrains[i],_controlvals[i],_controlvars[i])) + + _newstrains = [] + _vals = [] + _vars = [] + _controlvals = [[] for x in xrange(nCstrains)] + _controlvars = [[] for x in xrange(nCstrains)] + + for strain in _strains: + inall = True + for d in dicts: + if strain not in d: + inall = False + break + if inall: + _newstrains.append(strain) + _vals.append(dicts[0][strain][0]) + _vars.append(dicts[0][strain][1]) + for i in xrange(nCstrains): + _controlvals[i].append(dicts[i+1][strain][0]) + _controlvars[i].append(dicts[i+1][strain][1]) + + return _newstrains, _vals, _controlvals, _vars, _controlvars + + +#XZ, 6/15/2010: If there is no identical control traits, the returned list is empty. +#else, the returned list has two elements of control trait name. +def findIdenticalControlTraits ( controlVals, controlNames ): + nameOfIdenticalTraits = [] + + controlTraitNumber = len(controlVals) + + if controlTraitNumber > 1: + + #XZ: reset the precision of values and convert to string type + for oneTraitVal in controlVals: + for oneStrainVal in oneTraitVal: + oneStrainVal = '%.3f' % oneStrainVal + + for i, oneTraitVal in enumerate( controlVals ): + for j in range(i+1, controlTraitNumber): + if oneTraitVal == controlVals[j]: + nameOfIdenticalTraits.append(controlNames[i]) + nameOfIdenticalTraits.append(controlNames[j]) + + return nameOfIdenticalTraits + +#XZ, 6/15/2010: If there is no identical control traits, the returned list is empty. +#else, the returned list has two elements of control trait name. +#primaryVal is of list type. It contains value of primary trait. +#primaryName is of string type. +#controlVals is of list type. Each element is list too. Each element contain value of one control trait. +#controlNames is of list type. +def findIdenticalTraits (primaryVal, primaryName, controlVals, controlNames ): + nameOfIdenticalTraits = [] + + #XZ: reset the precision of values and convert to string type + for oneStrainVal in primaryVal: + oneStrainVal = '%.3f' % oneStrainVal + + for oneTraitVal in controlVals: + for oneStrainVal in oneTraitVal: + oneStrainVal = '%.3f' % oneStrainVal + + controlTraitNumber = len(controlVals) + + if controlTraitNumber > 1: + for i, oneTraitVal in enumerate( controlVals ): + for j in range(i+1, controlTraitNumber): + if oneTraitVal == controlVals[j]: + nameOfIdenticalTraits.append(controlNames[i]) + nameOfIdenticalTraits.append(controlNames[j]) + break + + if len(nameOfIdenticalTraits) == 0: + for i, oneTraitVal in enumerate( controlVals ): + if primaryVal == oneTraitVal: + nameOfIdenticalTraits.append(primaryName) + nameOfIdenticalTraits.append(controlNames[i]) + break + + return nameOfIdenticalTraits + + + +#XZ, 03/29/2010: The strains in primaryVal, controlVals, targetVals must be of the same number and in same order. +#XZ: No value in primaryVal and controlVals could be None. + +def determinePartialsByR (primaryVal, controlVals, targetVals, targetNames, method='p'): + + def compute_partial ( primaryVal, controlVals, targetVals, targetNames, method ): + + rpy2.robjects.r(""" +pcor.test <- function(x,y,z,use="mat",method="p",na.rm=T){ + # The partial correlation coefficient between x and y given z + # + # pcor.test is free and comes with ABSOLUTELY NO WARRANTY. + # + # x and y should be vectors + # + # z can be either a vector or a matrix + # + # use: There are two methods to calculate the partial correlation coefficient. + # One is by using variance-covariance matrix ("mat") and the other is by using recursive formula ("rec"). + # Default is "mat". + # + # method: There are three ways to calculate the correlation coefficient, + # which are Pearson's ("p"), Spearman's ("s"), and Kendall's ("k") methods. + # The last two methods which are Spearman's and Kendall's coefficient are based on the non-parametric analysis. + # Default is "p". + # + # na.rm: If na.rm is T, then all the missing samples are deleted from the whole dataset, which is (x,y,z). + # If not, the missing samples will be removed just when the correlation coefficient is calculated. + # However, the number of samples for the p-value is the number of samples after removing + # all the missing samples from the whole dataset. + # Default is "T". + + x <- c(x) + y <- c(y) + z <- as.data.frame(z) + + if(use == "mat"){ + p.use <- "Var-Cov matrix" + pcor = pcor.mat(x,y,z,method=method,na.rm=na.rm) + }else if(use == "rec"){ + p.use <- "Recursive formula" + pcor = pcor.rec(x,y,z,method=method,na.rm=na.rm) + }else{ + stop("use should be either rec or mat!\n") + } + + # print the method + if(gregexpr("p",method)[[1]][1] == 1){ + p.method <- "Pearson" + }else if(gregexpr("s",method)[[1]][1] == 1){ + p.method <- "Spearman" + }else if(gregexpr("k",method)[[1]][1] == 1){ + p.method <- "Kendall" + }else{ + stop("method should be pearson or spearman or kendall!\n") + } + + # sample number + n <- dim(na.omit(data.frame(x,y,z)))[1] + + # given variables' number + gn <- dim(z)[2] + + # p-value + if(p.method == "Kendall"){ + statistic <- pcor/sqrt(2*(2*(n-gn)+5)/(9*(n-gn)*(n-1-gn))) + p.value <- 2*pnorm(-abs(statistic)) + + }else{ + statistic <- pcor*sqrt((n-2-gn)/(1-pcor^2)) + p.value <- 2*pnorm(-abs(statistic)) + } + + data.frame(estimate=pcor,p.value=p.value,statistic=statistic,n=n,gn=gn,Method=p.method,Use=p.use) +} + +# By using var-cov matrix +pcor.mat <- function(x,y,z,method="p",na.rm=T){ + + x <- c(x) + y <- c(y) + z <- as.data.frame(z) + + if(dim(z)[2] == 0){ + stop("There should be given data\n") + } + + data <- data.frame(x,y,z) + + if(na.rm == T){ + data = na.omit(data) + } + + xdata <- na.omit(data.frame(data[,c(1,2)])) + Sxx <- cov(xdata,xdata,m=method) + + xzdata <- na.omit(data) + xdata <- data.frame(xzdata[,c(1,2)]) + zdata <- data.frame(xzdata[,-c(1,2)]) + Sxz <- cov(xdata,zdata,m=method) + + zdata <- na.omit(data.frame(data[,-c(1,2)])) + Szz <- cov(zdata,zdata,m=method) + + # is Szz positive definite? + zz.ev <- eigen(Szz)$values + if(min(zz.ev)[1]<0){ + stop("\'Szz\' is not positive definite!\n") + } + + # partial correlation + Sxx.z <- Sxx - Sxz %*% solve(Szz) %*% t(Sxz) + + rxx.z <- cov2cor(Sxx.z)[1,2] + + rxx.z +} + +# By using recursive formula +pcor.rec <- function(x,y,z,method="p",na.rm=T){ + # + + x <- c(x) + y <- c(y) + z <- as.data.frame(z) + + if(dim(z)[2] == 0){ + stop("There should be given data\n") + } + + data <- data.frame(x,y,z) + + if(na.rm == T){ + data = na.omit(data) + } + + # recursive formula + if(dim(z)[2] == 1){ + tdata <- na.omit(data.frame(data[,1],data[,2])) + rxy <- cor(tdata[,1],tdata[,2],m=method) + + tdata <- na.omit(data.frame(data[,1],data[,-c(1,2)])) + rxz <- cor(tdata[,1],tdata[,2],m=method) + + tdata <- na.omit(data.frame(data[,2],data[,-c(1,2)])) + ryz <- cor(tdata[,1],tdata[,2],m=method) + + rxy.z <- (rxy - rxz*ryz)/( sqrt(1-rxz^2)*sqrt(1-ryz^2) ) + + return(rxy.z) + }else{ + x <- c(data[,1]) + y <- c(data[,2]) + z0 <- c(data[,3]) + zc <- as.data.frame(data[,-c(1,2,3)]) + + rxy.zc <- pcor.rec(x,y,zc,method=method,na.rm=na.rm) + rxz0.zc <- pcor.rec(x,z0,zc,method=method,na.rm=na.rm) + ryz0.zc <- pcor.rec(y,z0,zc,method=method,na.rm=na.rm) + + rxy.z <- (rxy.zc - rxz0.zc*ryz0.zc)/( sqrt(1-rxz0.zc^2)*sqrt(1-ryz0.zc^2) ) + return(rxy.z) + } +} +""") + + R_pcorr_function = rpy2.robjects.r['pcor.test'] + R_corr_test = rpy2.robjects.r['cor.test'] + + primary = rpy2.robjects.FloatVector(range(len(primaryVal))) + for i in range(len(primaryVal)): + primary[i] = primaryVal[i] + + control = rpy2.robjects.r.matrix(rpy2.robjects.FloatVector( range(len(controlVals)*len(controlVals[0])) ), ncol=len(controlVals)) + for i in range(len(controlVals)): + for j in range(len(controlVals[0])): + control[i*len(controlVals[0]) + j] = controlVals[i][j] + + allcorrelations = [] + + for targetIndex, oneTargetVals in enumerate(targetVals): + + this_primary = None + this_control = None + this_target = None + + if None in oneTargetVals: + + goodIndex = [] + for i in range(len(oneTargetVals)): + if oneTargetVals[i] != None: + goodIndex.append(i) + + this_primary = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + this_primary[i] = primaryVal[goodIndex[i]] + + this_control = rpy2.robjects.r.matrix(rpy2.robjects.FloatVector( range(len(controlVals)*len(goodIndex)) ), ncol=len(controlVals)) + for i in range(len(controlVals)): + for j in range(len(goodIndex)): + this_control[i*len(goodIndex) + j] = controlVals[i][goodIndex[j]] + + this_target = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + this_target[i] = oneTargetVals[goodIndex[i]] + + else: + this_primary = primary + this_control = control + this_target = rpy2.robjects.FloatVector(range(len(oneTargetVals))) + for i in range(len(oneTargetVals)): + this_target[i] = oneTargetVals[i] + + one_name = targetNames[targetIndex] + one_N = len(this_primary) + + #calculate partial correlation + one_pc_coefficient = 'NA' + one_pc_p = 1 + + try: + if method == 's': + result = R_pcorr_function(this_primary, this_target, this_control, method='s') + else: + result = R_pcorr_function(this_primary, this_target, this_control) + + #XZ: In very few cases, the returned coefficient is nan. + #XZ: One way to detect nan is to compare the number to itself. NaN is always != NaN + if result[0][0] == result[0][0]: + one_pc_coefficient = result[0][0] + #XZ: when the coefficient value is 1 (primary trait and target trait are the same), + #XZ: occationally, the returned p value is nan instead of 0. + if result[1][0] == result[1][0]: + one_pc_p = result[1][0] + elif abs(one_pc_coefficient - 1) < 0.0000001: + one_pc_p = 0 + except: + pass + + #calculate zero order correlation + one_corr_coefficient = 0 + one_corr_p = 1 + + try: + if method == 's': + R_result = R_corr_test(this_primary, this_target, method='spearman') + else: + R_result = R_corr_test(this_primary, this_target) + + one_corr_coefficient = R_result[3][0] + one_corr_p = R_result[2][0] + except: + pass + + traitinfo = [ one_name, one_N, one_pc_coefficient, one_pc_p, one_corr_coefficient, one_corr_p ] + + allcorrelations.append(traitinfo) + + return allcorrelations + #End of function compute_partial + + + allcorrelations = [] + + target_trait_number = len(targetVals) + + if target_trait_number < 1000: + allcorrelations = compute_partial ( primaryVal, controlVals, targetVals, targetNames, method ) + else: + step = 1000 + job_number = math.ceil( float(target_trait_number)/step ) + + job_targetVals_lists = [] + job_targetNames_lists = [] + + for job_index in range( int(job_number) ): + starti = job_index*step + endi = min((job_index+1)*step, target_trait_number) + + one_job_targetVals_list = [] + one_job_targetNames_list = [] + + for i in range( starti, endi ): + one_job_targetVals_list.append( targetVals[i] ) + one_job_targetNames_list.append( targetNames[i] ) + + job_targetVals_lists.append( one_job_targetVals_list ) + job_targetNames_lists.append( one_job_targetNames_list ) + + ppservers = () + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + + jobs = [] + results = [] + + for i, one_job_targetVals_list in enumerate( job_targetVals_lists ): + one_job_targetNames_list = job_targetNames_lists[i] + #pay attention to modules from outside + jobs.append( job_server.submit(func=compute_partial, args=( primaryVal, controlVals, one_job_targetVals_list, one_job_targetNames_list, method), depfuncs=(), modules=("rpy2.robjects",)) ) + + for one_job in jobs: + one_result = one_job() + results.append( one_result ) + + for one_result in results: + for one_traitinfo in one_result: + allcorrelations.append( one_traitinfo ) + + return allcorrelations + + + +#XZ, April 30, 2010: The input primaryTrait and targetTrait are instance of webqtlTrait +#XZ: The primaryTrait and targetTrait should have executed retrieveData function +def calZeroOrderCorr (primaryTrait, targetTrait, method='pearson'): + + #primaryTrait.retrieveData() + + #there is no None value in primary_val + primary_strain, primary_val, primary_var = primaryTrait.exportInformative() + + #targetTrait.retrieveData() + + #there might be None value in target_val + target_val = targetTrait.exportData(primary_strain, type="val") + + R_primary = rpy2.robjects.FloatVector(range(len(primary_val))) + for i in range(len(primary_val)): + R_primary[i] = primary_val[i] + + N = len(target_val) + + if None in target_val: + goodIndex = [] + for i in range(len(target_val)): + if target_val[i] != None: + goodIndex.append(i) + + N = len(goodIndex) + + R_primary = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + R_primary[i] = primary_val[goodIndex[i]] + + R_target = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + R_target[i] = target_val[goodIndex[i]] + + else: + R_target = rpy2.robjects.FloatVector(range(len(target_val))) + for i in range(len(target_val)): + R_target[i] = target_val[i] + + R_corr_test = rpy2.robjects.r['cor.test'] + + if method == 'spearman': + R_result = R_corr_test(R_primary, R_target, method='spearman') + else: + R_result = R_corr_test(R_primary, R_target) + + corr_result = [] + corr_result.append( R_result[3][0] ) + corr_result.append( N ) + corr_result.append( R_result[2][0] ) + + return corr_result + +##################################################################################### +#Input: primaryValue(list): one list of expression values of one probeSet, +# targetValue(list): one list of expression values of one probeSet, +# method(string): indicate correlation method ('pearson' or 'spearman') +#Output: corr_result(list): first item is Correlation Value, second item is tissue number, +# third item is PValue +#Function: get correlation value,Tissue quantity ,p value result by using R; +#Note : This function is special case since both primaryValue and targetValue are from +#the same dataset. So the length of these two parameters is the same. They are pairs. +#Also, in the datatable TissueProbeSetData, all Tissue values are loaded based on +#the same tissue order +##################################################################################### + +def calZeroOrderCorrForTiss (primaryValue=[], targetValue=[], method='pearson'): + + R_primary = rpy2.robjects.FloatVector(range(len(primaryValue))) + N = len(primaryValue) + for i in range(len(primaryValue)): + R_primary[i] = primaryValue[i] + + R_target = rpy2.robjects.FloatVector(range(len(targetValue))) + for i in range(len(targetValue)): + R_target[i]=targetValue[i] + + R_corr_test = rpy2.robjects.r['cor.test'] + if method =='spearman': + R_result = R_corr_test(R_primary, R_target, method='spearman') + else: + R_result = R_corr_test(R_primary, R_target) + + corr_result =[] + corr_result.append( R_result[3][0]) + corr_result.append( N ) + corr_result.append( R_result[2][0]) + + return corr_result + + + + +def batchCalTissueCorr(primaryTraitValue=[], SymbolValueDict={}, method='pearson'): + + def cal_tissue_corr(primaryTraitValue, oneSymbolValueDict, method ): + + oneSymbolCorrDict = {} + oneSymbolPvalueDict = {} + + R_corr_test = rpy2.robjects.r['cor.test'] + + R_primary = rpy2.robjects.FloatVector(range(len(primaryTraitValue))) + + for i in range(len(primaryTraitValue)): + R_primary[i] = primaryTraitValue[i] + + for (oneTraitSymbol, oneTraitValue) in oneSymbolValueDict.iteritems(): + R_target = rpy2.robjects.FloatVector(range(len(oneTraitValue))) + for i in range(len(oneTraitValue)): + R_target[i] = oneTraitValue[i] + + if method =='spearman': + R_result = R_corr_test(R_primary, R_target, method='spearman') + else: + R_result = R_corr_test(R_primary, R_target) + + oneSymbolCorrDict[oneTraitSymbol] = R_result[3][0] + oneSymbolPvalueDict[oneTraitSymbol] = R_result[2][0] + + return(oneSymbolCorrDict, oneSymbolPvalueDict) + + + + symbolCorrDict = {} + symbolPvalueDict = {} + + items_number = len(SymbolValueDict) + + if items_number <= 1000: + symbolCorrDict, symbolPvalueDict = cal_tissue_corr(primaryTraitValue, SymbolValueDict, method) + else: + items_list = SymbolValueDict.items() + + step = 1000 + job_number = math.ceil( float(items_number)/step ) + + job_oneSymbolValueDict_list = [] + + for job_index in range( int(job_number) ): + starti = job_index*step + endi = min((job_index+1)*step, items_number) + + oneSymbolValueDict = {} + + for i in range( starti, endi ): + one_item = items_list[i] + one_symbol = one_item[0] + one_value = one_item[1] + oneSymbolValueDict[one_symbol] = one_value + + job_oneSymbolValueDict_list.append( oneSymbolValueDict ) + + + ppservers = () + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + + jobs = [] + results = [] + + for i, oneSymbolValueDict in enumerate( job_oneSymbolValueDict_list ): + + #pay attention to modules from outside + jobs.append( job_server.submit(func=cal_tissue_corr, args=(primaryTraitValue, oneSymbolValueDict, method), depfuncs=(), modules=("rpy2.robjects",)) ) + + for one_job in jobs: + one_result = one_job() + results.append( one_result ) + + for one_result in results: + oneSymbolCorrDict, oneSymbolPvalueDict = one_result + symbolCorrDict.update( oneSymbolCorrDict ) + symbolPvalueDict.update( oneSymbolPvalueDict ) + + return (symbolCorrDict, symbolPvalueDict) + +########################################################################### +#Input: cursor, GeneNameLst (list), TissueProbeSetFreezeId +#output: geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict (Dict) +#function: get multi dicts for short and long label functions, and for getSymbolValuePairDict and +# getGeneSymbolTissueValueDict to build dict to get CorrPvArray +#Note: If there are multiple probesets for one gene, select the one with highest mean. +########################################################################### +def getTissueProbeSetXRefInfo(cursor=None,GeneNameLst=[],TissueProbeSetFreezeId=0): + Symbols ="" + symbolList =[] + geneIdDict ={} + dataIdDict = {} + ChrDict = {} + MbDict = {} + descDict = {} + pTargetDescDict = {} + + count = len(GeneNameLst) + + # Added by NL 01/06/2011 + # Note that:inner join is necessary in this query to get distinct record in one symbol group with highest mean value + # Duo to the limit size of TissueProbeSetFreezeId table in DB, performance of inner join is acceptable. + if count==0: + query=''' + select t.Symbol,t.GeneId, t.DataId,t.Chr, t.Mb,t.description,t.Probe_Target_Description + from ( + select Symbol, max(Mean) as maxmean + from TissueProbeSetXRef + where TissueProbeSetFreezeId=%s and Symbol!='' and Symbol Is Not Null group by Symbol) + as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; + '''%TissueProbeSetFreezeId + + else: + for i, item in enumerate(GeneNameLst): + + if i == count-1: + Symbols += "'%s'" %item + else: + Symbols += "'%s'," %item + + Symbols = "("+ Symbols+")" + query=''' + select t.Symbol,t.GeneId, t.DataId,t.Chr, t.Mb,t.description,t.Probe_Target_Description + from ( + select Symbol, max(Mean) as maxmean + from TissueProbeSetXRef + where TissueProbeSetFreezeId=%s and Symbol in %s group by Symbol) + as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; + '''% (TissueProbeSetFreezeId,Symbols) + + try: + + cursor.execute(query) + results =cursor.fetchall() + resultCount = len(results) + # Key in all dicts is the lower-cased symbol + for i, item in enumerate(results): + symbol = item[0] + symbolList.append(symbol) + + key =symbol.lower() + geneIdDict[key]=item[1] + dataIdDict[key]=item[2] + ChrDict[key]=item[3] + MbDict[key]=item[4] + descDict[key]=item[5] + pTargetDescDict[key]=item[6] + + except: + symbolList = None + geneIdDict=None + dataIdDict=None + ChrDict=None + MbDict=None + descDict=None + pTargetDescDict=None + + return symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict + +########################################################################### +#Input: cursor, symbolList (list), dataIdDict(Dict) +#output: symbolValuepairDict (dictionary):one dictionary of Symbol and Value Pair, +# key is symbol, value is one list of expression values of one probeSet; +#function: get one dictionary whose key is gene symbol and value is tissue expression data (list type). +#Attention! All keys are lower case! +########################################################################### +def getSymbolValuePairDict(cursor=None,symbolList=None,dataIdDict={}): + symbolList = map(string.lower, symbolList) + symbolValuepairDict={} + valueList=[] + + for key in symbolList: + if dataIdDict.has_key(key): + DataId = dataIdDict[key] + + valueQuery = "select value from TissueProbeSetData where Id=%s" % DataId + try : + cursor.execute(valueQuery) + valueResults = cursor.fetchall() + for item in valueResults: + item =item[0] + valueList.append(item) + symbolValuepairDict[key] = valueList + valueList=[] + except: + symbolValuepairDict[key] = None + + return symbolValuepairDict + + +######################################################################################################## +#input: cursor, symbolList (list), dataIdDict(Dict): key is symbol +#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. +# key is symbol, value is one list of expression values of one probeSet. +#function: wrapper function for getSymbolValuePairDict function +# build gene symbol list if necessary, cut it into small lists if necessary, +# then call getSymbolValuePairDict function and merge the results. +######################################################################################################## + +def getGeneSymbolTissueValueDict(cursor=None,symbolList=None,dataIdDict={}): + limitNum=1000 + count = len(symbolList) + + SymbolValuePairDict = {} + + if count !=0 and count <=limitNum: + SymbolValuePairDict = getSymbolValuePairDict(cursor=cursor,symbolList=symbolList,dataIdDict=dataIdDict) + + elif count >limitNum: + SymbolValuePairDict={} + n = count/limitNum + start =0 + stop =0 + + for i in range(n): + stop =limitNum*(i+1) + gList1 = symbolList[start:stop] + PairDict1 = getSymbolValuePairDict(cursor=cursor,symbolList=gList1,dataIdDict=dataIdDict) + start =limitNum*(i+1) + + SymbolValuePairDict.update(PairDict1) + + if stop < count: + stop = count + gList2 = symbolList[start:stop] + PairDict2 = getSymbolValuePairDict(cursor=cursor,symbolList=gList2,dataIdDict=dataIdDict) + SymbolValuePairDict.update(PairDict2) + + return SymbolValuePairDict + +######################################################################################################## +#input: cursor, GeneNameLst (list), TissueProbeSetFreezeId(int) +#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. +# key is symbol, value is one list of expression values of one probeSet. +#function: wrapper function of getGeneSymbolTissueValueDict function +# for CorrelationPage.py +######################################################################################################## + +def getGeneSymbolTissueValueDictForTrait(cursor=None,GeneNameLst=[],TissueProbeSetFreezeId=0): + SymbolValuePairDict={} + symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict = getTissueProbeSetXRefInfo(cursor=cursor,GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) + if symbolList: + SymbolValuePairDict = getGeneSymbolTissueValueDict(cursor=cursor,symbolList=symbolList,dataIdDict=dataIdDict) + return SymbolValuePairDict + +######################################################################################################## +#Input: cursor(cursor): MySQL connnection cursor; +# priGeneSymbolList(list): one list of gene symbol; +# symbolValuepairDict(dictionary): one dictionary of Symbol and Value Pair, +# key is symbol, value is one list of expression values of one probeSet; +#Output: corrArray(array): array of Correlation Value, +# pvArray(array): array of PValue; +#Function: build corrArray, pvArray for display by calling calculation function:calZeroOrderCorrForTiss +######################################################################################################## + +def getCorrPvArray(cursor=None,priGeneSymbolList=[],symbolValuepairDict={}): + # setting initial value for corrArray, pvArray equal to 0 + Num = len(priGeneSymbolList) + + corrArray = [([0] * (Num))[:] for i in range(Num)] + pvArray = [([0] * (Num))[:] for i in range(Num)] + i = 0 + for pkey in priGeneSymbolList: + j = 0 + pkey = pkey.strip().lower()# key in symbolValuepairDict is low case + if symbolValuepairDict.has_key(pkey): + priValue = symbolValuepairDict[pkey] + for tkey in priGeneSymbolList: + tkey = tkey.strip().lower()# key in symbolValuepairDict is low case + if priValue and symbolValuepairDict.has_key(tkey): + tarValue = symbolValuepairDict[tkey] + + if tarValue: + if i>j: + # corrArray stores Pearson Correlation values + # pvArray stores Pearson P-Values + pcorr_result =calZeroOrderCorrForTiss(primaryValue=priValue,targetValue=tarValue) + corrArray[i][j] =pcorr_result[0] + pvArray[i][j] =pcorr_result[2] + elif i<j: + # corrArray stores Spearman Correlation values + # pvArray stores Spearman P-Values + scorr_result =calZeroOrderCorrForTiss(primaryValue=priValue,targetValue=tarValue,method='spearman') + corrArray[i][j] =scorr_result[0] + pvArray[i][j] =scorr_result[2] + else: + # on the diagonal line, correlation value is 1, P-Values is 0 + corrArray[i][j] =1 + pvArray[i][j] =0 + j+=1 + else: + corrArray[i][j] = None + pvArray[i][j] = None + j+=1 + else: + corrArray[i][j] = None + pvArray[i][j] = None + j+=1 + else: + corrArray[i][j] = None + pvArray[i][j] = None + + i+=1 + + return corrArray, pvArray + +######################################################################################################## +#Input: cursor(cursor): MySQL connnection cursor; +# primaryTraitSymbol(string): one gene symbol; +# TissueProbeSetFreezeId (int): Id of related TissueProbeSetFreeze +# method: '0' default value, Pearson Correlation; '1', Spearman Correlation +#Output: symbolCorrDict(Dict): Dict of Correlation Value, key is symbol +# symbolPvalueDict(Dict): Dict of PValue,key is symbol ; +#Function: build symbolCorrDict, symbolPvalueDict for display by calling calculation function:calZeroOrderCorrForTiss +######################################################################################################## +def calculateCorrOfAllTissueTrait(cursor=None, primaryTraitSymbol=None, TissueProbeSetFreezeId=None,method='0'): + + symbolCorrDict = {} + symbolPvalueDict = {} + + primaryTraitSymbolValueDict = getGeneSymbolTissueValueDictForTrait(cursor=cursor, GeneNameLst=[primaryTraitSymbol], TissueProbeSetFreezeId=TissueProbeSetFreezeId) + primaryTraitValue = primaryTraitSymbolValueDict.values()[0] + + SymbolValueDict = getGeneSymbolTissueValueDictForTrait(cursor=cursor, GeneNameLst=[], TissueProbeSetFreezeId=TissueProbeSetFreezeId) + + if method =='1': + symbolCorrDict, symbolPvalueDict = batchCalTissueCorr(primaryTraitValue,SymbolValueDict,method='spearman') + else: + symbolCorrDict, symbolPvalueDict = batchCalTissueCorr(primaryTraitValue,SymbolValueDict) + + + return (symbolCorrDict, symbolPvalueDict) |