From e7e3117695ef86a28df96cf32bfb66f6da2bc404 Mon Sep 17 00:00:00 2001 From: Zachary Sloan Date: Fri, 7 Sep 2012 17:08:36 -0500 Subject: Worked with passing form data to correlation page --- wqflask/base/webqtlFormData.py | 73 +- wqflask/wqflask/correlation/CorrelationPage.py | 2069 ++++++++++++++++++++ wqflask/wqflask/correlation/__init__.py | 0 wqflask/wqflask/correlation/correlationFunction.py | 923 +++++++++ wqflask/wqflask/show_trait/DataEditingPage.py | 4 +- wqflask/wqflask/show_trait/show_trait_page.py | 6 +- .../new/javascript/trait_data_and_analysis.coffee | 21 +- .../new/javascript/trait_data_and_analysis.js | 17 +- .../wqflask/templates/trait_data_and_analysis.html | 17 +- wqflask/wqflask/views.py | 10 +- 10 files changed, 3094 insertions(+), 46 deletions(-) create mode 100644 wqflask/wqflask/correlation/CorrelationPage.py create mode 100644 wqflask/wqflask/correlation/__init__.py create mode 100644 wqflask/wqflask/correlation/correlationFunction.py (limited to 'wqflask') diff --git a/wqflask/base/webqtlFormData.py b/wqflask/base/webqtlFormData.py index 06faacc0..a9e3b7d4 100755 --- a/wqflask/base/webqtlFormData.py +++ b/wqflask/base/webqtlFormData.py @@ -25,6 +25,10 @@ # Last updated by GeneNetwork Core Team 2010/10/20 #from mod_python import Cookie + +from __future__ import print_function +from pprint import pformat as pf + import string import os @@ -47,12 +51,25 @@ class webqtlFormData: #XZ: Attention! All attribute values must be picklable! - def __init__(self, start_vars = None, req = None, mod_python_session=None, FieldStorage_formdata=None): - - self.__dict__.update(start_vars) - - for item in self.attrs: - setattr(self,item, None) + def __init__(self, + start_vars = None, + req = None, + mod_python_session=None, + FieldStorage_formdata=None): + # Todo: rework this whole thing + print("in webqtlFormData start_vars are:", pf(start_vars)) + for item in webqtlFormData.attrs: + self.__dict__[item] = None + #self.__dict__.update(start_vars) + for item in start_vars: + self.__dict__[item] = start_vars[item] + print(" Now self.dict is:", pf(self.__dict__)) + #for item in self.attrs: + # if getattr(self, item, None): + # print("Setting item %s to None" % (item,)) + # self.attrs[item] = None + # else: + # self.attrs[item] = self.attrs[item].strip() try: self.remote_ip = req.connection.remote_ip @@ -84,31 +101,37 @@ class webqtlFormData: # if value != None: # setattr(self,item,string.strip(value)) - self.ppolar = "" - self.mpolar = "" + self.ppolar = None + self.mpolar = None + + print("[yellow] self.RISet is:", self.RISet) if self.RISet: - try: - # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py; - f1, f12, self.mpolar, self.ppolar = webqtlUtil.ParInfo[self.RISet] - except: - f1 = f12 = self.mpolar = self.ppolar = None - - try: - self.nperm = int(self.nperm) - self.nboot = int(self.nboot) - except: - self.nperm = 2000 #XZ: Rob asked to change the default value to 2000 - self.nboot = 2000 #XZ: Rob asked to change the default value to 2000 - + #try: + # # NL, 07/27/2010. ParInfo has been moved from webqtlForm.py to webqtlUtil.py; + _f1, _f12, self.mpolar, self.ppolar = webqtlUtil.ParInfo[self.RISet] + #except: + # f1 = f12 = self.mpolar = self.ppolar = None + + + def set_number(stringy): + return int(stringy) if stringy else 2000 # Rob asked to change the default value to 2000 + + self.nperm = set_number(self.nperm) + self.nboot = set_number(self.nboot) + + + #if self.allstrainlist: + # self.allstrainlist = map(string.strip, string.split(self.allstrainlist)) + print("self.allstrainlist is:", self.allstrainlist) if self.allstrainlist: - self.allstrainlist = map(string.strip, string.split(self.allstrainlist)) + self.allstrainlist = self.allstrainlist.split() + print("now self.allstrainlist is:", self.allstrainlist) #self.readGenotype() #self.readData() if self.RISet == 'BXD300': self.RISet = 'BXD' - else: - pass + def __getitem__(self, key): return self.__dict__[key] @@ -133,7 +156,7 @@ class webqtlFormData: self.RISet = 'BXD' else: pass - assert self.RISet + assert self.RISet #genotype_1 is Dataset Object without parents and f1 #genotype_2 is Dataset Object with parents and f1 (not for intercross) self.genotype_1 = reaper.Dataset() diff --git a/wqflask/wqflask/correlation/CorrelationPage.py b/wqflask/wqflask/correlation/CorrelationPage.py new file mode 100644 index 00000000..9caf6595 --- /dev/null +++ b/wqflask/wqflask/correlation/CorrelationPage.py @@ -0,0 +1,2069 @@ +## Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by NL 2011/02/11 +# Last updated by Christian Fernandez 2012/04/07 +# Refactored correlation calculation into smaller functions in preparation of +# separating html from existing code + +from __future__ import print_function + +import string +from math import * +import cPickle +import os +import time +#import pyXLWriter as xl +import pp +import math + +from pprint import pformat as pf + +from htmlgen import HTMLgen2 as HT +import reaper + +from base import webqtlConfig +from utility.THCell import THCell +from utility.TDCell import TDCell +from base.webqtlTrait import webqtlTrait +from base.webqtlDataset import webqtlDataset +from base.templatePage import templatePage +from utility import webqtlUtil +from dbFunction import webqtlDatabaseFunction +import utility.webqtlUtil #this is for parallel computing only. +import correlationFunction + +import logging +logging.basicConfig(filename="/tmp/gn_log", level=logging.INFO) +_log = logging.getLogger("correlation") + +METHOD_SAMPLE_PEARSON = "1" +METHOD_SAMPLE_RANK = "2" +METHOD_LIT = "3" +METHOD_TISSUE_PEARSON = "4" +METHOD_TISSUE_RANK = "5" + +TISSUE_METHODS = [METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] + +TISSUE_MOUSE_DB = 1 + +class AuthException(Exception): pass + + +class Trait(object): + def __init__(self, name, raw_values = None, lit_corr = None, tissue_corr = None, p_tissue = None): + self.name = name + self.raw_values = raw_values + self.lit_corr = lit_corr + self.tissue_corr = tissue_corr + self.p_tissue = p_tissue + self.correlation = 0 + self.p_value = 0 + + @staticmethod + def from_csv(line, data_start = 1): + name = line[0] + numbers = line[data_start:] + # _log.info(numbers) + numbers = [ float(number) for number in numbers ] + + return Trait(name, raw_values = numbers) + + def calculate_correlation(self, values, method): + """Calculate the correlation value and p value according to the method specified""" + + #ZS: This takes the list of values of the trait our selected trait is being correlated against and removes the values of the samples our trait has no value for + #There's probably a better way of dealing with this, but I'll have to ask Christian + updated_raw_values = [] + updated_values = [] + for i in range(len(values)): + if values[i] != "None": + updated_raw_values.append(self.raw_values[i]) + updated_values.append(values[i]) + + self.raw_values = updated_raw_values + values = updated_values + + if method == METHOD_SAMPLE_PEARSON or method == METHOD_LIT or method == METHOD_TISSUE_PEARSON: + corr,nOverlap = webqtlUtil.calCorrelation(self.raw_values, values, len(values)) + else: + corr,nOverlap = webqtlUtil.calCorrelationRank(self.raw_values, values, len(values)) + + self.correlation = corr + self.overlap = nOverlap + + if self.overlap < 3: + self.p_value = 1.0 + else: + #ZS - This is probably the wrong way to deal with this. Correlation values of 1.0 definitely exist (the trait correlated against itself), so zero division needs to br prevented. + if abs(self.correlation) >= 1.0: + self.p_value = 0.0 + else: + ZValue = 0.5*log((1.0+self.correlation)/(1.0-self.correlation)) + ZValue = ZValue*sqrt(self.overlap-3) + self.p_value = 2.0*(1.0 - reaper.normp(abs(ZValue))) + + + +#XZ, 01/14/2009: This method is for parallel computing only. +#XZ: It is supposed to be called when "Genetic Correlation, Pearson's r" (method 1) +#XZ: or "Genetic Correlation, Spearman's rho" (method 2) is selected +def compute_corr( input_nnCorr, input_trait, input_list, computing_method): + + allcorrelations = [] + + for line in input_list: + tokens = line.split('","') + tokens[-1] = tokens[-1][:-2] #remove the last " + tokens[0] = tokens[0][1:] #remove the first " + + traitdataName = tokens[0] + database_trait = tokens[1:] + + if computing_method == "1": #XZ: Pearson's r + corr,nOverlap = utility.webqtlUtil.calCorrelationText(input_trait, database_trait, input_nnCorr) + else: #XZ: Spearman's rho + corr,nOverlap = utility.webqtlUtil.calCorrelationRankText(input_trait, database_trait, input_nnCorr) + traitinfo = [traitdataName,corr,nOverlap] + allcorrelations.append(traitinfo) + + return allcorrelations + +def get_correlation_method_key(form_data): + #XZ, 09/28/2008: if user select "1", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "2", then display 2, 3 and 5. + #XZ, 09/28/2008: if user select "3", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "4", then display 1, 3 and 4. + #XZ, 09/28/2008: if user select "5", then display 2, 3 and 5. + + method = form_data.formdata.getvalue("method") + if method not in ["1", "2", "3" ,"4", "5"]: + return "1" + + return method + + +def get_custom_trait(form_data, cursor): + """Pulls the custom trait, if it exists, out of the form data""" + trait_name = form_data.formdata.getvalue('fullname') + + if trait_name: + trait = webqtlTrait(fullname=trait_name, cursor=cursor) + trait.retrieveInfo() + return trait + else: + return None + + +#XZ, 09/18/2008: get the information such as value, variance of the input strain names from the form. +def get_sample_data(form_data): + if form_data.allstrainlist: + mdpchoice = form_data.formdata.getvalue('MDPChoice') + #XZ, in HTML source code, it is "BXD Only" or "BXH only", and so on + if mdpchoice == "1": + strainlist = form_data.f1list + form_data.strainlist + #XZ, in HTML source code, it is "MDP Only" + elif mdpchoice == "2": + strainlist = [] + strainlist2 = form_data.f1list + form_data.strainlist + for strain in form_data.allstrainlist: + if strain not in strainlist2: + strainlist.append(strain) + #So called MDP Panel + if strainlist: + strainlist = form_data.f1list+form_data.parlist+strainlist + #XZ, in HTML source code, it is "All Cases" + else: + strainlist = form_data.allstrainlist + #XZ, 09/18/2008: put the trait data into dictionary form_data.allTraitData + form_data.readData(form_data.allstrainlist) + else: + mdpchoice = None + strainlist = form_data.strainlist + #XZ, 09/18/2008: put the trait data into dictionary form_data.allTraitData + form_data.readData() + + return strainlist + + +def get_mdp_choice(form_data): + if form_data.allstrainlist: + return form_data.formdata.getvalue("MDPChoice") + else: + return None + + +def get_species(fd, cursor): + #XZ, 3/16/2010: variable RISet must be pass by the form + RISet = fd.RISet + #XZ, 12/12/2008: get species infomation + species = webqtlDatabaseFunction.retrieveSpecies(cursor=cursor, RISet=RISet) + return species + + +def sortTraitCorrelations(traits, method="1"): + if method in TISSUE_METHODS: + traits.sort(key=lambda trait: trait.tissue_corr != None and abs(trait.tissue_corr), reverse=True) + elif method == METHOD_LIT: + traits.sort(key=lambda trait: trait.lit_corr != None and abs(trait.lit_corr), reverse=True) + else: + traits.sort(key=lambda trait: trait.correlation != None and abs(trait.correlation), reverse=True) + + return traits + + +def auth_user_for_db(db, cursor, target_db_name, privilege, username): + """Authorize a user for access to a database if that database is + confidential. A db (identified by a record in ProbeSetFreeze) contains a + list of authorized users who may access it, as well as its confidentiality + level. + + If the current user's privilege level is greater than 'user', ie: root or + admin, then they are automatically authed, otherwise, check the + AuthorizedUsers field for the presence of their name.""" + + if db.type == 'ProbeSet': + cursor.execute('SELECT Id, Name, FullName, confidentiality, AuthorisedUsers FROM ProbeSetFreeze WHERE Name = "%s"' % target_db_name) + indId, indName, indFullName, confidential, AuthorisedUsers = cursor.fetchall()[0] + + if confidential: + authorized = 0 + + #for the dataset that confidentiality is 1 + #1. 'admin' and 'root' can see all of the dataset + #2. 'user' can see the dataset that AuthorisedUsers contains his id(stored in the Id field of User table) + if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: + authorized = 1 + else: + if username in AuthorisedUsers.split(","): + authorized = 1 + + if not authorized: + raise AuthException("The %s database you selected is not open to the public at this time, please go back and select other database." % indFullName) + + +class CorrelationPage(templatePage): + + corrMinInformative = 4 + + PAGE_HEADING = "Correlation Table" + CORRELATION_METHODS = {"1" : "Genetic Correlation (Pearson's r)", + "2" : "Genetic Correlation (Spearman's rho)", + "3" : "SGO Literature Correlation", + "4" : "Tissue Correlation (Pearson's r)", + "5" : "Tissue Correlation (Spearman's rho)"} + + RANK_ORDERS = {"1": 0, "2": 1, "3": 0, "4": 0, "5": 1} + + + def error(self, message, error="Error", heading = None): + heading = heading or self.PAGE_HEADING + return templatePage.error(heading = heading, detail = [message], error=error) + + def __init__(self, fd): + print("in CorrelationPage __init__ fd is:", pf(fd.__dict__)) + # Call the superclass constructor + templatePage.__init__(self, fd) + print("in CorrelationPage __init__ now fd is:", pf(fd.__dict__)) + # Connect to the database + if not self.openMysql(): + return + + # Read the genotype from a file + if not fd.genotype: + fd.readGenotype() + + sample_list = get_sample_data(fd) + mdp_choice = get_mdp_choice(fd) # No idea what this is yet + self.species = get_species(fd, self.cursor) + + #XZ, 09/18/2008: get all information about the user selected database. + target_db_name = fd.formdata.getvalue('database') + self.target_db_name = target_db_name + + try: + self.db = webqtlDataset(target_db_name, self.cursor) + except: + detail = ["The database you just requested has not been established yet."] + self.error(detail) + return + + # Auth if needed + try: + auth_user_for_db(self.db, self.cursor, target_db_name, self.privilege, self.userName) + except AuthException, e: + detail = [e.message] + return self.error(detail) + + #XZ, 09/18/2008: filter out the strains that have no value. + self.sample_names, vals, vars, N = fd.informativeStrains(sample_list) + + #CF - If less than a minimum number of strains/cases in common, don't calculate anything + if len(self.sample_names) < self.corrMinInformative: + detail = ['Fewer than %d strain data were entered for %s data set. No calculation of correlation has been attempted.' % (self.corrMinInformative, fd.RISet)] + self.error(heading=PAGE_HEADING,detail=detail) + + + self.method = get_correlation_method_key(fd) + correlation_method = self.CORRELATION_METHODS[self.method] + rankOrder = self.RANK_ORDERS[self.method] + + # CF - Number of results returned + self.returnNumber = int(fd.formdata.getvalue('criteria')) + + self.record_count = 0 + + myTrait = get_custom_trait(fd, self.cursor) + + + # We will not get Literature Correlations if there is no GeneId because there is nothing to look against + self.gene_id = int(fd.formdata.getvalue('GeneId') or 0) + + # We will not get Tissue Correlations if there is no gene symbol because there is nothing to look against + self.trait_symbol = myTrait.symbol + + + #XZ, 12/12/2008: if the species is rat or human, translate the geneid to mouse geneid + self.input_trait_mouse_gene_id = self.translateToMouseGeneID(self.species, self.gene_id) + + #XZ: As of Nov/13/2010, this dataset is 'UTHSC Illumina V6.2 RankInv B6 D2 average CNS GI average (May 08)' + self.tissue_probeset_freeze_id = 1 + + traitList = self.correlate(vals) + + _log.info("Done doing correlation calculation") + +############################################################################################################################################ + + TD_LR = HT.TD(height=200,width="100%",bgColor='#eeeeee') + + mainfmName = webqtlUtil.genRandStr("fm_") + form = HT.Form(cgi= os.path.join(webqtlConfig.CGIDIR, webqtlConfig.SCRIPTFILE), enctype='multipart/form-data', name= mainfmName, submit=HT.Input(type='hidden')) + hddn = {'FormID': 'showDatabase', + 'ProbeSetID': '_', + 'database': self.target_db_name, + 'databaseFull': self.db.fullname, + 'CellID': '_', + 'RISet': fd.RISet, + 'identification': fd.identification} + + if myTrait: + hddn['fullname']=fd.formdata.getvalue('fullname') + if mdp_choice: + hddn['MDPChoice']=mdp_choice + + + #XZ, 09/18/2008: pass the trait data to next page by hidden parameters. + webqtlUtil.exportData(hddn, fd.allTraitData) + + if fd.incparentsf1: + hddn['incparentsf1']='ON' + + if fd.allstrainlist: + hddn['allstrainlist'] = string.join(fd.allstrainlist, ' ') + + + for key in hddn.keys(): + form.append(HT.Input(name=key, value=hddn[key], type='hidden')) + + #XZ, 11/21/2008: add two parameters to form + form.append(HT.Input(name="X_geneSymbol", value="", type='hidden')) + form.append(HT.Input(name="Y_geneSymbol", value="", type='hidden')) + + #XZ, 3/11/2010: add one parameter to record if the method is rank order. + form.append(HT.Input(name="rankOrder", value="%s" % rankOrder, type='hidden')) + + form.append(HT.Input(name="TissueProbeSetFreezeId", value="%s" % self.tissue_probeset_freeze_id, type='hidden')) + + #################################### + # generate the info on top of page # + #################################### + + info = self.getTopInfo(myTrait=myTrait, method=self.method, db=self.db, target_db_name=self.target_db_name, returnNumber=self.returnNumber, methodDict=self.CORRELATION_METHODS, totalTraits=traitList, identification=fd.identification ) + + ############## + # Excel file # + ############## + filename= webqtlUtil.genRandStr("Corr_") + xlsUrl = HT.Input(type='button', value = 'Download Table', onClick= "location.href='/tmp/%s.xls'" % filename, Class='button') + # Create a new Excel workbook + workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+filename)) + headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") + + #XZ, 3/18/2010: pay attention to the line number of header in this file. As of today, there are 7 lines. + worksheet = self.createExcelFileWithTitleAndFooter(workbook=workbook, identification=fd.identification, db=self.db, returnNumber=self.returnNumber) + + newrow = 7 + + +##################################################################### + + + #Select All, Deselect All, Invert Selection, Add to Collection + mintmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'showIntMap');" % mainfmName) + mintmap_img = HT.Image("/images/multiple_interval_mapping1_final.jpg", name='mintmap', alt="Multiple Interval Mapping", title="Multiple Interval Mapping", style="border:none;") + mintmap.append(mintmap_img) + mcorr = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'compCorr');" % mainfmName) + mcorr_img = HT.Image("/images/compare_correlates2_final.jpg", alt="Compare Correlates", title="Compare Correlates", style="border:none;") + mcorr.append(mcorr_img) + cormatrix = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'corMatrix');" % mainfmName) + cormatrix_img = HT.Image("/images/correlation_matrix1_final.jpg", alt="Correlation Matrix and PCA", title="Correlation Matrix and PCA", style="border:none;") + cormatrix.append(cormatrix_img) + networkGraph = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'networkGraph');" % mainfmName) + networkGraph_img = HT.Image("/images/network_graph1_final.jpg", name='mintmap', alt="Network Graphs", title="Network Graphs", style="border:none;") + networkGraph.append(networkGraph_img) + heatmap = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'heatmap');" % mainfmName) + heatmap_img = HT.Image("/images/heatmap2_final.jpg", name='mintmap', alt="QTL Heat Map and Clustering", title="QTL Heatmap and Clustering", style="border:none;") + heatmap.append(heatmap_img) + partialCorr = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'partialCorrInput');" % mainfmName) + partialCorr_img = HT.Image("/images/partial_correlation_final.jpg", name='partialCorr', alt="Partial Correlation", title="Partial Correlation", style="border:none;") + partialCorr.append(partialCorr_img) + addselect = HT.Href(url="#redirect", onClick="addRmvSelection('%s', document.getElementsByName('%s')[0], 'addToSelection');" % (fd.RISet, mainfmName)) + addselect_img = HT.Image("/images/add_collection1_final.jpg", name="addselect", alt="Add To Collection", title="Add To Collection", style="border:none;") + addselect.append(addselect_img) + selectall = HT.Href(url="#redirect", onClick="checkAll(document.getElementsByName('%s')[0]);" % mainfmName) + selectall_img = HT.Image("/images/select_all2_final.jpg", name="selectall", alt="Select All", title="Select All", style="border:none;") + selectall.append(selectall_img) + selectinvert = HT.Href(url="#redirect", onClick = "checkInvert(document.getElementsByName('%s')[0]);" % mainfmName) + selectinvert_img = HT.Image("/images/invert_selection2_final.jpg", name="selectinvert", alt="Invert Selection", title="Invert Selection", style="border:none;") + selectinvert.append(selectinvert_img) + reset = HT.Href(url="#redirect", onClick="checkNone(document.getElementsByName('%s')[0]); return false;" % mainfmName) + reset_img = HT.Image("/images/select_none2_final.jpg", alt="Select None", title="Select None", style="border:none;") + reset.append(reset_img) + selecttraits = HT.Input(type='button' ,name='selecttraits',value='Select Traits', onClick="checkTraits(this.form);",Class="button") + selectgt = HT.Input(type='text' ,name='selectgt',value='-1.0', size=6,maxlength=10,onChange="checkNumeric(this,1.0,'-1.0','gthan','greater than filed')") + selectlt = HT.Input(type='text' ,name='selectlt',value='1.0', size=6,maxlength=10,onChange="checkNumeric(this,-1.0,'1.0','lthan','less than field')") + selectandor = HT.Select(name='selectandor') + selectandor.append(('AND','and')) + selectandor.append(('OR','or')) + selectandor.selected.append('AND') + + + #External analysis tools + GCATButton = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'GCAT');" % mainfmName) + GCATButton_img = HT.Image("/images/GCAT_logo_final.jpg", name="GCAT", alt="GCAT", title="GCAT", style="border:none") + GCATButton.append(GCATButton_img) + + ODE = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'ODE');" % mainfmName) + ODE_img = HT.Image("/images/ODE_logo_final.jpg", name="ode", alt="ODE", title="ODE", style="border:none") + ODE.append(ODE_img) + + ''' + #XZ, 07/07/2010: I comment out this block of code. + WebGestaltScript = HT.Script(language="Javascript") + WebGestaltScript.append(""" +setTimeout('openWebGestalt()', 2000); +function openWebGestalt(){ +var thisForm = document['WebGestalt']; +makeWebGestaltTree(thisForm, '%s', %d, 'edag_only.php'); +} + """ % (mainfmName, len(traitList))) + ''' + + self.cursor.execute('SELECT GeneChip.GO_tree_value FROM GeneChip, ProbeFreeze, ProbeSetFreeze WHERE GeneChip.Id = ProbeFreeze.ChipId and ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id and ProbeSetFreeze.Name = "%s"' % self.db.name) + result = self.cursor.fetchone() + + if result: + GO_tree_value = result[0] + + if GO_tree_value: + + WebGestalt = HT.Href(url="#redirect", onClick="databaseFunc(document.getElementsByName('%s')[0], 'GOTree');" % mainfmName) + WebGestalt_img = HT.Image("/images/webgestalt_icon_final.jpg", name="webgestalt", alt="Gene Set Analysis Toolkit", title="Gene Set Analysis Toolkit", style="border:none") + WebGestalt.append(WebGestalt_img) + + hddnWebGestalt = { + 'id_list':'', + 'correlation':'', + 'id_value':'', + 'llid_list':'', + 'id_type':GO_tree_value, + 'idtype':'', + 'species':'', + 'list':'', + 'client':''} + + hddnWebGestalt['ref_type'] = hddnWebGestalt['id_type'] + hddnWebGestalt['cat_type'] = 'GO' + hddnWebGestalt['significancelevel'] = 'Top10' + + if self.species == 'rat': + hddnWebGestalt['org'] = 'Rattus norvegicus' + elif self.species == 'human': + hddnWebGestalt['org'] = 'Homo sapiens' + elif self.species == 'mouse': + hddnWebGestalt['org'] = 'Mus musculus' + else: + hddnWebGestalt['org'] = '' + + for key in hddnWebGestalt.keys(): + form.append(HT.Input(name=key, value=hddnWebGestalt[key], type='hidden')) + + + #Create tables with options, etc + + pageTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="100%", border=0, align="Left") + + containerTable = HT.TableLite(cellSpacing=0,cellPadding=0,width="90%",border=0, align="Left") + + + if not GO_tree_value: + optionsTable = HT.TableLite(cellSpacing=2, cellPadding=0,width="480", height="80", border=0, align="Left") + optionsTable.append(HT.TR(HT.TD(selectall), HT.TD(reset), HT.TD(selectinvert), HT.TD(addselect), HT.TD(GCATButton), HT.TD(ODE), align="left")) + optionsTable.append(HT.TR(HT.TD(" "*1,"Select"), HT.TD("Deselect"), HT.TD(" "*1,"Invert"), HT.TD(" "*3,"Add"), HT.TD("Gene Set"), HT.TD(" "*2,"GCAT"))) + else: + optionsTable = HT.TableLite(cellSpacing=2, cellPadding=0,width="560", height="80", border=0, align="Left") + optionsTable.append(HT.TR(HT.TD(selectall), HT.TD(reset), HT.TD(selectinvert), HT.TD(addselect), HT.TD(GCATButton), HT.TD(ODE), HT.TD(WebGestalt), align="left")) + optionsTable.append(HT.TR(HT.TD(" "*1,"Select"), HT.TD("Deselect"), HT.TD(" "*1,"Invert"), HT.TD(" "*3,"Add"), HT.TD("Gene Set"), HT.TD(" "*2,"GCAT"), HT.TD(" "*3, "ODE"))) + containerTable.append(HT.TR(HT.TD(optionsTable))) + + functionTable = HT.TableLite(cellSpacing=2,cellPadding=0,width="480",height="80", border=0, align="Left") + functionRow = HT.TR(HT.TD(networkGraph, width="16.7%"), HT.TD(cormatrix, width="16.7%"), HT.TD(partialCorr, width="16.7%"), HT.TD(mcorr, width="16.7%"), HT.TD(mintmap, width="16.7%"), HT.TD(heatmap), align="left") + labelRow = HT.TR(HT.TD(" "*1,HT.Text("Graph")), HT.TD(" "*1,HT.Text("Matrix")), HT.TD(" "*1,HT.Text("Partial")), HT.TD(HT.Text("Compare")), HT.TD(HT.Text("QTL Map")), HT.TD(HT.Text(text="Heat Map"))) + functionTable.append(functionRow, labelRow) + containerTable.append(HT.TR(HT.TD(functionTable), HT.BR())) + + #more_options = HT.Image("/images/more_options1_final.jpg", name='more_options', alt="Expand Options", title="Expand Options", style="border:none;", Class="toggleShowHide") + + #containerTable.append(HT.TR(HT.TD(more_options, HT.BR(), HT.BR()))) + + moreOptions = HT.Input(type='button',name='options',value='More Options', onClick="",Class="toggle") + fewerOptions = HT.Input(type='button',name='options',value='Fewer Options', onClick="",Class="toggle") + + """ + if (fd.formdata.getvalue('showHideOptions') == 'less'): + containerTable.append(HT.TR(HT.TD(" "), height="10"), HT.TR(HT.TD(HT.Div(fewerOptions, Class="toggleShowHide")))) + containerTable.append(HT.TR(HT.TD(" "))) + else: + containerTable.append(HT.TR(HT.TD(" "), height="10"), HT.TR(HT.TD(HT.Div(moreOptions, Class="toggleShowHide")))) + containerTable.append(HT.TR(HT.TD(" "))) + """ + + containerTable.append(HT.TR(HT.TD(HT.Span(selecttraits,' with r > ',selectgt, ' ',selectandor, ' r < ',selectlt,Class="bd1 cbddf fs11")), style="display:none;", Class="extra_options")) + + chrMenu = HT.Input(type='hidden',name='chromosomes',value='all') + + corrHeading = HT.Paragraph('Correlation Table', Class="title") + + + tblobj = {} + + if self.db.type=="Geno": + containerTable.append(HT.TR(HT.TD(xlsUrl, height=60))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + tblobj['header'], worksheet = self.getTableHeaderForGeno( method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForGeno(traitList=traitList, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript) + + workbook.close() + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + + pageTable.append(HT.TR(HT.TD(div))) + + form.append(HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + HT.P(), HT.P(), pageTable) + TD_LR.append(corrHeading, info, form, HT.P()) + + self.dict['body'] = str(TD_LR) + self.dict['js1'] = '' + self.dict['title'] = 'Correlation' + + elif self.db.type=="Publish": + + containerTable.append(HT.TR(HT.TD(xlsUrl, height=40))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + tblobj['header'], worksheet = self.getTableHeaderForPublish(method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForPublish(traitList=traitList, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript, species=self.species) + + workbook.close() + + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + # NL, 07/27/2010. genTableObj function has been moved from templatePage.py to webqtlUtil.py; + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1"), corrScript, Id="sortable") + + pageTable.append(HT.TR(HT.TD(div))) + + form.append( + HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + HT.P(), pageTable) + TD_LR.append(corrHeading, info, form, HT.P()) + + self.dict['body'] = str(TD_LR) + self.dict['js1'] = '' + self.dict['title'] = 'Correlation' + + + elif self.db.type=="ProbeSet": + tblobj['header'], worksheet = self.getTableHeaderForProbeSet(method=self.method, worksheet=worksheet, newrow=newrow, headingStyle=headingStyle) + newrow += 1 + + sortby = self.getSortByValue( calculationMethod = self.method ) + + corrScript = HT.Script(language="Javascript") + corrScript.append("var corrArray = new Array();") + + tblobj['body'], worksheet, corrScript = self.getTableBodyForProbeSet(traitList=traitList, primaryTrait=myTrait, formName=mainfmName, worksheet=worksheet, newrow=newrow, corrScript=corrScript, species=self.species) + + workbook.close() + objfile = open('%s.obj' % (webqtlConfig.TMPDIR+filename), 'wb') + cPickle.dump(tblobj, objfile) + objfile.close() + + #XZ: here is the table of traits + div = HT.Div(webqtlUtil.genTableObj(tblobj=tblobj, file=filename, sortby=sortby, tableID = "sortable", addIndex = "1", hiddenColumns=["Gene ID","Homologene ID"]), corrScript, Id="sortable") + + + #XZ, 01/12/2009: create database menu for 'Add Correlation' + self.cursor.execute(""" + select + ProbeSetFreeze.FullName, ProbeSetFreeze.Id, Tissue.name + from + ProbeSetFreeze, ProbeFreeze, ProbeSetFreeze as ps2, ProbeFreeze as p2, Tissue + where + ps2.Id = %d + and ps2.ProbeFreezeId = p2.Id + and ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id + and (ProbeFreeze.InbredSetId = p2.InbredSetId or (ProbeFreeze.InbredSetId in (1, 3) and p2.InbredSetId in (1, 3))) + and p2.ChipId = ProbeFreeze.ChipId + and ps2.Id != ProbeSetFreeze.Id + and ProbeFreeze.TissueId = Tissue.Id + and ProbeSetFreeze.public > %d + order by + ProbeFreeze.TissueId, ProbeSetFreeze.CreateTime desc + """ % (self.db.id, webqtlConfig.PUBLICTHRESH)) + + results = self.cursor.fetchall() + dbCustomizer = HT.Select(results, name = "customizer") + databaseMenuSub = preTissue = "" + for item in results: + TName, TId, TTissue = item + if TTissue != preTissue: + if databaseMenuSub: + dbCustomizer.append(databaseMenuSub) + databaseMenuSub = HT.Optgroup(label = '%s mRNA ------' % TTissue) + preTissue = TTissue + + databaseMenuSub.append(item[:2]) + if databaseMenuSub: + dbCustomizer.append(databaseMenuSub) + + #updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + #variables: filename, strainIds and vals are required by getquerystring function + strainIds=self.getStrainIds(species=self.species, strains=self.sample_names) + var1 = HT.Input(name="filename", value=filename, type='hidden') + var2 = HT.Input(name="strainIds", value=strainIds, type='hidden') + var3 = HT.Input(name="vals", value=vals, type='hidden') + customizerButton = HT.Input(type="button", Class="button", value="Add Correlation", onClick = "xmlhttpPost('%smain.py?FormID=AJAX_table', 'sortable', (getquerystring(this.form)))" % webqtlConfig.CGIDIR) + + containerTable.append(HT.TR(HT.TD(HT.Span(var1,var2,var3,customizerButton, "with", dbCustomizer, Class="bd1 cbddf fs11"), HT.BR(), HT.BR()), style="display:none;", Class="extra_options")) + + containerTable.append(HT.TR(HT.TD(xlsUrl, HT.BR(), HT.BR()))) + + pageTable.append(HT.TR(HT.TD(containerTable))) + + pageTable.append(HT.TR(HT.TD(div))) + + if self.species == 'human': + heatmap = "" + + form.append(HT.Input(name='ShowStrains',type='hidden', value =1), + HT.Input(name='ShowLine',type='hidden', value =1), + info, HT.BR(), pageTable, HT.BR()) + + TD_LR.append(corrHeading, form, HT.P()) + + + self.dict['body'] = str(TD_LR) + self.dict['title'] = 'Correlation' + # updated by NL. Delete function generateJavaScript, move js files to dhtml.js, webqtl.js and jqueryFunction.js + self.dict['js1'] = '' + self.dict['js2'] = 'onLoad="pageOffset()"' + self.dict['layer'] = self.generateWarningLayer() + else: + self.dict['body'] = "" + + +############################# +# # +# CorrelationPage Functions # +# # +############################# + + + def getSortByValue(self, calculationMethod): + + if calculationMethod == "1": + sortby = ("Sample p(r)", "up") + elif calculationMethod == "2": + sortby = ("Sample p(rho)", "up") + elif calculationMethod == "3": #XZ: literature correlation + sortby = ("Lit Corr","down") + elif calculationMethod == "4": #XZ: tissue correlation + sortby = ("Tissue r", "down") + elif calculationMethod == "5": + sortby = ("Tissue rho", "down") + + return sortby + + + + def generateWarningLayer(self): + + layerString = """ + + + + + """ + + return layerString + + + #XZ, 01/07/2009: In HTML code, the variable 'database' corresponds to the column 'Name' in database table. + def getFileName(self, target_db_name): ### dcrowell August 2008 + """Returns the name of the reference database file with which correlations are calculated. + Takes argument cursor which is a cursor object of any instance of a subclass of templatePage + Used by correlationPage""" + + query = 'SELECT Id, FullName FROM ProbeSetFreeze WHERE Name = "%s"' % target_db_name + self.cursor.execute(query) + result = self.cursor.fetchone() + Id = result[0] + FullName = result[1] + FullName = FullName.replace(' ','_') + FullName = FullName.replace('/','_') + + FileName = 'ProbeSetFreezeId_' + str(Id) + '_FullName_' + FullName + '.txt' + + return FileName + + + #XZ, 01/29/2009: I modified this function. + #XZ: Note that the type of StrainIds must be number, not string. + def getStrainIds(self, species=None, strains=[]): + StrainIds = [] + for item in strains: + self.cursor.execute('''SELECT Strain.Id FROM Strain, Species WHERE + Strain.Name="%s" and Strain.SpeciesId=Species.Id and Species.name = "%s" ''' % (item, species)) + Id = self.cursor.fetchone()[0] + StrainIds.append(Id) + + return StrainIds + + + #XZ, 12/12/2008: if the species is rat or human, translate the geneid to mouse geneid + #XZ, 12/12/2008: if the input geneid is 'None', return 0 + #XZ, 12/12/2008: if the input geneid has no corresponding mouse geneid, return 0 + def translateToMouseGeneID (self, species, geneid): + mouse_geneid = 0; + + #if input geneid is None, return 0. + if not geneid: + return mouse_geneid + + if species == 'mouse': + mouse_geneid = geneid + elif species == 'rat': + self.cursor.execute( "SELECT mouse FROM GeneIDXRef WHERE rat=%d" % int(geneid) ) + record = self.cursor.fetchone() + if record: + mouse_geneid = record[0] + elif species == 'human': + self.cursor.execute( "SELECT mouse FROM GeneIDXRef WHERE human=%d" % int(geneid) ) + record = self.cursor.fetchone() + if record: + mouse_geneid = record[0] + + return mouse_geneid + + + #XZ, 12/16/2008: the input geneid is of mouse type + def checkForLitInfo(self,geneId): + q = 'SELECT 1 FROM LCorrRamin3 WHERE GeneId1=%s LIMIT 1' % geneId + self.cursor.execute(q) + try: + x = self.cursor.fetchone() + if x: return True + else: raise + except: return False + + + #XZ, 12/16/2008: the input geneid is of mouse type + def checkSymbolForTissueCorr(self, tissueProbeSetFreezeId=0, symbol=""): + q = "SELECT 1 FROM TissueProbeSetXRef WHERE TissueProbeSetFreezeId=%s and Symbol='%s' LIMIT 1" % (tissueProbeSetFreezeId,symbol) + self.cursor.execute(q) + try: + x = self.cursor.fetchone() + if x: return True + else: raise + except: return False + + + + def fetchAllDatabaseData(self, species, GeneId, GeneSymbol, strains, db, method, returnNumber, tissueProbeSetFreezeId): + + StrainIds = [] + for item in strains: + self.cursor.execute('''SELECT Strain.Id FROM Strain, Species WHERE Strain.Name="%s" and Strain.SpeciesId=Species.Id and Species.name = "%s" ''' % (item, species)) + Id = self.cursor.fetchone()[0] + StrainIds.append('%d' % Id) + + # break it into smaller chunks so we don't overload the MySql server + nnn = len(StrainIds) / 25 + if len(StrainIds) % 25: + nnn += 1 + oridata = [] + + #XZ, 09/24/2008: build one temporary table that only contains the records associated with the input GeneId + tempTable = None + if GeneId and db.type == "ProbeSet": + if method == "3": + tempTable = self.getTempLiteratureTable(species=species, input_species_geneid=GeneId, returnNumber=returnNumber) + + if method == "4" or method == "5": + tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=GeneSymbol, TissueProbeSetFreezeId=TISSUE_MOUSE_DB, method=method, returnNumber=returnNumber) + + for step in range(nnn): + temp = [] + StrainIdstep = StrainIds[step*25:min(len(StrainIds), (step+1)*25)] + for item in StrainIdstep: temp.append('T%s.value' % item) + + if db.type == "Publish": + query = "SELECT PublishXRef.Id, " + dataStartPos = 1 + query += string.join(temp,', ') + query += ' FROM (PublishXRef, PublishFreeze)' + #XZ, 03/04/2009: Xiaodong changed Data to PublishData + for item in StrainIdstep: + query += 'left join PublishData as T%s on T%s.Id = PublishXRef.DataId and T%s.StrainId=%s\n' %(item,item,item,item) + query += "WHERE PublishXRef.InbredSetId = PublishFreeze.InbredSetId and PublishFreeze.Name = '%s'" % (db.name, ) + #XZ, 09/20/2008: extract literature correlation value together with gene expression values. + #XZ, 09/20/2008: notice the difference between the code in next block. + elif tempTable: + # we can get a little performance out of selecting our LitCorr here + # but also we need to do this because we are unconcerned with probes that have no geneId associated with them + # as we would not have litCorr data. + + if method == "3": + query = "SELECT %s.Name, %s.value," % (db.type,tempTable) + dataStartPos = 2 + if method == "4" or method == "5": + query = "SELECT %s.Name, %s.Correlation, %s.PValue," % (db.type,tempTable, tempTable) + dataStartPos = 3 + + query += string.join(temp,', ') + query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) + if method == "3": + query += ' LEFT JOIN %s ON %s.GeneId2=ProbeSet.GeneId ' % (tempTable,tempTable) + if method == "4" or method == "5": + query += ' LEFT JOIN %s ON %s.Symbol=ProbeSet.Symbol ' % (tempTable,tempTable) + #XZ, 03/04/2009: Xiaodong changed Data to %sData and changed parameters from %(item,item, db.type,item,item) to %(db.type, item,item, db.type,item,item) + for item in StrainIdstep: + query += 'left join %sData as T%s on T%s.Id = %sXRef.DataId and T%s.StrainId=%s\n' %(db.type, item,item, db.type,item,item) + + if method == "3": + query += "WHERE ProbeSet.GeneId IS NOT NULL AND %s.value IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable,db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + if method == "4" or method == "5": + query += "WHERE ProbeSet.Symbol IS NOT NULL AND %s.Correlation IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable,db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + else: + query = "SELECT %s.Name," % db.type + dataStartPos = 1 + query += string.join(temp,', ') + query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) + #XZ, 03/04/2009: Xiaodong changed Data to %sData and changed parameters from %(item,item, db.type,item,item) to %(db.type, item,item, db.type,item,item) + for item in StrainIdstep: + query += 'left join %sData as T%s on T%s.Id = %sXRef.DataId and T%s.StrainId=%s\n' %(db.type, item,item, db.type,item,item) + query += "WHERE %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + + self.cursor.execute(query) + results = self.cursor.fetchall() + oridata.append(results) + + datasize = len(oridata[0]) + traits = [] + # put all of the separate data together into a huge list of lists + for j in range(datasize): + traitdata = list(oridata[0][j]) + for i in range(1,nnn): + traitdata += list(oridata[i][j][dataStartPos:]) + + trait = Trait(traitdata[0], traitdata[dataStartPos:]) + + if method == METHOD_LIT: + trait.lit_corr = traitdata[1] + + if method in TISSUE_METHODS: + trait.tissue_corr = traitdata[1] + trait.p_tissue = traitdata[2] + + traits.append(trait) + + if tempTable: + self.cursor.execute( 'DROP TEMPORARY TABLE %s' % tempTable ) + + return traits + + + + + # XZ, 09/20/2008: This function creates TEMPORARY TABLE tmpTableName_2 and return its name. + # XZ, 09/20/2008: It stores top literature correlation values associated with the input geneId. + # XZ, 09/20/2008: Attention: In each row, the input geneId is always in column GeneId1. + #XZ, 12/16/2008: the input geneid can be of mouse, rat or human type + def getTempLiteratureTable(self, species, input_species_geneid, returnNumber): + # according to mysql the TEMPORARY TABLE name should not have to be unique because + # it is only available to the current connection. This program will be invoked via command line, but if it + # were to be invoked over mod_python this could cuase problems. mod_python will keep the connection alive + # in its executing threads ( i think) so there is a potential for the table not being dropped between users. + #XZ, 01/29/2009: To prevent the potential risk, I generate random table names and drop the tables after use them. + + + # the 'input_species_geneid' could be rat or human geneid, need to translate it to mouse geneid + translated_mouse_geneid = self.translateToMouseGeneID (species, input_species_geneid) + + tmpTableName_1 = webqtlUtil.genRandStr(prefix="LITERATURE") + + q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName_1 + q2 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId1,GeneId2,value FROM LCorrRamin3 WHERE GeneId1=%s' % (tmpTableName_1, translated_mouse_geneid) + q3 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId2,GeneId1,value FROM LCorrRamin3 WHERE GeneId2=%s AND GeneId1!=%s' % (tmpTableName_1, translated_mouse_geneid,translated_mouse_geneid) + for x in [q1,q2,q3]: self.cursor.execute(x) + + #XZ, 09/23/2008: Just use the top records insteard of using all records + tmpTableName_2 = webqtlUtil.genRandStr(prefix="TOPLITERATURE") + + q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName_2 + self.cursor.execute(q1) + q2 = 'SELECT GeneId1, GeneId2, value FROM %s ORDER BY value DESC' % tmpTableName_1 + self.cursor.execute(q2) + result = self.cursor.fetchall() + + counter = 0 #this is to count how many records being inserted into table + for one_row in result: + mouse_geneid1, mouse_geneid2, lit_corr_alue = one_row + + #mouse_geneid1 has been tested before, now should test if mouse_geneid2 has corresponding geneid in other species + translated_species_geneid = 0 + if species == 'mouse': + translated_species_geneid = mouse_geneid2 + elif species == 'rat': + self.cursor.execute( "SELECT rat FROM GeneIDXRef WHERE mouse=%d" % int(mouse_geneid2) ) + record = self.cursor.fetchone() + if record: + translated_species_geneid = record[0] + elif species == 'human': + self.cursor.execute( "SELECT human FROM GeneIDXRef WHERE mouse=%d" % int(mouse_geneid2) ) + record = self.cursor.fetchone() + if record: + translated_species_geneid = record[0] + + if translated_species_geneid: + self.cursor.execute( 'INSERT INTO %s (GeneId1, GeneId2, value) VALUES (%d,%d,%f)' % (tmpTableName_2, int(input_species_geneid),int(translated_species_geneid), float(lit_corr_alue)) ) + counter = counter + 1 + + #pay attention to the number + if (counter > 2*returnNumber): + break + + self.cursor.execute('DROP TEMPORARY TABLE %s' % tmpTableName_1) + + return tmpTableName_2 + + + + #XZ, 09/23/2008: In tissue correlation tables, there is no record of GeneId1 == GeneId2 + #XZ, 09/24/2008: Note that the correlation value can be negative. + def getTempTissueCorrTable(self, primaryTraitSymbol="", TissueProbeSetFreezeId=0, method="", returnNumber=0): + + def cmpTissCorrAbsoluteValue(A, B): + try: + if abs(A[1]) < abs(B[1]): return 1 + elif abs(A[1]) == abs(B[1]): + return 0 + else: return -1 + except: + return 0 + + symbolCorrDict, symbolPvalueDict = self.calculateCorrOfAllTissueTrait(primaryTraitSymbol=primaryTraitSymbol, TissueProbeSetFreezeId=TISSUE_MOUSE_DB, method=method) + + symbolCorrList = symbolCorrDict.items() + + symbolCorrList.sort(cmpTissCorrAbsoluteValue) + symbolCorrList = symbolCorrList[0 : 2*returnNumber] + + tmpTableName = webqtlUtil.genRandStr(prefix="TOPTISSUE") + + q1 = 'CREATE TEMPORARY TABLE %s (Symbol varchar(100) PRIMARY KEY, Correlation float, PValue float)' % tmpTableName + self.cursor.execute(q1) + + for one_pair in symbolCorrList: + one_symbol = one_pair[0] + one_corr = one_pair[1] + one_p_value = symbolPvalueDict[one_symbol] + + self.cursor.execute( "INSERT INTO %s (Symbol, Correlation, PValue) VALUES ('%s',%f,%f)" % (tmpTableName, one_symbol, float(one_corr), float(one_p_value)) ) + + return tmpTableName + + + #XZ, 01/09/2009: This function was created by David Crowell. Xiaodong cleaned up and modified it. + def fetchLitCorrelations(self, species, GeneId, db, returnNumber): ### Used to generate Lit Correlations when calculations are done from text file. dcrowell August 2008 + """Uses getTempLiteratureTable to generate table of literatire correlations. This function then gathers that data and + pairs it with the TraitID string. Takes as its arguments a formdata instance, and a database instance. + Returns a dictionary of 'TraitID':'LitCorr' for the requested correlation""" + + tempTable = self.getTempLiteratureTable(species=species, input_species_geneid=GeneId, returnNumber=returnNumber) + + query = "SELECT %s.Name, %s.value" % (db.type,tempTable) + query += ' FROM (%s, %sXRef, %sFreeze)' % (db.type, db.type, db.type) + query += ' LEFT JOIN %s ON %s.GeneId2=ProbeSet.GeneId ' % (tempTable,tempTable) + query += "WHERE ProbeSet.GeneId IS NOT NULL AND %s.value IS NOT NULL AND %sXRef.%sFreezeId = %sFreeze.Id and %sFreeze.Name = '%s' and %s.Id = %sXRef.%sId order by %s.Id" % (tempTable, db.type, db.type, db.type, db.type, db.name, db.type, db.type, db.type, db.type) + + self.cursor.execute(query) + results = self.cursor.fetchall() + + litCorrDict = {} + + for entry in results: + traitName,litcorr = entry + litCorrDict[traitName] = litcorr + + self.cursor.execute('DROP TEMPORARY TABLE %s' % tempTable) + + return litCorrDict + + + + #XZ, 01/09/2009: Xiaodong created this function. + def fetchTissueCorrelations(self, db, primaryTraitSymbol="", TissueProbeSetFreezeId=0, method="", returnNumber = 0): + """Uses getTempTissueCorrTable to generate table of tissue correlations. This function then gathers that data and + pairs it with the TraitID string. Takes as its arguments a formdata instance, and a database instance. + Returns a dictionary of 'TraitID':(tissueCorr, tissuePValue) for the requested correlation""" + + + tempTable = self.getTempTissueCorrTable(primaryTraitSymbol=primaryTraitSymbol, TissueProbeSetFreezeId=TISSUE_MOUSE_DB, method=method, returnNumber=returnNumber) + + query = "SELECT ProbeSet.Name, %s.Correlation, %s.PValue" % (tempTable, tempTable) + query += ' FROM (ProbeSet, ProbeSetXRef, ProbeSetFreeze)' + query += ' LEFT JOIN %s ON %s.Symbol=ProbeSet.Symbol ' % (tempTable,tempTable) + query += "WHERE ProbeSetFreeze.Name = '%s' and ProbeSetFreeze.Id=ProbeSetXRef.ProbeSetFreezeId and ProbeSet.Id = ProbeSetXRef.ProbeSetId and ProbeSet.Symbol IS NOT NULL AND %s.Correlation IS NOT NULL" % (db.name, tempTable) + + self.cursor.execute(query) + results = self.cursor.fetchall() + + tissueCorrDict = {} + + for entry in results: + traitName, tissueCorr, tissuePValue = entry + tissueCorrDict[traitName] = (tissueCorr, tissuePValue) + + self.cursor.execute('DROP TEMPORARY TABLE %s' % tempTable) + + return tissueCorrDict + + + + #XZ, 01/13/2008 + def getLiteratureCorrelationByList(self, input_trait_mouse_geneid=None, species=None, traitList=None): + + tmpTableName = webqtlUtil.genRandStr(prefix="LITERATURE") + + q1 = 'CREATE TEMPORARY TABLE %s (GeneId1 int(12) unsigned, GeneId2 int(12) unsigned PRIMARY KEY, value double)' % tmpTableName + q2 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId1,GeneId2,value FROM LCorrRamin3 WHERE GeneId1=%s' % (tmpTableName, input_trait_mouse_geneid) + q3 = 'INSERT INTO %s (GeneId1, GeneId2, value) SELECT GeneId2,GeneId1,value FROM LCorrRamin3 WHERE GeneId2=%s AND GeneId1!=%s' % (tmpTableName, input_trait_mouse_geneid, input_trait_mouse_geneid) + + for x in [q1,q2,q3]: + self.cursor.execute(x) + + for thisTrait in traitList: + try: + if thisTrait.geneid: + thisTrait.mouse_geneid = self.translateToMouseGeneID(species, thisTrait.geneid) + else: + thisTrait.mouse_geneid = 0 + except: + thisTrait.mouse_geneid = 0 + + if thisTrait.mouse_geneid and str(thisTrait.mouse_geneid).find(";") == -1: + try: + self.cursor.execute("SELECT value FROM %s WHERE GeneId2 = %s" % (tmpTableName, thisTrait.mouse_geneid)) + result = self.cursor.fetchone() + if result: + thisTrait.LCorr = result[0] + else: + thisTrait.LCorr = None + except: + thisTrait.LCorr = None + else: + thisTrait.LCorr = None + + self.cursor.execute("DROP TEMPORARY TABLE %s" % tmpTableName) + + return traitList + + def get_trait(self, cached, vals): + + if cached: + _log.info("Using the fast method because the file exists") + lit_corrs = {} + tissue_corrs = {} + use_lit = False + if self.method == METHOD_LIT: + lit_corrs = self.fetchLitCorrelations(species=self.species, GeneId=self.gene_id, db=self.db, returnNumber=self.returnNumber) + use_lit = True + + use_tissue_corr = False + if self.method in TISSUE_METHODS: + tissue_corrs = self.fetchTissueCorrelations(db=self.db, primaryTraitSymbol=self.trait_symbol, TissueProbeSetFreezeId=TISSUE_MOUSE_DB, method=self.method, returnNumber = self.returnNumber) + use_tissue_corr = True + + DatabaseFileName = self.getFileName( target_db_name=self.target_db_name ) + datasetFile = open(webqtlConfig.TEXTDIR+DatabaseFileName,'r') + + #XZ, 01/08/2009: read the first line + line = datasetFile.readline() + cached_sample_names = webqtlUtil.readLineCSV(line)[1:] + + #XZ, 01/08/2009: This step is critical. It is necessary for this new method. + #XZ: The original function fetchAllDatabaseData uses all strains stored in variable _strains to + #XZ: retrieve the values of each strain from database in real time. + #XZ: The new method uses all strains stored in variable dataset_strains to create a new variable + #XZ: _newvals. _newvals has the same length as dataset_strains. The items in _newvals is in + #XZ: the same order of items in dataset_strains. The value of each item in _newvals is either + #XZ: the value of correspinding strain in _vals or 'None'. + new_vals = [] + for name in cached_sample_names: + if name in self.sample_names: + new_vals.append(float(vals[self.sample_names.index(name)])) + else: + new_vals.append('None') + + nnCorr = len(new_vals) + + #XZ, 01/14/2009: If literature corr or tissue corr is selected, + #XZ: there is no need to use parallel computing. + + traits = [] + data_start = 1 + for line in datasetFile: + raw_trait = webqtlUtil.readLineCSV(line) + trait = Trait.from_csv(raw_trait, data_start) + trait.lit_corr = lit_corrs.get(trait.name) + trait.tissue_corr, trait.p_tissue = tissue_corrs.get(trait.name, (None, None)) + traits.append(trait) + + return traits, new_vals + + else: + _log.info("Using the slow method for correlation") + + _log.info("Fetching from database") + traits = self.fetchAllDatabaseData(species=self.species, GeneId=self.gene_id, GeneSymbol=self.trait_symbol, strains=self.sample_names, db=self.db, method=self.method, returnNumber=self.returnNumber, tissueProbeSetFreezeId= self.tissue_probeset_freeze_id) + _log.info("Done fetching from database") + totalTraits = len(traits) #XZ, 09/18/2008: total trait number + + return traits, vals + + + def do_parallel_correlation(self): + _log.info("Invoking parallel computing") + input_line_list = datasetFile.readlines() + _log.info("Read lines from the file") + all_line_number = len(input_line_list) + + step = 1000 + job_number = math.ceil( float(all_line_number)/step ) + + job_input_lists = [] + + _log.info("Configuring jobs") + + for job_index in range( int(job_number) ): + starti = job_index*step + endi = min((job_index+1)*step, all_line_number) + + one_job_input_list = [] + + for i in range( starti, endi ): + one_job_input_list.append( input_line_list[i] ) + + job_input_lists.append( one_job_input_list ) + + _log.info("Creating pp servers") + + ppservers = () + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + + _log.info("Done creating servers") + + jobs = [] + results = [] + + _log.info("Starting parallel computation, submitting jobs") + for one_job_input_list in job_input_lists: #pay attention to modules from outside + jobs.append( job_server.submit(func=compute_corr, args=(nnCorr, _newvals, one_job_input_list, self.method), depfuncs=(), modules=("utility.webqtlUtil",)) ) + _log.info("Done submitting jobs") + + for one_job in jobs: + one_result = one_job() + results.append( one_result ) + + _log.info("Acquiring results") + + for one_result in results: + for one_traitinfo in one_result: + allcorrelations.append( one_traitinfo ) + + _log.info("Appending the results") + + datasetFile.close() + totalTraits = len(allcorrelations) + _log.info("Done correlating using the fast method") + + + def correlate(self, vals): + + correlations = [] + + #XZ: Use the fast method only for probeset dataset, and this dataset must have been created. + #XZ: Otherwise, use original method + _log.info("Entering correlation") + + db_filename = self.getFileName( target_db_name=self.target_db_name ) + + cache_available = db_filename in os.listdir(webqtlConfig.TEXTDIR) + + # If the cache file exists, do a cached correlation for probeset data + if self.db.type == "ProbeSet": +# if self.method in [METHOD_SAMPLE_PEARSON, METHOD_SAMPLE_RANK] and cache_available: +# traits = do_parallel_correlation() +# +# else: + + (traits, vals) = self.get_trait(cache_available, vals) + + for trait in traits: + trait.calculate_correlation(vals, self.method) + + self.record_count = len(traits) #ZS: This isn't a good way to get this value, so I need to change it later + + #XZ, 3/31/2010: Theoretically, we should create one function 'comTissueCorr' + #to compare each trait by their tissue corr p values. + #But because the tissue corr p values are generated by permutation test, + #the top ones always have p value 0. So comparing p values actually does nothing. + #In addition, for the tissue data in our database, the N is always the same. + #So it's safe to compare with tissue corr statistic value. + #That's the same as literature corr. + #if self.method in [METHOD_LIT, METHOD_TISSUE_PEARSON, METHOD_TISSUE_RANK] and self.gene_id: + # traits.sort(webqtlUtil.cmpLitCorr) + #else: + #if self.method in TISSUE_METHODS: + # sort(traits, key=lambda A: math.fabs(A.tissue_corr)) + #elif self.method == METHOD_LIT: + # traits.sort(traits, key=lambda A: math.fabs(A.lit_corr)) + #else: + traits = sortTraitCorrelations(traits, self.method) + + # Strip to the top N correlations + traits = traits[:min(self.returnNumber, len(traits))] + + addLiteratureCorr = False + addTissueCorr = False + + trait_list = [] + for trait in traits: + db_trait = webqtlTrait(db=self.db, name=trait.name, cursor=self.cursor) + db_trait.retrieveInfo( QTL='Yes' ) + + db_trait.Name = trait.name + db_trait.corr = trait.correlation + db_trait.nOverlap = trait.overlap + db_trait.corrPValue = trait.p_value + + # NL, 07/19/2010 + # js function changed, add a new parameter rankOrder for js function 'showTissueCorrPlot' + db_trait.RANK_ORDER = self.RANK_ORDERS[self.method] + + #XZ, 26/09/2008: Method is 4 or 5. Have fetched tissue corr, but no literature correlation yet. + if self.method in TISSUE_METHODS: + db_trait.tissueCorr = trait.tissue_corr + db_trait.tissuePValue = trait.p_tissue + addTissueCorr = True + + + #XZ, 26/09/2008: Method is 3, Have fetched literature corr, but no tissue corr yet. + elif self.method == METHOD_LIT: + db_trait.LCorr = trait.lit_corr + db_trait.mouse_geneid = self.translateToMouseGeneID(self.species, db_trait.geneid) + addLiteratureCorr = True + + #XZ, 26/09/2008: Method is 1 or 2. Have NOT fetched literature corr and tissue corr yet. + # Phenotype data will not have geneid, and neither will some probes + # we need to handle this because we will get an attribute error + else: + if self.input_trait_mouse_gene_id and self.db.type=="ProbeSet": + addLiteratureCorr = True + if self.trait_symbol and self.db.type=="ProbeSet": + addTissueCorr = True + + trait_list.append(db_trait) + + if addLiteratureCorr: + trait_list = self.getLiteratureCorrelationByList(self.input_trait_mouse_gene_id, + self.species, trait_list) + if addTissueCorr: + trait_list = self.getTissueCorrelationByList( + primaryTraitSymbol = self.trait_symbol, + traitList = trait_list, + TissueProbeSetFreezeId = TISSUE_MOUSE_DB, + method=self.method) + + return trait_list + + + def calculateCorrOfAllTissueTrait(self, primaryTraitSymbol=None, TissueProbeSetFreezeId=None, method=None): + + symbolCorrDict = {} + symbolPvalueDict = {} + + primaryTraitSymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=[primaryTraitSymbol], TissueProbeSetFreezeId=TISSUE_MOUSE_DB) + primaryTraitValue = primaryTraitSymbolValueDict.values()[0] + + SymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=[], TissueProbeSetFreezeId=TISSUE_MOUSE_DB) + + if method in ["2","5"]: + symbolCorrDict, symbolPvalueDict = correlationFunction.batchCalTissueCorr(primaryTraitValue,SymbolValueDict,method='spearman') + else: + symbolCorrDict, symbolPvalueDict = correlationFunction.batchCalTissueCorr(primaryTraitValue,SymbolValueDict) + + + return (symbolCorrDict, symbolPvalueDict) + + + + #XZ, 10/13/2010 + def getTissueCorrelationByList(self, primaryTraitSymbol=None, traitList=None, TissueProbeSetFreezeId=None, method=None): + + primaryTraitSymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=[primaryTraitSymbol], TissueProbeSetFreezeId=TISSUE_MOUSE_DB) + + if primaryTraitSymbol.lower() in primaryTraitSymbolValueDict: + primaryTraitValue = primaryTraitSymbolValueDict[primaryTraitSymbol.lower()] + + geneSymbolList = [] + + for thisTrait in traitList: + if hasattr(thisTrait, 'symbol'): + geneSymbolList.append(thisTrait.symbol) + + SymbolValueDict = correlationFunction.getGeneSymbolTissueValueDictForTrait(cursor=self.cursor, GeneNameLst=geneSymbolList, TissueProbeSetFreezeId=TISSUE_MOUSE_DB) + + for thisTrait in traitList: + if hasattr(thisTrait, 'symbol') and thisTrait.symbol and thisTrait.symbol.lower() in SymbolValueDict: + oneTraitValue = SymbolValueDict[thisTrait.symbol.lower()] + if method in ["2","5"]: + result = correlationFunction.calZeroOrderCorrForTiss( primaryTraitValue, oneTraitValue, method='spearman' ) + else: + result = correlationFunction.calZeroOrderCorrForTiss( primaryTraitValue, oneTraitValue) + thisTrait.tissueCorr = result[0] + thisTrait.tissuePValue = result[2] + else: + thisTrait.tissueCorr = None + thisTrait.tissuePValue = None + else: + for thisTrait in traitList: + thisTrait.tissueCorr = None + thisTrait.tissuePValue = None + + return traitList + + + def getTopInfo(self, myTrait=None, method=None, db=None, target_db_name=None, returnNumber=None, methodDict=None, totalTraits=None, identification=None ): + + if myTrait: + if method in ["1","2"]: #genetic correlation + info = HT.Paragraph("Values of Record %s in the " % myTrait.getGivenName(), HT.Href(text=myTrait.db.fullname,url=webqtlConfig.INFOPAGEHREF % myTrait.db.name,target="_blank", Class="fwn"), + " database were compared to all %d records in the " % self.record_count, HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank", Class="fwn"), + ' database. The top %d correlations ranked by the %s are displayed.' % (returnNumber,methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.') + else: + #myTrait.retrieveInfo()#need to know geneid and symbol + if method == "3":#literature correlation + searchDBName = "Literature Correlation" + searchDBLink = "/correlationAnnotation.html#literatureCorr" + else: #tissue correlation + searchDBName = "Tissue Correlation" + searchDBLink = "/correlationAnnotation.html#tissueCorr" + info = HT.Paragraph("Your input record %s in the " % myTrait.getGivenName(), HT.Href(text=myTrait.db.fullname,url=webqtlConfig.INFOPAGEHREF % myTrait.db.name,target="_blank", Class="fwn"), + " database corresponds to ", + HT.Href(text='gene Id %s, and gene symbol %s' % (myTrait.geneid, myTrait.symbol), target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" % myTrait.geneid, Class="fs12 fwn"), + '. GN ranked all genes in the ', HT.Href(text=searchDBName,url=searchDBLink,target="_blank", Class="fwn"),' database by the %s.' % methodDict[method], + ' The top %d probes or probesets in the ' % returnNumber, HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank", Class="fwn"), + ' database corresponding to the top genes ranked by the %s are displayed.' %( methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.' ) + + elif identification: + info = HT.Paragraph('Values of %s were compared to all %d traits in ' % (identification, self.record_count), + HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank",Class="fwn"), + ' database. The TOP %d correlations ranked by the %s are displayed.' % (returnNumber,methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.') + + else: + info = HT.Paragraph('Trait values were compared to all values in ', + HT.Href(text=db.fullname,url=webqtlConfig.INFOPAGEHREF % target_db_name,target="_blank",Class="fwn"), + ' database. The TOP %d correlations ranked by the %s are displayed.' % (returnNumber,methodDict[method]), + ' You can resort this list using the small arrowheads in the top row.') + + if db.type=="Geno": + info.append(HT.BR(),HT.BR(),'Clicking on the Locus will open the genotypes data for that locus. Click on the correlation to see a scatter plot of the trait data.') + elif db.type=="Publish": + info.append(HT.BR(),HT.BR(),'Clicking on the record ID will open the published phenotype data for that publication. Click on the correlation to see a scatter plot of the trait data. ') + elif db.type=="ProbeSet": + info.append(HT.BR(),'Click the correlation values to generate scatter plots. Select the Record ID to open the Trait Data and Analysis form. Select the symbol to open NCBI Entrez.') + else: + pass + + + return info + + + def createExcelFileWithTitleAndFooter(self, workbook=None, identification=None, db=None, returnNumber=None): + + worksheet = workbook.add_worksheet() + + titleStyle = workbook.add_format(align = 'left', bold = 0, size=14, border = 1, border_color="gray") + + ##Write title Info + # Modified by Hongqiang Li + worksheet.write([1, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle) + worksheet.write([1, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle) + worksheet.write([2, 0], "Trait : %s" % identification, titleStyle) + worksheet.write([3, 0], "Database : %s" % db.fullname, titleStyle) + worksheet.write([4, 0], "Date : %s" % time.strftime("%B %d, %Y", time.gmtime()), titleStyle) + worksheet.write([5, 0], "Time : %s GMT" % time.strftime("%H:%M ", time.gmtime()), titleStyle) + worksheet.write([6, 0], "Status of data ownership: Possibly unpublished data; please see %s/statusandContact.html for details on sources, ownership, and usage of these data." % webqtlConfig.PORTADDR, titleStyle) + #Write footer info + worksheet.write([9 + returnNumber, 0], "Funding for The GeneNetwork: NIAAA (U01AA13499, U24AA13513), NIDA, NIMH, and NIAAA (P20-DA21131), NCI MMHCC (U01CA105417), and NCRR (U01NR 105417)", titleStyle) + worksheet.write([10 + returnNumber, 0], "PLEASE RETAIN DATA SOURCE INFORMATION WHENEVER POSSIBLE", titleStyle) + + return worksheet + + + def getTableHeaderForGeno(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Record', HT.BR(), 'ID', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Record ID', idx=1), + THCell(HT.TD('Location', HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Location (Chr and Mb)', idx=2), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample r", idx=3), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=4), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(r)", idx=5)]] + + for ncol, item in enumerate(['Record ID', 'Location (Chr, Mb)', 'Sample r', 'N Cases', 'Sample p(r)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb"), sort=0), + THCell(HT.TD('Record', HT.BR(), 'ID', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Record ID', idx=1), + THCell(HT.TD('Location', HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text='Location (Chr and Mb)', idx=2), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample rho", idx=3), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=4), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(rho)", idx=5)]] + + for ncol, item in enumerate(['Record ID', 'Location (Chr, Mb)', 'Sample rho', 'N Cases', 'Sample p(rho)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + + return tblobj_header, worksheet + + + def getTableBodyForGeno(self, traitList, formName=None, worksheet=None, newrow=None, corrScript=None): + + tblobj_body = [] + + for thisTrait in traitList: + tr = [] + + trId = str(thisTrait) + + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.corr)) + + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"),align="left", Class="fs12 fwn ffl b1 c222"), text=thisTrait.name, val=thisTrait.name.upper())) + + #XZ: trait_location_value is used for sorting + trait_location_repr = '--' + trait_location_value = 1000000 + + if thisTrait.chr and thisTrait.mb: + try: + trait_location_value = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if thisTrait.chr.upper() == 'X': + trait_location_value = 20*1000 + thisTrait.mb + else: + trait_location_value = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + trait_location_repr = 'Chr%s: %.6f' % (thisTrait.chr, float(thisTrait.mb) ) + + tr.append(TDCell(HT.TD(trait_location_repr, Class="fs12 fwn b1 c222", nowrap="on"), trait_location_repr, trait_location_value)) + + + repr='%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlot('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"), Class="fs12 fwn ffl b1 c222", nowrap='ON', align='right'),repr,abs(thisTrait.corr))) + + repr = '%d' % thisTrait.nOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222",align='right'),repr,thisTrait.nOverlap)) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, trait_location_repr, thisTrait.corr, thisTrait.nOverlap, thisTrait.corrPValue]): + worksheet.write([newrow, ncol], item) + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getTableHeaderForPublish(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Record ID", idx=1), + THCell(HT.TD('Phenotype', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Phenotype", idx=2), + THCell(HT.TD('Authors', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Authors", idx=3), + THCell(HT.TD('Year', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Year", idx=4), + THCell(HT.TD('Max',HT.BR(), 'LRS', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS", idx=5), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS Location", idx=6), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample r", idx=7), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=8), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(r)", idx=9)]] + + for ncol, item in enumerate(["Record", "Phenotype", "Authors", "Year", "Pubmed Id", "Max LRS", "Max LRS Location (Chr: Mb)", "Sample r", "N Cases", "Sample p(r)"]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Record ID", idx=1), + THCell(HT.TD('Phenotype', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Phenotype", idx=2), + THCell(HT.TD('Authors', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Authors", idx=3), + THCell(HT.TD('Year', HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Year", idx=4), + THCell(HT.TD('Max',HT.BR(), 'LRS', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS", idx=5), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap="on"), text="Max LRS Location", idx=6), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample rho", idx=7), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=8), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(rho)", idx=9)]] + + for ncol, item in enumerate(["Record", "Phenotype", "Authors", "Year", "Pubmed Id", "Max LRS", "Max LRS Location (Chr: Mb)", "Sample rho", "N Cases", "Sample p(rho)"]): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + + return tblobj_header, worksheet + + + def getTableBodyForPublish(self, traitList, formName=None, worksheet=None, newrow=None, corrScript=None, species=''): + + tblobj_body = [] + + for thisTrait in traitList: + tr = [] + + trId = str(thisTrait) + + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.corr)) + + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn"), nowrap="yes",align="center", Class="fs12 fwn b1 c222"),str(thisTrait.name), thisTrait.name)) + + PhenotypeString = thisTrait.post_publication_description + if thisTrait.confidential: + if not webqtlUtil.hasAccessToConfidentialPhenotypeTrait(privilege=self.privilege, userName=self.userName, authorized_users=thisTrait.authorized_users): + PhenotypeString = thisTrait.pre_publication_description + + tr.append(TDCell(HT.TD(PhenotypeString, Class="fs12 fwn b1 c222"), PhenotypeString, PhenotypeString.upper())) + + tr.append(TDCell(HT.TD(thisTrait.authors, Class="fs12 fwn b1 c222 fsI"),thisTrait.authors, thisTrait.authors.strip().upper())) + + try: + PubMedLinkText = myear = repr = int(thisTrait.year) + except: + PubMedLinkText = repr = "--" + myear = 0 + if thisTrait.pubmed_id: + PubMedLink = HT.Href(text= repr,url= webqtlConfig.PUBMEDLINK_URL % thisTrait.pubmed_id,target='_blank', Class="fs12 fwn") + else: + PubMedLink = repr + + tr.append(TDCell(HT.TD(PubMedLink, Class="fs12 fwn b1 c222", align='center'), repr, myear)) + + #LRS and its location + LRS_score_repr = '--' + LRS_score_value = 0 + LRS_location_repr = '--' + LRS_location_value = 1000000 + LRS_flag = 1 + + #Max LRS and its Locus location + if thisTrait.lrs and thisTrait.locus: + self.cursor.execute(""" + select Geno.Chr, Geno.Mb from Geno, Species + where Species.Name = '%s' and + Geno.Name = '%s' and + Geno.SpeciesId = Species.Id + """ % (species, thisTrait.locus)) + result = self.cursor.fetchone() + + if result: + if result[0] and result[1]: + LRS_Chr = result[0] + LRS_Mb = result[1] + + #XZ: LRS_location_value is used for sorting + try: + LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) + except: + if LRS_Chr.upper() == 'X': + LRS_location_value = 20*1000 + float(LRS_Mb) + else: + LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) + + + LRS_score_repr = '%3.1f' % thisTrait.lrs + LRS_score_value = thisTrait.lrs + LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb) ) + LRS_flag = 0 + + #tr.append(TDCell(HT.TD(HT.Href(text=LRS_score_repr,url="javascript:showIntervalMapping('%s', '%s : %s')" % (formName, thisTrait.db.shortname, thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn ffl b1 c222", align='right', nowrap="on"),LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222", align='right', nowrap="on"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222"), LRS_location_repr, LRS_location_value)) + + if LRS_flag: + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222"), LRS_location_repr, LRS_location_value)) + + repr = '%3.4f' % thisTrait.corr + tr.append(TDCell(HT.TD(HT.Href(text=repr,url="javascript:showCorrPlot('%s', '%s')" % (formName,thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn b1 c222", align='right',nowrap="on"), repr, abs(thisTrait.corr))) + + repr = '%d' % thisTrait.nOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.nOverlap)) + + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, PhenotypeString, thisTrait.authors, thisTrait.year, thisTrait.pubmed_id, LRS_score_repr, LRS_location_repr, thisTrait.corr, thisTrait.nOverlap, thisTrait.corrPValue]): + worksheet.write([newrow, ncol], item) + newrow += 1 + + return tblobj_body, worksheet, corrScript + + + def getTableHeaderForProbeSet(self, method=None, worksheet=None, newrow=None, headingStyle=None): + + tblobj_header = [] + + if method in ["1","3","4"]: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Record ID", idx=1), + THCell(HT.TD('Gene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Gene ID", idx=2), + THCell(HT.TD('Homologene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Homologene ID", idx=3), + THCell(HT.TD('Symbol',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Symbol", idx=4), + THCell(HT.TD('Description',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Description", idx=5), + THCell(HT.TD('Location',HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Location (Chr: Mb)", idx=6), + THCell(HT.TD('Mean',HT.BR(),'Expr',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Mean Expr", idx=7), + THCell(HT.TD('Max',HT.BR(),'LRS',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS", idx=8), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS Location (Chr: Mb)", idx=9), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample r", idx=10), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=11), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(r)", idx=12), + THCell(HT.TD(HT.Href( + text = HT.Span('Lit',HT.BR(), 'Corr', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#literatureCorr"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Lit Corr", idx=13), + #XZ, 09/22/2008: tissue correlation + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'r', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue r", idx=14), + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'p(r)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue p(r)", idx=15)]] + + for ncol, item in enumerate(['Record', 'Gene ID', 'Homologene ID', 'Symbol', 'Description', 'Location (Chr: Mb)', 'Mean Expr', 'Max LRS', 'Max LRS Location (Chr: Mb)', 'Sample r', 'N Cases', 'Sample p(r)', 'Lit Corr', 'Tissue r', 'Tissue p(r)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + else: + tblobj_header = [[THCell(HT.TD(' ', Class="fs13 fwb ffl b1 cw cbrb",nowrap='ON'), sort=0), + THCell(HT.TD('Record',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Record ID", idx=1), + THCell(HT.TD('Gene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Gene ID", idx=2), + THCell(HT.TD('Homologene',HT.BR(), 'ID',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Homologene ID", idx=3), + THCell(HT.TD('Symbol',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Symbol", idx=4), + THCell(HT.TD('Description',HT.BR(),HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Description", idx=5), + THCell(HT.TD('Location',HT.BR(), 'Chr and Mb', HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Location (Chr: Mb)", idx=6), + THCell(HT.TD('Mean',HT.BR(),'Expr',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="Mean Expr", idx=7), + THCell(HT.TD('Max',HT.BR(),'LRS',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS", idx=8), + THCell(HT.TD('Max LRS Location',HT.BR(),'Chr and Mb',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Max LRS Location (Chr: Mb)", idx=9), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample rho", idx=10), + THCell(HT.TD('N',HT.BR(),'Cases',HT.BR(), Class="fs13 fwb ffl b1 cw cbrb"), text="N Cases", idx=11), + THCell(HT.TD(HT.Href( + text = HT.Span('Sample',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#genetic_p_rho"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Sample p(rho)", idx=12), + THCell(HT.TD(HT.Href( + text = HT.Span('Lit',HT.BR(), 'Corr', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#literatureCorr"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Lit Corr", idx=13), + #XZ, 09/22/2008: tissue correlation + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'rho', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue rho", idx=14), + THCell(HT.TD(HT.Href( + text = HT.Span('Tissue',HT.BR(), 'p(rho)', HT.Sup(' ?', style="color:#f00"),HT.BR(), Class="fs13 fwb ffl cw"), + target = '_blank', + url = "/correlationAnnotation.html#tissue_p_r"), + Class="fs13 fwb ffl b1 cw cbrb", nowrap='ON'), text="Tissue p(rho)", idx=15)]] + + for ncol, item in enumerate(['Record ID', 'Gene ID', 'Homologene ID', 'Symbol', 'Description', 'Location (Chr: Mb)', 'Mean Expr', 'Max LRS', 'Max LRS Location (Chr: Mb)', 'Sample rho', 'N Cases', 'Sample p(rho)', 'Lit Corr', 'Tissue rho', 'Tissue p(rho)']): + worksheet.write([newrow, ncol], item, headingStyle) + worksheet.set_column([ncol, ncol], 2*len(item)) + + return tblobj_header, worksheet + + + def getTableBodyForProbeSet(self, traitList=[], primaryTrait=None, formName=None, worksheet=None, newrow=None, corrScript=None, species=''): + + tblobj_body = [] + + for thisTrait in traitList: + + if thisTrait.symbol: + pass + else: + thisTrait.symbol = "--" + + if thisTrait.geneid: + symbolurl = HT.Href(text=thisTrait.symbol,target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?db=gene&cmd=Retrieve&dopt=Graphics&list_uids=%s" % thisTrait.geneid, Class="fs12 fwn") + else: + symbolurl = HT.Href(text=thisTrait.symbol,target='_blank',url="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?CMD=search&DB=gene&term=%s" % thisTrait.symbol, Class="fs12 fwn") + + tr = [] + + trId = str(thisTrait) + + corrScript.append('corrArray["%s"] = {corr:%1.4f};' % (trId, thisTrait.corr)) + + #XZ, 12/08/2008: checkbox + tr.append(TDCell(HT.TD(HT.Input(type="checkbox", Class="checkbox", name="searchResult",value=trId, onClick="highlight(this)"), nowrap="on", Class="fs12 fwn ffl b1 c222"), text=trId)) + + #XZ, 12/08/2008: probeset name + tr.append(TDCell(HT.TD(HT.Href(text=thisTrait.name,url="javascript:showTrait('%s', '%s')" % (formName,thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn b1 c222"), thisTrait.name, thisTrait.name.upper())) + + #XZ, 12/08/2008: gene id + if thisTrait.geneid: + tr.append(TDCell(None, thisTrait.geneid, val=999)) + else: + tr.append(TDCell(None, thisTrait.geneid, val=999)) + + #XZ, 12/08/2008: homologene id + if thisTrait.homologeneid: + tr.append(TDCell("", thisTrait.homologeneid, val=999)) + else: + tr.append(TDCell("", thisTrait.homologeneid, val=999)) + + #XZ, 12/08/2008: gene symbol + tr.append(TDCell(HT.TD(symbolurl, Class="fs12 fwn b1 c222 fsI"),thisTrait.symbol, thisTrait.symbol.upper())) + + #XZ, 12/08/2008: description + #XZ, 06/05/2009: Rob asked to add probe target description + description_string = str(thisTrait.description).strip() + target_string = str(thisTrait.probe_target_description).strip() + + description_display = '' + + if len(description_string) > 1 and description_string != 'None': + description_display = description_string + else: + description_display = thisTrait.symbol + + if len(description_display) > 1 and description_display != 'N/A' and len(target_string) > 1 and target_string != 'None': + description_display = description_display + '; ' + target_string.strip() + + tr.append(TDCell(HT.TD(description_display, Class="fs12 fwn b1 c222"), description_display, description_display)) + + #XZ: trait_location_value is used for sorting + trait_location_repr = '--' + trait_location_value = 1000000 + + if thisTrait.chr and thisTrait.mb: + try: + trait_location_value = int(thisTrait.chr)*1000 + thisTrait.mb + except: + if thisTrait.chr.upper() == 'X': + trait_location_value = 20*1000 + thisTrait.mb + else: + trait_location_value = ord(str(thisTrait.chr).upper()[0])*1000 + thisTrait.mb + + trait_location_repr = 'Chr%s: %.6f' % (thisTrait.chr, float(thisTrait.mb) ) + + tr.append(TDCell(HT.TD(trait_location_repr, Class="fs12 fwn b1 c222", nowrap="on"), trait_location_repr, trait_location_value)) + + """ + #XZ, 12/08/2008: chromosome number + #XZ, 12/10/2008: use Mbvalue to sort chromosome + tr.append(TDCell( HT.TD(thisTrait.chr, Class="fs12 fwn b1 c222", align='right'), thisTrait.chr, Mbvalue) ) + + #XZ, 12/08/2008: Rob wants 6 digit precision, and we have to deal with that the mb could be None + if not thisTrait.mb: + tr.append(TDCell(HT.TD(thisTrait.mb, Class="fs12 fwn b1 c222",align='right'), thisTrait.mb, Mbvalue)) + else: + tr.append(TDCell(HT.TD('%.6f' % thisTrait.mb, Class="fs12 fwn b1 c222", align='right'), thisTrait.mb, Mbvalue)) + """ + + + + #XZ, 01/12/08: This SQL query is much faster. + self.cursor.execute(""" + select ProbeSetXRef.mean from ProbeSetXRef, ProbeSet + where ProbeSetXRef.ProbeSetFreezeId = %d and + ProbeSet.Id = ProbeSetXRef.ProbeSetId and + ProbeSet.Name = '%s' + """ % (thisTrait.db.id, thisTrait.name)) + result = self.cursor.fetchone() + if result: + if result[0]: + mean = result[0] + else: + mean=0 + else: + mean = 0 + + #XZ, 06/05/2009: It is neccessary to turn on nowrap + repr = "%2.3f" % mean + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right', nowrap='ON'),repr, mean)) + + #LRS and its location + LRS_score_repr = '--' + LRS_score_value = 0 + LRS_location_repr = '--' + LRS_location_value = 1000000 + LRS_flag = 1 + + #Max LRS and its Locus location + if thisTrait.lrs and thisTrait.locus: + self.cursor.execute(""" + select Geno.Chr, Geno.Mb from Geno, Species + where Species.Name = '%s' and + Geno.Name = '%s' and + Geno.SpeciesId = Species.Id + """ % (species, thisTrait.locus)) + result = self.cursor.fetchone() + + if result: + if result[0] and result[1]: + LRS_Chr = result[0] + LRS_Mb = result[1] + + #XZ: LRS_location_value is used for sorting + try: + LRS_location_value = int(LRS_Chr)*1000 + float(LRS_Mb) + except: + if LRS_Chr.upper() == 'X': + LRS_location_value = 20*1000 + float(LRS_Mb) + else: + LRS_location_value = ord(str(LRS_chr).upper()[0])*1000 + float(LRS_Mb) + + + LRS_score_repr = '%3.1f' % thisTrait.lrs + LRS_score_value = thisTrait.lrs + LRS_location_repr = 'Chr%s: %.6f' % (LRS_Chr, float(LRS_Mb) ) + LRS_flag = 0 + + #tr.append(TDCell(HT.TD(HT.Href(text=LRS_score_repr,url="javascript:showIntervalMapping('%s', '%s : %s')" % (formName, thisTrait.db.shortname, thisTrait.name), Class="fs12 fwn"), Class="fs12 fwn ffl b1 c222", align='right', nowrap="on"),LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222", align='right', nowrap="on"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222", nowrap="on"), LRS_location_repr, LRS_location_value)) + + if LRS_flag: + tr.append(TDCell(HT.TD(LRS_score_repr, Class="fs12 fwn b1 c222"), LRS_score_repr, LRS_score_value)) + tr.append(TDCell(HT.TD(LRS_location_repr, Class="fs12 fwn b1 c222"), LRS_location_repr, LRS_location_value)) + + + #XZ, 12/08/2008: generic correlation + repr='%3.3f' % thisTrait.corr + tr.append(TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlot('%s', '%s')" % (formName, thisTrait.name), Class="fs12 fwn ffl"), Class="fs12 fwn ffl b1 c222", align='right'),repr,abs(thisTrait.corr))) + + #XZ, 12/08/2008: number of overlaped cases + repr = '%d' % thisTrait.nOverlap + tr.append(TDCell(HT.TD(repr, Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.nOverlap)) + + #XZ, 12/08/2008: p value of genetic correlation + repr = webqtlUtil.SciFloat(thisTrait.corrPValue) + tr.append(TDCell(HT.TD(repr,nowrap='ON', Class="fs12 fwn ffl b1 c222", align='right'),repr,thisTrait.corrPValue)) + + #XZ, 12/08/2008: literature correlation + LCorr = 0.0 + LCorrStr = "--" + if hasattr(thisTrait, 'LCorr') and thisTrait.LCorr: + LCorr = thisTrait.LCorr + LCorrStr = "%2.3f" % thisTrait.LCorr + tr.append(TDCell(HT.TD(LCorrStr, Class="fs12 fwn b1 c222", align='right'), LCorrStr, abs(LCorr))) + + #XZ, 09/22/2008: tissue correlation. + TCorr = 0.0 + TCorrStr = "--" + #XZ, 11/20/2008: need to pass two geneids: input_trait_mouse_geneid and thisTrait.mouse_geneid + if hasattr(thisTrait, 'tissueCorr') and thisTrait.tissueCorr: + TCorr = thisTrait.tissueCorr + TCorrStr = "%2.3f" % thisTrait.tissueCorr + # NL, 07/19/2010: add a new parameter rankOrder for js function 'showTissueCorrPlot' + rankOrder = self.RANK_ORDERS[self.method] + TCorrPlotURL = "javascript:showTissueCorrPlot('%s','%s','%s',%d)" %(formName, primaryTrait.symbol, thisTrait.symbol,rankOrder) + tr.append(TDCell(HT.TD(HT.Href(text=TCorrStr, url=TCorrPlotURL, Class="fs12 fwn ff1"), Class="fs12 fwn ff1 b1 c222", align='right'), TCorrStr, abs(TCorr))) + else: + tr.append(TDCell(HT.TD(TCorrStr, Class="fs12 fwn b1 c222", align='right'), TCorrStr, abs(TCorr))) + + #XZ, 12/08/2008: p value of tissue correlation + TPValue = 1.0 + TPValueStr = "--" + if hasattr(thisTrait, 'tissueCorr') and thisTrait.tissuePValue: #XZ, 09/22/2008: thisTrait.tissuePValue can't be used here because it could be 0 + TPValue = thisTrait.tissuePValue + TPValueStr = "%2.3f" % thisTrait.tissuePValue + tr.append(TDCell(HT.TD(TPValueStr, Class="fs12 fwn b1 c222", align='right'), TPValueStr, TPValue)) + + tblobj_body.append(tr) + + for ncol, item in enumerate([thisTrait.name, thisTrait.geneid, thisTrait.homologeneid, thisTrait.symbol, thisTrait.description, trait_location_repr, mean, LRS_score_repr, LRS_location_repr, thisTrait.corr, thisTrait.nOverlap, thisTrait.corrPValue, LCorr, TCorr, TPValue]): + worksheet.write([newrow, ncol], item) + + newrow += 1 + + return tblobj_body, worksheet, corrScript diff --git a/wqflask/wqflask/correlation/__init__.py b/wqflask/wqflask/correlation/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wqflask/wqflask/correlation/correlationFunction.py b/wqflask/wqflask/correlation/correlationFunction.py new file mode 100644 index 00000000..4d62a468 --- /dev/null +++ b/wqflask/wqflask/correlation/correlationFunction.py @@ -0,0 +1,923 @@ +# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License +# as published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# This program is available from Source Forge: at GeneNetwork Project +# (sourceforge.net/projects/genenetwork/). +# +# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) +# at rwilliams@uthsc.edu and xzhou15@uthsc.edu +# +# +# +# This module is used by GeneNetwork project (www.genenetwork.org) +# +# Created by GeneNetwork Core Team 2010/08/10 +# +# Last updated by NL 2011/03/23 + + +import math +import rpy2.robjects +import pp +import string + +from utility import webqtlUtil +from base.webqtlTrait import webqtlTrait +from dbFunction import webqtlDatabaseFunction + + + +#XZ: The input 'controls' is String. It contains the full name of control traits. +#XZ: The input variable 'strainlst' is List. It contains the strain names of primary trait. +#XZ: The returned tcstrains is the list of list [[],[]...]. So are tcvals and tcvars. The last returned parameter is list of numbers. +#XZ, 03/29/2010: For each returned control trait, there is no None value in it. +def controlStrains(controls, strainlst): + + controls = controls.split(',') + + cvals = {} + for oneTraitName in controls: + oneTrait = webqtlTrait(fullname=oneTraitName, cursor=webqtlDatabaseFunction.getCursor() ) + oneTrait.retrieveData() + cvals[oneTraitName] = oneTrait.data + + tcstrains = [] + tcvals = [] + tcvars = [] + + for oneTraitName in controls: + strains = [] + vals = [] + vars = [] + + for _strain in strainlst: + if cvals[oneTraitName].has_key(_strain): + _val = cvals[oneTraitName][_strain].val + if _val != None: + strains.append(_strain) + vals.append(_val) + vars.append(None) + + tcstrains.append(strains) + tcvals.append(vals) + tcvars.append(vars) + + return tcstrains, tcvals, tcvars, [len(x) for x in tcstrains] + + + +#XZ, 03/29/2010: After execution of functon "controlStrains" and "fixStrains", primary trait and control traits have the same strains and in the same order. There is no 'None' value in them. +def fixStrains(_strains,_controlstrains,_vals,_controlvals,_vars,_controlvars): + """Corrects strains, vals, and vars so that all contrain only those strains common + to the reference trait and all control traits.""" + + def dictify(strains,vals,vars): + subdict = {} + for i in xrange(len(strains)): + subdict[strains[i]] = (vals[i],vars[i]) + return subdict + + #XZ: The 'dicts' is a list of dictionary. The first element is the dictionary of reference trait. The rest elements are for control traits. + dicts = [] + dicts.append(dictify(_strains,_vals,_vars)) + + nCstrains = len(_controlstrains) + for i in xrange(nCstrains): + dicts.append(dictify(_controlstrains[i],_controlvals[i],_controlvars[i])) + + _newstrains = [] + _vals = [] + _vars = [] + _controlvals = [[] for x in xrange(nCstrains)] + _controlvars = [[] for x in xrange(nCstrains)] + + for strain in _strains: + inall = True + for d in dicts: + if strain not in d: + inall = False + break + if inall: + _newstrains.append(strain) + _vals.append(dicts[0][strain][0]) + _vars.append(dicts[0][strain][1]) + for i in xrange(nCstrains): + _controlvals[i].append(dicts[i+1][strain][0]) + _controlvars[i].append(dicts[i+1][strain][1]) + + return _newstrains, _vals, _controlvals, _vars, _controlvars + + +#XZ, 6/15/2010: If there is no identical control traits, the returned list is empty. +#else, the returned list has two elements of control trait name. +def findIdenticalControlTraits ( controlVals, controlNames ): + nameOfIdenticalTraits = [] + + controlTraitNumber = len(controlVals) + + if controlTraitNumber > 1: + + #XZ: reset the precision of values and convert to string type + for oneTraitVal in controlVals: + for oneStrainVal in oneTraitVal: + oneStrainVal = '%.3f' % oneStrainVal + + for i, oneTraitVal in enumerate( controlVals ): + for j in range(i+1, controlTraitNumber): + if oneTraitVal == controlVals[j]: + nameOfIdenticalTraits.append(controlNames[i]) + nameOfIdenticalTraits.append(controlNames[j]) + + return nameOfIdenticalTraits + +#XZ, 6/15/2010: If there is no identical control traits, the returned list is empty. +#else, the returned list has two elements of control trait name. +#primaryVal is of list type. It contains value of primary trait. +#primaryName is of string type. +#controlVals is of list type. Each element is list too. Each element contain value of one control trait. +#controlNames is of list type. +def findIdenticalTraits (primaryVal, primaryName, controlVals, controlNames ): + nameOfIdenticalTraits = [] + + #XZ: reset the precision of values and convert to string type + for oneStrainVal in primaryVal: + oneStrainVal = '%.3f' % oneStrainVal + + for oneTraitVal in controlVals: + for oneStrainVal in oneTraitVal: + oneStrainVal = '%.3f' % oneStrainVal + + controlTraitNumber = len(controlVals) + + if controlTraitNumber > 1: + for i, oneTraitVal in enumerate( controlVals ): + for j in range(i+1, controlTraitNumber): + if oneTraitVal == controlVals[j]: + nameOfIdenticalTraits.append(controlNames[i]) + nameOfIdenticalTraits.append(controlNames[j]) + break + + if len(nameOfIdenticalTraits) == 0: + for i, oneTraitVal in enumerate( controlVals ): + if primaryVal == oneTraitVal: + nameOfIdenticalTraits.append(primaryName) + nameOfIdenticalTraits.append(controlNames[i]) + break + + return nameOfIdenticalTraits + + + +#XZ, 03/29/2010: The strains in primaryVal, controlVals, targetVals must be of the same number and in same order. +#XZ: No value in primaryVal and controlVals could be None. + +def determinePartialsByR (primaryVal, controlVals, targetVals, targetNames, method='p'): + + def compute_partial ( primaryVal, controlVals, targetVals, targetNames, method ): + + rpy2.robjects.r(""" +pcor.test <- function(x,y,z,use="mat",method="p",na.rm=T){ + # The partial correlation coefficient between x and y given z + # + # pcor.test is free and comes with ABSOLUTELY NO WARRANTY. + # + # x and y should be vectors + # + # z can be either a vector or a matrix + # + # use: There are two methods to calculate the partial correlation coefficient. + # One is by using variance-covariance matrix ("mat") and the other is by using recursive formula ("rec"). + # Default is "mat". + # + # method: There are three ways to calculate the correlation coefficient, + # which are Pearson's ("p"), Spearman's ("s"), and Kendall's ("k") methods. + # The last two methods which are Spearman's and Kendall's coefficient are based on the non-parametric analysis. + # Default is "p". + # + # na.rm: If na.rm is T, then all the missing samples are deleted from the whole dataset, which is (x,y,z). + # If not, the missing samples will be removed just when the correlation coefficient is calculated. + # However, the number of samples for the p-value is the number of samples after removing + # all the missing samples from the whole dataset. + # Default is "T". + + x <- c(x) + y <- c(y) + z <- as.data.frame(z) + + if(use == "mat"){ + p.use <- "Var-Cov matrix" + pcor = pcor.mat(x,y,z,method=method,na.rm=na.rm) + }else if(use == "rec"){ + p.use <- "Recursive formula" + pcor = pcor.rec(x,y,z,method=method,na.rm=na.rm) + }else{ + stop("use should be either rec or mat!\n") + } + + # print the method + if(gregexpr("p",method)[[1]][1] == 1){ + p.method <- "Pearson" + }else if(gregexpr("s",method)[[1]][1] == 1){ + p.method <- "Spearman" + }else if(gregexpr("k",method)[[1]][1] == 1){ + p.method <- "Kendall" + }else{ + stop("method should be pearson or spearman or kendall!\n") + } + + # sample number + n <- dim(na.omit(data.frame(x,y,z)))[1] + + # given variables' number + gn <- dim(z)[2] + + # p-value + if(p.method == "Kendall"){ + statistic <- pcor/sqrt(2*(2*(n-gn)+5)/(9*(n-gn)*(n-1-gn))) + p.value <- 2*pnorm(-abs(statistic)) + + }else{ + statistic <- pcor*sqrt((n-2-gn)/(1-pcor^2)) + p.value <- 2*pnorm(-abs(statistic)) + } + + data.frame(estimate=pcor,p.value=p.value,statistic=statistic,n=n,gn=gn,Method=p.method,Use=p.use) +} + +# By using var-cov matrix +pcor.mat <- function(x,y,z,method="p",na.rm=T){ + + x <- c(x) + y <- c(y) + z <- as.data.frame(z) + + if(dim(z)[2] == 0){ + stop("There should be given data\n") + } + + data <- data.frame(x,y,z) + + if(na.rm == T){ + data = na.omit(data) + } + + xdata <- na.omit(data.frame(data[,c(1,2)])) + Sxx <- cov(xdata,xdata,m=method) + + xzdata <- na.omit(data) + xdata <- data.frame(xzdata[,c(1,2)]) + zdata <- data.frame(xzdata[,-c(1,2)]) + Sxz <- cov(xdata,zdata,m=method) + + zdata <- na.omit(data.frame(data[,-c(1,2)])) + Szz <- cov(zdata,zdata,m=method) + + # is Szz positive definite? + zz.ev <- eigen(Szz)$values + if(min(zz.ev)[1]<0){ + stop("\'Szz\' is not positive definite!\n") + } + + # partial correlation + Sxx.z <- Sxx - Sxz %*% solve(Szz) %*% t(Sxz) + + rxx.z <- cov2cor(Sxx.z)[1,2] + + rxx.z +} + +# By using recursive formula +pcor.rec <- function(x,y,z,method="p",na.rm=T){ + # + + x <- c(x) + y <- c(y) + z <- as.data.frame(z) + + if(dim(z)[2] == 0){ + stop("There should be given data\n") + } + + data <- data.frame(x,y,z) + + if(na.rm == T){ + data = na.omit(data) + } + + # recursive formula + if(dim(z)[2] == 1){ + tdata <- na.omit(data.frame(data[,1],data[,2])) + rxy <- cor(tdata[,1],tdata[,2],m=method) + + tdata <- na.omit(data.frame(data[,1],data[,-c(1,2)])) + rxz <- cor(tdata[,1],tdata[,2],m=method) + + tdata <- na.omit(data.frame(data[,2],data[,-c(1,2)])) + ryz <- cor(tdata[,1],tdata[,2],m=method) + + rxy.z <- (rxy - rxz*ryz)/( sqrt(1-rxz^2)*sqrt(1-ryz^2) ) + + return(rxy.z) + }else{ + x <- c(data[,1]) + y <- c(data[,2]) + z0 <- c(data[,3]) + zc <- as.data.frame(data[,-c(1,2,3)]) + + rxy.zc <- pcor.rec(x,y,zc,method=method,na.rm=na.rm) + rxz0.zc <- pcor.rec(x,z0,zc,method=method,na.rm=na.rm) + ryz0.zc <- pcor.rec(y,z0,zc,method=method,na.rm=na.rm) + + rxy.z <- (rxy.zc - rxz0.zc*ryz0.zc)/( sqrt(1-rxz0.zc^2)*sqrt(1-ryz0.zc^2) ) + return(rxy.z) + } +} +""") + + R_pcorr_function = rpy2.robjects.r['pcor.test'] + R_corr_test = rpy2.robjects.r['cor.test'] + + primary = rpy2.robjects.FloatVector(range(len(primaryVal))) + for i in range(len(primaryVal)): + primary[i] = primaryVal[i] + + control = rpy2.robjects.r.matrix(rpy2.robjects.FloatVector( range(len(controlVals)*len(controlVals[0])) ), ncol=len(controlVals)) + for i in range(len(controlVals)): + for j in range(len(controlVals[0])): + control[i*len(controlVals[0]) + j] = controlVals[i][j] + + allcorrelations = [] + + for targetIndex, oneTargetVals in enumerate(targetVals): + + this_primary = None + this_control = None + this_target = None + + if None in oneTargetVals: + + goodIndex = [] + for i in range(len(oneTargetVals)): + if oneTargetVals[i] != None: + goodIndex.append(i) + + this_primary = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + this_primary[i] = primaryVal[goodIndex[i]] + + this_control = rpy2.robjects.r.matrix(rpy2.robjects.FloatVector( range(len(controlVals)*len(goodIndex)) ), ncol=len(controlVals)) + for i in range(len(controlVals)): + for j in range(len(goodIndex)): + this_control[i*len(goodIndex) + j] = controlVals[i][goodIndex[j]] + + this_target = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + this_target[i] = oneTargetVals[goodIndex[i]] + + else: + this_primary = primary + this_control = control + this_target = rpy2.robjects.FloatVector(range(len(oneTargetVals))) + for i in range(len(oneTargetVals)): + this_target[i] = oneTargetVals[i] + + one_name = targetNames[targetIndex] + one_N = len(this_primary) + + #calculate partial correlation + one_pc_coefficient = 'NA' + one_pc_p = 1 + + try: + if method == 's': + result = R_pcorr_function(this_primary, this_target, this_control, method='s') + else: + result = R_pcorr_function(this_primary, this_target, this_control) + + #XZ: In very few cases, the returned coefficient is nan. + #XZ: One way to detect nan is to compare the number to itself. NaN is always != NaN + if result[0][0] == result[0][0]: + one_pc_coefficient = result[0][0] + #XZ: when the coefficient value is 1 (primary trait and target trait are the same), + #XZ: occationally, the returned p value is nan instead of 0. + if result[1][0] == result[1][0]: + one_pc_p = result[1][0] + elif abs(one_pc_coefficient - 1) < 0.0000001: + one_pc_p = 0 + except: + pass + + #calculate zero order correlation + one_corr_coefficient = 0 + one_corr_p = 1 + + try: + if method == 's': + R_result = R_corr_test(this_primary, this_target, method='spearman') + else: + R_result = R_corr_test(this_primary, this_target) + + one_corr_coefficient = R_result[3][0] + one_corr_p = R_result[2][0] + except: + pass + + traitinfo = [ one_name, one_N, one_pc_coefficient, one_pc_p, one_corr_coefficient, one_corr_p ] + + allcorrelations.append(traitinfo) + + return allcorrelations + #End of function compute_partial + + + allcorrelations = [] + + target_trait_number = len(targetVals) + + if target_trait_number < 1000: + allcorrelations = compute_partial ( primaryVal, controlVals, targetVals, targetNames, method ) + else: + step = 1000 + job_number = math.ceil( float(target_trait_number)/step ) + + job_targetVals_lists = [] + job_targetNames_lists = [] + + for job_index in range( int(job_number) ): + starti = job_index*step + endi = min((job_index+1)*step, target_trait_number) + + one_job_targetVals_list = [] + one_job_targetNames_list = [] + + for i in range( starti, endi ): + one_job_targetVals_list.append( targetVals[i] ) + one_job_targetNames_list.append( targetNames[i] ) + + job_targetVals_lists.append( one_job_targetVals_list ) + job_targetNames_lists.append( one_job_targetNames_list ) + + ppservers = () + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + + jobs = [] + results = [] + + for i, one_job_targetVals_list in enumerate( job_targetVals_lists ): + one_job_targetNames_list = job_targetNames_lists[i] + #pay attention to modules from outside + jobs.append( job_server.submit(func=compute_partial, args=( primaryVal, controlVals, one_job_targetVals_list, one_job_targetNames_list, method), depfuncs=(), modules=("rpy2.robjects",)) ) + + for one_job in jobs: + one_result = one_job() + results.append( one_result ) + + for one_result in results: + for one_traitinfo in one_result: + allcorrelations.append( one_traitinfo ) + + return allcorrelations + + + +#XZ, April 30, 2010: The input primaryTrait and targetTrait are instance of webqtlTrait +#XZ: The primaryTrait and targetTrait should have executed retrieveData function +def calZeroOrderCorr (primaryTrait, targetTrait, method='pearson'): + + #primaryTrait.retrieveData() + + #there is no None value in primary_val + primary_strain, primary_val, primary_var = primaryTrait.exportInformative() + + #targetTrait.retrieveData() + + #there might be None value in target_val + target_val = targetTrait.exportData(primary_strain, type="val") + + R_primary = rpy2.robjects.FloatVector(range(len(primary_val))) + for i in range(len(primary_val)): + R_primary[i] = primary_val[i] + + N = len(target_val) + + if None in target_val: + goodIndex = [] + for i in range(len(target_val)): + if target_val[i] != None: + goodIndex.append(i) + + N = len(goodIndex) + + R_primary = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + R_primary[i] = primary_val[goodIndex[i]] + + R_target = rpy2.robjects.FloatVector(range(len(goodIndex))) + for i in range(len(goodIndex)): + R_target[i] = target_val[goodIndex[i]] + + else: + R_target = rpy2.robjects.FloatVector(range(len(target_val))) + for i in range(len(target_val)): + R_target[i] = target_val[i] + + R_corr_test = rpy2.robjects.r['cor.test'] + + if method == 'spearman': + R_result = R_corr_test(R_primary, R_target, method='spearman') + else: + R_result = R_corr_test(R_primary, R_target) + + corr_result = [] + corr_result.append( R_result[3][0] ) + corr_result.append( N ) + corr_result.append( R_result[2][0] ) + + return corr_result + +##################################################################################### +#Input: primaryValue(list): one list of expression values of one probeSet, +# targetValue(list): one list of expression values of one probeSet, +# method(string): indicate correlation method ('pearson' or 'spearman') +#Output: corr_result(list): first item is Correlation Value, second item is tissue number, +# third item is PValue +#Function: get correlation value,Tissue quantity ,p value result by using R; +#Note : This function is special case since both primaryValue and targetValue are from +#the same dataset. So the length of these two parameters is the same. They are pairs. +#Also, in the datatable TissueProbeSetData, all Tissue values are loaded based on +#the same tissue order +##################################################################################### + +def calZeroOrderCorrForTiss (primaryValue=[], targetValue=[], method='pearson'): + + R_primary = rpy2.robjects.FloatVector(range(len(primaryValue))) + N = len(primaryValue) + for i in range(len(primaryValue)): + R_primary[i] = primaryValue[i] + + R_target = rpy2.robjects.FloatVector(range(len(targetValue))) + for i in range(len(targetValue)): + R_target[i]=targetValue[i] + + R_corr_test = rpy2.robjects.r['cor.test'] + if method =='spearman': + R_result = R_corr_test(R_primary, R_target, method='spearman') + else: + R_result = R_corr_test(R_primary, R_target) + + corr_result =[] + corr_result.append( R_result[3][0]) + corr_result.append( N ) + corr_result.append( R_result[2][0]) + + return corr_result + + + + +def batchCalTissueCorr(primaryTraitValue=[], SymbolValueDict={}, method='pearson'): + + def cal_tissue_corr(primaryTraitValue, oneSymbolValueDict, method ): + + oneSymbolCorrDict = {} + oneSymbolPvalueDict = {} + + R_corr_test = rpy2.robjects.r['cor.test'] + + R_primary = rpy2.robjects.FloatVector(range(len(primaryTraitValue))) + + for i in range(len(primaryTraitValue)): + R_primary[i] = primaryTraitValue[i] + + for (oneTraitSymbol, oneTraitValue) in oneSymbolValueDict.iteritems(): + R_target = rpy2.robjects.FloatVector(range(len(oneTraitValue))) + for i in range(len(oneTraitValue)): + R_target[i] = oneTraitValue[i] + + if method =='spearman': + R_result = R_corr_test(R_primary, R_target, method='spearman') + else: + R_result = R_corr_test(R_primary, R_target) + + oneSymbolCorrDict[oneTraitSymbol] = R_result[3][0] + oneSymbolPvalueDict[oneTraitSymbol] = R_result[2][0] + + return(oneSymbolCorrDict, oneSymbolPvalueDict) + + + + symbolCorrDict = {} + symbolPvalueDict = {} + + items_number = len(SymbolValueDict) + + if items_number <= 1000: + symbolCorrDict, symbolPvalueDict = cal_tissue_corr(primaryTraitValue, SymbolValueDict, method) + else: + items_list = SymbolValueDict.items() + + step = 1000 + job_number = math.ceil( float(items_number)/step ) + + job_oneSymbolValueDict_list = [] + + for job_index in range( int(job_number) ): + starti = job_index*step + endi = min((job_index+1)*step, items_number) + + oneSymbolValueDict = {} + + for i in range( starti, endi ): + one_item = items_list[i] + one_symbol = one_item[0] + one_value = one_item[1] + oneSymbolValueDict[one_symbol] = one_value + + job_oneSymbolValueDict_list.append( oneSymbolValueDict ) + + + ppservers = () + # Creates jobserver with automatically detected number of workers + job_server = pp.Server(ppservers=ppservers) + + jobs = [] + results = [] + + for i, oneSymbolValueDict in enumerate( job_oneSymbolValueDict_list ): + + #pay attention to modules from outside + jobs.append( job_server.submit(func=cal_tissue_corr, args=(primaryTraitValue, oneSymbolValueDict, method), depfuncs=(), modules=("rpy2.robjects",)) ) + + for one_job in jobs: + one_result = one_job() + results.append( one_result ) + + for one_result in results: + oneSymbolCorrDict, oneSymbolPvalueDict = one_result + symbolCorrDict.update( oneSymbolCorrDict ) + symbolPvalueDict.update( oneSymbolPvalueDict ) + + return (symbolCorrDict, symbolPvalueDict) + +########################################################################### +#Input: cursor, GeneNameLst (list), TissueProbeSetFreezeId +#output: geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict (Dict) +#function: get multi dicts for short and long label functions, and for getSymbolValuePairDict and +# getGeneSymbolTissueValueDict to build dict to get CorrPvArray +#Note: If there are multiple probesets for one gene, select the one with highest mean. +########################################################################### +def getTissueProbeSetXRefInfo(cursor=None,GeneNameLst=[],TissueProbeSetFreezeId=0): + Symbols ="" + symbolList =[] + geneIdDict ={} + dataIdDict = {} + ChrDict = {} + MbDict = {} + descDict = {} + pTargetDescDict = {} + + count = len(GeneNameLst) + + # Added by NL 01/06/2011 + # Note that:inner join is necessary in this query to get distinct record in one symbol group with highest mean value + # Duo to the limit size of TissueProbeSetFreezeId table in DB, performance of inner join is acceptable. + if count==0: + query=''' + select t.Symbol,t.GeneId, t.DataId,t.Chr, t.Mb,t.description,t.Probe_Target_Description + from ( + select Symbol, max(Mean) as maxmean + from TissueProbeSetXRef + where TissueProbeSetFreezeId=%s and Symbol!='' and Symbol Is Not Null group by Symbol) + as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; + '''%TissueProbeSetFreezeId + + else: + for i, item in enumerate(GeneNameLst): + + if i == count-1: + Symbols += "'%s'" %item + else: + Symbols += "'%s'," %item + + Symbols = "("+ Symbols+")" + query=''' + select t.Symbol,t.GeneId, t.DataId,t.Chr, t.Mb,t.description,t.Probe_Target_Description + from ( + select Symbol, max(Mean) as maxmean + from TissueProbeSetXRef + where TissueProbeSetFreezeId=%s and Symbol in %s group by Symbol) + as x inner join TissueProbeSetXRef as t on t.Symbol = x.Symbol and t.Mean = x.maxmean; + '''% (TissueProbeSetFreezeId,Symbols) + + try: + + cursor.execute(query) + results =cursor.fetchall() + resultCount = len(results) + # Key in all dicts is the lower-cased symbol + for i, item in enumerate(results): + symbol = item[0] + symbolList.append(symbol) + + key =symbol.lower() + geneIdDict[key]=item[1] + dataIdDict[key]=item[2] + ChrDict[key]=item[3] + MbDict[key]=item[4] + descDict[key]=item[5] + pTargetDescDict[key]=item[6] + + except: + symbolList = None + geneIdDict=None + dataIdDict=None + ChrDict=None + MbDict=None + descDict=None + pTargetDescDict=None + + return symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict + +########################################################################### +#Input: cursor, symbolList (list), dataIdDict(Dict) +#output: symbolValuepairDict (dictionary):one dictionary of Symbol and Value Pair, +# key is symbol, value is one list of expression values of one probeSet; +#function: get one dictionary whose key is gene symbol and value is tissue expression data (list type). +#Attention! All keys are lower case! +########################################################################### +def getSymbolValuePairDict(cursor=None,symbolList=None,dataIdDict={}): + symbolList = map(string.lower, symbolList) + symbolValuepairDict={} + valueList=[] + + for key in symbolList: + if dataIdDict.has_key(key): + DataId = dataIdDict[key] + + valueQuery = "select value from TissueProbeSetData where Id=%s" % DataId + try : + cursor.execute(valueQuery) + valueResults = cursor.fetchall() + for item in valueResults: + item =item[0] + valueList.append(item) + symbolValuepairDict[key] = valueList + valueList=[] + except: + symbolValuepairDict[key] = None + + return symbolValuepairDict + + +######################################################################################################## +#input: cursor, symbolList (list), dataIdDict(Dict): key is symbol +#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. +# key is symbol, value is one list of expression values of one probeSet. +#function: wrapper function for getSymbolValuePairDict function +# build gene symbol list if necessary, cut it into small lists if necessary, +# then call getSymbolValuePairDict function and merge the results. +######################################################################################################## + +def getGeneSymbolTissueValueDict(cursor=None,symbolList=None,dataIdDict={}): + limitNum=1000 + count = len(symbolList) + + SymbolValuePairDict = {} + + if count !=0 and count <=limitNum: + SymbolValuePairDict = getSymbolValuePairDict(cursor=cursor,symbolList=symbolList,dataIdDict=dataIdDict) + + elif count >limitNum: + SymbolValuePairDict={} + n = count/limitNum + start =0 + stop =0 + + for i in range(n): + stop =limitNum*(i+1) + gList1 = symbolList[start:stop] + PairDict1 = getSymbolValuePairDict(cursor=cursor,symbolList=gList1,dataIdDict=dataIdDict) + start =limitNum*(i+1) + + SymbolValuePairDict.update(PairDict1) + + if stop < count: + stop = count + gList2 = symbolList[start:stop] + PairDict2 = getSymbolValuePairDict(cursor=cursor,symbolList=gList2,dataIdDict=dataIdDict) + SymbolValuePairDict.update(PairDict2) + + return SymbolValuePairDict + +######################################################################################################## +#input: cursor, GeneNameLst (list), TissueProbeSetFreezeId(int) +#output: SymbolValuePairDict(dictionary):one dictionary of Symbol and Value Pair. +# key is symbol, value is one list of expression values of one probeSet. +#function: wrapper function of getGeneSymbolTissueValueDict function +# for CorrelationPage.py +######################################################################################################## + +def getGeneSymbolTissueValueDictForTrait(cursor=None,GeneNameLst=[],TissueProbeSetFreezeId=0): + SymbolValuePairDict={} + symbolList,geneIdDict,dataIdDict,ChrDict,MbDict,descDict,pTargetDescDict = getTissueProbeSetXRefInfo(cursor=cursor,GeneNameLst=GeneNameLst,TissueProbeSetFreezeId=TissueProbeSetFreezeId) + if symbolList: + SymbolValuePairDict = getGeneSymbolTissueValueDict(cursor=cursor,symbolList=symbolList,dataIdDict=dataIdDict) + return SymbolValuePairDict + +######################################################################################################## +#Input: cursor(cursor): MySQL connnection cursor; +# priGeneSymbolList(list): one list of gene symbol; +# symbolValuepairDict(dictionary): one dictionary of Symbol and Value Pair, +# key is symbol, value is one list of expression values of one probeSet; +#Output: corrArray(array): array of Correlation Value, +# pvArray(array): array of PValue; +#Function: build corrArray, pvArray for display by calling calculation function:calZeroOrderCorrForTiss +######################################################################################################## + +def getCorrPvArray(cursor=None,priGeneSymbolList=[],symbolValuepairDict={}): + # setting initial value for corrArray, pvArray equal to 0 + Num = len(priGeneSymbolList) + + corrArray = [([0] * (Num))[:] for i in range(Num)] + pvArray = [([0] * (Num))[:] for i in range(Num)] + i = 0 + for pkey in priGeneSymbolList: + j = 0 + pkey = pkey.strip().lower()# key in symbolValuepairDict is low case + if symbolValuepairDict.has_key(pkey): + priValue = symbolValuepairDict[pkey] + for tkey in priGeneSymbolList: + tkey = tkey.strip().lower()# key in symbolValuepairDict is low case + if priValue and symbolValuepairDict.has_key(tkey): + tarValue = symbolValuepairDict[tkey] + + if tarValue: + if i>j: + # corrArray stores Pearson Correlation values + # pvArray stores Pearson P-Values + pcorr_result =calZeroOrderCorrForTiss(primaryValue=priValue,targetValue=tarValue) + corrArray[i][j] =pcorr_result[0] + pvArray[i][j] =pcorr_result[2] + elif i console.log("Found Show Outliers") $('#show_hide_outliers').val("Hide Outliers") console.log("Should be now Hide Outliers") - + + ### Calculate Correlations Code ### @@ -165,6 +166,17 @@ $ -> $('select[name=corr_method]').change(on_corr_method_change) + on_corr_submit = -> + console.log("in beginning of on_corr_submit") + values = $('#trait_data_form').serialize() + console.log("in on_corr_submit, values are:", values) + $.ajax "/corr_compute", + type: 'GET' + dataType: 'html' + data: values + + $('#corr_compute').click(on_corr_submit) + ### End Calculate Correlations Code ### @@ -177,8 +189,9 @@ $ -> _.mixin(_.str.exports()); # Add string fuctions directly to underscore $('#value_table').change(edit_data_change) console.log("loaded") - console.log("basic_table is:", basic_table) - make_table() - edit_data_change() # Set the values at the beginning + #console.log("basic_table is:", basic_table) + # Add back following two lines later + #make_table() + #edit_data_change() # Set the values at the beginning #$("#all-mean").html('foobar8') console.log("end") diff --git a/wqflask/wqflask/static/new/javascript/trait_data_and_analysis.js b/wqflask/wqflask/static/new/javascript/trait_data_and_analysis.js index 96b245ea..0de0297b 100644 --- a/wqflask/wqflask/static/new/javascript/trait_data_and_analysis.js +++ b/wqflask/wqflask/static/new/javascript/trait_data_and_analysis.js @@ -10,7 +10,7 @@ }; $(function() { - var edit_data_change, hide_tabs, make_table, on_corr_method_change, process_id, show_hide_outliers, stats_mdp_change, update_stat_values; + var edit_data_change, hide_tabs, make_table, on_corr_method_change, on_corr_submit, process_id, show_hide_outliers, stats_mdp_change, update_stat_values; hide_tabs = function(start) { var x, _i, _results; _results = []; @@ -199,6 +199,18 @@ return $('#' + corr_method + "_r_desc").show().effect("highlight"); }; $('select[name=corr_method]').change(on_corr_method_change); + on_corr_submit = function() { + var values; + console.log("in beginning of on_corr_submit"); + values = $('#trait_data_form').serialize(); + console.log("in on_corr_submit, values are:", values); + return $.ajax("/corr_compute", { + type: 'GET', + dataType: 'html', + data: values + }); + }; + $('#corr_compute').click(on_corr_submit); /* End Calculate Correlations Code */ @@ -209,9 +221,6 @@ _.mixin(_.str.exports()); $('#value_table').change(edit_data_change); console.log("loaded"); - console.log("basic_table is:", basic_table); - make_table(); - edit_data_change(); return console.log("end"); }); diff --git a/wqflask/wqflask/templates/trait_data_and_analysis.html b/wqflask/wqflask/templates/trait_data_and_analysis.html index f6917a90..a5d0e05c 100644 --- a/wqflask/wqflask/templates/trait_data_and_analysis.html +++ b/wqflask/wqflask/templates/trait_data_and_analysis.html @@ -8,7 +8,7 @@ @@ -1308,9 +1311,9 @@ - --> --> diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 1ed3c1fd..114ec458 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -8,6 +8,7 @@ from flask import render_template, request from wqflask import search_results from wqflask.show_trait import show_trait_page +from wqflask.correlation import CorrelationPage from wqflask.dataSharing import SharingInfo, SharingInfoPage @@ -74,7 +75,14 @@ def showDatabaseBXD(): print("showDatabaseBXD template_vars:", pf(template_vars.__dict__)) return render_template("trait_data_and_analysis.html", **template_vars.__dict__) - +@app.route("/corr_compute") +def corr_compute(): + print("In corr_compute") + fd = webqtlFormData.webqtlFormData(request.args) + print("Have fd") + template_vars = CorrelationPage.CorrelationPage(fd) + print("Made it to rendering") + return render_template("corr_compute.html", **template_vars.__dict__) # Todo: Can we simplify this? -Sam def sharing_info_page(): -- cgit v1.2.3
-
+
{# @@ -51,9 +51,9 @@ #} - {% for key in hddn %} - - {% endfor %} + {% for key in hddn %} + + {% endfor %}
Trait Data and Analysis  for Record ID 1441186_at @@ -770,7 +770,9 @@ Pearson    Spearman Rank

-

+ +

+ The Sample Correlation is computed @@ -801,6 +803,7 @@ Pearson and Spearman Rank correlations have been computed for all pairs of genes
using data from mouse samples.
 +