Merge branch 'testing' of https://github.com/genenetwork/genenetwork2 into production

11 files changed, 152 insertions, 2075 deletions

diff --git a/wqflask/utility/AJAX_table.py b/wqflask/utility/AJAX_table.py
deleted file mode 100644
index d70acfcd..00000000
--- a/wqflask/utility/AJAX_table.py
+++ /dev/null
@@ -1,155 +0,0 @@
-# Copyright (C) University of Tennessee Health Science Center, Memphis, TN.
-#
-# This program is free software: you can redistribute it and/or modify it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
-# See the GNU Affero General Public License for more details.
-#
-# This program is available from Source Forge: at GeneNetwork Project
-# (sourceforge.net/projects/genenetwork/).
-#
-# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010)
-# at rwilliams@uthsc.edu and xzhou15@uthsc.edu
-#
-#
-#
-# This module is used by GeneNetwork project (www.genenetwork.org)
-#
-# Created by GeneNetwork Core Team 2010/08/10
-#
-# Last updated by GeneNetwork Core Team 2010/10/20
-
-import cPickle
-import os
-import MySQLdb
-import time
-import pyXLWriter as xl
-
-from htmlgen import HTMLgen2 as HT
-
-from base import webqtlConfig
-from THCell import THCell
-from TDCell import TDCell
-import webqtlUtil
-
-
-class AJAX_table:
-    def __init__(self, fd):
-        file = fd.formdata.getfirst("file", "")
-        sort = fd.formdata.getfirst("sort", "")
-        order = fd.formdata.getfirst("order", "up")
-        cmd = fd.formdata.getfirst("cmd", "")
-        tableID = fd.formdata.getfirst("tableID", "")
-        addIndex = fd.formdata.getfirst("addIndex", "1")
-        hiddenColumnsString = fd.formdata.getfirst("hiddenColumns", "")
-        hiddenColumns = hiddenColumnsString.split(',')
-
-        try:
-            fp = open(os.path.join(webqtlConfig.TMPDIR, file + '.obj'), 'rb')
-            tblobj = cPickle.load(fp)
-            fp.close()
-
-            if cmd == 'addCorr':
-                dbId = int(fd.formdata.getfirst("db"))
-                dbFullName = fd.formdata.getfirst("dbname")
-                trait = fd.formdata.getfirst("trait")
-                form = fd.formdata.getfirst("form")
-                ids = fd.formdata.getfirst("ids")
-                vals = fd.formdata.getfirst("vals")
-                ids = eval(ids)
-                nnCorr = len(ids)
-                vals = eval(vals)
-
-                workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+file))
-                worksheet = workbook.add_worksheet()
-
-                logger.warning("Creating new MySQLdb cursor (this method is OBSOLETE!)")
-
-                con = MySQLdb.Connect(db=webqtlConfig.DB_NAME,host=webqtlConfig.MYSQL_SERVER, user=webqtlConfig.DB_USER,passwd=webqtlConfig.DB_PASSWD)
-                cursor = con.cursor()
-
-                cursor.execute("Select name, ShortName from ProbeSetFreeze where Id = %s", dbId)
-                dbName, dbShortName = cursor.fetchone()
-
-                tblobj['header'][0].append(
-                        THCell(HT.TD(dbShortName, Class="fs11 ffl b1 cw cbrb"),
-                        text="%s" % dbShortName, idx=tblobj['header'][0][-1].idx + 1),
-                )
-
-                headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white")
-                for i, item in enumerate(tblobj['header'][0]):
-                    if (i > 0):
-                        worksheet.write([8, i-1], item.text, headingStyle)
-                        worksheet.set_column([i-1, i-1], 2*len(item.text))
-
-                for i, row in enumerate(tblobj['body']):
-                    ProbeSetId = row[1].text
-                    #XZ, 03/02/2009: Xiaodong changed Data to ProbeSetData
-                    cursor.execute("""
-                            Select ProbeSetData.StrainId, ProbeSetData.Value
-                            From ProbeSetData, ProbeSetXRef, ProbeSet
-                            where ProbeSetXRef.ProbeSetFreezeId = %d AND
-                                    ProbeSetXRef.DataId = ProbeSetData.Id AND
-                                    ProbeSetXRef.ProbeSetId = ProbeSet.Id AND
-                                    ProbeSet.Name = '%s'
-                    """ % (dbId, ProbeSetId))
-                    results = cursor.fetchall()
-                    vdict = {}
-                    for item in results:
-                        vdict[item[0]] = item[1]
-                    newvals = []
-                    for id in ids:
-                        if vdict.has_key(id):
-                            newvals.append(vdict[id])
-                        else:
-                            newvals.append(None)
-                    corr,nOverlap= webqtlUtil.calCorrelation(newvals,vals,nnCorr)
-                    repr = '%0.4f' % corr
-                    row.append(
-                            TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlotThird('%s', '%s', '%s')" % (form, dbName, ProbeSetId), Class="fs11 fwn ffl"), " / ", nOverlap, Class="fs11 fwn ffl b1 c222", align="middle"),repr,abs(corr))
-                    )
-
-                    last_row=0
-                    for j, item in enumerate(tblobj['body'][i]):
-                        if (j > 0):
-                            worksheet.write([9+i, j-1], item.text)
-                            last_row = 9+i
-                    last_row += 1
-
-                titleStyle = workbook.add_format(align = 'left', bold = 0, size=14, border = 1, border_color="gray")
-                ##Write title Info
-                # Modified by Hongqiang Li
-                worksheet.write([0, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle)
-                worksheet.write([1, 0], "Trait : %s" % trait, titleStyle)
-                worksheet.write([2, 0], "Database : %s" % dbFullName, titleStyle)
-                worksheet.write([3, 0], "Date : %s" % time.strftime("%B %d, %Y", time.gmtime()), titleStyle)
-                worksheet.write([4, 0], "Time : %s GMT" % time.strftime("%H:%M ", time.gmtime()), titleStyle)
-                worksheet.write([5, 0], "Status of data ownership: Possibly unpublished data; please see %s/statusandContact.html for details on sources, ownership, and usage of these data." % webqtlConfig.PORTADDR, titleStyle)
-                #Write footer info
-                worksheet.write([1 + last_row, 0], "Funding for The GeneNetwork: NIAAA (U01AA13499, U24AA13513), NIDA, NIMH, and NIAAA (P20-DA21131), NCI MMHCC (U01CA105417), and NCRR (U01NR 105417)", titleStyle)
-                worksheet.write([2 + last_row, 0], "PLEASE RETAIN DATA SOURCE INFORMATION WHENEVER POSSIBLE", titleStyle)
-
-                cursor.close()
-                workbook.close()
-
-                objfile = open(os.path.join(webqtlConfig.TMPDIR, file + '.obj'), 'wb')
-                cPickle.dump(tblobj, objfile)
-                objfile.close()
-            else:
-                pass
-
-            self.value = str(webqtlUtil.genTableObj(tblobj=tblobj, file=file, sortby=(sort, order), tableID = tableID, addIndex = addIndex, hiddenColumns = hiddenColumns))
-
-        except:
-            self.value = "<span class='fs16 fwb cr ffl'>The table is no longer available on this server</span>"
-
-    def __str__(self):
-        return self.value
-
-    def write(self):
-        return str(self)
diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py
index d60e2bb2..529cd117 100644
--- a/wqflask/utility/Plot.py
+++ b/wqflask/utility/Plot.py
@@ -36,11 +36,9 @@ from numarray import linear_algebra as la
 from numarray import ones, array, dot, swapaxes
 
 import reaper
-# sys.path.append("..") Never in a running webserver
-from basicStatistics import corestats
 
-import svg
 import webqtlUtil
+import corestats
 from base import webqtlConfig
 
 import utility.logger
@@ -83,202 +81,6 @@ def frange(start, end=None, inc=1.0):
         L[i] = start + i * inc
     return L
 
-
-def gammln(xx):
-    cof=[76.18009173,-86.50532033,24.01409822,-1.231739516,0.120858003e-2,-0.536382e-5]
-    x=xx-1.0
-    tmp=x+5.5
-    tmp -=(x+0.5)*log(tmp)
-    ser=1.0
-    for item in cof:
-        x+=1.0
-        ser+=item/x
-
-    return -tmp+log(2.50662827465*ser)
-
-
-def gser(a,x):
-    gln=gammln(a)
-    ITMAX=100
-    EPS=3.0e-7
-
-    if x<=0.0:
-        gamser=0.0
-        return [gamser,gln]
-    else:
-        ap=a
-        sum=1.0/a
-        dele=sum
-        for i in range(1,ITMAX+1):
-            ap+=1.0
-            dele*=x/ap
-            sum+=dele
-            if abs(dele)<abs(sum)*EPS:
-                gamser=sum*exp(-x+a*log(x)-gln)
-                return [gamser,gln]
-    return None
-
-def gcf(a,x):
-    ITMAX=100
-    EPS=3.0e-7
-    gold=0.0
-    fac=1
-    b1=1.0
-    b0=0.0
-    a0=1.0
-    gln=gammln(a)
-
-    a1=x
-    for n in range(1,ITMAX+1):
-        an=n+0.0
-        ana=an-a
-        a0=(a1+a0*ana)*fac
-        b0=(b1+b0*ana)*fac
-        anf=an*fac
-        a1=x*a0+anf*a1
-        b1=x*b0+anf*b1
-        if (a1):
-            fac=1.0/a1
-            g=b1*fac
-            if abs((g-gold)/g)<EPS:
-                gammcf=exp(-x+a*log(x)-gln)*g
-                return [gammcf,gln]
-            gold=g
-    return None
-
-def gammp(a,x):
-    if x<0.0 or a<=0.0:
-        return None
-    if x<(a+1.0):
-        a=gser(a,x)[0]
-        return a
-    else:
-        a=gcf(a,x)[0]
-        return 1.0-a
-def U(n):
-    x=pow(0.5,1.0/n)
-    m=[1-x]
-    for i in range(2,n):
-        a=(i-0.3175)/(n+0.365)
-        m.append(a)
-    m.append(x)
-    return m
-
-def erf(x):
-    if x<0.0:
-        return -gammp(0.5,x*x)
-    else:
-        return gammp(0.5,x*x)
-
-def erfcc(x):
-    z=abs(x)
-    t=1.0/(1.0+0.5*z)
-    ans=t*exp(-z*z-1.26551223+t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277)))))))))
-    if x>=0.0:
-        return ans
-    else:
-        return 2.0-ans
-
-def calMeanVar(data):
-    n=len(data)
-    if n<2:
-        return None
-    else:
-        sum=reduce(lambda x,y:x+y,data,0.0)
-        mean=sum/n
-        z=data[:]
-        for i in range(n):
-            z[i]=z[i]-mean
-        variance=reduce(lambda x,y:x+y*y,z,0.0)
-        variance /= n-1
-        variance =sqrt(variance)
-        for i in range(n):
-            z[i]=z[i]/variance
-        return z
-
-def inverseCumul(p):
-    #Coefficients in rational approximations.
-    a = [-3.969683028665376e+01,2.209460984245205e+02,-2.759285104469687e+02,1.383577518672690e+02,-3.066479806614716e+01,2.506628277459239e+00]
-
-    b = [-5.447609879822406e+01,1.615858368580409e+02,-1.556989798598866e+02,6.680131188771972e+01,-1.328068155288572e+01]
-
-    c = [-7.784894002430293e-03,-3.223964580411365e-01,-2.400758277161838e+00,-2.549732539343734e+00,4.374664141464968e+00,2.938163982698783e+00]
-
-    d =  [7.784695709041462e-03,3.224671290700398e-01,2.445134137142996e+00,3.754408661907416e+00]
-
-    #Define break-points.
-
-    p_low  = 0.02425
-    p_high = 1 - p_low
-
-    #Rational approximation for lower region.
-
-    if p > 0 and p < p_low:
-        q = sqrt(-2*log(p))
-        x = (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
-
-
-    #Rational approximation for central region.
-
-    elif p>= p_low and p <= p_high:
-        q = p - 0.5
-        r = q*q
-        x = (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q /(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1)
-
-    #Rational approximation for upper region.
-
-    elif p>p_high and  p < 1:
-        q = sqrt(-2*log(1-p))
-        x = -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
-
-    else:
-        return None
-
-    if p>0 and p < 1:
-        e = 0.5 * erfcc(-x/sqrt(2)) - p
-        u = e * sqrt(2*pi) * exp(x*x/2)
-        x = x - u/(1 + x*u/2)
-        return x
-    else:
-        return None
-
-def gmean(lst):
-    N = len(lst)
-    if N == 0:
-        return 0
-    else:
-        return (reduce(lambda x,y: x+y, lst, 0.0))/N
-
-def gmedian(lst2):
-    lst = lst2[:]
-    N = len(lst)
-    if N == 0:
-        return 0
-    else:
-        lst.sort()
-        if N % 2 == 0:
-            return (lst[N/2]+lst[(N-2)/2])/2.0
-        else:
-            return lst[(N-1)/2]
-
-def gpercentile(lst2, np):
-    """Obsolete - use percentile in corestats instead"""
-    lst = lst2[:]
-    N = len(lst)
-    if N == 0 or np > 100 or np < 0:
-        return None
-    else:
-        lst.sort()
-        pNadd1 = (np/100.0)*N
-        k = int(pNadd1)
-        d = pNadd1 - k
-        if k == 0:
-            return lst[0]
-        elif k >= N-1:
-            return lst[N-1]
-        else:
-            return lst[k-1] + d*(lst[k] - lst[k-1])
-
 def find_outliers(vals):
     """Calculates the upper and lower bounds of a set of sample/case values
 
@@ -315,165 +117,6 @@ def find_outliers(vals):
     logger.debug(pf(locals()))
     return upper_bound, lower_bound
 
-
-def plotBoxPlot(canvas, data, offset= (40, 40, 40, 40), XLabel="Category", YLabel="Value"):
-    xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset
-    plotWidth = canvas.size[0] - xLeftOffset - xRightOffset
-    plotHeight = canvas.size[1] - yTopOffset - yBottomOffset
-    iValues = []
-    for item in data:
-        for item2 in item[1]:
-            try:
-                iValues.append(item2[1])
-            except:
-                iValues.append(item2)
-
-    #draw frame
-    max_Y = max(iValues)
-    min_Y = min(iValues)
-    scaleY = detScale(min_Y, max_Y)
-    Yll = scaleY[0]
-    Yur = scaleY[1]
-    nStep = scaleY[2]
-    stepY = (Yur - Yll)/nStep
-    stepYPixel = plotHeight/(nStep)
-    canvas.drawRect(plotWidth+xLeftOffset, plotHeight + yTopOffset, xLeftOffset, yTopOffset)
-
-    ##draw Y Scale
-    YYY = Yll
-    YCoord = plotHeight + yTopOffset
-    scaleFont=pid.Font(ttf="cour",size=11,bold=1)
-    for i in range(nStep+1):
-        strY = cformat(d=YYY, rank=0)
-        YCoord = max(YCoord, yTopOffset)
-        canvas.drawLine(xLeftOffset,YCoord,xLeftOffset-5,YCoord)
-        canvas.drawString(strY, xLeftOffset -30,YCoord +5,font=scaleFont)
-        YYY += stepY
-        YCoord -= stepYPixel
-
-    ##draw X Scale
-    stepX = plotWidth/len(data)
-    XCoord = xLeftOffset + 0.5*stepX
-    YCoord = plotHeight + yTopOffset
-    scaleFont = pid.Font(ttf="tahoma",size=12,bold=0)
-    labelFont = pid.Font(ttf="tahoma",size=13,bold=0)
-    for item in data:
-        itemname, itemvalue = item
-        canvas.drawLine(XCoord, YCoord,XCoord, YCoord+5, color=pid.black)
-        canvas.drawString(itemname, XCoord - canvas.stringWidth(itemname,font=labelFont)/2.0,\
-        YCoord +20,font=labelFont)
-
-        nValue = len(itemvalue)
-        catValue = []
-        for item2 in itemvalue:
-            try:
-                tstrain, tvalue = item2
-            except:
-                tvalue = item2
-            if nValue <= 4:
-                canvas.drawCross(XCoord, plotHeight + yTopOffset - (tvalue-Yll)*plotHeight/(Yur - Yll), color=pid.red,size=5)
-            else:
-                catValue.append(tvalue)
-        if catValue != []:
-            catMean = gmean(catValue)
-            catMedian = gmedian(catValue)
-            lowHinge = gpercentile(catValue, 25)
-            upHinge = gpercentile(catValue, 75)
-            Hstep = 1.5*(upHinge - lowHinge)
-
-            outlier = []
-            extrem = []
-
-            upperAdj = None
-            for item in catValue:
-                if item >= upHinge + 2*Hstep:
-                    extrem.append(item)
-                elif item >= upHinge + Hstep:
-                    outlier.append(item)
-                elif item > upHinge and  item < upHinge + Hstep:
-                    if upperAdj == None or item > upperAdj:
-                        upperAdj = item
-                else:
-                    pass
-            lowerAdj = None
-            for item in catValue:
-                if item <= lowHinge - 2*Hstep:
-                    extrem.append(item)
-                elif item <= lowHinge - Hstep:
-                    outlier.append(item)
-                if item < lowHinge and  item > lowHinge - Hstep:
-                    if lowerAdj == None or item < lowerAdj:
-                        lowerAdj = item
-                    else:
-                        pass
-            canvas.drawRect(XCoord-20, plotHeight + yTopOffset - (lowHinge-Yll)*plotHeight/(Yur - Yll), \
-                    XCoord+20, plotHeight + yTopOffset - (upHinge-Yll)*plotHeight/(Yur - Yll))
-            canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (catMedian-Yll)*plotHeight/(Yur - Yll), \
-                    XCoord+20, plotHeight + yTopOffset - (catMedian-Yll)*plotHeight/(Yur - Yll))
-            if upperAdj != None:
-                canvas.drawLine(XCoord, plotHeight + yTopOffset - (upHinge-Yll)*plotHeight/(Yur - Yll), \
-                XCoord, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll))
-                canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll), \
-                XCoord+20, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll))
-            if lowerAdj != None:
-                canvas.drawLine(XCoord, plotHeight + yTopOffset - (lowHinge-Yll)*plotHeight/(Yur - Yll), \
-                XCoord, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll))
-                canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll), \
-                XCoord+20, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll))
-
-            outlierFont = pid.Font(ttf="cour",size=12,bold=0)
-            if outlier != []:
-                for item in outlier:
-                    yc = plotHeight + yTopOffset - (item-Yll)*plotHeight/(Yur - Yll)
-                    #canvas.drawEllipse(XCoord-3, yc-3, XCoord+3, yc+3)
-                    canvas.drawString('o', XCoord-3, yc+5, font=outlierFont, color=pid.orange)
-            if extrem != []:
-                for item in extrem:
-                    yc = plotHeight + yTopOffset - (item-Yll)*plotHeight/(Yur - Yll)
-                    #canvas.drawEllipse(XCoord-3, yc-3, XCoord+3, yc+3)
-                    canvas.drawString('*', XCoord-3, yc+6, font=outlierFont, color=pid.red)
-
-            canvas.drawCross(XCoord, plotHeight + yTopOffset - (catMean-Yll)*plotHeight/(Yur - Yll), \
-                    color=pid.blue,size=3)
-            #print(catMean, catMedian, cat25per, cat75per)
-            pass
-
-        XCoord += stepX
-
-    labelFont=pid.Font(ttf="verdana",size=18,bold=0)
-    canvas.drawString(XLabel, xLeftOffset + (plotWidth -canvas.stringWidth(XLabel,font=labelFont))/2.0, \
-    YCoord +40, font=labelFont)
-    canvas.drawString(YLabel,xLeftOffset-40, YCoord-(plotHeight -canvas.stringWidth(YLabel,font=labelFont))/2.0,\
-     font=labelFont, angle =90)
-
-def plotSecurity(canvas, text="12345"):
-    if not text:
-        return
-
-    plotWidth = canvas.size[0]
-    plotHeight = canvas.size[1]
-    if plotHeight<=0 or plotWidth<=0:
-        return
-
-    bgColor = pid.Color(0.6+0.4*random.random(), 0.6+0.4*random.random(), 0.6+0.4*random.random())
-    canvas.drawRect(0,0,plotWidth,plotHeight, edgeColor=bgColor, fillColor=bgColor)
-
-    for i in range(30):
-        randomColor = pid.Color(0.6+0.4*random.random(), 0.6+0.4*random.random(), 0.6+0.4*random.random())
-        scaleFont=pid.Font(ttf="cour",size=random.choice(range(20, 50)))
-        canvas.drawString(random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'),
-                int(random.random()*plotWidth), int(random.random()*plotHeight), font=scaleFont,
-                color=randomColor, angle=random.choice(range(-45, 50)))
-
-    step = (plotWidth-20)/len(text)
-    startX = 20
-    for item in text:
-        randomColor = pid.Color(0.6*random.random(),0.6*random.random(), 0.6*random.random())
-        scaleFont=pid.Font(ttf="verdana",size=random.choice(range(50, 60)),bold=1)
-        canvas.drawString(item, startX, plotHeight/2-10, font=scaleFont,
-                color=randomColor, angle=random.choice(range(-45, 50)))
-        startX += step
-
 # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py)
 # or the first object returned by direct (pair-scan) permu function (called by DirectPlotPage.py)
 def plotBar(canvas, data, barColor=pid.blue, axesColor=pid.black, labelColor=pid.black, XLabel=None, YLabel=None, title=None, offset= (60, 20, 40, 40), zoom = 1):
@@ -561,542 +204,6 @@ def plotBar(canvas, data, barColor=pid.blue, axesColor=pid.black, labelColor=pid
        canvas.drawString(title,xLeftOffset+(plotWidth-canvas.stringWidth(title,font=labelFont))/2.0,
                20,font=labelFont,color=labelColor)
 
-def plotBarText(canvas, data, label, variance=None, barColor=pid.blue, axesColor=pid.black, labelColor=pid.black, XLabel=None, YLabel=None, title=None, sLabel = None, offset= (80, 20, 40, 100), barSpace = 2, zoom = 1):
-    xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset
-    plotWidth = canvas.size[0] - xLeftOffset - xRightOffset
-    plotHeight = canvas.size[1] - yTopOffset - yBottomOffset
-    if plotHeight<=0 or plotWidth<=0:
-       return
-
-    NNN = len(data)
-    if NNN < 2 or NNN != len(label):
-       return
-    if variance and len(variance)!=NNN:
-       variance = []
-
-    Y2 = data[:]
-    if variance:
-       for i in range(NNN):
-           if variance[i]:
-               Y2 += [data[i]-variance[i]]
-
-    #Y axis
-    YLow, YTop, stepY = detScale(min(Y2), max(Y2))
-    YScale = plotHeight/(YTop - YLow)
-
-    if YLow < 0  and  YTop > 0:
-       drawZero = 1
-    else:
-       drawZero = 0
-
-    #X axis
-    X = range(NNN)
-    Xll= 0
-    Xur= NNN-1
-
-
-    if drawZero:
-       YZero = yTopOffset+plotHeight-YScale*(0-YLow)
-       canvas.drawLine(xLeftOffset, YZero, xLeftOffset+plotWidth, YZero)
-    else:
-       YZero = yTopOffset+plotHeight
-    #draw data
-    spaceWidth = barSpace
-    if spaceWidth < 1:
-       spaceWidth = 1
-    barWidth = int((plotWidth - (NNN-1.0)*spaceWidth)/NNN)
-
-    xc= xLeftOffset
-    scaleFont=pid.Font(ttf="verdana",size=11,bold=0)
-    for i in range(NNN):
-       yc = yTopOffset+plotHeight-(data[i]-YLow)*YScale
-       canvas.drawRect(xc,YZero,xc+barWidth-1, yc, edgeColor=barColor,fillColor=barColor)
-       if variance and variance[i]:
-           varlen = variance[i]*YScale
-           if yc-varlen < yTopOffset:
-               topYd = yTopOffset
-           else:
-               topYd = yc-varlen
-               canvas.drawLine(xc+barWidth/2-2,yc-varlen,xc+barWidth/2+2,yc-varlen,color=pid.red)
-           canvas.drawLine(xc+barWidth/2,yc+varlen,xc+barWidth/2,topYd,color=pid.red)
-           canvas.drawLine(xc+barWidth/2-2,yc+varlen,xc+barWidth/2+2,yc+varlen,color=pid.red)
-       strX = label[i]
-       canvas.drawString(strX,xc+barWidth/2.0+2,yTopOffset+plotHeight+2+canvas.stringWidth(strX,font=scaleFont),font=scaleFont,angle=90)
-       xc += barWidth + spaceWidth
-
-    #draw drawing region
-    canvas.drawRect(xLeftOffset, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight)
-
-    #draw Y scale
-    scaleFont=pid.Font(ttf="cour",size=16,bold=1)
-    y=YLow
-    for i in range(stepY+1):
-       yc=yTopOffset+plotHeight-(y-YLow)*YScale
-       canvas.drawLine(xLeftOffset,yc,xLeftOffset-5,yc, color=axesColor)
-       strY = cformat(d=y, rank=0)
-       canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)-6,yc+5,font=scaleFont)
-       y+= (YTop - YLow)/stepY
-
-    #draw label
-    labelFont=pid.Font(ttf="verdana",size=17,bold=0)
-    if XLabel:
-       canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0,yTopOffset+plotHeight+65,font=labelFont,color=labelColor)
-
-    if YLabel:
-       canvas.drawString(YLabel,xLeftOffset-50, yTopOffset+plotHeight-(plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0,font=labelFont,color=labelColor,angle=90)
-
-    labelFont=pid.Font(ttf="verdana",size=18,bold=0)
-    if title:
-       canvas.drawString(title,xLeftOffset,yTopOffset-15,font=labelFont,color=labelColor)
-
-    return
-
-#def plotXY(canvas, dataX, dataY, rank=0, dataLabel=[], plotColor = pid.black, axesColor=pid.black, labelColor=pid.black, lineSize="thin", lineColor=pid.grey, idFont="arial", idColor=pid.blue, idSize="14", symbolColor=pid.black, symbolType="circle", filled="yes", symbolSize="tiny", XLabel=None, YLabel=None, title=None, fitcurve=None, connectdot=1, displayR=None, loadingPlot = 0, offset= (80, 20, 40, 60), zoom = 1, specialCases=[], showLabel = 1, bufferSpace = 15):
-#    'displayR : correlation scatter plot, loadings : loading plot'
-#
-#    dataXRanked, dataYRanked = webqtlUtil.calRank(dataX, dataY, len(dataX))
-#
-#    #get ID font size
-#    idFontSize = int(idSize)
-#
-#    #If filled is yes, set fill color
-#    if filled == "yes":
-#        fillColor = symbolColor
-#    else:
-#        fillColor = None
-#
-#    if symbolSize == "large":
-#        sizeModifier = 7
-#        fontModifier = 12
-#    elif symbolSize == "medium":
-#        sizeModifier = 5
-#        fontModifier = 8
-#    elif symbolSize == "small":
-#        sizeModifier = 3
-#        fontModifier = 3
-#    else:
-#        sizeModifier = 1
-#        fontModifier = -1
-#
-#    if rank == 0:    # Pearson correlation
-#        bufferSpace = 0
-#        dataXPrimary = dataX
-#        dataYPrimary = dataY
-#        dataXAlt = dataXRanked    #Values used just for printing the other corr type to the graph image
-#        dataYAlt = dataYRanked    #Values used just for printing the other corr type to the graph image
-#    else:    # Spearman correlation: Switching Ranked and Unranked X and Y values
-#        dataXPrimary = dataXRanked
-#        dataYPrimary = dataYRanked
-#        dataXAlt = dataX    #Values used just for printing the other corr type to the graph image
-#        dataYAlt = dataY    #Values used just for printing the other corr type to the graph image
-#
-#    xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset
-#    plotWidth = canvas.size[0] - xLeftOffset - xRightOffset
-#    plotHeight = canvas.size[1] - yTopOffset - yBottomOffset
-#    if plotHeight<=0 or plotWidth<=0:
-#        return
-#    if len(dataXPrimary) < 1 or  len(dataXPrimary) != len(dataYPrimary) or (dataLabel and len(dataXPrimary) != len(dataLabel)):
-#        return
-#
-#    max_X=max(dataXPrimary)
-#    min_X=min(dataXPrimary)
-#    max_Y=max(dataYPrimary)
-#    min_Y=min(dataYPrimary)
-#
-#    #for some reason I forgot why I need to do this
-#    if loadingPlot:
-#        min_X = min(-0.1,min_X)
-#        max_X = max(0.1,max_X)
-#        min_Y = min(-0.1,min_Y)
-#        max_Y = max(0.1,max_Y)
-#
-#    xLow, xTop, stepX=detScale(min_X,max_X)
-#    yLow, yTop, stepY=detScale(min_Y,max_Y)
-#    xScale = plotWidth/(xTop-xLow)
-#    yScale = plotHeight/(yTop-yLow)
-#
-#    #draw drawing region
-#    canvas.drawRect(xLeftOffset-bufferSpace, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight+bufferSpace)
-#    canvas.drawRect(xLeftOffset-bufferSpace+1, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight+bufferSpace-1)
-#
-#    #calculate data points
-#    data = map(lambda X, Y: (X, Y), dataXPrimary, dataYPrimary)
-#    xCoord = map(lambda X, Y: ((X-xLow)*xScale + xLeftOffset, yTopOffset+plotHeight-(Y-yLow)*yScale), dataXPrimary, dataYPrimary)
-#
-#    labelFont=pid.Font(ttf=idFont,size=idFontSize,bold=0)
-#
-#    if loadingPlot:
-#        xZero = -xLow*xScale+xLeftOffset
-#        yZero = yTopOffset+plotHeight+yLow*yScale
-#        for point in xCoord:
-#            canvas.drawLine(xZero,yZero,point[0],point[1],color=pid.red)
-#    else:
-#        if connectdot:
-#            canvas.drawPolygon(xCoord,edgeColor=plotColor,closed=0)
-#        else:
-#            pass
-#
-#    symbolFont = pid.Font(ttf="fnt_bs", size=12+fontModifier,bold=0)
-#
-#    for i, item in enumerate(xCoord):
-#        if dataLabel and dataLabel[i] in specialCases:
-#            canvas.drawRect(item[0]-3, item[1]-3, item[0]+3, item[1]+3, edgeColor=pid.green)
-#            #canvas.drawCross(item[0],item[1],color=pid.blue,size=5)
-#        else:
-#            if symbolType == "vertRect":
-#                canvas.drawRect(x1=item[0]-sizeModifier+2,y1=item[1]-sizeModifier-2, x2=item[0]+sizeModifier-1,y2=item[1]+sizeModifier+2, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor)
-#            elif (symbolType == "circle" and filled != "yes"):
-#                canvas.drawString(":", item[0]-canvas.stringWidth(":",font=symbolFont)/2+1,item[1]+2,color=symbolColor, font=symbolFont)
-#            elif (symbolType == "circle" and filled == "yes"):
-#                canvas.drawString("5", item[0]-canvas.stringWidth("5",font=symbolFont)/2+1,item[1]+2,color=symbolColor, font=symbolFont)
-#            elif symbolType == "horiRect":
-#                canvas.drawRect(x1=item[0]-sizeModifier-1,y1=item[1]-sizeModifier+3, x2=item[0]+sizeModifier+3,y2=item[1]+sizeModifier-2, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor)
-#            elif (symbolType == "square"):
-#                canvas.drawRect(x1=item[0]-sizeModifier+1,y1=item[1]-sizeModifier-4, x2=item[0]+sizeModifier+2,y2=item[1]+sizeModifier-3, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor)
-#            elif (symbolType == "diamond" and filled != "yes"):
-#                canvas.drawString(",", item[0]-canvas.stringWidth(",",font=symbolFont)/2+2, item[1]+6, font=symbolFont, color=symbolColor)
-#            elif (symbolType == "diamond" and filled == "yes"):
-#                canvas.drawString("D", item[0]-canvas.stringWidth("D",font=symbolFont)/2+2, item[1]+6, font=symbolFont, color=symbolColor)
-#            elif symbolType == "4-star":
-#                canvas.drawString("l", item[0]-canvas.stringWidth("l",font=symbolFont)/2+1, item[1]+3, font=symbolFont, color=symbolColor)
-#            elif symbolType == "3-star":
-#                canvas.drawString("k", item[0]-canvas.stringWidth("k",font=symbolFont)/2+1, item[1]+3, font=symbolFont, color=symbolColor)
-#            else:
-#                canvas.drawCross(item[0],item[1]-2,color=symbolColor, size=sizeModifier+2)
-#
-#        if showLabel and dataLabel:
-#            if (symbolType == "vertRect" or symbolType == "diamond"):
-#                labelGap = 15
-#            elif (symbolType == "4-star" or symbolType == "3-star"):
-#                labelGap = 12
-#            else:
-#                labelGap = 11
-#            canvas.drawString(dataLabel[i], item[0]- canvas.stringWidth(dataLabel[i],
-#                    font=labelFont)/2 + 1, item[1]+(labelGap+sizeModifier+(idFontSize-12)), font=labelFont, color=idColor)
-#
-#    #draw scale
-#    scaleFont=pid.Font(ttf="cour",size=16,bold=1)
-#
-#
-#    x=xLow
-#    for i in range(stepX+1):
-#        xc=xLeftOffset+(x-xLow)*xScale
-#        if ((x == 0) & (rank == 1)):
-#            pass
-#        else:
-#            canvas.drawLine(xc,yTopOffset+plotHeight + bufferSpace,xc,yTopOffset+plotHeight+5 + bufferSpace, color=axesColor)
-#        strX = cformat(d=x, rank=rank)
-#        if ((strX == "0") & (rank == 1)):
-#            pass
-#        else:
-#            canvas.drawString(strX,xc-canvas.stringWidth(strX,font=scaleFont)/2,yTopOffset+plotHeight+20 + bufferSpace,font=scaleFont)
-#        x+= (xTop - xLow)/stepX
-#
-#    y=yLow
-#    for i in range(stepY+1):
-#        yc=yTopOffset+plotHeight-(y-yLow)*yScale
-#        if ((y == 0) & (rank == 1)):
-#            pass
-#        else:
-#            canvas.drawLine(xLeftOffset - bufferSpace,yc,xLeftOffset-5 - bufferSpace,yc, color=axesColor)
-#        strY = cformat(d=y, rank=rank)
-#        if ((strY == "0") & (rank == 1)):
-#            pass
-#        else:
-#            canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)- 10 - bufferSpace,yc+4,font=scaleFont)
-#        y+= (yTop - yLow)/stepY
-#
-#    #draw label
-#
-#    labelFont=pid.Font(ttf="verdana",size=canvas.size[0]/45,bold=0)
-#    titleFont=pid.Font(ttf="verdana",size=canvas.size[0]/40,bold=0)
-#
-#    if (rank == 1 and not title):
-#        canvas.drawString("Spearman Rank Correlation", xLeftOffset-canvas.size[0]*.025+(plotWidth-canvas.stringWidth("Spearman Rank Correlation",font=titleFont))/2.0,
-#                                          25,font=titleFont,color=labelColor)
-#    elif (rank == 0 and not title):
-#        canvas.drawString("Pearson Correlation", xLeftOffset-canvas.size[0]*.025+(plotWidth-canvas.stringWidth("Pearson Correlation",font=titleFont))/2.0,
-#                                          25,font=titleFont,color=labelColor)
-#
-#    if XLabel:
-#        canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0,
-#                yTopOffset+plotHeight+yBottomOffset-25,font=labelFont,color=labelColor)
-#
-#    if YLabel:
-#        canvas.drawString(YLabel, xLeftOffset-65, yTopOffset+plotHeight- (plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0,
-#                font=labelFont,color=labelColor,angle=90)
-#
-#    labelFont=pid.Font(ttf="verdana",size=20,bold=0)
-#    if title:
-#        canvas.drawString(title,xLeftOffset+(plotWidth-canvas.stringWidth(title,font=labelFont))/2.0,
-#                20,font=labelFont,color=labelColor)
-#
-#    if fitcurve:
-#        import sys
-#        sys.argv = [ "mod_python" ]
-#        #from numarray import linear_algebra as la
-#        #from numarray import ones, array, dot, swapaxes
-#        fitYY = array(dataYPrimary)
-#        fitXX = array([ones(len(dataXPrimary)),dataXPrimary])
-#        AA = dot(fitXX,swapaxes(fitXX,0,1))
-#        BB = dot(fitXX,fitYY)
-#        bb = la.linear_least_squares(AA,BB)[0]
-#
-#        xc1 = xLeftOffset
-#        yc1 = yTopOffset+plotHeight-(bb[0]+bb[1]*xLow-yLow)*yScale
-#        if yc1 > yTopOffset+plotHeight:
-#            yc1 = yTopOffset+plotHeight
-#            xc1 = (yLow-bb[0])/bb[1]
-#            xc1=(xc1-xLow)*xScale+xLeftOffset
-#        elif yc1 < yTopOffset:
-#            yc1 = yTopOffset
-#            xc1 = (yTop-bb[0])/bb[1]
-#            xc1=(xc1-xLow)*xScale+xLeftOffset
-#        else:
-#            pass
-#
-#        xc2 = xLeftOffset + plotWidth
-#        yc2 = yTopOffset+plotHeight-(bb[0]+bb[1]*xTop-yLow)*yScale
-#        if yc2 > yTopOffset+plotHeight:
-#            yc2 = yTopOffset+plotHeight
-#            xc2 = (yLow-bb[0])/bb[1]
-#            xc2=(xc2-xLow)*xScale+xLeftOffset
-#        elif yc2 < yTopOffset:
-#            yc2 = yTopOffset
-#            xc2 = (yTop-bb[0])/bb[1]
-#            xc2=(xc2-xLow)*xScale+xLeftOffset
-#        else:
-#            pass
-#
-#        canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace,xc2,yc2,color=lineColor)
-#        if lineSize == "medium":
-#            canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace+1,xc2,yc2+1,color=lineColor)
-#        if lineSize == "thick":
-#            canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace+1,xc2,yc2+1,color=lineColor)
-#            canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace-1,xc2,yc2-1,color=lineColor)
-#
-#
-#    if displayR:
-#        labelFont=pid.Font(ttf="trebuc",size=canvas.size[0]/60,bold=0)
-#        NNN = len(dataX)
-#        corr = webqtlUtil.calCorrelation(dataXPrimary,dataYPrimary,NNN)[0]
-#
-#        if NNN < 3:
-#            corrPValue = 1.0
-#        else:
-#            if abs(corr) >= 1.0:
-#                corrPValue = 0.0
-#            else:
-#                ZValue = 0.5*log((1.0+corr)/(1.0-corr))
-#                ZValue = ZValue*sqrt(NNN-3)
-#                corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue)))
-#
-#        NStr = "N = %d" % NNN
-#        strLenN = canvas.stringWidth(NStr,font=labelFont)
-#
-#        if rank == 1:
-#            if corrPValue < 0.0000000000000001:
-#                corrStr = "Rho = %1.3f P < 1.00 E-16" % (corr)
-#            else:
-#                corrStr = "Rho = %1.3f P = %3.2E" % (corr, corrPValue)
-#        else:
-#            if corrPValue < 0.0000000000000001:
-#                corrStr = "r = %1.3f P < 1.00 E-16" % (corr)
-#            else:
-#                corrStr = "r = %1.3f P = %3.2E" % (corr, corrPValue)
-#        strLen = canvas.stringWidth(corrStr,font=labelFont)
-#
-#        canvas.drawString(NStr,xLeftOffset,yTopOffset-10,font=labelFont,color=labelColor)
-#        canvas.drawString(corrStr,xLeftOffset+plotWidth-strLen,yTopOffset-10,font=labelFont,color=labelColor)
-#
-#    return xCoord
-
-def plotXYSVG(drawSpace, dataX, dataY, rank=0, dataLabel=[], plotColor = "black", axesColor="black", labelColor="black", symbolColor="red", XLabel=None, YLabel=None, title=None, fitcurve=None, connectdot=1, displayR=None, loadingPlot = 0, offset= (80, 20, 40, 60), zoom = 1, specialCases=[], showLabel = 1):
-    'displayR : correlation scatter plot, loadings : loading plot'
-
-    dataXRanked, dataYRanked = webqtlUtil.calRank(dataX, dataY, len(dataX))
-
-    # Switching Ranked and Unranked X and Y values if a Spearman Rank Correlation
-    if rank == 0:
-        dataXPrimary = dataX
-        dataYPrimary = dataY
-        dataXAlt = dataXRanked
-        dataYAlt = dataYRanked
-
-    else:
-        dataXPrimary = dataXRanked
-        dataYPrimary = dataYRanked
-        dataXAlt = dataX
-        dataYAlt = dataY
-
-
-
-    xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset
-    plotWidth = drawSpace.attributes['width'] - xLeftOffset - xRightOffset
-    plotHeight = drawSpace.attributes['height'] - yTopOffset - yBottomOffset
-    if plotHeight<=0 or plotWidth<=0:
-        return
-    if len(dataXPrimary) < 1 or  len(dataXPrimary) != len(dataYPrimary) or (dataLabel and len(dataXPrimary) != len(dataLabel)):
-        return
-
-    max_X=max(dataXPrimary)
-    min_X=min(dataXPrimary)
-    max_Y=max(dataYPrimary)
-    min_Y=min(dataYPrimary)
-
-    #for some reason I forgot why I need to do this
-    if loadingPlot:
-        min_X = min(-0.1,min_X)
-        max_X = max(0.1,max_X)
-        min_Y = min(-0.1,min_Y)
-        max_Y = max(0.1,max_Y)
-
-    xLow, xTop, stepX=detScale(min_X,max_X)
-    yLow, yTop, stepY=detScale(min_Y,max_Y)
-    xScale = plotWidth/(xTop-xLow)
-    yScale = plotHeight/(yTop-yLow)
-
-    #draw drawing region
-    r = svg.rect(xLeftOffset, yTopOffset, plotWidth,  plotHeight, 'none', axesColor, 1)
-    drawSpace.addElement(r)
-
-    #calculate data points
-    data = map(lambda X, Y: (X, Y), dataXPrimary, dataYPrimary)
-    xCoord = map(lambda X, Y: ((X-xLow)*xScale + xLeftOffset, yTopOffset+plotHeight-(Y-yLow)*yScale), dataXPrimary, dataYPrimary)
-    labelFontF = "verdana"
-    labelFontS = 11
-
-    if loadingPlot:
-        xZero = -xLow*xScale+xLeftOffset
-        yZero = yTopOffset+plotHeight+yLow*yScale
-        for point in xCoord:
-            drawSpace.addElement(svg.line(xZero,yZero,point[0],point[1], "red", 1))
-    else:
-        if connectdot:
-            pass
-            #drawSpace.drawPolygon(xCoord,edgeColor=plotColor,closed=0)
-        else:
-            pass
-
-    for i, item in enumerate(xCoord):
-        if dataLabel and dataLabel[i] in specialCases:
-            drawSpace.addElement(svg.rect(item[0]-3, item[1]-3, 6, 6, "none", "green", 0.5))
-            #drawSpace.drawCross(item[0],item[1],color=pid.blue,size=5)
-        else:
-            drawSpace.addElement(svg.line(item[0],item[1]+5,item[0],item[1]-5,symbolColor,1))
-            drawSpace.addElement(svg.line(item[0]+5,item[1],item[0]-5,item[1],symbolColor,1))
-        if showLabel and dataLabel:
-            pass
-            drawSpace.addElement(svg.text(item[0], item[1]+14, dataLabel[i], labelFontS,
-                    labelFontF, text_anchor="middle", style="stroke:blue;stroke-width:0.5;"))
-            #canvas.drawString(, item[0]- canvas.stringWidth(dataLabel[i],
-            #       font=labelFont)/2, item[1]+14, font=labelFont, color=pid.blue)
-
-    #draw scale
-    #scaleFont=pid.Font(ttf="cour",size=14,bold=1)
-    x=xLow
-    for i in range(stepX+1):
-        xc=xLeftOffset+(x-xLow)*xScale
-        drawSpace.addElement(svg.line(xc,yTopOffset+plotHeight,xc,yTopOffset+plotHeight+5, axesColor, 1))
-        strX = cformat(d=x, rank=rank)
-        drawSpace.addElement(svg.text(xc,yTopOffset+plotHeight+20,strX,13, "courier", text_anchor="middle"))
-        x+= (xTop - xLow)/stepX
-
-    y=yLow
-    for i in range(stepY+1):
-        yc=yTopOffset+plotHeight-(y-yLow)*yScale
-        drawSpace.addElement(svg.line(xLeftOffset,yc,xLeftOffset-5,yc, axesColor, 1))
-        strY = cformat(d=y, rank=rank)
-        drawSpace.addElement(svg.text(xLeftOffset-10,yc+5,strY,13, "courier", text_anchor="end"))
-        y+= (yTop - yLow)/stepY
-
-    #draw label
-    labelFontF = "verdana"
-    labelFontS = 17
-    if XLabel:
-        drawSpace.addElement(svg.text(xLeftOffset+plotWidth/2.0,
-                yTopOffset+plotHeight+yBottomOffset-10,XLabel,
-                labelFontS, labelFontF, text_anchor="middle"))
-
-    if YLabel:
-        drawSpace.addElement(svg.text(xLeftOffset-50,
-                 yTopOffset+plotHeight/2,YLabel,
-                labelFontS, labelFontF, text_anchor="middle", style="writing-mode:tb-rl", transform="rotate(270 %d %d)" % (xLeftOffset-50,  yTopOffset+plotHeight/2)))
-        #drawSpace.drawString(YLabel, xLeftOffset-50, yTopOffset+plotHeight- (plotHeight-drawSpace.stringWidth(YLabel,font=labelFont))/2.0,
-        #       font=labelFont,color=labelColor,angle=90)
-
-
-    if fitcurve:
-        sys.argv = [ "mod_python" ]
-        #from numarray import linear_algebra as la
-        #from numarray import ones, array, dot, swapaxes
-        fitYY = array(dataYPrimary)
-        fitXX = array([ones(len(dataXPrimary)),dataXPrimary])
-        AA = dot(fitXX,swapaxes(fitXX,0,1))
-        BB = dot(fitXX,fitYY)
-        bb = la.linear_least_squares(AA,BB)[0]
-
-        xc1 = xLeftOffset
-        yc1 = yTopOffset+plotHeight-(bb[0]+bb[1]*xLow-yLow)*yScale
-        if yc1 > yTopOffset+plotHeight:
-            yc1 = yTopOffset+plotHeight
-            xc1 = (yLow-bb[0])/bb[1]
-            xc1=(xc1-xLow)*xScale+xLeftOffset
-        elif yc1 < yTopOffset:
-            yc1 = yTopOffset
-            xc1 = (yTop-bb[0])/bb[1]
-            xc1=(xc1-xLow)*xScale+xLeftOffset
-        else:
-            pass
-
-        xc2 = xLeftOffset + plotWidth
-        yc2 = yTopOffset+plotHeight-(bb[0]+bb[1]*xTop-yLow)*yScale
-        if yc2 > yTopOffset+plotHeight:
-            yc2 = yTopOffset+plotHeight
-            xc2 = (yLow-bb[0])/bb[1]
-            xc2=(xc2-xLow)*xScale+xLeftOffset
-        elif yc2 < yTopOffset:
-            yc2 = yTopOffset
-            xc2 = (yTop-bb[0])/bb[1]
-            xc2=(xc2-xLow)*xScale+xLeftOffset
-        else:
-            pass
-
-        drawSpace.addElement(svg.line(xc1,yc1,xc2,yc2,"green", 1))
-
-    if displayR:
-        labelFontF = "trebuc"
-        labelFontS = 14
-        NNN = len(dataX)
-
-        corr = webqtlUtil.calCorrelation(dataXPrimary,dataYPrimary,NNN)[0]
-
-        if NNN < 3:
-            corrPValue = 1.0
-        else:
-            if abs(corr) >= 1.0:
-                corrPValue = 0.0
-            else:
-                ZValue = 0.5*log((1.0+corr)/(1.0-corr))
-                ZValue = ZValue*sqrt(NNN-3)
-                corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue)))
-
-        NStr = "N of Cases=%d" % NNN
-
-        if rank == 1:
-            corrStr = "Spearman's r=%1.3f P=%3.2E" % (corr, corrPValue)
-        else:
-            corrStr = "Pearson's r=%1.3f P=%3.2E" % (corr, corrPValue)
-
-        drawSpace.addElement(svg.text(xLeftOffset,yTopOffset-10,NStr,
-                labelFontS, labelFontF, text_anchor="start"))
-        drawSpace.addElement(svg.text(xLeftOffset+plotWidth,yTopOffset-25,corrStr,
-                labelFontS, labelFontF, text_anchor="end"))
-    """
-    """
-    return
-
-
 # This function determines the scale of the plot
 def detScaleOld(min,max):
     if min>=max:
@@ -1114,7 +221,7 @@ def detScaleOld(min,max):
         high=c*ceil(max/c)
         return [low,high,round((high-low)/c)]
 
-def detScale(min=0,max=0,bufferSpace=3):
+def detScale(min=0,max=0):
 
     if min>=max:
         return None
@@ -1151,57 +258,9 @@ def detScale(min=0,max=0,bufferSpace=3):
 
         return [low,high,n]
 
-
-
-def colorSpectrumOld(n):
-    if n == 1:
-        return [pid.Color(1,0,0)]
-    elif n == 2:
-        return [pid.Color(1,0,0),pid.Color(0,0,1)]
-    elif n == 3:
-        return [pid.Color(1,0,0),pid.Color(0,1,0),pid.Color(0,0,1)]
-    else:
-        step = 2.0/(n-1)
-        red = 1.0
-        green = 0.0
-        blue = 0.0
-        colors = [pid.Color(red,green,blue)]
-        i = 1
-        greenpeak = 0
-        while i < n:
-            if red >= step:
-                red -= step
-                green += step
-                if green >= 1.0:
-                    greenpeak = 1
-                    blue += green -1.0
-                    green = 1.0
-            else:
-                red = 0.0
-                if greenpeak:
-                    green -= step
-                    blue += step
-                else:
-                    green += step
-                if green >= 1.0:
-                    greenpeak = 1
-                    blue += green -1.0
-                    green = 2.0 -green
-                elif green < 0.0:
-                    green = 0.0
-                else:
-                    pass
-            colors.append(pid.Color(red,green,blue))
-            i += 1
-        return colors
-
-
-
-
 def bluefunc(x):
     return 1.0 / (1.0 + exp(-10*(x-0.6)))
 
-
 def redfunc(x):
     return 1.0 / (1.0 + exp(10*(x-0.5)))
 
@@ -1230,52 +289,10 @@ def colorSpectrum(n=100):
     out2.append(out[-1])
     return out2
 
-
-def colorSpectrumSVG(n=100):
-    multiple = 10
-    if n == 1:
-        return ["rgb(255,0,0)"]
-    elif n == 2:
-        return ["rgb(255,0,0)","rgb(0,0,255)"]
-    elif n == 3:
-        return ["rgb(255,0,0)","rgb(0,255,0)","rgb(0,0,255)"]
-    N = n*multiple
-    out = [None]*N;
-    for i in range(N):
-        x = float(i)/N
-        out[i] = "rgb(%d, %d, %d)" % (redfunc(x)*255, greenfunc(x)*255, bluefunc(x)*255);
-    out2 = [out[0]]
-    step = N/float(n-1)
-    j = 0
-    for i in range(n-2):
-        j += step
-        out2.append(out[int(j)])
-    out2.append(out[-1])
-    return out2
-
-
-def BWSpectrum(n=100):
-    multiple = 10
-    if n == 1:
-        return [pid.Color(0,0,0)]
-    elif n == 2:
-        return [pid.Color(0,0,0),pid.Color(1,1,1)]
-    elif n == 3:
-        return [pid.Color(0,0,0),pid.Color(0.5,0.5,0.5),pid.Color(1,1,1)]
-
-    step = 1.0/n
-    x = 0.0
-    out = []
-    for i in range(n):
-        out.append(pid.Color(x,x,x));
-        x += step
-    return out
-
-
 def _test():
     import doctest
     doctest.testmod()
 
 
 if __name__=="__main__":
-    _test()
+    _test()
\ No newline at end of file
diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py
index a3bb85e9..b628a0a4 100644
--- a/wqflask/utility/after.py
+++ b/wqflask/utility/after.py
@@ -13,10 +13,4 @@ def after_this_request(f):
     if not hasattr(g, 'after_request_callbacks'):
         g.after_request_callbacks = []
     g.after_request_callbacks.append(f)
-    return f
-
-@app.after_request
-def call_after_request_callbacks(response):
-    for callback in getattr(g, 'after_request_callbacks', ()):
-        callback(response)
-    return response
+    return f
\ No newline at end of file
diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py
index 8c97370d..8f1c916b 100644
--- a/wqflask/utility/benchmark.py
+++ b/wqflask/utility/benchmark.py
@@ -11,7 +11,7 @@ logger = getLogger(__name__ )
 class Bench(object):
     entries = collections.OrderedDict()
 
-    def __init__(self, name=None, write_output=True):
+    def __init__(self, name=None, write_output=LOG_BENCH):
         self.name = name
         self.write_output = write_output
 
diff --git a/wqflask/utility/corestats.py b/wqflask/utility/corestats.py
new file mode 100644
index 00000000..67ca3ad3
--- /dev/null
+++ b/wqflask/utility/corestats.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+
+#  corestats.py (COREy STATS)
+#  Copyright (c) 2006-2007, Corey Goldberg (corey@goldb.org)
+#
+#    statistical calculation class
+#    for processing numeric sequences
+#
+#  license: GNU LGPL
+#
+#  This library is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2.1 of the License, or (at your option) any later version.
+
+import sys
+
+#ZS: Should switch to using some third party library for this; maybe scipy has an equivalent
+class Stats:
+
+    def __init__(self, sequence):
+        # sequence of numbers we will process
+        # convert all items to floats for numerical processing
+        self.sequence = [float(item) for item in sequence]
+
+    def sum(self):
+        if len(self.sequence) < 1:
+            return None
+        else:
+            return sum(self.sequence)
+
+    def count(self):
+        return len(self.sequence)
+
+    def min(self):
+        if len(self.sequence) < 1:
+            return None
+        else:
+            return min(self.sequence)
+
+    def max(self):
+        if len(self.sequence) < 1:
+            return None
+        else:
+            return max(self.sequence)
+
+    def avg(self):
+        if len(self.sequence) < 1:
+            return None
+        else:
+            return sum(self.sequence) / len(self.sequence)
+
+    def stdev(self):
+        if len(self.sequence) < 1:
+            return None
+        else:
+            avg = self.avg()
+            sdsq = sum([(i - avg) ** 2 for i in self.sequence])
+            stdev = (sdsq / (len(self.sequence) - 1)) ** .5
+            return stdev
+
+    def percentile(self, percentile):
+        if len(self.sequence) < 1:
+            value = None
+        elif (percentile >= 100):
+            sys.stderr.write('ERROR: percentile must be < 100.  you supplied: %s\n'% percentile)
+            value = None
+        else:
+            element_idx = int(len(self.sequence) * (percentile / 100.0))
+            self.sequence.sort()
+            value = self.sequence[element_idx]
+        return value
+
+# Sample script using this class:
+# -------------------------------------------
+#    #!/usr/bin/env python
+#    import corestats
+#
+#    sequence = [1, 2.5, 7, 13.4, 8.0]
+#    stats = corestats.Stats(sequence)
+#    print stats.avg()
+#    print stats.percentile(90)
+# -------------------------------------------
\ No newline at end of file
diff --git a/wqflask/utility/corr_result_helpers.py b/wqflask/utility/corr_result_helpers.py
index ef644d85..b543c589 100644
--- a/wqflask/utility/corr_result_helpers.py
+++ b/wqflask/utility/corr_result_helpers.py
@@ -15,7 +15,7 @@ def normalize_values(a_values, b_values):
     a_new = []
     b_new = []
     for counter in range(min_length):
-        if a_values[counter] and b_values[counter]:
+        if (a_values[counter] or a_values[counter] == 0) and (b_values[counter] or b_values[counter] == 0):
             a_new.append(a_values[counter])
             b_new.append(b_values[counter])
 
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index 1dba357d..15cdd0bc 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -1,3 +1,44 @@
+# Elasticsearch support
+#
+# Some helpful commands to view the database:
+#
+# You can test the server being up with
+#
+#   curl -H 'Content-Type: application/json' http://localhost:9200
+#
+# List all indices
+#
+#   curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v'
+#
+# To see the users index 'table'
+#
+#   curl http://localhost:9200/users
+#
+# To list all user ids
+#
+# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d '
+# {
+#     "query" : {
+#         "match_all" : {}
+#     },
+#     "stored_fields": []
+# }'
+#
+# To view a record
+#
+#   curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d '
+#   {
+#     "query" : {
+#       "match" : { "email_address": "pjotr2017@thebird.nl"}
+#     }
+#   }'
+#
+#
+# To delete the users index and data (dangerous!)
+#
+#   curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users'
+
+
 from elasticsearch import Elasticsearch, TransportError
 import logging
 
@@ -7,11 +48,11 @@ logger = getLogger(__name__)
 from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
 
 def test_elasticsearch_connection():
-    es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+ELASTICSEARCH_PORT+'/'], verify_certs=True)
+    es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+str(ELASTICSEARCH_PORT)+'/'], verify_certs=True)
     if not es.ping():
         logger.warning("Elasticsearch is DOWN")
 
-def get_elasticsearch_connection():
+def get_elasticsearch_connection(for_user=True):
     """Return a connection to ES. Returns None on failure"""
     logger.info("get_elasticsearch_connection")
     es = None
@@ -22,17 +63,30 @@ def get_elasticsearch_connection():
 
         es = Elasticsearch([{
             "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
-        }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
+        }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
+
+        if for_user:
+            setup_users_index(es)
 
         es_logger = logging.getLogger("elasticsearch")
         es_logger.setLevel(logging.INFO)
         es_logger.addHandler(logging.NullHandler())
-    except:
-        logger.error("Failed to get elasticsearch connection")
+    except Exception as e:
+        logger.error("Failed to get elasticsearch connection", e)
         es = None
 
     return es
 
+def setup_users_index(es_connection):
+    if es_connection:
+        index_settings = {
+            "properties": {
+                "email_address": {
+                    "type": "keyword"}}}
+
+        es_connection.indices.create(index='users', ignore=400)
+        es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local")
+
 def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"):
     return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type)
 
diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py
index 7149e560..af306731 100644
--- a/wqflask/utility/genofile_parser.py
+++ b/wqflask/utility/genofile_parser.py
@@ -72,7 +72,7 @@ class ConvertGenoFile(object):
         yield row
 
   def process_csv(self):
-    for row_count, row in enumerate(self.process_rows()):
+    for row in self.process_rows():
       row_items = row.split("\t")
 
       this_marker = Marker()
diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py
index 128706df..510b1041 100644
--- a/wqflask/utility/logger.py
+++ b/wqflask/utility/logger.py
@@ -33,7 +33,7 @@ from pprint import pformat as pf
 from inspect import stack
 import datetime
 
-from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL, LOG_FORMAT
+from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL
 
 class GNLogger:
     """A logger class with some additional functionality, such as
diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py
index 512bc9e6..db13b9d1 100644
--- a/wqflask/utility/svg.py
+++ b/wqflask/utility/svg.py
@@ -1029,7 +1029,7 @@ class drawing:
         try:
             xv.feed(svg)
         except:
-            raise "SVG is not well formed, see messages above"
+            raise Exception("SVG is not well formed, see messages above")
         else:
             print "SVG well formed"
 if __name__=='__main__':
diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py
index 4fc978f5..83fa90b7 100644
--- a/wqflask/utility/webqtlUtil.py
+++ b/wqflask/utility/webqtlUtil.py
@@ -34,9 +34,6 @@ from htmlgen import HTMLgen2 as HT
 
 from base import webqtlConfig
 
-
-
-
 # NL, 07/27/2010. moved from webqtlForm.py
 #Dict of Parents and F1 information, In the order of [F1, Mat, Pat]
 ParInfo ={
@@ -64,173 +61,10 @@ ParInfo ={
 'SXM':['SMF1', 'MSF1', 'Steptoe','Morex']
 }
 
-
-# NL, 07/27/2010. moved from template.py
-IMGSTEP1 = HT.Image('/images/step1.gif', alt='STEP 1',border=0) #XZ, Only be used in inputPage.py
-IMGSTEP2 = HT.Image('/images/step2.gif', alt='STEP 2',border=0) #XZ, Only be used in inputPage.py
-IMGSTEP3 = HT.Image('/images/step3.gif', alt='STEP 3',border=0) #XZ, Only be used in inputPage.py
-IMGNEXT = HT.Image('/images/arrowdown.gif', alt='NEXT',border=0) #XZ, Only be used in inputPage.py
-
-IMGASC = HT.Image("/images/sortup.gif", border=0)
-IMGASCON = HT.Image("/images/sortupon.gif", border=0)
-IMGDESC = HT.Image("/images/sortdown.gif", border=0)
-IMGDESCON = HT.Image("/images/sortdownon.gif", border=0)
-
-"""
-IMGASC = HT.Image("/images/sortup_icon.gif", border=0)
-IMGASCON = HT.Image("/images/sortupon.gif", border=0)
-IMGDESC = HT.Image("/images/sortdown_icon.gif", border=0)
-IMGDESCON = HT.Image("/images/sortdownon.gif", border=0)
-IMG_UNSORTED = HT.Image("/images/unsorted_icon.gif", border=0)
-"""
-
-PROGRESSBAR = HT.Image('/images/waitAnima2.gif', alt='checkblue',align="middle",border=0)
-
 #########################################
 #      Accessory Functions
 #########################################
 
-def inverseCumul(p):
-    #Coefficients in rational approximations.
-    a = [-3.969683028665376e+01,2.209460984245205e+02,-2.759285104469687e+02,1.383577518672690e+02,-3.066479806614716e+01,2.506628277459239e+00]
-
-    b = [-5.447609879822406e+01,1.615858368580409e+02,-1.556989798598866e+02,6.680131188771972e+01,-1.328068155288572e+01]
-
-    c = [-7.784894002430293e-03,-3.223964580411365e-01,-2.400758277161838e+00,-2.549732539343734e+00,4.374664141464968e+00,2.938163982698783e+00]
-
-    d =  [7.784695709041462e-03,3.224671290700398e-01,2.445134137142996e+00,3.754408661907416e+00]
-
-    #Define break-points.
-
-    p_low  = 0.02425
-    p_high = 1 - p_low
-
-    #Rational approximation for lower region.
-
-    if p > 0 and p < p_low:
-        q = sqrt(-2*log(p))
-        x = (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
-
-
-    #Rational approximation for central region.
-
-    elif p>= p_low and p <= p_high:
-        q = p - 0.5
-        r = q*q
-        x = (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q /(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1)
-
-    #Rational approximation for upper region.
-
-    elif p>p_high and  p < 1:
-        q = sqrt(-2*log(1-p))
-        x = -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1)
-
-    else:
-        return None
-
-    if p>0 and p < 1:
-        e = 0.5 * erfcc(-x/sqrt(2)) - p
-        u = e * sqrt(2*pi) * exp(x*x/2)
-        x = x - u/(1 + x*u/2)
-        return x
-    else:
-        return None
-
-def erfcc(x):
-	z=abs(x)
-	t=1.0/(1.0+0.5*z)
-	ans=t*exp(-z*z-1.26551223+t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277)))))))))
-	if x>=0.0:
-		return ans
-	else:
-		return 2.0-ans
-
-def U(n):
-	x=pow(0.5,1.0/n)
-	m=[1-x]
-	for i in range(2,n):
-		a=(i-0.3175)/(n+0.365)
-		m.append(a)
-	m.append(x)
-	return m
-
-def decodeEscape(str):
-    a = str
-    pattern = re.compile('(%[0-9A-Fa-f][0-9A-Fa-f])')
-    match = pattern.findall(a)
-    matched = []
-    for item in match:
-        if item not in matched:
-            a = a.replace(item, '%c' % eval("0x"+item[-2:]))
-        matched.append(item)
-    return a
-
-def exportData(hddn, tdata, NP = None):
-    for key in tdata.keys():
-        _val, _var, _N = tdata[key].val, tdata[key].var, tdata[key].N
-        if _val != None:
-            hddn[key] = _val
-            if _var != None:
-                hddn['V'+key] = _var
-            if NP and _N != None:
-                hddn['N'+key] = _N
-
-def genShortStrainName(RISet='', input_strainName=''):
-    #aliasStrainDict = {'C57BL/6J':'B6','DBA/2J':'D2'}
-    strainName = input_strainName
-    if RISet != 'AXBXA':
-        if RISet == 'BXD300':
-            this_RISet = 'BXD'
-        elif RISet == 'BDF2-2005':
-            this_RISet = 'CASE05_'
-        else:
-            this_RISet = RISet
-        strainName = string.replace(strainName,this_RISet,'')
-        strainName = string.replace(strainName,'CASE','')
-        try:
-            strainName = "%02d" % int(strainName)
-        except:
-            pass
-    else:
-        strainName = string.replace(strainName,'AXB','A')
-        strainName = string.replace(strainName,'BXA','B')
-        try:
-            strainName = strainName[0] + "%02d" % int(strainName[1:])
-        except:
-            pass
-    return strainName
-
-def toInt(in_str):
-    "Converts an arbitrary string to an unsigned integer"
-    start = -1
-    end = -1
-    for i, char in enumerate(in_str):
-        if char >= '0' and char <= '9':
-            if start < 0:
-                start = i
-            end = i+1
-        else:
-            if start >= 0:
-                break
-    if start < end:
-        return int(in_str[start:end])
-    else:
-        return  -1
-
-def transpose(m):
-    'transpose a matrix'
-    n = len(m)
-    return [[m[j][i] for i in range(len(m[0])) for j in range(n)][k*n:k*n+n] for k in range(len(m[0]))]
-
-def asymTranspose(m):
-    'transpose a matrix'
-    t = max(map(len, m))
-    n = len(m)
-    m2 = [["-"]]*n
-    for i in range(n):
-        m2[i] = m[i] + [""]*(t- len(m[i]))
-    return [[m2[j][i] for i in range(len(m2[0])) for j in range(n)][k*n:k*n+n] for k in range(len(m2[0]))]
-
 def genRandStr(prefix = "", length=8, chars=string.letters+string.digits):
     from random import choice
     _str = prefix[:]
@@ -238,84 +72,6 @@ def genRandStr(prefix = "", length=8, chars=string.letters+string.digits):
         _str += choice(chars)
     return _str
 
-def generate_session():
-    import sha
-    return sha.new(str(time.time())).hexdigest()
-
-def cvt2Dict(x):
-    tmp = {}
-    for key in x.keys():
-        tmp[key] = x[key]
-    return tmp
-
-def dump_session(session_obj, filename):
-    "It seems mod python can only cPickle most basic data type"
-    import cPickle
-    session_file = open(filename, 'wb')
-    #try:
-    #       pass
-    #except:
-    #       pass
-    cPickle.dump(session_obj, session_file)
-    session_file.close()
-
-def StringAsFloat(str):
-    'Converts string to float but catches any exception and returns None'
-    try:
-        return float(str)
-    except:
-        return None
-
-def IntAsFloat(str):
-    'Converts string to Int but catches any exception and returns None'
-    try:
-        return int(str)
-    except:
-        return None
-
-def FloatAsFloat(flt):
-    'Converts float to string but catches any exception and returns None'
-    try:
-        return float("%2.3f" % flt)
-    except:
-        return None
-
-def RemoveZero(flt):
-    'Converts string to float but catches any exception and returns None'
-    try:
-        if abs(flt) < 1e-6:
-            return None
-        else:
-            return flt
-    except:
-        return None
-
-
-def SciFloat(d):
-    'Converts string to float but catches any exception and returns None'
-
-    try:
-        if abs(d) <= 1.0e-4:
-            return "%1.2e" % d
-        else:
-            return "%1.5f" % d
-    except:
-        return None
-
-###To be removed
-def FloatList2String(lst):
-    'Converts float list to string but catches any exception and returns None'
-    tt=''
-    try:
-        for item in lst:
-            if item == None:
-                tt += 'X '
-            else:
-                tt += '%f ' % item
-        return tt
-    except:
-        return ""
-
 def ListNotNull(lst):
     '''Obsolete - Use built in function any (or all or whatever)
 
@@ -328,427 +84,6 @@ def ListNotNull(lst):
             return 1
     return None
 
-###To be removed
-def FileDataProcess(str):
-    'Remove the description text from the input file if theres any'
-    i=0
-    while i<len(str):
-        if str[i]<'\x7f' and str[i]>'\x20':
-            break
-        else:
-            i+=1
-    str=str[i:]
-    str=string.join(string.split(str,'\000'),'')
-    i=string.find(str,"*****")
-    if i>-1:
-        return str[i+5:]
-    else:
-        return str
-
-def rank(a,lst,offset=0):
-    """Calculate the integer rank of a number in an array, can be used to calculate p-value"""
-    n = len(lst)
-    if n == 2:
-        if a <lst[0]:
-            return offset
-        elif a > lst[1]:
-            return offset + 2
-        else:
-            return offset +1
-    elif n == 1:
-        if a <lst[0]:
-            return offset
-        else:
-            return offset +1
-    elif n== 0:
-        return offset
-    else:
-        mid = n/2
-        if a < lst[mid]:
-            return rank(a,lst[:mid-1],offset)
-        else:
-            return rank(a,lst[mid:],offset+mid)
-
-def cmpScanResult(A,B):
-    try:
-        if A.LRS > B.LRS:
-            return 1
-        elif A.LRS == B.LRS:
-            return 0
-        else:
-            return -1
-    except:
-        return 0
-
-
-def cmpScanResult2(A,B):
-    try:
-        if A.LRS < B.LRS:
-            return 1
-        elif A.LRS == B.LRS:
-            return 0
-        else:
-            return -1
-    except:
-        return 0
-
-def cmpOrder(A,B):
-    try:
-        if A[1] < B[1]:
-            return -1
-        elif A[1] == B[1]:
-            return 0
-        else:
-            return 1
-    except:
-        return 0
-
-def cmpOrder2(A,B):
-    try:
-        if A[-1] < B[-1]:
-            return -1
-        elif A[-1] == B[-1]:
-            return 0
-        else:
-            return 1
-    except:
-        return 0
-
-
-
-
-def calRank(xVals, yVals, N): ###  Zach Sloan, February 4 2010
-    """
-    Returns a ranked set of X and Y values. These are used when generating
-    a Spearman scatterplot. Bear in mind that this sets values equal to each
-    other as the same rank.
-    """
-    XX = []
-    YY = []
-    X = [0]*len(xVals)
-    Y = [0]*len(yVals)
-    j = 0
-
-    for i in range(len(xVals)):
-
-        if xVals[i] != None and yVals[i] != None:
-            XX.append((j, xVals[i]))
-            YY.append((j, yVals[i]))
-            j = j + 1
-
-    NN = len(XX)
-
-    XX.sort(cmpOrder2)
-    YY.sort(cmpOrder2)
-
-    j = 1
-    rank = 0.0
-
-    while j < NN:
-
-        if XX[j][1] != XX[j-1][1]:
-            X[XX[j-1][0]] = j
-            j = j+1
-
-        else:
-            jt = j+1
-            ji = j
-            for jt in range(j+1, NN):
-                if (XX[jt][1] != XX[j-1][1]):
-                    break
-            rank = 0.5*(j+jt)
-            for ji in range(j-1, jt):
-                X[XX[ji][0]] = rank
-            if (jt == NN-1):
-                if (XX[jt][1] == XX[j-1][1]):
-                    X[XX[NN-1][0]] = rank
-            j = jt+1
-
-    if j == NN:
-        if X[XX[NN-1][0]] == 0:
-            X[XX[NN-1][0]] = NN
-
-    j = 1
-    rank = 0.0
-
-    while j < NN:
-
-        if YY[j][1] != YY[j-1][1]:
-            Y[YY[j-1][0]] = j
-            j = j+1
-        else:
-            jt = j+1
-            ji = j
-            for jt in range(j+1, NN):
-                if (YY[jt][1] != YY[j-1][1]):
-                    break
-            rank = 0.5*(j+jt)
-            for ji in range(j-1, jt):
-                Y[YY[ji][0]] = rank
-            if (jt == NN-1):
-                if (YY[jt][1] == YY[j-1][1]):
-                    Y[YY[NN-1][0]] = rank
-            j = jt+1
-
-    if j == NN:
-        if Y[YY[NN-1][0]] == 0:
-            Y[YY[NN-1][0]] = NN
-
-    return (X,Y)
-
-def calCorrelationRank(xVals,yVals,N):
-    """
-    Calculated Spearman Ranked Correlation. The algorithm works
-    by setting all tied ranks to the average of those ranks (for
-    example, if ranks 5-10 all have the same value, each will be set
-    to rank 7.5).
-    """
-
-    XX = []
-    YY = []
-    j = 0
-
-    for i in range(len(xVals)):
-        if (xVals[i]!= None and yVals[i]!= None) and (xVals[i] != "None" and yVals[i] != "None"):
-            XX.append((j,xVals[i]))
-            YY.append((j,yVals[i]))
-            j = j+1
-
-    NN = len(XX)
-    if NN <6:
-        return (0.0,NN)
-    XX.sort(cmpOrder2)
-    YY.sort(cmpOrder2)
-    X = [0]*NN
-    Y = [0]*NN
-
-    j = 1
-    rank = 0.0
-    t = 0.0
-    sx = 0.0
-
-    while j < NN:
-
-        if XX[j][1] != XX[j-1][1]:
-            X[XX[j-1][0]] = j
-            j = j+1
-
-        else:
-            jt = j+1
-            ji = j
-            for jt in range(j+1, NN):
-                if (XX[jt][1] != XX[j-1][1]):
-                    break
-            rank = 0.5*(j+jt)
-            for ji in range(j-1, jt):
-                X[XX[ji][0]] = rank
-            t = jt-j
-            sx = sx + (t*t*t-t)
-            if (jt == NN-1):
-                if (XX[jt][1] == XX[j-1][1]):
-                    X[XX[NN-1][0]] = rank
-            j = jt+1
-
-    if j == NN:
-        if X[XX[NN-1][0]] == 0:
-            X[XX[NN-1][0]] = NN
-
-    j = 1
-    rank = 0.0
-    t = 0.0
-    sy = 0.0
-
-    while j < NN:
-
-        if YY[j][1] != YY[j-1][1]:
-            Y[YY[j-1][0]] = j
-            j = j+1
-        else:
-            jt = j+1
-            ji = j
-            for jt in range(j+1, NN):
-                if (YY[jt][1] != YY[j-1][1]):
-                    break
-            rank = 0.5*(j+jt)
-            for ji in range(j-1, jt):
-                Y[YY[ji][0]] = rank
-            t = jt - j
-            sy = sy + (t*t*t-t)
-            if (jt == NN-1):
-                if (YY[jt][1] == YY[j-1][1]):
-                    Y[YY[NN-1][0]] = rank
-            j = jt+1
-
-    if j == NN:
-        if Y[YY[NN-1][0]] == 0:
-            Y[YY[NN-1][0]] = NN
-
-    D = 0.0
-
-    for i in range(NN):
-        D += (X[i]-Y[i])*(X[i]-Y[i])
-
-    fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN))
-
-    return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN)
-
-
-def calCorrelationRankText(dbdata,userdata,N): ### dcrowell = David Crowell, July 2008
-    """Calculates correlation ranks with data formatted from the text file.
-    dbdata, userdata are lists of strings.  N is an int.  Returns a float.
-    Used by correlationPage"""
-    XX = []
-    YY = []
-    j = 0
-    for i in range(N):
-        if (dbdata[i]!= None and userdata[i]!=None) and (dbdata[i]!= 'None' and userdata[i]!='None'):
-            XX.append((j,float(dbdata[i])))
-            YY.append((j,float(userdata[i])))
-            j += 1
-    NN = len(XX)
-    if NN <6:
-        return (0.0,NN)
-    XX.sort(cmpOrder2)
-    YY.sort(cmpOrder2)
-    X = [0]*NN
-    Y = [0]*NN
-
-    j = 1
-    rank = 0.0
-    t = 0.0
-    sx = 0.0
-
-    while j < NN:
-
-        if XX[j][1] != XX[j-1][1]:
-            X[XX[j-1][0]] = j
-            j = j+1
-
-        else:
-            jt = j+1
-            ji = j
-            for jt in range(j+1, NN):
-                if (XX[jt][1] != XX[j-1][1]):
-                    break
-            rank = 0.5*(j+jt)
-            for ji in range(j-1, jt):
-                X[XX[ji][0]] = rank
-            t = jt-j
-            sx = sx + (t*t*t-t)
-            if (jt == NN-1):
-                if (XX[jt][1] == XX[j-1][1]):
-                    X[XX[NN-1][0]] = rank
-            j = jt+1
-
-    if j == NN:
-        if X[XX[NN-1][0]] == 0:
-            X[XX[NN-1][0]] = NN
-
-    j = 1
-    rank = 0.0
-    t = 0.0
-    sy = 0.0
-
-    while j < NN:
-
-        if YY[j][1] != YY[j-1][1]:
-            Y[YY[j-1][0]] = j
-            j = j+1
-        else:
-            jt = j+1
-            ji = j
-            for jt in range(j+1, NN):
-                if (YY[jt][1] != YY[j-1][1]):
-                    break
-            rank = 0.5*(j+jt)
-            for ji in range(j-1, jt):
-                Y[YY[ji][0]] = rank
-            t = jt - j
-            sy = sy + (t*t*t-t)
-            if (jt == NN-1):
-                if (YY[jt][1] == YY[j-1][1]):
-                    Y[YY[NN-1][0]] = rank
-            j = jt+1
-
-    if j == NN:
-        if Y[YY[NN-1][0]] == 0:
-            Y[YY[NN-1][0]] = NN
-
-    D = 0.0
-
-    for i in range(NN):
-        D += (X[i]-Y[i])*(X[i]-Y[i])
-
-    fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN))
-
-    return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN)
-
-
-
-def calCorrelation(dbdata,userdata,N):
-    X = []
-    Y = []
-    for i in range(N):
-        if dbdata[i]!= None and userdata[i]!= None:
-            X.append(dbdata[i])
-            Y.append(userdata[i])
-    NN = len(X)
-    if NN <6:
-        return (0.0,NN)
-    sx = reduce(lambda x,y:x+y,X,0.0)
-    sy = reduce(lambda x,y:x+y,Y,0.0)
-    meanx = sx/NN
-    meany = sy/NN
-    xyd = 0.0
-    sxd = 0.0
-    syd = 0.0
-    for i in range(NN):
-        xyd += (X[i] - meanx)*(Y[i]-meany)
-        sxd += (X[i] - meanx)*(X[i] - meanx)
-        syd += (Y[i] - meany)*(Y[i] - meany)
-    try:
-        corr = xyd/(sqrt(sxd)*sqrt(syd))
-    except:
-        corr = 0
-    return (corr,NN)
-
-def calCorrelationText(dbdata,userdata,N): ### dcrowell July 2008
-    """Calculates correlation coefficients with values formatted from text files. dbdata, userdata are lists of strings.  N is an int.  Returns a float
-    Used by correlationPage"""
-    X = []
-    Y = []
-    for i in range(N):
-        #if (dbdata[i]!= None and userdata[i]!= None) and (dbdata[i]!= 'None' and userdata[i]!= 'None'):
-        #               X.append(float(dbdata[i]))
-        #               Y.append(float(userdata[i]))
-        if dbdata[i] == None or dbdata[i] == 'None' or userdata[i] == None or userdata[i] == 'None':
-            continue
-        else:
-            X.append(float(dbdata[i]))
-            Y.append(float(userdata[i]))
-    NN = len(X)
-    if NN <6:
-        return (0.0,NN)
-    sx = sum(X)
-    sy = sum(Y)
-    meanx = sx/float(NN)
-    meany = sy/float(NN)
-    xyd = 0.0
-    sxd = 0.0
-    syd = 0.0
-    for i in range(NN):
-        x1 = X[i]-meanx
-        y1 = Y[i]-meany
-        xyd += x1*y1
-        sxd += x1**2
-        syd += y1**2
-    try:
-        corr = xyd/(sqrt(sxd)*sqrt(syd))
-    except:
-        corr = 0
-    return (corr,NN)
-
-
 def readLineCSV(line): ### dcrowell July 2008
     """Parses a CSV string of text and returns a list containing each element as a string.
     Used by correlationPage"""
@@ -757,45 +92,6 @@ def readLineCSV(line): ### dcrowell July 2008
     returnList[0]=returnList[0][1:]
     return returnList
 
-
-def cmpCorr(A,B):
-    try:
-        if abs(A[1]) < abs(B[1]):
-            return 1
-        elif abs(A[1]) == abs(B[1]):
-            return 0
-        else:
-            return -1
-    except:
-        return 0
-
-def cmpLitCorr(A,B):
-    try:
-        if abs(A[3]) < abs(B[3]): return 1
-        elif abs(A[3]) == abs(B[3]):
-            if abs(A[1]) < abs(B[1]): return 1
-            elif abs(A[1]) == abs(B[1]): return 0
-            else: return -1
-        else: return -1
-    except:
-        return 0
-
-def cmpPValue(A,B):
-    try:
-        if A.corrPValue < B.corrPValue:
-            return -1
-        elif A.corrPValue == B.corrPValue:
-            if abs(A.corr) > abs(B.corr):
-                return -1
-            elif abs(A.corr) < abs(B.corr):
-                return 1
-            else:
-                return 0
-        else:
-            return 1
-    except:
-        return 0
-
 def cmpEigenValue(A,B):
     try:
         if A[0] > B[0]:
@@ -807,80 +103,6 @@ def cmpEigenValue(A,B):
     except:
         return 0
 
-
-def cmpLRSFull(A,B):
-    try:
-        if A[0] < B[0]:
-            return -1
-        elif A[0] == B[0]:
-            return 0
-        else:
-            return 1
-    except:
-        return 0
-
-def cmpLRSInteract(A,B):
-    try:
-        if A[1] < B[1]:
-            return -1
-        elif A[1] == B[1]:
-            return 0
-        else:
-            return 1
-    except:
-        return 0
-
-
-def cmpPos(A,B):
-    try:
-        try:
-            AChr = int(A.chr)
-        except:
-            AChr = 20
-        try:
-            BChr = int(B.chr)
-        except:
-            BChr = 20
-        if AChr > BChr:
-            return 1
-        elif AChr == BChr:
-            if A.mb > B.mb:
-                return 1
-            if A.mb == B.mb:
-                return 0
-            else:
-                return -1
-        else:
-            return -1
-    except:
-        return 0
-
-def cmpGenoPos(A,B):
-    try:
-        A1 = A.chr
-        B1 = B.chr
-        try:
-            A1 = int(A1)
-        except:
-            A1 = 25
-        try:
-            B1 = int(B1)
-        except:
-            B1 = 25
-        if A1 > B1:
-            return 1
-        elif A1 == B1:
-            if A.mb > B.mb:
-                return 1
-            if A.mb == B.mb:
-                return 0
-            else:
-                return -1
-        else:
-            return -1
-    except:
-        return 0
-
 def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users):
     access_to_confidential_phenotype_trait = 0
     if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']:
@@ -889,142 +111,4 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users)
         AuthorizedUsersList=map(string.strip, string.split(authorized_users, ','))
         if AuthorizedUsersList.__contains__(userName):
             access_to_confidential_phenotype_trait = 1
-    return access_to_confidential_phenotype_trait
-
-
-class VisualizeException(Exception):
-    def __init__(self, message):
-        self.message = message
-    def __str__(self):
-        return self.message
-
-# safeConvert : (string -> A) -> A -> A
-# to convert a string to type A, using the supplied default value
-# if the given conversion function doesn't work
-def safeConvert(f, value, default):
-    try:
-        return f(value)
-    except:
-        return default
-
-# safeFloat : string -> float -> float
-# to convert a string to a float safely
-def safeFloat(value, default):
-    return safeConvert(float, value, default)
-
-# safeInt: string -> int -> int
-# to convert a string to an int safely
-def safeInt(value, default):
-    return safeConvert(int, value, default)
-
-# safeString : string -> (arrayof string) -> string -> string
-# if a string is not in a list of strings to pick a default value
-# for that string
-def safeString(value, validChoices, default):
-    if value in validChoices:
-        return value
-    else:
-        return default
-
-# yesNoToInt: string -> int
-# map "yes" -> 1 and "no" -> 0
-def yesNoToInt(value):
-    if value == "yes":
-        return 1
-    elif value == "no":
-        return 0
-    else:
-        return None
-
-# IntToYesNo: int -> string
-# map 1 -> "yes" and 0 -> "no"
-def intToYesNo(value):
-    if value == 1:
-        return "yes"
-    elif value == 0:
-        return "no"
-    else:
-        return None
-
-def formatField(name):
-    name = name.replace("_", " ")
-    name = name.title()
-    #name = name.replace("Mb Mm6", "Mb");
-    return name.replace("Id", "ID")
-
-#XZ, 03/27/2009: This function is very specific.
-#It is used by AJAX_table.py, correlationPage.py and dataPage.py
-
-
-def genTableObj(tblobj=None, file="", sortby = ("", ""), tableID = "sortable", addIndex = "1", hiddenColumns=[]):
-    header = tblobj['header']
-    body = tblobj['body']
-    field, order = sortby
-
-    #ZAS 9/12/2011 - The hiddenColumns array needs to be converted into a string so they can be placed into the javascript of each up/down button
-    hiddenColumnsString = ",".join(hiddenColumns)
-
-    tbl = HT.TableLite(Class="collap b2", cellspacing=1, cellpadding=5)
-
-    hiddenColumnIdx = [] #indices of columns to hide
-    idx = -1
-    last_idx = 0 #ZS: This is the index of the last item in the regular table header (without any extra parameters). It is used to determine the index of each extra parameter.
-    for row in header:
-        hr = HT.TR()
-        for i, item in enumerate(row):
-            if (item.text == '') or (item.text not in hiddenColumns):
-                if item.sort and item.text:
-                    down = HT.Href("javascript:xmlhttpPost('%smain.py?FormID=AJAX_table', '%s', 'sort=%s&order=down&file=%s&tableID=%s&addIndex=%s&hiddenColumns=%s')" % (webqtlConfig.CGIDIR, tableID, item.text, file, tableID, addIndex, hiddenColumnsString),IMGDESC)
-                    up = HT.Href("javascript:xmlhttpPost('%smain.py?FormID=AJAX_table', '%s', 'sort=%s&order=up&file=%s&tableID=%s&addIndex=%s&hiddenColumns=%s')" % (webqtlConfig.CGIDIR, tableID, item.text, file, tableID, addIndex, hiddenColumnsString),IMGASC)
-                    if item.text == field:
-                        idx = item.idx
-                        last_idx = idx
-                        if order == 'up':
-                            up = IMGASCON
-                        elif order == 'down':
-                            down = IMGDESCON
-                    item.html.append(HT.Div(up, down, style="float: bottom;"))
-                hr.append(item.html)
-            else:
-                hiddenColumnIdx.append(i)
-        tbl.append(hr)
-
-    for i, row in enumerate(body):
-        for j, item in enumerate(row):
-            if order == 'down':
-                if (item.val == '' or item.val == 'x' or item.val == 'None'):
-                    item.val = 0
-            if order == 'up':
-                if (item.val == '' or item.val == 'x' or item.val == 'None'):
-                    item.val = 'zzzzz'
-
-    if idx >= 0:
-        if order == 'down':
-            body.sort(lambda A, B: cmp(B[idx].val, A[idx].val), key=natsort_key)
-        elif order == 'up':
-            body.sort(lambda A, B: cmp(A[idx].val, B[idx].val), key=natsort_key)
-        else:
-            pass
-
-    for i, row in enumerate(body):
-        hr = HT.TR(Id = row[0].text)
-        for j, item in enumerate(row):
-            if (j not in hiddenColumnIdx):
-                if j == 0:
-                    if addIndex == "1":
-                        item.html.contents = [i+1] + item.html.contents
-                hr.append(item.html)
-        tbl.append(hr)
-
-    return tbl
-
-def natsort_key(string):
-    r = []
-    for c in string:
-        try:
-            c = int(c)
-            try: r[-1] = r[-1] * 10 + c
-            except: r.append(c)
-        except:
-            r.append(c)
-    return r
+    return access_to_confidential_phenotype_trait
\ No newline at end of file