diff options
Diffstat (limited to 'wqflask/utility')
| -rw-r--r-- | wqflask/utility/AJAX_table.py | 155 | ||||
| -rw-r--r-- | wqflask/utility/Plot.py | 1087 | ||||
| -rw-r--r-- | wqflask/utility/after.py | 8 | ||||
| -rw-r--r-- | wqflask/utility/authentication_tools.py | 130 | ||||
| -rw-r--r-- | wqflask/utility/benchmark.py | 2 | ||||
| -rw-r--r-- | wqflask/utility/chunks.py | 63 | ||||
| -rw-r--r-- | wqflask/utility/corestats.py | 83 | ||||
| -rw-r--r-- | wqflask/utility/corr_result_helpers.py | 26 | ||||
| -rw-r--r-- | wqflask/utility/elasticsearch_tools.py | 112 | ||||
| -rw-r--r-- | wqflask/utility/formatting.py | 29 | ||||
| -rw-r--r-- | wqflask/utility/gen_geno_ob.py | 181 | ||||
| -rw-r--r-- | wqflask/utility/genofile_parser.py | 2 | ||||
| -rw-r--r-- | wqflask/utility/helper_functions.py | 30 | ||||
| -rw-r--r-- | wqflask/utility/hmac.py | 40 | ||||
| -rw-r--r-- | wqflask/utility/logger.py | 4 | ||||
| -rw-r--r-- | wqflask/utility/pillow_utils.py | 27 | ||||
| -rw-r--r-- | wqflask/utility/redis_tools.py | 309 | ||||
| -rw-r--r-- | wqflask/utility/startup_config.py | 39 | ||||
| -rw-r--r-- | wqflask/utility/svg.py | 2 | ||||
| -rw-r--r-- | wqflask/utility/tools.py | 132 | ||||
| -rw-r--r-- | wqflask/utility/type_checking.py | 42 | ||||
| -rw-r--r-- | wqflask/utility/webqtlUtil.py | 925 |
22 files changed, 1180 insertions, 2248 deletions
diff --git a/wqflask/utility/AJAX_table.py b/wqflask/utility/AJAX_table.py deleted file mode 100644 index d70acfcd..00000000 --- a/wqflask/utility/AJAX_table.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (C) University of Tennessee Health Science Center, Memphis, TN. -# -# This program is free software: you can redistribute it and/or modify it -# under the terms of the GNU Affero General Public License -# as published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. -# See the GNU Affero General Public License for more details. -# -# This program is available from Source Forge: at GeneNetwork Project -# (sourceforge.net/projects/genenetwork/). -# -# Contact Drs. Robert W. Williams and Xiaodong Zhou (2010) -# at rwilliams@uthsc.edu and xzhou15@uthsc.edu -# -# -# -# This module is used by GeneNetwork project (www.genenetwork.org) -# -# Created by GeneNetwork Core Team 2010/08/10 -# -# Last updated by GeneNetwork Core Team 2010/10/20 - -import cPickle -import os -import MySQLdb -import time -import pyXLWriter as xl - -from htmlgen import HTMLgen2 as HT - -from base import webqtlConfig -from THCell import THCell -from TDCell import TDCell -import webqtlUtil - - -class AJAX_table: - def __init__(self, fd): - file = fd.formdata.getfirst("file", "") - sort = fd.formdata.getfirst("sort", "") - order = fd.formdata.getfirst("order", "up") - cmd = fd.formdata.getfirst("cmd", "") - tableID = fd.formdata.getfirst("tableID", "") - addIndex = fd.formdata.getfirst("addIndex", "1") - hiddenColumnsString = fd.formdata.getfirst("hiddenColumns", "") - hiddenColumns = hiddenColumnsString.split(',') - - try: - fp = open(os.path.join(webqtlConfig.TMPDIR, file + '.obj'), 'rb') - tblobj = cPickle.load(fp) - fp.close() - - if cmd == 'addCorr': - dbId = int(fd.formdata.getfirst("db")) - dbFullName = fd.formdata.getfirst("dbname") - trait = fd.formdata.getfirst("trait") - form = fd.formdata.getfirst("form") - ids = fd.formdata.getfirst("ids") - vals = fd.formdata.getfirst("vals") - ids = eval(ids) - nnCorr = len(ids) - vals = eval(vals) - - workbook = xl.Writer('%s.xls' % (webqtlConfig.TMPDIR+file)) - worksheet = workbook.add_worksheet() - - logger.warning("Creating new MySQLdb cursor (this method is OBSOLETE!)") - - con = MySQLdb.Connect(db=webqtlConfig.DB_NAME,host=webqtlConfig.MYSQL_SERVER, user=webqtlConfig.DB_USER,passwd=webqtlConfig.DB_PASSWD) - cursor = con.cursor() - - cursor.execute("Select name, ShortName from ProbeSetFreeze where Id = %s", dbId) - dbName, dbShortName = cursor.fetchone() - - tblobj['header'][0].append( - THCell(HT.TD(dbShortName, Class="fs11 ffl b1 cw cbrb"), - text="%s" % dbShortName, idx=tblobj['header'][0][-1].idx + 1), - ) - - headingStyle = workbook.add_format(align = 'center', bold = 1, border = 1, size=13, fg_color = 0x1E, color="white") - for i, item in enumerate(tblobj['header'][0]): - if (i > 0): - worksheet.write([8, i-1], item.text, headingStyle) - worksheet.set_column([i-1, i-1], 2*len(item.text)) - - for i, row in enumerate(tblobj['body']): - ProbeSetId = row[1].text - #XZ, 03/02/2009: Xiaodong changed Data to ProbeSetData - cursor.execute(""" - Select ProbeSetData.StrainId, ProbeSetData.Value - From ProbeSetData, ProbeSetXRef, ProbeSet - where ProbeSetXRef.ProbeSetFreezeId = %d AND - ProbeSetXRef.DataId = ProbeSetData.Id AND - ProbeSetXRef.ProbeSetId = ProbeSet.Id AND - ProbeSet.Name = '%s' - """ % (dbId, ProbeSetId)) - results = cursor.fetchall() - vdict = {} - for item in results: - vdict[item[0]] = item[1] - newvals = [] - for id in ids: - if vdict.has_key(id): - newvals.append(vdict[id]) - else: - newvals.append(None) - corr,nOverlap= webqtlUtil.calCorrelation(newvals,vals,nnCorr) - repr = '%0.4f' % corr - row.append( - TDCell(HT.TD(HT.Href(text=repr, url="javascript:showCorrPlotThird('%s', '%s', '%s')" % (form, dbName, ProbeSetId), Class="fs11 fwn ffl"), " / ", nOverlap, Class="fs11 fwn ffl b1 c222", align="middle"),repr,abs(corr)) - ) - - last_row=0 - for j, item in enumerate(tblobj['body'][i]): - if (j > 0): - worksheet.write([9+i, j-1], item.text) - last_row = 9+i - last_row += 1 - - titleStyle = workbook.add_format(align = 'left', bold = 0, size=14, border = 1, border_color="gray") - ##Write title Info - # Modified by Hongqiang Li - worksheet.write([0, 0], "Citations: Please see %s/reference.html" % webqtlConfig.PORTADDR, titleStyle) - worksheet.write([1, 0], "Trait : %s" % trait, titleStyle) - worksheet.write([2, 0], "Database : %s" % dbFullName, titleStyle) - worksheet.write([3, 0], "Date : %s" % time.strftime("%B %d, %Y", time.gmtime()), titleStyle) - worksheet.write([4, 0], "Time : %s GMT" % time.strftime("%H:%M ", time.gmtime()), titleStyle) - worksheet.write([5, 0], "Status of data ownership: Possibly unpublished data; please see %s/statusandContact.html for details on sources, ownership, and usage of these data." % webqtlConfig.PORTADDR, titleStyle) - #Write footer info - worksheet.write([1 + last_row, 0], "Funding for The GeneNetwork: NIAAA (U01AA13499, U24AA13513), NIDA, NIMH, and NIAAA (P20-DA21131), NCI MMHCC (U01CA105417), and NCRR (U01NR 105417)", titleStyle) - worksheet.write([2 + last_row, 0], "PLEASE RETAIN DATA SOURCE INFORMATION WHENEVER POSSIBLE", titleStyle) - - cursor.close() - workbook.close() - - objfile = open(os.path.join(webqtlConfig.TMPDIR, file + '.obj'), 'wb') - cPickle.dump(tblobj, objfile) - objfile.close() - else: - pass - - self.value = str(webqtlUtil.genTableObj(tblobj=tblobj, file=file, sortby=(sort, order), tableID = tableID, addIndex = addIndex, hiddenColumns = hiddenColumns)) - - except: - self.value = "<span class='fs16 fwb cr ffl'>The table is no longer available on this server</span>" - - def __str__(self): - return self.value - - def write(self): - return str(self) diff --git a/wqflask/utility/Plot.py b/wqflask/utility/Plot.py index d60e2bb2..b9b71129 100644 --- a/wqflask/utility/Plot.py +++ b/wqflask/utility/Plot.py @@ -26,26 +26,29 @@ from __future__ import print_function -import piddle as pid -from pprint import pformat as pf +from PIL import ImageColor +from PIL import ImageDraw +from PIL import ImageFont from math import * -import random -import sys, os -from numarray import linear_algebra as la -from numarray import ones, array, dot, swapaxes -import reaper -# sys.path.append("..") Never in a running webserver -from basicStatistics import corestats - -import svg -import webqtlUtil +import corestats from base import webqtlConfig - +from utility.pillow_utils import draw_rotated_text import utility.logger logger = utility.logger.getLogger(__name__ ) +# ---- Define common colours ---- # +BLUE = ImageColor.getrgb("blue") +BLACK = ImageColor.getrgb("black") +# ---- END: Define common colours ---- # + +# ---- FONT FILES ---- # +VERDANA_FILE = "./wqflask/static/fonts/verdana.ttf" +COUR_FILE = "./wqflask/static/fonts/courbd.ttf" +TAHOMA_FILE = "./wqflask/static/fonts/tahoma.ttf" +# ---- END: FONT FILES ---- # + def cformat(d, rank=0): 'custom string format' strD = "%2.6f" % d @@ -83,202 +86,6 @@ def frange(start, end=None, inc=1.0): L[i] = start + i * inc return L - -def gammln(xx): - cof=[76.18009173,-86.50532033,24.01409822,-1.231739516,0.120858003e-2,-0.536382e-5] - x=xx-1.0 - tmp=x+5.5 - tmp -=(x+0.5)*log(tmp) - ser=1.0 - for item in cof: - x+=1.0 - ser+=item/x - - return -tmp+log(2.50662827465*ser) - - -def gser(a,x): - gln=gammln(a) - ITMAX=100 - EPS=3.0e-7 - - if x<=0.0: - gamser=0.0 - return [gamser,gln] - else: - ap=a - sum=1.0/a - dele=sum - for i in range(1,ITMAX+1): - ap+=1.0 - dele*=x/ap - sum+=dele - if abs(dele)<abs(sum)*EPS: - gamser=sum*exp(-x+a*log(x)-gln) - return [gamser,gln] - return None - -def gcf(a,x): - ITMAX=100 - EPS=3.0e-7 - gold=0.0 - fac=1 - b1=1.0 - b0=0.0 - a0=1.0 - gln=gammln(a) - - a1=x - for n in range(1,ITMAX+1): - an=n+0.0 - ana=an-a - a0=(a1+a0*ana)*fac - b0=(b1+b0*ana)*fac - anf=an*fac - a1=x*a0+anf*a1 - b1=x*b0+anf*b1 - if (a1): - fac=1.0/a1 - g=b1*fac - if abs((g-gold)/g)<EPS: - gammcf=exp(-x+a*log(x)-gln)*g - return [gammcf,gln] - gold=g - return None - -def gammp(a,x): - if x<0.0 or a<=0.0: - return None - if x<(a+1.0): - a=gser(a,x)[0] - return a - else: - a=gcf(a,x)[0] - return 1.0-a -def U(n): - x=pow(0.5,1.0/n) - m=[1-x] - for i in range(2,n): - a=(i-0.3175)/(n+0.365) - m.append(a) - m.append(x) - return m - -def erf(x): - if x<0.0: - return -gammp(0.5,x*x) - else: - return gammp(0.5,x*x) - -def erfcc(x): - z=abs(x) - t=1.0/(1.0+0.5*z) - ans=t*exp(-z*z-1.26551223+t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277))))))))) - if x>=0.0: - return ans - else: - return 2.0-ans - -def calMeanVar(data): - n=len(data) - if n<2: - return None - else: - sum=reduce(lambda x,y:x+y,data,0.0) - mean=sum/n - z=data[:] - for i in range(n): - z[i]=z[i]-mean - variance=reduce(lambda x,y:x+y*y,z,0.0) - variance /= n-1 - variance =sqrt(variance) - for i in range(n): - z[i]=z[i]/variance - return z - -def inverseCumul(p): - #Coefficients in rational approximations. - a = [-3.969683028665376e+01,2.209460984245205e+02,-2.759285104469687e+02,1.383577518672690e+02,-3.066479806614716e+01,2.506628277459239e+00] - - b = [-5.447609879822406e+01,1.615858368580409e+02,-1.556989798598866e+02,6.680131188771972e+01,-1.328068155288572e+01] - - c = [-7.784894002430293e-03,-3.223964580411365e-01,-2.400758277161838e+00,-2.549732539343734e+00,4.374664141464968e+00,2.938163982698783e+00] - - d = [7.784695709041462e-03,3.224671290700398e-01,2.445134137142996e+00,3.754408661907416e+00] - - #Define break-points. - - p_low = 0.02425 - p_high = 1 - p_low - - #Rational approximation for lower region. - - if p > 0 and p < p_low: - q = sqrt(-2*log(p)) - x = (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - - #Rational approximation for central region. - - elif p>= p_low and p <= p_high: - q = p - 0.5 - r = q*q - x = (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q /(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1) - - #Rational approximation for upper region. - - elif p>p_high and p < 1: - q = sqrt(-2*log(1-p)) - x = -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - else: - return None - - if p>0 and p < 1: - e = 0.5 * erfcc(-x/sqrt(2)) - p - u = e * sqrt(2*pi) * exp(x*x/2) - x = x - u/(1 + x*u/2) - return x - else: - return None - -def gmean(lst): - N = len(lst) - if N == 0: - return 0 - else: - return (reduce(lambda x,y: x+y, lst, 0.0))/N - -def gmedian(lst2): - lst = lst2[:] - N = len(lst) - if N == 0: - return 0 - else: - lst.sort() - if N % 2 == 0: - return (lst[N/2]+lst[(N-2)/2])/2.0 - else: - return lst[(N-1)/2] - -def gpercentile(lst2, np): - """Obsolete - use percentile in corestats instead""" - lst = lst2[:] - N = len(lst) - if N == 0 or np > 100 or np < 0: - return None - else: - lst.sort() - pNadd1 = (np/100.0)*N - k = int(pNadd1) - d = pNadd1 - k - if k == 0: - return lst[0] - elif k >= N-1: - return lst[N-1] - else: - return lst[k-1] + d*(lst[k] - lst[k-1]) - def find_outliers(vals): """Calculates the upper and lower bounds of a set of sample/case values @@ -296,8 +103,6 @@ def find_outliers(vals): """ - logger.debug("xerxes vals is:", pf(vals)) - if vals: #logger.debug("vals is:", pf(vals)) stats = corestats.Stats(vals) @@ -312,171 +117,12 @@ def find_outliers(vals): upper_bound = None lower_bound = None - logger.debug(pf(locals())) return upper_bound, lower_bound - -def plotBoxPlot(canvas, data, offset= (40, 40, 40, 40), XLabel="Category", YLabel="Value"): - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - iValues = [] - for item in data: - for item2 in item[1]: - try: - iValues.append(item2[1]) - except: - iValues.append(item2) - - #draw frame - max_Y = max(iValues) - min_Y = min(iValues) - scaleY = detScale(min_Y, max_Y) - Yll = scaleY[0] - Yur = scaleY[1] - nStep = scaleY[2] - stepY = (Yur - Yll)/nStep - stepYPixel = plotHeight/(nStep) - canvas.drawRect(plotWidth+xLeftOffset, plotHeight + yTopOffset, xLeftOffset, yTopOffset) - - ##draw Y Scale - YYY = Yll - YCoord = plotHeight + yTopOffset - scaleFont=pid.Font(ttf="cour",size=11,bold=1) - for i in range(nStep+1): - strY = cformat(d=YYY, rank=0) - YCoord = max(YCoord, yTopOffset) - canvas.drawLine(xLeftOffset,YCoord,xLeftOffset-5,YCoord) - canvas.drawString(strY, xLeftOffset -30,YCoord +5,font=scaleFont) - YYY += stepY - YCoord -= stepYPixel - - ##draw X Scale - stepX = plotWidth/len(data) - XCoord = xLeftOffset + 0.5*stepX - YCoord = plotHeight + yTopOffset - scaleFont = pid.Font(ttf="tahoma",size=12,bold=0) - labelFont = pid.Font(ttf="tahoma",size=13,bold=0) - for item in data: - itemname, itemvalue = item - canvas.drawLine(XCoord, YCoord,XCoord, YCoord+5, color=pid.black) - canvas.drawString(itemname, XCoord - canvas.stringWidth(itemname,font=labelFont)/2.0,\ - YCoord +20,font=labelFont) - - nValue = len(itemvalue) - catValue = [] - for item2 in itemvalue: - try: - tstrain, tvalue = item2 - except: - tvalue = item2 - if nValue <= 4: - canvas.drawCross(XCoord, plotHeight + yTopOffset - (tvalue-Yll)*plotHeight/(Yur - Yll), color=pid.red,size=5) - else: - catValue.append(tvalue) - if catValue != []: - catMean = gmean(catValue) - catMedian = gmedian(catValue) - lowHinge = gpercentile(catValue, 25) - upHinge = gpercentile(catValue, 75) - Hstep = 1.5*(upHinge - lowHinge) - - outlier = [] - extrem = [] - - upperAdj = None - for item in catValue: - if item >= upHinge + 2*Hstep: - extrem.append(item) - elif item >= upHinge + Hstep: - outlier.append(item) - elif item > upHinge and item < upHinge + Hstep: - if upperAdj == None or item > upperAdj: - upperAdj = item - else: - pass - lowerAdj = None - for item in catValue: - if item <= lowHinge - 2*Hstep: - extrem.append(item) - elif item <= lowHinge - Hstep: - outlier.append(item) - if item < lowHinge and item > lowHinge - Hstep: - if lowerAdj == None or item < lowerAdj: - lowerAdj = item - else: - pass - canvas.drawRect(XCoord-20, plotHeight + yTopOffset - (lowHinge-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (upHinge-Yll)*plotHeight/(Yur - Yll)) - canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (catMedian-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (catMedian-Yll)*plotHeight/(Yur - Yll)) - if upperAdj != None: - canvas.drawLine(XCoord, plotHeight + yTopOffset - (upHinge-Yll)*plotHeight/(Yur - Yll), \ - XCoord, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll)) - canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (upperAdj-Yll)*plotHeight/(Yur - Yll)) - if lowerAdj != None: - canvas.drawLine(XCoord, plotHeight + yTopOffset - (lowHinge-Yll)*plotHeight/(Yur - Yll), \ - XCoord, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll)) - canvas.drawLine(XCoord-20, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll), \ - XCoord+20, plotHeight + yTopOffset - (lowerAdj-Yll)*plotHeight/(Yur - Yll)) - - outlierFont = pid.Font(ttf="cour",size=12,bold=0) - if outlier != []: - for item in outlier: - yc = plotHeight + yTopOffset - (item-Yll)*plotHeight/(Yur - Yll) - #canvas.drawEllipse(XCoord-3, yc-3, XCoord+3, yc+3) - canvas.drawString('o', XCoord-3, yc+5, font=outlierFont, color=pid.orange) - if extrem != []: - for item in extrem: - yc = plotHeight + yTopOffset - (item-Yll)*plotHeight/(Yur - Yll) - #canvas.drawEllipse(XCoord-3, yc-3, XCoord+3, yc+3) - canvas.drawString('*', XCoord-3, yc+6, font=outlierFont, color=pid.red) - - canvas.drawCross(XCoord, plotHeight + yTopOffset - (catMean-Yll)*plotHeight/(Yur - Yll), \ - color=pid.blue,size=3) - #print(catMean, catMedian, cat25per, cat75per) - pass - - XCoord += stepX - - labelFont=pid.Font(ttf="verdana",size=18,bold=0) - canvas.drawString(XLabel, xLeftOffset + (plotWidth -canvas.stringWidth(XLabel,font=labelFont))/2.0, \ - YCoord +40, font=labelFont) - canvas.drawString(YLabel,xLeftOffset-40, YCoord-(plotHeight -canvas.stringWidth(YLabel,font=labelFont))/2.0,\ - font=labelFont, angle =90) - -def plotSecurity(canvas, text="12345"): - if not text: - return - - plotWidth = canvas.size[0] - plotHeight = canvas.size[1] - if plotHeight<=0 or plotWidth<=0: - return - - bgColor = pid.Color(0.6+0.4*random.random(), 0.6+0.4*random.random(), 0.6+0.4*random.random()) - canvas.drawRect(0,0,plotWidth,plotHeight, edgeColor=bgColor, fillColor=bgColor) - - for i in range(30): - randomColor = pid.Color(0.6+0.4*random.random(), 0.6+0.4*random.random(), 0.6+0.4*random.random()) - scaleFont=pid.Font(ttf="cour",size=random.choice(range(20, 50))) - canvas.drawString(random.choice('abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'), - int(random.random()*plotWidth), int(random.random()*plotHeight), font=scaleFont, - color=randomColor, angle=random.choice(range(-45, 50))) - - step = (plotWidth-20)/len(text) - startX = 20 - for item in text: - randomColor = pid.Color(0.6*random.random(),0.6*random.random(), 0.6*random.random()) - scaleFont=pid.Font(ttf="verdana",size=random.choice(range(50, 60)),bold=1) - canvas.drawString(item, startX, plotHeight/2-10, font=scaleFont, - color=randomColor, angle=random.choice(range(-45, 50))) - startX += step - # parameter: data is either object returned by reaper permutation function (called by MarkerRegressionPage.py) # or the first object returned by direct (pair-scan) permu function (called by DirectPlotPage.py) -def plotBar(canvas, data, barColor=pid.blue, axesColor=pid.black, labelColor=pid.black, XLabel=None, YLabel=None, title=None, offset= (60, 20, 40, 40), zoom = 1): +def plotBar(canvas, data, barColor=BLUE, axesColor=BLACK, labelColor=BLACK, XLabel=None, YLabel=None, title=None, offset= (60, 20, 40, 40), zoom = 1): + im_drawer = ImageDraw.Draw(canvas) xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset plotWidth = canvas.size[0] - xLeftOffset - xRightOffset @@ -523,579 +169,67 @@ def plotBar(canvas, data, barColor=pid.blue, axesColor=pid.black, labelColor=pid if count: xc = (dataXY[i]-xLow)*xScale+xLeftOffset yc =-(count-yLow)*yScale+yTopOffset+plotHeight - canvas.drawRect(xc+2,yc,xc+barWidth-2,yTopOffset+plotHeight,edgeColor=barColor,fillColor=barColor) + im_drawer.rectangle( + xy=((xc+2,yc),(xc+barWidth-2,yTopOffset+plotHeight)), + outline=barColor, fill=barColor) #draw drawing region - canvas.drawRect(xLeftOffset, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight) + im_drawer.rectangle( + xy=((xLeftOffset, yTopOffset), (xLeftOffset+plotWidth, yTopOffset+plotHeight)) + ) #draw scale - scaleFont=pid.Font(ttf="cour",size=11,bold=1) + scaleFont=ImageFont.truetype(font=COUR_FILE,size=11) x=xLow for i in range(int(stepX)+1): xc=xLeftOffset+(x-xLow)*xScale - canvas.drawLine(xc,yTopOffset+plotHeight,xc,yTopOffset+plotHeight+5, color=axesColor) + im_drawer.line( + xy=((xc,yTopOffset+plotHeight),(xc,yTopOffset+plotHeight+5)), + fill=axesColor) strX = cformat(d=x, rank=0) - canvas.drawString(strX,xc-canvas.stringWidth(strX,font=scaleFont)/2,yTopOffset+plotHeight+14,font=scaleFont) + im_drawer.text( + text=strX, + xy=(xc-im_drawer.textsize(strX,font=scaleFont)[0]/2, + yTopOffset+plotHeight+14),font=scaleFont) x+= (xTop - xLow)/stepX y=yLow for i in range(int(stepY)+1): yc=yTopOffset+plotHeight-(y-yLow)*yScale - canvas.drawLine(xLeftOffset,yc,xLeftOffset-5,yc, color=axesColor) + im_drawer.line(xy=((xLeftOffset,yc),(xLeftOffset-5,yc)), fill=axesColor) strY = "%d" %y - canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)-6,yc+5,font=scaleFont) + im_drawer.text( + text=strY, + xy=(xLeftOffset-im_drawer.textsize(strY,font=scaleFont)[0]-6,yc+5), + font=scaleFont) y+= (yTop - yLow)/stepY #draw label - labelFont=pid.Font(ttf="tahoma",size=17,bold=0) - if XLabel: - canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0, - yTopOffset+plotHeight+yBottomOffset-10,font=labelFont,color=labelColor) - - if YLabel: - canvas.drawString(YLabel, 19, yTopOffset+plotHeight-(plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0, - font=labelFont,color=labelColor,angle=90) - - labelFont=pid.Font(ttf="verdana",size=16,bold=0) - if title: - canvas.drawString(title,xLeftOffset+(plotWidth-canvas.stringWidth(title,font=labelFont))/2.0, - 20,font=labelFont,color=labelColor) - -def plotBarText(canvas, data, label, variance=None, barColor=pid.blue, axesColor=pid.black, labelColor=pid.black, XLabel=None, YLabel=None, title=None, sLabel = None, offset= (80, 20, 40, 100), barSpace = 2, zoom = 1): - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = canvas.size[0] - xLeftOffset - xRightOffset - plotHeight = canvas.size[1] - yTopOffset - yBottomOffset - if plotHeight<=0 or plotWidth<=0: - return - - NNN = len(data) - if NNN < 2 or NNN != len(label): - return - if variance and len(variance)!=NNN: - variance = [] - - Y2 = data[:] - if variance: - for i in range(NNN): - if variance[i]: - Y2 += [data[i]-variance[i]] - - #Y axis - YLow, YTop, stepY = detScale(min(Y2), max(Y2)) - YScale = plotHeight/(YTop - YLow) - - if YLow < 0 and YTop > 0: - drawZero = 1 - else: - drawZero = 0 - - #X axis - X = range(NNN) - Xll= 0 - Xur= NNN-1 - - - if drawZero: - YZero = yTopOffset+plotHeight-YScale*(0-YLow) - canvas.drawLine(xLeftOffset, YZero, xLeftOffset+plotWidth, YZero) - else: - YZero = yTopOffset+plotHeight - #draw data - spaceWidth = barSpace - if spaceWidth < 1: - spaceWidth = 1 - barWidth = int((plotWidth - (NNN-1.0)*spaceWidth)/NNN) - - xc= xLeftOffset - scaleFont=pid.Font(ttf="verdana",size=11,bold=0) - for i in range(NNN): - yc = yTopOffset+plotHeight-(data[i]-YLow)*YScale - canvas.drawRect(xc,YZero,xc+barWidth-1, yc, edgeColor=barColor,fillColor=barColor) - if variance and variance[i]: - varlen = variance[i]*YScale - if yc-varlen < yTopOffset: - topYd = yTopOffset - else: - topYd = yc-varlen - canvas.drawLine(xc+barWidth/2-2,yc-varlen,xc+barWidth/2+2,yc-varlen,color=pid.red) - canvas.drawLine(xc+barWidth/2,yc+varlen,xc+barWidth/2,topYd,color=pid.red) - canvas.drawLine(xc+barWidth/2-2,yc+varlen,xc+barWidth/2+2,yc+varlen,color=pid.red) - strX = label[i] - canvas.drawString(strX,xc+barWidth/2.0+2,yTopOffset+plotHeight+2+canvas.stringWidth(strX,font=scaleFont),font=scaleFont,angle=90) - xc += barWidth + spaceWidth - - #draw drawing region - canvas.drawRect(xLeftOffset, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight) - - #draw Y scale - scaleFont=pid.Font(ttf="cour",size=16,bold=1) - y=YLow - for i in range(stepY+1): - yc=yTopOffset+plotHeight-(y-YLow)*YScale - canvas.drawLine(xLeftOffset,yc,xLeftOffset-5,yc, color=axesColor) - strY = cformat(d=y, rank=0) - canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)-6,yc+5,font=scaleFont) - y+= (YTop - YLow)/stepY - - #draw label - labelFont=pid.Font(ttf="verdana",size=17,bold=0) + labelFont=ImageFont.truetype(font=TAHOMA_FILE,size=17) if XLabel: - canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0,yTopOffset+plotHeight+65,font=labelFont,color=labelColor) + im_drawer.text( + text=XLabel, + xy=(xLeftOffset+( + plotWidth-im_drawer.textsize(XLabel,font=labelFont)[0])/2.0, + yTopOffset+plotHeight+yBottomOffset-10), + font=labelFont,fill=labelColor) if YLabel: - canvas.drawString(YLabel,xLeftOffset-50, yTopOffset+plotHeight-(plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0,font=labelFont,color=labelColor,angle=90) - - labelFont=pid.Font(ttf="verdana",size=18,bold=0) + draw_rotated_text(canvas, text=YLabel, + xy=(19, + yTopOffset+plotHeight-( + plotHeight-im_drawer.textsize( + YLabel,font=labelFont)[0])/2.0), + font=labelFont, fill=labelColor, angle=90) + + labelFont=ImageFont.truetype(font=VERDANA_FILE,size=16) if title: - canvas.drawString(title,xLeftOffset,yTopOffset-15,font=labelFont,color=labelColor) - - return - -#def plotXY(canvas, dataX, dataY, rank=0, dataLabel=[], plotColor = pid.black, axesColor=pid.black, labelColor=pid.black, lineSize="thin", lineColor=pid.grey, idFont="arial", idColor=pid.blue, idSize="14", symbolColor=pid.black, symbolType="circle", filled="yes", symbolSize="tiny", XLabel=None, YLabel=None, title=None, fitcurve=None, connectdot=1, displayR=None, loadingPlot = 0, offset= (80, 20, 40, 60), zoom = 1, specialCases=[], showLabel = 1, bufferSpace = 15): -# 'displayR : correlation scatter plot, loadings : loading plot' -# -# dataXRanked, dataYRanked = webqtlUtil.calRank(dataX, dataY, len(dataX)) -# -# #get ID font size -# idFontSize = int(idSize) -# -# #If filled is yes, set fill color -# if filled == "yes": -# fillColor = symbolColor -# else: -# fillColor = None -# -# if symbolSize == "large": -# sizeModifier = 7 -# fontModifier = 12 -# elif symbolSize == "medium": -# sizeModifier = 5 -# fontModifier = 8 -# elif symbolSize == "small": -# sizeModifier = 3 -# fontModifier = 3 -# else: -# sizeModifier = 1 -# fontModifier = -1 -# -# if rank == 0: # Pearson correlation -# bufferSpace = 0 -# dataXPrimary = dataX -# dataYPrimary = dataY -# dataXAlt = dataXRanked #Values used just for printing the other corr type to the graph image -# dataYAlt = dataYRanked #Values used just for printing the other corr type to the graph image -# else: # Spearman correlation: Switching Ranked and Unranked X and Y values -# dataXPrimary = dataXRanked -# dataYPrimary = dataYRanked -# dataXAlt = dataX #Values used just for printing the other corr type to the graph image -# dataYAlt = dataY #Values used just for printing the other corr type to the graph image -# -# xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset -# plotWidth = canvas.size[0] - xLeftOffset - xRightOffset -# plotHeight = canvas.size[1] - yTopOffset - yBottomOffset -# if plotHeight<=0 or plotWidth<=0: -# return -# if len(dataXPrimary) < 1 or len(dataXPrimary) != len(dataYPrimary) or (dataLabel and len(dataXPrimary) != len(dataLabel)): -# return -# -# max_X=max(dataXPrimary) -# min_X=min(dataXPrimary) -# max_Y=max(dataYPrimary) -# min_Y=min(dataYPrimary) -# -# #for some reason I forgot why I need to do this -# if loadingPlot: -# min_X = min(-0.1,min_X) -# max_X = max(0.1,max_X) -# min_Y = min(-0.1,min_Y) -# max_Y = max(0.1,max_Y) -# -# xLow, xTop, stepX=detScale(min_X,max_X) -# yLow, yTop, stepY=detScale(min_Y,max_Y) -# xScale = plotWidth/(xTop-xLow) -# yScale = plotHeight/(yTop-yLow) -# -# #draw drawing region -# canvas.drawRect(xLeftOffset-bufferSpace, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight+bufferSpace) -# canvas.drawRect(xLeftOffset-bufferSpace+1, yTopOffset, xLeftOffset+plotWidth, yTopOffset+plotHeight+bufferSpace-1) -# -# #calculate data points -# data = map(lambda X, Y: (X, Y), dataXPrimary, dataYPrimary) -# xCoord = map(lambda X, Y: ((X-xLow)*xScale + xLeftOffset, yTopOffset+plotHeight-(Y-yLow)*yScale), dataXPrimary, dataYPrimary) -# -# labelFont=pid.Font(ttf=idFont,size=idFontSize,bold=0) -# -# if loadingPlot: -# xZero = -xLow*xScale+xLeftOffset -# yZero = yTopOffset+plotHeight+yLow*yScale -# for point in xCoord: -# canvas.drawLine(xZero,yZero,point[0],point[1],color=pid.red) -# else: -# if connectdot: -# canvas.drawPolygon(xCoord,edgeColor=plotColor,closed=0) -# else: -# pass -# -# symbolFont = pid.Font(ttf="fnt_bs", size=12+fontModifier,bold=0) -# -# for i, item in enumerate(xCoord): -# if dataLabel and dataLabel[i] in specialCases: -# canvas.drawRect(item[0]-3, item[1]-3, item[0]+3, item[1]+3, edgeColor=pid.green) -# #canvas.drawCross(item[0],item[1],color=pid.blue,size=5) -# else: -# if symbolType == "vertRect": -# canvas.drawRect(x1=item[0]-sizeModifier+2,y1=item[1]-sizeModifier-2, x2=item[0]+sizeModifier-1,y2=item[1]+sizeModifier+2, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor) -# elif (symbolType == "circle" and filled != "yes"): -# canvas.drawString(":", item[0]-canvas.stringWidth(":",font=symbolFont)/2+1,item[1]+2,color=symbolColor, font=symbolFont) -# elif (symbolType == "circle" and filled == "yes"): -# canvas.drawString("5", item[0]-canvas.stringWidth("5",font=symbolFont)/2+1,item[1]+2,color=symbolColor, font=symbolFont) -# elif symbolType == "horiRect": -# canvas.drawRect(x1=item[0]-sizeModifier-1,y1=item[1]-sizeModifier+3, x2=item[0]+sizeModifier+3,y2=item[1]+sizeModifier-2, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor) -# elif (symbolType == "square"): -# canvas.drawRect(x1=item[0]-sizeModifier+1,y1=item[1]-sizeModifier-4, x2=item[0]+sizeModifier+2,y2=item[1]+sizeModifier-3, edgeColor=symbolColor, edgeWidth=1, fillColor=fillColor) -# elif (symbolType == "diamond" and filled != "yes"): -# canvas.drawString(",", item[0]-canvas.stringWidth(",",font=symbolFont)/2+2, item[1]+6, font=symbolFont, color=symbolColor) -# elif (symbolType == "diamond" and filled == "yes"): -# canvas.drawString("D", item[0]-canvas.stringWidth("D",font=symbolFont)/2+2, item[1]+6, font=symbolFont, color=symbolColor) -# elif symbolType == "4-star": -# canvas.drawString("l", item[0]-canvas.stringWidth("l",font=symbolFont)/2+1, item[1]+3, font=symbolFont, color=symbolColor) -# elif symbolType == "3-star": -# canvas.drawString("k", item[0]-canvas.stringWidth("k",font=symbolFont)/2+1, item[1]+3, font=symbolFont, color=symbolColor) -# else: -# canvas.drawCross(item[0],item[1]-2,color=symbolColor, size=sizeModifier+2) -# -# if showLabel and dataLabel: -# if (symbolType == "vertRect" or symbolType == "diamond"): -# labelGap = 15 -# elif (symbolType == "4-star" or symbolType == "3-star"): -# labelGap = 12 -# else: -# labelGap = 11 -# canvas.drawString(dataLabel[i], item[0]- canvas.stringWidth(dataLabel[i], -# font=labelFont)/2 + 1, item[1]+(labelGap+sizeModifier+(idFontSize-12)), font=labelFont, color=idColor) -# -# #draw scale -# scaleFont=pid.Font(ttf="cour",size=16,bold=1) -# -# -# x=xLow -# for i in range(stepX+1): -# xc=xLeftOffset+(x-xLow)*xScale -# if ((x == 0) & (rank == 1)): -# pass -# else: -# canvas.drawLine(xc,yTopOffset+plotHeight + bufferSpace,xc,yTopOffset+plotHeight+5 + bufferSpace, color=axesColor) -# strX = cformat(d=x, rank=rank) -# if ((strX == "0") & (rank == 1)): -# pass -# else: -# canvas.drawString(strX,xc-canvas.stringWidth(strX,font=scaleFont)/2,yTopOffset+plotHeight+20 + bufferSpace,font=scaleFont) -# x+= (xTop - xLow)/stepX -# -# y=yLow -# for i in range(stepY+1): -# yc=yTopOffset+plotHeight-(y-yLow)*yScale -# if ((y == 0) & (rank == 1)): -# pass -# else: -# canvas.drawLine(xLeftOffset - bufferSpace,yc,xLeftOffset-5 - bufferSpace,yc, color=axesColor) -# strY = cformat(d=y, rank=rank) -# if ((strY == "0") & (rank == 1)): -# pass -# else: -# canvas.drawString(strY,xLeftOffset-canvas.stringWidth(strY,font=scaleFont)- 10 - bufferSpace,yc+4,font=scaleFont) -# y+= (yTop - yLow)/stepY -# -# #draw label -# -# labelFont=pid.Font(ttf="verdana",size=canvas.size[0]/45,bold=0) -# titleFont=pid.Font(ttf="verdana",size=canvas.size[0]/40,bold=0) -# -# if (rank == 1 and not title): -# canvas.drawString("Spearman Rank Correlation", xLeftOffset-canvas.size[0]*.025+(plotWidth-canvas.stringWidth("Spearman Rank Correlation",font=titleFont))/2.0, -# 25,font=titleFont,color=labelColor) -# elif (rank == 0 and not title): -# canvas.drawString("Pearson Correlation", xLeftOffset-canvas.size[0]*.025+(plotWidth-canvas.stringWidth("Pearson Correlation",font=titleFont))/2.0, -# 25,font=titleFont,color=labelColor) -# -# if XLabel: -# canvas.drawString(XLabel,xLeftOffset+(plotWidth-canvas.stringWidth(XLabel,font=labelFont))/2.0, -# yTopOffset+plotHeight+yBottomOffset-25,font=labelFont,color=labelColor) -# -# if YLabel: -# canvas.drawString(YLabel, xLeftOffset-65, yTopOffset+plotHeight- (plotHeight-canvas.stringWidth(YLabel,font=labelFont))/2.0, -# font=labelFont,color=labelColor,angle=90) -# -# labelFont=pid.Font(ttf="verdana",size=20,bold=0) -# if title: -# canvas.drawString(title,xLeftOffset+(plotWidth-canvas.stringWidth(title,font=labelFont))/2.0, -# 20,font=labelFont,color=labelColor) -# -# if fitcurve: -# import sys -# sys.argv = [ "mod_python" ] -# #from numarray import linear_algebra as la -# #from numarray import ones, array, dot, swapaxes -# fitYY = array(dataYPrimary) -# fitXX = array([ones(len(dataXPrimary)),dataXPrimary]) -# AA = dot(fitXX,swapaxes(fitXX,0,1)) -# BB = dot(fitXX,fitYY) -# bb = la.linear_least_squares(AA,BB)[0] -# -# xc1 = xLeftOffset -# yc1 = yTopOffset+plotHeight-(bb[0]+bb[1]*xLow-yLow)*yScale -# if yc1 > yTopOffset+plotHeight: -# yc1 = yTopOffset+plotHeight -# xc1 = (yLow-bb[0])/bb[1] -# xc1=(xc1-xLow)*xScale+xLeftOffset -# elif yc1 < yTopOffset: -# yc1 = yTopOffset -# xc1 = (yTop-bb[0])/bb[1] -# xc1=(xc1-xLow)*xScale+xLeftOffset -# else: -# pass -# -# xc2 = xLeftOffset + plotWidth -# yc2 = yTopOffset+plotHeight-(bb[0]+bb[1]*xTop-yLow)*yScale -# if yc2 > yTopOffset+plotHeight: -# yc2 = yTopOffset+plotHeight -# xc2 = (yLow-bb[0])/bb[1] -# xc2=(xc2-xLow)*xScale+xLeftOffset -# elif yc2 < yTopOffset: -# yc2 = yTopOffset -# xc2 = (yTop-bb[0])/bb[1] -# xc2=(xc2-xLow)*xScale+xLeftOffset -# else: -# pass -# -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace,xc2,yc2,color=lineColor) -# if lineSize == "medium": -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace+1,xc2,yc2+1,color=lineColor) -# if lineSize == "thick": -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace+1,xc2,yc2+1,color=lineColor) -# canvas.drawLine(xc1 - bufferSpace,yc1 + bufferSpace-1,xc2,yc2-1,color=lineColor) -# -# -# if displayR: -# labelFont=pid.Font(ttf="trebuc",size=canvas.size[0]/60,bold=0) -# NNN = len(dataX) -# corr = webqtlUtil.calCorrelation(dataXPrimary,dataYPrimary,NNN)[0] -# -# if NNN < 3: -# corrPValue = 1.0 -# else: -# if abs(corr) >= 1.0: -# corrPValue = 0.0 -# else: -# ZValue = 0.5*log((1.0+corr)/(1.0-corr)) -# ZValue = ZValue*sqrt(NNN-3) -# corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue))) -# -# NStr = "N = %d" % NNN -# strLenN = canvas.stringWidth(NStr,font=labelFont) -# -# if rank == 1: -# if corrPValue < 0.0000000000000001: -# corrStr = "Rho = %1.3f P < 1.00 E-16" % (corr) -# else: -# corrStr = "Rho = %1.3f P = %3.2E" % (corr, corrPValue) -# else: -# if corrPValue < 0.0000000000000001: -# corrStr = "r = %1.3f P < 1.00 E-16" % (corr) -# else: -# corrStr = "r = %1.3f P = %3.2E" % (corr, corrPValue) -# strLen = canvas.stringWidth(corrStr,font=labelFont) -# -# canvas.drawString(NStr,xLeftOffset,yTopOffset-10,font=labelFont,color=labelColor) -# canvas.drawString(corrStr,xLeftOffset+plotWidth-strLen,yTopOffset-10,font=labelFont,color=labelColor) -# -# return xCoord - -def plotXYSVG(drawSpace, dataX, dataY, rank=0, dataLabel=[], plotColor = "black", axesColor="black", labelColor="black", symbolColor="red", XLabel=None, YLabel=None, title=None, fitcurve=None, connectdot=1, displayR=None, loadingPlot = 0, offset= (80, 20, 40, 60), zoom = 1, specialCases=[], showLabel = 1): - 'displayR : correlation scatter plot, loadings : loading plot' - - dataXRanked, dataYRanked = webqtlUtil.calRank(dataX, dataY, len(dataX)) - - # Switching Ranked and Unranked X and Y values if a Spearman Rank Correlation - if rank == 0: - dataXPrimary = dataX - dataYPrimary = dataY - dataXAlt = dataXRanked - dataYAlt = dataYRanked - - else: - dataXPrimary = dataXRanked - dataYPrimary = dataYRanked - dataXAlt = dataX - dataYAlt = dataY - - - - xLeftOffset, xRightOffset, yTopOffset, yBottomOffset = offset - plotWidth = drawSpace.attributes['width'] - xLeftOffset - xRightOffset - plotHeight = drawSpace.attributes['height'] - yTopOffset - yBottomOffset - if plotHeight<=0 or plotWidth<=0: - return - if len(dataXPrimary) < 1 or len(dataXPrimary) != len(dataYPrimary) or (dataLabel and len(dataXPrimary) != len(dataLabel)): - return - - max_X=max(dataXPrimary) - min_X=min(dataXPrimary) - max_Y=max(dataYPrimary) - min_Y=min(dataYPrimary) - - #for some reason I forgot why I need to do this - if loadingPlot: - min_X = min(-0.1,min_X) - max_X = max(0.1,max_X) - min_Y = min(-0.1,min_Y) - max_Y = max(0.1,max_Y) - - xLow, xTop, stepX=detScale(min_X,max_X) - yLow, yTop, stepY=detScale(min_Y,max_Y) - xScale = plotWidth/(xTop-xLow) - yScale = plotHeight/(yTop-yLow) - - #draw drawing region - r = svg.rect(xLeftOffset, yTopOffset, plotWidth, plotHeight, 'none', axesColor, 1) - drawSpace.addElement(r) - - #calculate data points - data = map(lambda X, Y: (X, Y), dataXPrimary, dataYPrimary) - xCoord = map(lambda X, Y: ((X-xLow)*xScale + xLeftOffset, yTopOffset+plotHeight-(Y-yLow)*yScale), dataXPrimary, dataYPrimary) - labelFontF = "verdana" - labelFontS = 11 - - if loadingPlot: - xZero = -xLow*xScale+xLeftOffset - yZero = yTopOffset+plotHeight+yLow*yScale - for point in xCoord: - drawSpace.addElement(svg.line(xZero,yZero,point[0],point[1], "red", 1)) - else: - if connectdot: - pass - #drawSpace.drawPolygon(xCoord,edgeColor=plotColor,closed=0) - else: - pass - - for i, item in enumerate(xCoord): - if dataLabel and dataLabel[i] in specialCases: - drawSpace.addElement(svg.rect(item[0]-3, item[1]-3, 6, 6, "none", "green", 0.5)) - #drawSpace.drawCross(item[0],item[1],color=pid.blue,size=5) - else: - drawSpace.addElement(svg.line(item[0],item[1]+5,item[0],item[1]-5,symbolColor,1)) - drawSpace.addElement(svg.line(item[0]+5,item[1],item[0]-5,item[1],symbolColor,1)) - if showLabel and dataLabel: - pass - drawSpace.addElement(svg.text(item[0], item[1]+14, dataLabel[i], labelFontS, - labelFontF, text_anchor="middle", style="stroke:blue;stroke-width:0.5;")) - #canvas.drawString(, item[0]- canvas.stringWidth(dataLabel[i], - # font=labelFont)/2, item[1]+14, font=labelFont, color=pid.blue) - - #draw scale - #scaleFont=pid.Font(ttf="cour",size=14,bold=1) - x=xLow - for i in range(stepX+1): - xc=xLeftOffset+(x-xLow)*xScale - drawSpace.addElement(svg.line(xc,yTopOffset+plotHeight,xc,yTopOffset+plotHeight+5, axesColor, 1)) - strX = cformat(d=x, rank=rank) - drawSpace.addElement(svg.text(xc,yTopOffset+plotHeight+20,strX,13, "courier", text_anchor="middle")) - x+= (xTop - xLow)/stepX - - y=yLow - for i in range(stepY+1): - yc=yTopOffset+plotHeight-(y-yLow)*yScale - drawSpace.addElement(svg.line(xLeftOffset,yc,xLeftOffset-5,yc, axesColor, 1)) - strY = cformat(d=y, rank=rank) - drawSpace.addElement(svg.text(xLeftOffset-10,yc+5,strY,13, "courier", text_anchor="end")) - y+= (yTop - yLow)/stepY - - #draw label - labelFontF = "verdana" - labelFontS = 17 - if XLabel: - drawSpace.addElement(svg.text(xLeftOffset+plotWidth/2.0, - yTopOffset+plotHeight+yBottomOffset-10,XLabel, - labelFontS, labelFontF, text_anchor="middle")) - - if YLabel: - drawSpace.addElement(svg.text(xLeftOffset-50, - yTopOffset+plotHeight/2,YLabel, - labelFontS, labelFontF, text_anchor="middle", style="writing-mode:tb-rl", transform="rotate(270 %d %d)" % (xLeftOffset-50, yTopOffset+plotHeight/2))) - #drawSpace.drawString(YLabel, xLeftOffset-50, yTopOffset+plotHeight- (plotHeight-drawSpace.stringWidth(YLabel,font=labelFont))/2.0, - # font=labelFont,color=labelColor,angle=90) - - - if fitcurve: - sys.argv = [ "mod_python" ] - #from numarray import linear_algebra as la - #from numarray import ones, array, dot, swapaxes - fitYY = array(dataYPrimary) - fitXX = array([ones(len(dataXPrimary)),dataXPrimary]) - AA = dot(fitXX,swapaxes(fitXX,0,1)) - BB = dot(fitXX,fitYY) - bb = la.linear_least_squares(AA,BB)[0] - - xc1 = xLeftOffset - yc1 = yTopOffset+plotHeight-(bb[0]+bb[1]*xLow-yLow)*yScale - if yc1 > yTopOffset+plotHeight: - yc1 = yTopOffset+plotHeight - xc1 = (yLow-bb[0])/bb[1] - xc1=(xc1-xLow)*xScale+xLeftOffset - elif yc1 < yTopOffset: - yc1 = yTopOffset - xc1 = (yTop-bb[0])/bb[1] - xc1=(xc1-xLow)*xScale+xLeftOffset - else: - pass - - xc2 = xLeftOffset + plotWidth - yc2 = yTopOffset+plotHeight-(bb[0]+bb[1]*xTop-yLow)*yScale - if yc2 > yTopOffset+plotHeight: - yc2 = yTopOffset+plotHeight - xc2 = (yLow-bb[0])/bb[1] - xc2=(xc2-xLow)*xScale+xLeftOffset - elif yc2 < yTopOffset: - yc2 = yTopOffset - xc2 = (yTop-bb[0])/bb[1] - xc2=(xc2-xLow)*xScale+xLeftOffset - else: - pass - - drawSpace.addElement(svg.line(xc1,yc1,xc2,yc2,"green", 1)) - - if displayR: - labelFontF = "trebuc" - labelFontS = 14 - NNN = len(dataX) - - corr = webqtlUtil.calCorrelation(dataXPrimary,dataYPrimary,NNN)[0] - - if NNN < 3: - corrPValue = 1.0 - else: - if abs(corr) >= 1.0: - corrPValue = 0.0 - else: - ZValue = 0.5*log((1.0+corr)/(1.0-corr)) - ZValue = ZValue*sqrt(NNN-3) - corrPValue = 2.0*(1.0 - reaper.normp(abs(ZValue))) - - NStr = "N of Cases=%d" % NNN - - if rank == 1: - corrStr = "Spearman's r=%1.3f P=%3.2E" % (corr, corrPValue) - else: - corrStr = "Pearson's r=%1.3f P=%3.2E" % (corr, corrPValue) - - drawSpace.addElement(svg.text(xLeftOffset,yTopOffset-10,NStr, - labelFontS, labelFontF, text_anchor="start")) - drawSpace.addElement(svg.text(xLeftOffset+plotWidth,yTopOffset-25,corrStr, - labelFontS, labelFontF, text_anchor="end")) - """ - """ - return - + im_drawer.text( + text=title, + xy=(xLeftOffset+(plotWidth-im_drawer.textsize( + title,font=labelFont)[0])/2.0, + 20), + font=labelFont,fill=labelColor) # This function determines the scale of the plot def detScaleOld(min,max): @@ -1114,7 +248,7 @@ def detScaleOld(min,max): high=c*ceil(max/c) return [low,high,round((high-low)/c)] -def detScale(min=0,max=0,bufferSpace=3): +def detScale(min=0,max=0): if min>=max: return None @@ -1151,57 +285,9 @@ def detScale(min=0,max=0,bufferSpace=3): return [low,high,n] - - -def colorSpectrumOld(n): - if n == 1: - return [pid.Color(1,0,0)] - elif n == 2: - return [pid.Color(1,0,0),pid.Color(0,0,1)] - elif n == 3: - return [pid.Color(1,0,0),pid.Color(0,1,0),pid.Color(0,0,1)] - else: - step = 2.0/(n-1) - red = 1.0 - green = 0.0 - blue = 0.0 - colors = [pid.Color(red,green,blue)] - i = 1 - greenpeak = 0 - while i < n: - if red >= step: - red -= step - green += step - if green >= 1.0: - greenpeak = 1 - blue += green -1.0 - green = 1.0 - else: - red = 0.0 - if greenpeak: - green -= step - blue += step - else: - green += step - if green >= 1.0: - greenpeak = 1 - blue += green -1.0 - green = 2.0 -green - elif green < 0.0: - green = 0.0 - else: - pass - colors.append(pid.Color(red,green,blue)) - i += 1 - return colors - - - - def bluefunc(x): return 1.0 / (1.0 + exp(-10*(x-0.6))) - def redfunc(x): return 1.0 / (1.0 + exp(10*(x-0.5))) @@ -1211,39 +297,21 @@ def greenfunc(x): def colorSpectrum(n=100): multiple = 10 if n == 1: - return [pid.Color(1,0,0)] - elif n == 2: - return [pid.Color(1,0,0),pid.Color(0,0,1)] - elif n == 3: - return [pid.Color(1,0,0),pid.Color(0,1,0),pid.Color(0,0,1)] - N = n*multiple - out = [None]*N; - for i in range(N): - x = float(i)/N - out[i] = pid.Color(redfunc(x), greenfunc(x), bluefunc(x)); - out2 = [out[0]] - step = N/float(n-1) - j = 0 - for i in range(n-2): - j += step - out2.append(out[int(j)]) - out2.append(out[-1]) - return out2 - - -def colorSpectrumSVG(n=100): - multiple = 10 - if n == 1: - return ["rgb(255,0,0)"] + return [ImageColor.getrgb("rgb(100%,0%,0%)")] elif n == 2: - return ["rgb(255,0,0)","rgb(0,0,255)"] + return [ImageColor.getrgb("100%,0%,0%)"), + ImageColor.getrgb("rgb(0%,0%,100%)")] elif n == 3: - return ["rgb(255,0,0)","rgb(0,255,0)","rgb(0,0,255)"] + return [ImageColor.getrgb("rgb(100%,0%,0%)"), + ImageColor.getrgb("rgb(0%,100%,0%)"), + ImageColor.getrgb("rgb(0%,0%,100%)")] N = n*multiple out = [None]*N; for i in range(N): x = float(i)/N - out[i] = "rgb(%d, %d, %d)" % (redfunc(x)*255, greenfunc(x)*255, bluefunc(x)*255); + out[i] = ImageColor.getrgb("rgb({}%,{}%,{}%".format( + *[int(i*100) for i in ( + redfunc(x), greenfunc(x), bluefunc(x))])) out2 = [out[0]] step = N/float(n-1) j = 0 @@ -1253,25 +321,6 @@ def colorSpectrumSVG(n=100): out2.append(out[-1]) return out2 - -def BWSpectrum(n=100): - multiple = 10 - if n == 1: - return [pid.Color(0,0,0)] - elif n == 2: - return [pid.Color(0,0,0),pid.Color(1,1,1)] - elif n == 3: - return [pid.Color(0,0,0),pid.Color(0.5,0.5,0.5),pid.Color(1,1,1)] - - step = 1.0/n - x = 0.0 - out = [] - for i in range(n): - out.append(pid.Color(x,x,x)); - x += step - return out - - def _test(): import doctest doctest.testmod() diff --git a/wqflask/utility/after.py b/wqflask/utility/after.py index a3bb85e9..b628a0a4 100644 --- a/wqflask/utility/after.py +++ b/wqflask/utility/after.py @@ -13,10 +13,4 @@ def after_this_request(f): if not hasattr(g, 'after_request_callbacks'): g.after_request_callbacks = [] g.after_request_callbacks.append(f) - return f - -@app.after_request -def call_after_request_callbacks(response): - for callback in getattr(g, 'after_request_callbacks', ()): - callback(response) - return response + return f
\ No newline at end of file diff --git a/wqflask/utility/authentication_tools.py b/wqflask/utility/authentication_tools.py new file mode 100644 index 00000000..3553b92b --- /dev/null +++ b/wqflask/utility/authentication_tools.py @@ -0,0 +1,130 @@ +from __future__ import absolute_import, print_function, division + +import json +import requests + +from base import data_set, webqtlConfig + +from utility import hmac +from utility.redis_tools import get_redis_conn, get_resource_info, get_resource_id, add_resource +Redis = get_redis_conn() + +from flask import Flask, g, redirect, url_for + +import logging +logger = logging.getLogger(__name__ ) + +def check_resource_availability(dataset, trait_id=None): + + #At least for now assume temporary entered traits are accessible + if type(dataset) == str: + return webqtlConfig.DEFAULT_PRIVILEGES + if dataset.type == "Temp": + return webqtlConfig.DEFAULT_PRIVILEGES + + resource_id = get_resource_id(dataset, trait_id) + + if resource_id: #ZS: This should never be false, but it's technically possible if a non-Temp dataset somehow had a type other than Publish/ProbeSet/Geno + resource_info = get_resource_info(resource_id) + if not resource_info: #ZS: If resource isn't already in redis, add it with default privileges + resource_info = add_new_resource(dataset, trait_id) + + #ZS: Check if super-user - we should probably come up with some way to integrate this into the proxy + if g.user_session.user_id in Redis.smembers("super_users"): + return webqtlConfig.SUPER_PRIVILEGES + + response = None + + the_url = "http://localhost:8080/available?resource={}&user={}".format(resource_id, g.user_session.user_id) + try: + response = json.loads(requests.get(the_url).content) + except: + response = resource_info['default_mask'] + + return response + +def add_new_resource(dataset, trait_id=None): + resource_ob = { + 'owner_id' : "none", # webqtlConfig.DEFAULT_OWNER_ID, + 'default_mask': webqtlConfig.DEFAULT_PRIVILEGES, + 'group_masks' : {} + } + + if dataset.type == "Publish": + resource_ob['name'] = get_group_code(dataset) + "_" + str(trait_id) + resource_ob['data'] = { + 'dataset': dataset.id, + 'trait' : trait_id + } + resource_ob['type'] = 'dataset-publish' + elif dataset.type == "Geno": + resource_ob['name'] = dataset.name + resource_ob['data'] = { + 'dataset': dataset.id + } + resource_ob['type'] = 'dataset-geno' + else: + resource_ob['name'] = dataset.name + resource_ob['data'] = { + 'dataset': dataset.id + } + resource_ob['type'] = 'dataset-probeset' + + resource_info = add_resource(resource_ob, update=False) + + return resource_info + +def get_group_code(dataset): + results = g.db.execute("SELECT InbredSetCode from InbredSet where Name='{}'".format(dataset.group.name)).fetchone() + if results[0]: + return results[0] + else: + return "" + +def check_admin(resource_id=None): + the_url = "http://localhost:8080/available?resource={}&user={}".format(resource_id, g.user_session.user_id) + try: + response = json.loads(requests.get(the_url).content)['admin'] + except: + resource_info = get_resource_info(resource_id) + response = resource_info['default_mask']['admin'] + + if 'edit-admins' in response: + return "edit-admins" + elif 'edit-access' in response: + return "edit-access" + else: + return "not-admin" + +def check_owner(dataset=None, trait_id=None, resource_id=None): + if resource_id: + resource_info = get_resource_info(resource_id) + if g.user_session.user_id == resource_info['owner_id']: + return resource_id + else: + resource_id = get_resource_id(dataset, trait_id) + if resource_id: + resource_info = get_resource_info(resource_id) + if g.user_session.user_id == resource_info['owner_id']: + return resource_id + + return False + +def check_owner_or_admin(dataset=None, trait_id=None, resource_id=None): + if not resource_id: + if dataset.type == "Temp": + return "not-admin" + else: + resource_id = get_resource_id(dataset, trait_id) + + if g.user_session.user_id in Redis.smembers("super_users"): + return "owner" + + resource_info = get_resource_info(resource_id) + if resource_info: + if g.user_session.user_id == resource_info['owner_id']: + return "owner" + else: + return check_admin(resource_id) + + return "not-admin" diff --git a/wqflask/utility/benchmark.py b/wqflask/utility/benchmark.py index 8c97370d..8f1c916b 100644 --- a/wqflask/utility/benchmark.py +++ b/wqflask/utility/benchmark.py @@ -11,7 +11,7 @@ logger = getLogger(__name__ ) class Bench(object): entries = collections.OrderedDict() - def __init__(self, name=None, write_output=True): + def __init__(self, name=None, write_output=LOG_BENCH): self.name = name self.write_output = write_output diff --git a/wqflask/utility/chunks.py b/wqflask/utility/chunks.py index b0e33c08..d91b9bf4 100644 --- a/wqflask/utility/chunks.py +++ b/wqflask/utility/chunks.py @@ -31,66 +31,3 @@ def divide_into_chunks(the_list, number_chunks): chunks.append(the_list[counter:counter+chunksize]) return chunks - -def _confirm_chunk(original, result): - all_chunked = [] - for chunk in result: - all_chunked.extend(chunk) - print("length of all chunked:", len(all_chunked)) - assert original == all_chunked, "You didn't chunk right" - - -def _chunk_test(divide_func): - import random - random.seed(7) - - number_exact = 0 - total_amount_off = 0 - - for test in range(1, 1001): - print("\n\ntest:", test) - number_chunks = random.randint(1, 20) - number_elements = random.randint(0, 100) - the_list = list(range(1, number_elements)) - result = divide_func(the_list, number_chunks) - - print("Dividing list of length {} into approximately {} chunks - got {} chunks".format( - len(the_list), number_chunks, len(result))) - print("result:", result) - - _confirm_chunk(the_list, result) - - amount_off = abs(number_chunks - len(result)) - if amount_off == 0: - number_exact += 1 - else: - total_amount_off += amount_off - - - print("\n{} exact out of {} [Total amount off: {}]".format(number_exact, - test, - total_amount_off)) - assert number_exact == 558 - assert total_amount_off == 1580 - return number_exact, total_amount_off - - -def _main(): - info = dict() - #funcs = (("sam", sam_divide_into_chunks), ("zach", zach_divide_into_chunks)) - funcs = (("only one", divide_into_chunks),) - for name, func in funcs: - start = time.time() - number_exact, total_amount_off = _chunk_test(func) - took = time.time() - start - info[name] = dict(number_exact=number_exact, - total_amount_off=total_amount_off, - took=took) - - print("info is:", info) - -if __name__ == '__main__': - _main() - print("\nConfirming doctests...") - import doctest - doctest.testmod() diff --git a/wqflask/utility/corestats.py b/wqflask/utility/corestats.py new file mode 100644 index 00000000..67ca3ad3 --- /dev/null +++ b/wqflask/utility/corestats.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +# corestats.py (COREy STATS) +# Copyright (c) 2006-2007, Corey Goldberg (corey@goldb.org) +# +# statistical calculation class +# for processing numeric sequences +# +# license: GNU LGPL +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. + +import sys + +#ZS: Should switch to using some third party library for this; maybe scipy has an equivalent +class Stats: + + def __init__(self, sequence): + # sequence of numbers we will process + # convert all items to floats for numerical processing + self.sequence = [float(item) for item in sequence] + + def sum(self): + if len(self.sequence) < 1: + return None + else: + return sum(self.sequence) + + def count(self): + return len(self.sequence) + + def min(self): + if len(self.sequence) < 1: + return None + else: + return min(self.sequence) + + def max(self): + if len(self.sequence) < 1: + return None + else: + return max(self.sequence) + + def avg(self): + if len(self.sequence) < 1: + return None + else: + return sum(self.sequence) / len(self.sequence) + + def stdev(self): + if len(self.sequence) < 1: + return None + else: + avg = self.avg() + sdsq = sum([(i - avg) ** 2 for i in self.sequence]) + stdev = (sdsq / (len(self.sequence) - 1)) ** .5 + return stdev + + def percentile(self, percentile): + if len(self.sequence) < 1: + value = None + elif (percentile >= 100): + sys.stderr.write('ERROR: percentile must be < 100. you supplied: %s\n'% percentile) + value = None + else: + element_idx = int(len(self.sequence) * (percentile / 100.0)) + self.sequence.sort() + value = self.sequence[element_idx] + return value + +# Sample script using this class: +# ------------------------------------------- +# #!/usr/bin/env python +# import corestats +# +# sequence = [1, 2.5, 7, 13.4, 8.0] +# stats = corestats.Stats(sequence) +# print stats.avg() +# print stats.percentile(90) +# -------------------------------------------
\ No newline at end of file diff --git a/wqflask/utility/corr_result_helpers.py b/wqflask/utility/corr_result_helpers.py index ef644d85..ea3ababf 100644 --- a/wqflask/utility/corr_result_helpers.py +++ b/wqflask/utility/corr_result_helpers.py @@ -14,15 +14,11 @@ def normalize_values(a_values, b_values): min_length = min(len(a_values), len(b_values)) a_new = [] b_new = [] - for counter in range(min_length): - if a_values[counter] and b_values[counter]: - a_new.append(a_values[counter]) - b_new.append(b_values[counter]) - - num_overlap = len(a_new) - assert num_overlap == len(b_new), "Lengths should be the same" - - return a_new, b_new, num_overlap + for a, b in zip(a_values, b_values): + if not (a == None or b == None): + a_new.append(a) + b_new.append(b) + return a_new, b_new, len(a_new) def common_keys(a_samples, b_samples): @@ -37,20 +33,10 @@ def common_keys(a_samples, b_samples): def normalize_values_with_samples(a_samples, b_samples): common_samples = common_keys(a_samples, b_samples) - a_new = {} b_new = {} for sample in common_samples: a_new[sample] = a_samples[sample] b_new[sample] = b_samples[sample] - num_overlap = len(a_new) - assert num_overlap == len(b_new), "Lengths should be the same" - - return a_new, b_new, num_overlap - - - -if __name__ == '__main__': - import doctest - doctest.testmod()
\ No newline at end of file + return a_new, b_new, len(a_new) diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py new file mode 100644 index 00000000..15cdd0bc --- /dev/null +++ b/wqflask/utility/elasticsearch_tools.py @@ -0,0 +1,112 @@ +# Elasticsearch support +# +# Some helpful commands to view the database: +# +# You can test the server being up with +# +# curl -H 'Content-Type: application/json' http://localhost:9200 +# +# List all indices +# +# curl -H 'Content-Type: application/json' 'localhost:9200/_cat/indices?v' +# +# To see the users index 'table' +# +# curl http://localhost:9200/users +# +# To list all user ids +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match_all" : {} +# }, +# "stored_fields": [] +# }' +# +# To view a record +# +# curl -H 'Content-Type: application/json' http://localhost:9200/users/local/_search?pretty=true -d ' +# { +# "query" : { +# "match" : { "email_address": "pjotr2017@thebird.nl"} +# } +# }' +# +# +# To delete the users index and data (dangerous!) +# +# curl -XDELETE -H 'Content-Type: application/json' 'localhost:9200/users' + + +from elasticsearch import Elasticsearch, TransportError +import logging + +from utility.logger import getLogger +logger = getLogger(__name__) + +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + +def test_elasticsearch_connection(): + es = Elasticsearch(['http://'+ELASTICSEARCH_HOST+":"+str(ELASTICSEARCH_PORT)+'/'], verify_certs=True) + if not es.ping(): + logger.warning("Elasticsearch is DOWN") + +def get_elasticsearch_connection(for_user=True): + """Return a connection to ES. Returns None on failure""" + logger.info("get_elasticsearch_connection") + es = None + try: + assert(ELASTICSEARCH_HOST) + assert(ELASTICSEARCH_PORT) + logger.info("ES HOST",ELASTICSEARCH_HOST) + + es = Elasticsearch([{ + "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT + }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None + + if for_user: + setup_users_index(es) + + es_logger = logging.getLogger("elasticsearch") + es_logger.setLevel(logging.INFO) + es_logger.addHandler(logging.NullHandler()) + except Exception as e: + logger.error("Failed to get elasticsearch connection", e) + es = None + + return es + +def setup_users_index(es_connection): + if es_connection: + index_settings = { + "properties": { + "email_address": { + "type": "keyword"}}} + + es_connection.indices.create(index='users', ignore=400) + es_connection.indices.put_mapping(body=index_settings, index="users", doc_type="local") + +def get_user_by_unique_column(es, column_name, column_value, index="users", doc_type="local"): + return get_item_by_unique_column(es, column_name, column_value, index=index, doc_type=doc_type) + +def save_user(es, user, user_id): + es_save_data(es, "users", "local", user, user_id) + +def get_item_by_unique_column(es, column_name, column_value, index, doc_type): + item_details = None + try: + response = es.search( + index = index, doc_type = doc_type, body = { + "query": { "match": { column_name: column_value } } + }) + if len(response["hits"]["hits"]) > 0: + item_details = response["hits"]["hits"][0]["_source"] + except TransportError as te: + pass + return item_details + +def es_save_data(es, index, doc_type, data_item, data_id,): + from time import sleep + es.create(index, doc_type, body=data_item, id=data_id) + sleep(1) # Delay 1 second to allow indexing diff --git a/wqflask/utility/formatting.py b/wqflask/utility/formatting.py index e53dda22..1da3e9b7 100644 --- a/wqflask/utility/formatting.py +++ b/wqflask/utility/formatting.py @@ -28,21 +28,20 @@ def numify(number, singular=None, plural=None): '12,334 hippopotami' """ - num_repr = {1 : "one", - 2 : "two", - 3 : "three", - 4 : "four", - 5 : "five", - 6 : "six", - 7 : "seven", - 8 : "eight", - 9 : "nine", - 10 : "ten", - 11 : "eleven", - 12 : "twelve"} - - #Below line commented out cause doesn't work in Python 2.4 - #assert all((singular, plural)) or not any((singular, plural)), "Need to pass two words or none" + num_repr = {0: "zero", + 1: "one", + 2: "two", + 3: "three", + 4: "four", + 5: "five", + 6: "six", + 7: "seven", + 8: "eight", + 9: "nine", + 10: "ten", + 11: "eleven", + 12: "twelve"} + if number == 1: word = singular else: diff --git a/wqflask/utility/gen_geno_ob.py b/wqflask/utility/gen_geno_ob.py new file mode 100644 index 00000000..23b0b650 --- /dev/null +++ b/wqflask/utility/gen_geno_ob.py @@ -0,0 +1,181 @@ +from __future__ import absolute_import, division, print_function + +import utility.logger +logger = utility.logger.getLogger(__name__ ) + +class genotype(object): + """ + Replacement for reaper.Dataset so we can remove qtlreaper use while still generating mapping output figure + """ + + def __init__(self, filename): + self.group = None + self.type = "riset" + self.prgy = [] + self.nprgy = 0 + self.mat = -1 + self.pat = 1 + self.het = 0 + self.unk = "U" + self.filler = False + self.mb_exists = False + + #ZS: This is because I'm not sure if some files switch the column that contains Mb/cM positions; might be unnecessary + self.cm_column = 2 + self.mb_column = 3 + + self.chromosomes = [] + + self.read_file(filename) + + def __iter__(self): + return iter(self.chromosomes) + + def __getitem__(self, index): + return self.chromosomes[index] + + def __len__(self): + return len(self.chromosomes) + + def read_rdata_output(self, qtl_results): + #ZS: This is necessary because R/qtl requires centimorgan marker positions, which it normally gets from the .geno file, but that doesn't exist for HET3-ITP (which only has RData), so it needs to read in the marker cM positions from the results + self.chromosomes = [] #ZS: Overwriting since the .geno file's contents are just placeholders + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for marker in qtl_results: + locus = Locus(self) + if (str(marker['chr']) != this_chr) and this_chr != "X": #ZS: This is really awkward but works as a temporary fix + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = str(marker['chr']) + if this_chr == "20": + this_chr = "X" + chr_ob = Chr(this_chr, self) + if 'chr' in marker: + locus.chr = str(marker['chr']) + if 'name' in marker: + locus.name = marker['name'] + if 'Mb' in marker: + locus.Mb = marker['Mb'] + if 'cM' in marker: + locus.cM = marker['cM'] + chr_ob.loci.append(locus) + + self.chromosomes.append(chr_ob) + + return self + + def read_file(self, filename): + with open(filename, 'r') as geno_file: + lines = geno_file.readlines() + + this_chr = "" #ZS: This is so it can track when the chromosome changes as it iterates through markers + chr_ob = None + for line in lines: + if line[0] == "#": + continue + elif line[0] == "@": + label = line.split(":")[0][1:] + if label == "name": + self.group = line.split(":")[1].strip() + elif label == "filler": + if line.split(":")[1].strip() == "yes": + self.filler = True + elif label == "type": + self.type = line.split(":")[1].strip() + elif label == "mat": + self.mat = line.split(":")[1].strip() + elif label == "pat": + self.pat = line.split(":")[1].strip() + elif label == "het": + self.het = line.split(":")[1].strip() + elif label == "unk": + self.unk = line.split(":")[1].strip() + else: + continue + elif line[:3] == "Chr": + header_row = line.split("\t") + if header_row[2] == "Mb": + self.mb_exists = True + self.mb_column = 2 + self.cm_column = 3 + elif header_row[3] == "Mb": + self.mb_exists = True + self.mb_column = 3 + elif header_row[2] == "cM": + self.cm_column = 2 + + if self.mb_exists: + self.prgy = header_row[4:] + else: + self.prgy = header_row[3:] + self.nprgy = len(self.prgy) + else: + if line.split("\t")[0] != this_chr: + if this_chr != "": + self.chromosomes.append(chr_ob) + this_chr = line.split("\t")[0] + chr_ob = Chr(line.split("\t")[0], self) + chr_ob.add_marker(line.split("\t")) + + self.chromosomes.append(chr_ob) + +class Chr(object): + def __init__(self, name, geno_ob): + self.name = name + self.loci = [] + self.mb_exists = geno_ob.mb_exists + self.cm_column = geno_ob.cm_column + self.mb_column = geno_ob.mb_column + self.geno_ob = geno_ob + + def __iter__(self): + return iter(self.loci) + + def __getitem__(self, index): + return self.loci[index] + + def __len__(self): + return len(self.loci) + + def add_marker(self, marker_row): + self.loci.append(Locus(self.geno_ob, marker_row)) + +class Locus(object): + def __init__(self, geno_ob, marker_row = None): + self.chr = None + self.name = None + self.cM = None + self.Mb = None + self.genotype = [] + if marker_row: + self.chr = marker_row[0] + self.name = marker_row[1] + try: + self.cM = float(marker_row[geno_ob.cm_column]) + except: + self.cM = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else 0 + try: + self.Mb = float(marker_row[geno_ob.mb_column]) if geno_ob.mb_exists else None + except: + self.Mb = self.cM + + geno_table = { + geno_ob.mat: -1, + geno_ob.pat: 1, + geno_ob.het: 0, + geno_ob.unk: "U" + } + + self.genotype = [] + if geno_ob.mb_exists: + start_pos = 4 + else: + start_pos = 3 + + for allele in marker_row[start_pos:]: + if allele in geno_table.keys(): + self.genotype.append(geno_table[allele]) + else: #ZS: Some genotype appears that isn't specified in the metadata, make it unknown + self.genotype.append("U")
\ No newline at end of file diff --git a/wqflask/utility/genofile_parser.py b/wqflask/utility/genofile_parser.py index 7149e560..af306731 100644 --- a/wqflask/utility/genofile_parser.py +++ b/wqflask/utility/genofile_parser.py @@ -72,7 +72,7 @@ class ConvertGenoFile(object): yield row def process_csv(self): - for row_count, row in enumerate(self.process_rows()): + for row in self.process_rows(): row_items = row.split("\t") this_marker = Marker() diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py index cf16879f..9ce809b6 100644 --- a/wqflask/utility/helper_functions.py +++ b/wqflask/utility/helper_functions.py @@ -1,24 +1,29 @@ from __future__ import absolute_import, print_function, division -from base.trait import GeneralTrait from base import data_set +from base.trait import create_trait from base.species import TheSpecies -from wqflask import user_manager +from utility import hmac from flask import Flask, g import logging logger = logging.getLogger(__name__ ) - def get_species_dataset_trait(self, start_vars): #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype" - self.dataset = data_set.create_dataset(start_vars['dataset']) + if "temp_trait" in start_vars.keys(): + if start_vars['temp_trait'] == "True": + self.dataset = data_set.create_dataset(dataset_name = "Temp", dataset_type = "Temp", group_name = start_vars['group']) + else: + self.dataset = data_set.create_dataset(start_vars['dataset']) + else: + self.dataset = data_set.create_dataset(start_vars['dataset']) logger.debug("After creating dataset") self.species = TheSpecies(dataset=self.dataset) logger.debug("After creating species") - self.this_trait = GeneralTrait(dataset=self.dataset, + self.this_trait = create_trait(dataset=self.dataset, name=start_vars['trait_id'], cellid=None, get_qtl_info=True) @@ -28,22 +33,25 @@ def get_species_dataset_trait(self, start_vars): #self.dataset.group.read_genotype_file() #self.genotype = self.dataset.group.genotype - def get_trait_db_obs(self, trait_db_list): if isinstance(trait_db_list, basestring): trait_db_list = trait_db_list.split(",") self.trait_list = [] for trait in trait_db_list: - data, _separator, hmac = trait.rpartition(':') + data, _separator, hmac_string = trait.rpartition(':') data = data.strip() - assert hmac==user_manager.actual_hmac_creation(data), "Data tampering?" + assert hmac_string==hmac.hmac_creation(data), "Data tampering?" trait_name, dataset_name = data.split(":") - dataset_ob = data_set.create_dataset(dataset_name) - trait_ob = GeneralTrait(dataset=dataset_ob, + if dataset_name == "Temp": + dataset_ob = data_set.create_dataset(dataset_name=dataset_name, dataset_type="Temp", group_name=trait_name.split("_")[2]) + else: + dataset_ob = data_set.create_dataset(dataset_name) + trait_ob = create_trait(dataset=dataset_ob, name=trait_name, cellid=None) - self.trait_list.append((trait_ob, dataset_ob)) + if trait_ob: + self.trait_list.append((trait_ob, dataset_ob)) def get_species_groups(): diff --git a/wqflask/utility/hmac.py b/wqflask/utility/hmac.py new file mode 100644 index 00000000..b08be97e --- /dev/null +++ b/wqflask/utility/hmac.py @@ -0,0 +1,40 @@ +from __future__ import print_function, division, absolute_import + +import hmac +import hashlib + +from flask import url_for + +from wqflask import app + +def hmac_creation(stringy): + """Helper function to create the actual hmac""" + + secret = app.config['SECRET_HMAC_CODE'] + + hmaced = hmac.new(secret, stringy, hashlib.sha1) + hm = hmaced.hexdigest() + # ZS: Leaving the below comment here to ask Pjotr about + # "Conventional wisdom is that you don't lose much in terms of security if you throw away up to half of the output." + # http://www.w3.org/QA/2009/07/hmac_truncation_in_xml_signatu.html + hm = hm[:20] + return hm + +def data_hmac(stringy): + """Takes arbitray data string and appends :hmac so we know data hasn't been tampered with""" + return stringy + ":" + hmac_creation(stringy) + +def url_for_hmac(endpoint, **values): + """Like url_for but adds an hmac at the end to insure the url hasn't been tampered with""" + + url = url_for(endpoint, **values) + + hm = hmac_creation(url) + if '?' in url: + combiner = "&" + else: + combiner = "?" + return url + combiner + "hm=" + hm + +app.jinja_env.globals.update(url_for_hmac=url_for_hmac, + data_hmac=data_hmac)
\ No newline at end of file diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py index bacb0aa4..510b1041 100644 --- a/wqflask/utility/logger.py +++ b/wqflask/utility/logger.py @@ -33,7 +33,7 @@ from pprint import pformat as pf from inspect import stack import datetime -from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL, LOG_FORMAT +from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL class GNLogger: """A logger class with some additional functionality, such as @@ -72,7 +72,7 @@ LOG_LEVEL_DEBUG (NYI). def warning(self,*args): """Call logging.warning for multiple args""" self.collect(self.logger.warning,*args) - self.logger.warning(self.collect(*args)) + # self.logger.warning(self.collect(*args)) def error(self,*args): """Call logging.error for multiple args""" diff --git a/wqflask/utility/pillow_utils.py b/wqflask/utility/pillow_utils.py new file mode 100644 index 00000000..0c2ce7af --- /dev/null +++ b/wqflask/utility/pillow_utils.py @@ -0,0 +1,27 @@ +from PIL import Image, ImageColor, ImageDraw, ImageFont + +from utility.tools import TEMPDIR + +import utility.logger +logger = utility.logger.getLogger(__name__ ) + +BLACK = ImageColor.getrgb("black") +WHITE = ImageColor.getrgb("white") + +# def draw_rotated_text(canvas: Image, text: str, font: ImageFont, xy: tuple, fill: ImageColor=BLACK, angle: int=-90): +def draw_rotated_text(canvas, text, font, xy, fill=BLACK, angle=-90): + # type: (Image, str, ImageFont, tuple, ImageColor, int) + """Utility function draw rotated text""" + tmp_img = Image.new("RGBA", font.getsize(text), color=(0,0,0,0)) + draw_text = ImageDraw.Draw(tmp_img) + draw_text.text(text=text, xy=(0,0), font=font, fill=fill) + tmp_img2 = tmp_img.rotate(angle, expand=1) + tmp_img2.save("/{0}/{1}.png".format(TEMPDIR, text), format="png") + canvas.paste(im=tmp_img2, box=tuple([int(i) for i in xy])) + +# def draw_open_polygon(canvas: Image, xy: tuple, fill: ImageColor=WHITE, outline: ImageColor=BLACK): +def draw_open_polygon(canvas, xy, fill=None, outline=BLACK, width=0): + # type: (Image, tuple, ImageColor, ImageColor) + draw_ctx = ImageDraw.Draw(canvas) + draw_ctx.polygon(xy, fill=fill) + draw_ctx.line(xy, fill=outline, width=width) diff --git a/wqflask/utility/redis_tools.py b/wqflask/utility/redis_tools.py new file mode 100644 index 00000000..81ba04ea --- /dev/null +++ b/wqflask/utility/redis_tools.py @@ -0,0 +1,309 @@ +from __future__ import print_function, division, absolute_import + +import uuid +import simplejson as json +import datetime + +import redis # used for collections + +import logging + +from flask import (render_template, flash) + +from utility import hmac + +from utility.logger import getLogger +logger = getLogger(__name__) + +def get_redis_conn(): + Redis = redis.StrictRedis(port=6379) + return Redis + +Redis = get_redis_conn() + +def is_redis_available(): + try: + Redis.ping() + except: + return False + return True + +def get_user_id(column_name, column_value): + user_list = Redis.hgetall("users") + key_list = [] + for key in user_list: + user_ob = json.loads(user_list[key]) + if column_name in user_ob and user_ob[column_name] == column_value: + return key + + return None + +def get_user_by_unique_column(column_name, column_value): + item_details = None + + user_list = Redis.hgetall("users") + if column_name != "user_id": + for key in user_list: + user_ob = json.loads(user_list[key]) + if column_name in user_ob and user_ob[column_name] == column_value: + item_details = user_ob + else: + item_details = json.loads(user_list[column_value]) + + return item_details + +def get_users_like_unique_column(column_name, column_value): + """ + Like previous function, but this only checks if the input is a subset of a field and can return multiple results + """ + matched_users = [] + + if column_value != "": + user_list = Redis.hgetall("users") + if column_name != "user_id": + for key in user_list: + user_ob = json.loads(user_list[key]) + if "user_id" not in user_ob: + set_user_attribute(key, "user_id", key) + user_ob["user_id"] = key + if column_name in user_ob: + if column_value in user_ob[column_name]: + matched_users.append(user_ob) + else: + matched_users.append(json.loads(user_list[column_value])) + + return matched_users + +# def search_users_by_unique_column(column_name, column_value): + +def set_user_attribute(user_id, column_name, column_value): + user_info = json.loads(Redis.hget("users", user_id)) + user_info[column_name] = column_value + + Redis.hset("users", user_id, json.dumps(user_info)) + +def get_user_collections(user_id): + collections = None + collections = Redis.hget("collections", user_id) + + if collections: + return json.loads(collections) + else: + return [] + +def save_user(user, user_id): + Redis.hset("users", user_id, json.dumps(user)) + +def save_collections(user_id, collections_ob): + Redis.hset("collections", user_id, collections_ob) + +def save_verification_code(user_email, code): + Redis.hset("verification_codes", code, user_email) + +def check_verification_code(code): + email_address = None + user_details = None + email_address = Redis.hget("verification_codes", code) + + if email_address: + user_details = get_user_by_unique_column('email_address', email_address) + if user_details: + return user_details + else: + return None + else: + return None + +def get_user_groups(user_id): + #ZS: Get the groups where a user is an admin or a member and return lists corresponding to those two sets of groups + admin_group_ids = [] #ZS: Group IDs where user is an admin + user_group_ids = [] #ZS: Group IDs where user is a regular user + groups_list = Redis.hgetall("groups") + for key in groups_list: + try: + group_ob = json.loads(groups_list[key]) + group_admins = set(group_ob['admins']) + group_members = set(group_ob['members']) + if user_id in group_admins: + admin_group_ids.append(group_ob['id']) + elif user_id in group_members: + user_group_ids.append(group_ob['id']) + else: + continue + except: + continue + + admin_groups = [] + user_groups = [] + for the_id in admin_group_ids: + admin_groups.append(get_group_info(the_id)) + for the_id in user_group_ids: + user_groups.append(get_group_info(the_id)) + + return admin_groups, user_groups + +def get_group_info(group_id): + group_json = Redis.hget("groups", group_id) + group_info = None + if group_json: + group_info = json.loads(group_json) + + return group_info + +def get_group_by_unique_column(column_name, column_value): + """ Get group by column; not sure if there's a faster way to do this """ + + matched_groups = [] + + all_group_list = Redis.hgetall("groups") + for key in all_group_list: + group_info = json.loads(all_group_list[key]) + if column_name == "admins" or column_name == "members": #ZS: Since these fields are lists, search in the list + if column_value in group_info[column_name]: + matched_groups.append(group_info) + else: + if group_info[column_name] == column_value: + matched_groups.append(group_info) + + return matched_groups + +def get_groups_like_unique_column(column_name, column_value): + """ + Like previous function, but this only checks if the input is a subset of a field and can return multiple results + """ + matched_groups = [] + + if column_value != "": + group_list = Redis.hgetall("groups") + if column_name != "group_id": + for key in group_list: + group_info = json.loads(group_list[key]) + if column_name == "admins" or column_name == "members": #ZS: Since these fields are lists, search in the list + if column_value in group_info[column_name]: + matched_groups.append(group_info) + else: + if column_name in group_info: + if column_value in group_info[column_name]: + matched_groups.append(group_info) + else: + matched_groups.append(json.loads(group_list[column_value])) + + return matched_groups + +def create_group(admin_user_ids, member_user_ids = [], group_name = "Default Group Name"): + group_id = str(uuid.uuid4()) + new_group = { + "id" : group_id, + "admins": admin_user_ids, + "members" : member_user_ids, + "name" : group_name, + "created_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p'), + "changed_timestamp": datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + } + + Redis.hset("groups", group_id, json.dumps(new_group)) + + return new_group + +def delete_group(user_id, group_id): + #ZS: If user is an admin of a group, remove it from the groups hash + group_info = get_group_info(group_id) + if user_id in group_info["admins"]: + Redis.hdel("groups", group_id) + return get_user_groups(user_id) + else: + None + +def add_users_to_group(user_id, group_id, user_emails = [], admins = False): #ZS "admins" is just to indicate whether the users should be added to the groups admins or regular users set + group_info = get_group_info(group_id) + if user_id in group_info["admins"]: #ZS: Just to make sure that the user is an admin for the group, even though they shouldn't be able to reach this point unless they are + if admins: + group_users = set(group_info["admins"]) + else: + group_users = set(group_info["members"]) + + for email in user_emails: + user_id = get_user_id("email_address", email) + group_users.add(user_id) + + if admins: + group_info["admins"] = list(group_users) + else: + group_info["members"] = list(group_users) + + group_info["changed_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + Redis.hset("groups", group_id, json.dumps(group_info)) + return group_info + else: + return None + +def remove_users_from_group(user_id, users_to_remove_ids, group_id, user_type = "members"): #ZS: User type is because I assume admins can remove other admins + group_info = get_group_info(group_id) + + if user_id in group_info["admins"]: + users_to_remove_set = set(users_to_remove_ids) + if user_type == "admins" and user_id in users_to_remove_set: #ZS: Make sure an admin can't remove themselves from a group, since I imagine we don't want groups to be able to become admin-less + users_to_remove_set.remove(user_id) + group_users = set(group_info[user_type]) + group_users -= users_to_remove_set + group_info[user_type] = list(group_users) + group_info["changed_timestamp"] = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p') + Redis.hset("groups", group_id, json.dumps(group_info)) + +def change_group_name(user_id, group_id, new_name): + group_info = get_group_info(group_id) + if user_id in group_info["admins"]: + group_info["name"] = new_name + Redis.hset("groups", group_id, json.dumps(group_info)) + return group_info + else: + return None + +def get_resources(): + resource_list = Redis.hgetall("resources") + return resource_list + +def get_resource_id(dataset, trait_id=None): + resource_id = False + if dataset.type == "Publish": + if trait_id: + resource_id = hmac.hmac_creation("{}:{}:{}".format('dataset-publish', dataset.id, trait_id)) + elif dataset.type == "ProbeSet": + resource_id = hmac.hmac_creation("{}:{}".format('dataset-probeset', dataset.id)) + elif dataset.type == "Geno": + resource_id = hmac.hmac_creation("{}:{}".format('dataset-geno', dataset.id)) + + return resource_id + +def get_resource_info(resource_id): + resource_info = Redis.hget("resources", resource_id) + if resource_info: + return json.loads(resource_info) + else: + return None + +def add_resource(resource_info, update=True): + if 'trait' in resource_info['data']: + resource_id = hmac.hmac_creation('{}:{}:{}'.format(str(resource_info['type']), str(resource_info['data']['dataset']), str(resource_info['data']['trait']))) + else: + resource_id = hmac.hmac_creation('{}:{}'.format(str(resource_info['type']), str(resource_info['data']['dataset']))) + + if update or not Redis.hexists("resources", resource_id): + Redis.hset("resources", resource_id, json.dumps(resource_info)) + + return resource_info + +def add_access_mask(resource_id, group_id, access_mask): + the_resource = get_resource_info(resource_id) + the_resource['group_masks'][group_id] = access_mask + + Redis.hset("resources", resource_id, json.dumps(the_resource)) + + return the_resource + +def change_resource_owner(resource_id, new_owner_id): + the_resource= get_resource_info(resource_id) + the_resource['owner_id'] = new_owner_id + + Redis.delete("resource") + Redis.hset("resources", resource_id, json.dumps(the_resource))
\ No newline at end of file diff --git a/wqflask/utility/startup_config.py b/wqflask/utility/startup_config.py new file mode 100644 index 00000000..817284dd --- /dev/null +++ b/wqflask/utility/startup_config.py @@ -0,0 +1,39 @@ + +from wqflask import app +from utility.tools import WEBSERVER_MODE, show_settings, get_setting_int, get_setting, get_setting_bool + +import utility.logger +logger = utility.logger.getLogger(__name__ ) + +BLUE = '\033[94m' +GREEN = '\033[92m' +BOLD = '\033[1m' +ENDC = '\033[0m' + +def app_config(): + app.config['SESSION_TYPE'] = 'filesystem' + if not app.config.get('SECRET_KEY'): + import os + app.config['SECRET_KEY'] = str(os.urandom(24)) + + mode = WEBSERVER_MODE + if mode == "DEV" or mode == "DEBUG": + app.config['TEMPLATES_AUTO_RELOAD'] = True + # if mode == "DEBUG": + # app.config['EXPLAIN_TEMPLATE_LOADING'] = True <--- use overriding app param instead + print("==========================================") + show_settings() + + port = get_setting_int("SERVER_PORT") + + if get_setting_bool("USE_GN_SERVER"): + print("GN2 API server URL is ["+BLUE+get_setting("GN_SERVER_URL")+ENDC+"]") + import requests + page = requests.get(get_setting("GN_SERVER_URL")) + if page.status_code != 200: + raise Exception("API server not found!") + + # import utility.elasticsearch_tools as es + # es.test_elasticsearch_connection() + + print("GN2 is running. Visit %s[http://localhost:%s/%s](%s)" % (BLUE,str(port),ENDC,get_setting("WEBSERVER_URL"))) diff --git a/wqflask/utility/svg.py b/wqflask/utility/svg.py index 512bc9e6..db13b9d1 100644 --- a/wqflask/utility/svg.py +++ b/wqflask/utility/svg.py @@ -1029,7 +1029,7 @@ class drawing: try: xv.feed(svg) except: - raise "SVG is not well formed, see messages above" + raise Exception("SVG is not well formed, see messages above") else: print "SVG well formed" if __name__=='__main__': diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index c28c617a..77db5d53 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -13,6 +13,11 @@ logger = logging.getLogger(__name__ ) OVERRIDES = {} +def app_set(command_id, value): + """Set application wide value""" + app.config.setdefault(command_id,value) + return value + def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with valid_path is a function checking whether the @@ -40,15 +45,16 @@ def get_setting(command_id,guess=None): def value(command): if command: # sys.stderr.write("Found "+command+"\n") + app_set(command_id,command) return command else: return None # ---- Check whether environment exists - logger.debug("Looking for "+command_id+"\n") + # print("Looking for "+command_id+"\n") command = value(os.environ.get(command_id)) if command is None or command == "": - command = OVERRIDES.get(command_id) + command = OVERRIDES.get(command_id) # currently not in use if command is None: # ---- Check whether setting exists in app command = value(app.config.get(command_id)) @@ -57,7 +63,7 @@ def get_setting(command_id,guess=None): if command is None or command == "": # print command raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).') - logger.debug("Set "+command_id+"="+str(command)) + # print("Set "+command_id+"="+str(command)) return command def get_setting_bool(id): @@ -89,14 +95,29 @@ def valid_path(dir): return dir return None -def pylmm_command(guess=None): - return valid_bin(get_setting("PYLMM_COMMAND",guess)) +def js_path(module=None): + """ + Find the JS module in the two paths + """ + try_gn = get_setting("JS_GN_PATH")+"/"+module + if valid_path(try_gn): + return try_gn + try_guix = get_setting("JS_GUIX_PATH")+"/"+module + if valid_path(try_guix): + return try_guix + raise "No JS path found for "+module+" (if not in Guix check JS_GN_PATH)" + +def reaper_command(guess=None): + return get_setting("REAPER_COMMAND",guess) def gemma_command(guess=None): - return valid_bin(get_setting("GEMMA_COMMAND",guess)) + return assert_bin(get_setting("GEMMA_COMMAND",guess)) + +def gemma_wrapper_command(guess=None): + return assert_bin(get_setting("GEMMA_WRAPPER_COMMAND",guess)) def plink_command(guess=None): - return valid_bin(get_setting("PLINK_COMMAND",guess)) + return assert_bin(get_setting("PLINK_COMMAND",guess)) def flat_file_exists(subdir): base = get_setting("GENENETWORK_FILES") @@ -108,6 +129,11 @@ def flat_files(subdir=None): return assert_dir(base+"/"+subdir) return assert_dir(base) +def assert_bin(fn): + if not valid_bin(fn): + raise Exception("ERROR: can not find binary "+fn) + return fn + def assert_dir(dir): if not valid_path(dir): raise Exception("ERROR: can not find directory "+dir) @@ -121,9 +147,14 @@ def assert_writable_dir(dir): fh.close() os.remove(fn) except IOError: - raise Exception('Unable to write test.txt to directory ' + dir ) + raise Exception('Unable to write test.txt to directory ' + dir) return dir +def assert_file(fn): + if not valid_file(fn): + raise Exception('Unable to find file '+fn) + return fn + def mk_dir(dir): if not valid_path(dir): os.makedirs(dir) @@ -148,6 +179,9 @@ def locate(name, subdir=None): if subdir: sys.stderr.write(subdir) raise Exception("Can not locate "+name+" in "+base) +def locate_phewas(name, subdir=None): + return locate(name,'/phewas/'+subdir) + def locate_ignore_error(name, subdir=None): """ Locate a static flat file in the GENENETWORK_FILES environment. @@ -186,7 +220,7 @@ def show_settings(): logger.info(OVERRIDES) logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC) - print "runserver.py: ****** Webserver configuration ******" + print "runserver.py: ****** Webserver configuration - k,v pairs from app.config ******" keylist = app.config.keys() keylist.sort() for k in keylist: @@ -199,40 +233,74 @@ def show_settings(): # Cached values GN_VERSION = get_setting('GN_VERSION') HOME = get_setting('HOME') +SERVER_PORT = get_setting('SERVER_PORT') WEBSERVER_MODE = get_setting('WEBSERVER_MODE') +GN2_BASE_URL = get_setting('GN2_BASE_URL') +GN2_BRANCH_URL = get_setting('GN2_BRANCH_URL') GN_SERVER_URL = get_setting('GN_SERVER_URL') +SERVER_PORT = get_setting_int('SERVER_PORT') SQL_URI = get_setting('SQL_URI') LOG_LEVEL = get_setting('LOG_LEVEL') LOG_LEVEL_DEBUG = get_setting_int('LOG_LEVEL_DEBUG') LOG_SQL = get_setting_bool('LOG_SQL') -LOG_SQLALCHEMY = get_setting_bool('LOG_SQLALCHEMY') +LOG_SQL_ALCHEMY = get_setting_bool('LOG_SQL_ALCHEMY') LOG_BENCH = get_setting_bool('LOG_BENCH') LOG_FORMAT = "%(message)s" # not yet in use USE_REDIS = get_setting_bool('USE_REDIS') USE_GN_SERVER = get_setting_bool('USE_GN_SERVER') GENENETWORK_FILES = get_setting('GENENETWORK_FILES') -TEMP_TRAITS = get_setting('TEMP_TRAITS') - -PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = gemma_command() -GEMMA_RESULTS_PATH = get_setting('GEMMA_RESULTS_PATH') -PLINK_COMMAND = plink_command() +JS_GUIX_PATH = get_setting('JS_GUIX_PATH') +assert_dir(JS_GUIX_PATH) +JS_GN_PATH = get_setting('JS_GN_PATH') +# assert_dir(JS_GN_PATH) + +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') +if GITHUB_CLIENT_ID != 'UNKNOWN' and GITHUB_CLIENT_SECRET: + GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id=" + \ + GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET + GITHUB_API_URL = get_setting('GITHUB_API_URL') + +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') +ORCID_AUTH_URL = None +if ORCID_CLIENT_ID != 'UNKNOWN' and ORCID_CLIENT_SECRET: + ORCID_AUTH_URL = "https://orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id=" + \ + ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET + ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') + +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') +# import utility.elasticsearch_tools as es +# es.test_elasticsearch_connection() + +SMTP_CONNECT = get_setting('SMTP_CONNECT') +SMTP_USERNAME = get_setting('SMTP_USERNAME') +SMTP_PASSWORD = get_setting('SMTP_PASSWORD') + +REAPER_COMMAND = app_set("REAPER_COMMAND",reaper_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +assert(GEMMA_COMMAND is not None) +PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) +GEMMA_WRAPPER_COMMAND = gemma_wrapper_command() TEMPDIR = tempdir() # defaults to UNIX TMPDIR +assert_dir(TEMPDIR) + +# ---- Handle specific JS modules +JS_GUIX_PATH = get_setting("JS_GUIX_PATH") +assert_dir(JS_GUIX_PATH) +assert_dir(JS_GUIX_PATH+'/cytoscape-panzoom') + +CSS_PATH = JS_GUIX_PATH # The CSS is bundled together with the JS +# assert_dir(JS_PATH) + +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("javascript-twitter-post-fetcher")) +assert_dir(JS_TWITTER_POST_FETCHER_PATH) +assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") + +JS_CYTOSCAPE_PATH = get_setting("JS_CYTOSCAPE_PATH",js_path("cytoscape")) +assert_dir(JS_CYTOSCAPE_PATH) +assert_file(JS_CYTOSCAPE_PATH+'/cytoscape.min.js') -from six import string_types - -if os.environ.get('WQFLASK_OVERRIDES'): - jsonfn = get_setting('WQFLASK_OVERRIDES') - logger.error("WQFLASK_OVERRIDES: %s" % jsonfn) - with open(jsonfn) as data_file: - overrides = json.load(data_file) - for k in overrides: - cmd = overrides[k] - if isinstance(cmd, string_types): - OVERRIDES[k] = eval(cmd) - else: - OVERRIDES[k] = cmd - logger.debug(OVERRIDES) - -assert_dir(get_setting("TWITTER_POST_FETCHER_JS_PATH")) +# assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData") diff --git a/wqflask/utility/type_checking.py b/wqflask/utility/type_checking.py new file mode 100644 index 00000000..f15b17e2 --- /dev/null +++ b/wqflask/utility/type_checking.py @@ -0,0 +1,42 @@ +# Type checking functions + +def is_float(value): + try: + float(value) + return True + except: + return False + +def is_int(value): + try: + int(value) + return True + except: + return False + +def is_str(value): + if value is None: + return False + try: + str(value) + return True + except: + return False + +def get_float(vars,name,default=None): + if name in vars: + if is_float(vars[name]): + return float(vars[name]) + return default + +def get_int(vars,name,default=None): + if name in vars: + if is_int(vars[name]): + return float(vars[name]) + return default + +def get_string(vars,name,default=None): + if name in vars: + if not vars[name] is None: + return str(vars[name]) + return default diff --git a/wqflask/utility/webqtlUtil.py b/wqflask/utility/webqtlUtil.py index 4fc978f5..53661ae4 100644 --- a/wqflask/utility/webqtlUtil.py +++ b/wqflask/utility/webqtlUtil.py @@ -30,13 +30,8 @@ import re import math from math import * -from htmlgen import HTMLgen2 as HT - from base import webqtlConfig - - - # NL, 07/27/2010. moved from webqtlForm.py #Dict of Parents and F1 information, In the order of [F1, Mat, Pat] ParInfo ={ @@ -57,180 +52,18 @@ ParInfo ={ 'AXB':['ABF1', 'BAF1', 'C57BL/6J', 'A/J'], 'BXA':['BAF1', 'ABF1', 'C57BL/6J', 'A/J'], 'LXS':['LSF1', 'SLF1', 'ISS', 'ILS'], -'HXBBXH':['HSRBNF1', 'BNHSRF1', 'BN', 'HSR'], +'HXBBXH':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'], 'BayXSha':['BayXShaF1', 'ShaXBayF1', 'Bay-0','Shahdara'], 'ColXBur':['ColXBurF1', 'BurXColF1', 'Col-0','Bur-0'], 'ColXCvi':['ColXCviF1', 'CviXColF1', 'Col-0','Cvi'], -'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'] +'SXM':['SMF1', 'MSF1', 'Steptoe','Morex'], +'HRDP':['SHR_BNF1', 'BN_SHRF1', 'BN-Lx/Cub', 'SHR/OlaIpcv'] } - -# NL, 07/27/2010. moved from template.py -IMGSTEP1 = HT.Image('/images/step1.gif', alt='STEP 1',border=0) #XZ, Only be used in inputPage.py -IMGSTEP2 = HT.Image('/images/step2.gif', alt='STEP 2',border=0) #XZ, Only be used in inputPage.py -IMGSTEP3 = HT.Image('/images/step3.gif', alt='STEP 3',border=0) #XZ, Only be used in inputPage.py -IMGNEXT = HT.Image('/images/arrowdown.gif', alt='NEXT',border=0) #XZ, Only be used in inputPage.py - -IMGASC = HT.Image("/images/sortup.gif", border=0) -IMGASCON = HT.Image("/images/sortupon.gif", border=0) -IMGDESC = HT.Image("/images/sortdown.gif", border=0) -IMGDESCON = HT.Image("/images/sortdownon.gif", border=0) - -""" -IMGASC = HT.Image("/images/sortup_icon.gif", border=0) -IMGASCON = HT.Image("/images/sortupon.gif", border=0) -IMGDESC = HT.Image("/images/sortdown_icon.gif", border=0) -IMGDESCON = HT.Image("/images/sortdownon.gif", border=0) -IMG_UNSORTED = HT.Image("/images/unsorted_icon.gif", border=0) -""" - -PROGRESSBAR = HT.Image('/images/waitAnima2.gif', alt='checkblue',align="middle",border=0) - ######################################### # Accessory Functions ######################################### -def inverseCumul(p): - #Coefficients in rational approximations. - a = [-3.969683028665376e+01,2.209460984245205e+02,-2.759285104469687e+02,1.383577518672690e+02,-3.066479806614716e+01,2.506628277459239e+00] - - b = [-5.447609879822406e+01,1.615858368580409e+02,-1.556989798598866e+02,6.680131188771972e+01,-1.328068155288572e+01] - - c = [-7.784894002430293e-03,-3.223964580411365e-01,-2.400758277161838e+00,-2.549732539343734e+00,4.374664141464968e+00,2.938163982698783e+00] - - d = [7.784695709041462e-03,3.224671290700398e-01,2.445134137142996e+00,3.754408661907416e+00] - - #Define break-points. - - p_low = 0.02425 - p_high = 1 - p_low - - #Rational approximation for lower region. - - if p > 0 and p < p_low: - q = sqrt(-2*log(p)) - x = (((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) / ((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - - #Rational approximation for central region. - - elif p>= p_low and p <= p_high: - q = p - 0.5 - r = q*q - x = (((((a[0]*r+a[1])*r+a[2])*r+a[3])*r+a[4])*r+a[5])*q /(((((b[0]*r+b[1])*r+b[2])*r+b[3])*r+b[4])*r+1) - - #Rational approximation for upper region. - - elif p>p_high and p < 1: - q = sqrt(-2*log(1-p)) - x = -(((((c[0]*q+c[1])*q+c[2])*q+c[3])*q+c[4])*q+c[5]) /((((d[0]*q+d[1])*q+d[2])*q+d[3])*q+1) - - else: - return None - - if p>0 and p < 1: - e = 0.5 * erfcc(-x/sqrt(2)) - p - u = e * sqrt(2*pi) * exp(x*x/2) - x = x - u/(1 + x*u/2) - return x - else: - return None - -def erfcc(x): - z=abs(x) - t=1.0/(1.0+0.5*z) - ans=t*exp(-z*z-1.26551223+t*(1.00002368+t*(0.37409196+t*(0.09678418+t*(-0.18628806+t*(0.27886807+t*(-1.13520398+t*(1.48851587+t*(-0.82215223+t*0.17087277))))))))) - if x>=0.0: - return ans - else: - return 2.0-ans - -def U(n): - x=pow(0.5,1.0/n) - m=[1-x] - for i in range(2,n): - a=(i-0.3175)/(n+0.365) - m.append(a) - m.append(x) - return m - -def decodeEscape(str): - a = str - pattern = re.compile('(%[0-9A-Fa-f][0-9A-Fa-f])') - match = pattern.findall(a) - matched = [] - for item in match: - if item not in matched: - a = a.replace(item, '%c' % eval("0x"+item[-2:])) - matched.append(item) - return a - -def exportData(hddn, tdata, NP = None): - for key in tdata.keys(): - _val, _var, _N = tdata[key].val, tdata[key].var, tdata[key].N - if _val != None: - hddn[key] = _val - if _var != None: - hddn['V'+key] = _var - if NP and _N != None: - hddn['N'+key] = _N - -def genShortStrainName(RISet='', input_strainName=''): - #aliasStrainDict = {'C57BL/6J':'B6','DBA/2J':'D2'} - strainName = input_strainName - if RISet != 'AXBXA': - if RISet == 'BXD300': - this_RISet = 'BXD' - elif RISet == 'BDF2-2005': - this_RISet = 'CASE05_' - else: - this_RISet = RISet - strainName = string.replace(strainName,this_RISet,'') - strainName = string.replace(strainName,'CASE','') - try: - strainName = "%02d" % int(strainName) - except: - pass - else: - strainName = string.replace(strainName,'AXB','A') - strainName = string.replace(strainName,'BXA','B') - try: - strainName = strainName[0] + "%02d" % int(strainName[1:]) - except: - pass - return strainName - -def toInt(in_str): - "Converts an arbitrary string to an unsigned integer" - start = -1 - end = -1 - for i, char in enumerate(in_str): - if char >= '0' and char <= '9': - if start < 0: - start = i - end = i+1 - else: - if start >= 0: - break - if start < end: - return int(in_str[start:end]) - else: - return -1 - -def transpose(m): - 'transpose a matrix' - n = len(m) - return [[m[j][i] for i in range(len(m[0])) for j in range(n)][k*n:k*n+n] for k in range(len(m[0]))] - -def asymTranspose(m): - 'transpose a matrix' - t = max(map(len, m)) - n = len(m) - m2 = [["-"]]*n - for i in range(n): - m2[i] = m[i] + [""]*(t- len(m[i])) - return [[m2[j][i] for i in range(len(m2[0])) for j in range(n)][k*n:k*n+n] for k in range(len(m2[0]))] - def genRandStr(prefix = "", length=8, chars=string.letters+string.digits): from random import choice _str = prefix[:] @@ -238,84 +71,6 @@ def genRandStr(prefix = "", length=8, chars=string.letters+string.digits): _str += choice(chars) return _str -def generate_session(): - import sha - return sha.new(str(time.time())).hexdigest() - -def cvt2Dict(x): - tmp = {} - for key in x.keys(): - tmp[key] = x[key] - return tmp - -def dump_session(session_obj, filename): - "It seems mod python can only cPickle most basic data type" - import cPickle - session_file = open(filename, 'wb') - #try: - # pass - #except: - # pass - cPickle.dump(session_obj, session_file) - session_file.close() - -def StringAsFloat(str): - 'Converts string to float but catches any exception and returns None' - try: - return float(str) - except: - return None - -def IntAsFloat(str): - 'Converts string to Int but catches any exception and returns None' - try: - return int(str) - except: - return None - -def FloatAsFloat(flt): - 'Converts float to string but catches any exception and returns None' - try: - return float("%2.3f" % flt) - except: - return None - -def RemoveZero(flt): - 'Converts string to float but catches any exception and returns None' - try: - if abs(flt) < 1e-6: - return None - else: - return flt - except: - return None - - -def SciFloat(d): - 'Converts string to float but catches any exception and returns None' - - try: - if abs(d) <= 1.0e-4: - return "%1.2e" % d - else: - return "%1.5f" % d - except: - return None - -###To be removed -def FloatList2String(lst): - 'Converts float list to string but catches any exception and returns None' - tt='' - try: - for item in lst: - if item == None: - tt += 'X ' - else: - tt += '%f ' % item - return tt - except: - return "" - def ListNotNull(lst): '''Obsolete - Use built in function any (or all or whatever) @@ -328,427 +83,6 @@ def ListNotNull(lst): return 1 return None -###To be removed -def FileDataProcess(str): - 'Remove the description text from the input file if theres any' - i=0 - while i<len(str): - if str[i]<'\x7f' and str[i]>'\x20': - break - else: - i+=1 - str=str[i:] - str=string.join(string.split(str,'\000'),'') - i=string.find(str,"*****") - if i>-1: - return str[i+5:] - else: - return str - -def rank(a,lst,offset=0): - """Calculate the integer rank of a number in an array, can be used to calculate p-value""" - n = len(lst) - if n == 2: - if a <lst[0]: - return offset - elif a > lst[1]: - return offset + 2 - else: - return offset +1 - elif n == 1: - if a <lst[0]: - return offset - else: - return offset +1 - elif n== 0: - return offset - else: - mid = n/2 - if a < lst[mid]: - return rank(a,lst[:mid-1],offset) - else: - return rank(a,lst[mid:],offset+mid) - -def cmpScanResult(A,B): - try: - if A.LRS > B.LRS: - return 1 - elif A.LRS == B.LRS: - return 0 - else: - return -1 - except: - return 0 - - -def cmpScanResult2(A,B): - try: - if A.LRS < B.LRS: - return 1 - elif A.LRS == B.LRS: - return 0 - else: - return -1 - except: - return 0 - -def cmpOrder(A,B): - try: - if A[1] < B[1]: - return -1 - elif A[1] == B[1]: - return 0 - else: - return 1 - except: - return 0 - -def cmpOrder2(A,B): - try: - if A[-1] < B[-1]: - return -1 - elif A[-1] == B[-1]: - return 0 - else: - return 1 - except: - return 0 - - - - -def calRank(xVals, yVals, N): ### Zach Sloan, February 4 2010 - """ - Returns a ranked set of X and Y values. These are used when generating - a Spearman scatterplot. Bear in mind that this sets values equal to each - other as the same rank. - """ - XX = [] - YY = [] - X = [0]*len(xVals) - Y = [0]*len(yVals) - j = 0 - - for i in range(len(xVals)): - - if xVals[i] != None and yVals[i] != None: - XX.append((j, xVals[i])) - YY.append((j, yVals[i])) - j = j + 1 - - NN = len(XX) - - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - - j = 1 - rank = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - return (X,Y) - -def calCorrelationRank(xVals,yVals,N): - """ - Calculated Spearman Ranked Correlation. The algorithm works - by setting all tied ranks to the average of those ranks (for - example, if ranks 5-10 all have the same value, each will be set - to rank 7.5). - """ - - XX = [] - YY = [] - j = 0 - - for i in range(len(xVals)): - if (xVals[i]!= None and yVals[i]!= None) and (xVals[i] != "None" and yVals[i] != "None"): - XX.append((j,xVals[i])) - YY.append((j,yVals[i])) - j = j+1 - - NN = len(XX) - if NN <6: - return (0.0,NN) - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - X = [0]*NN - Y = [0]*NN - - j = 1 - rank = 0.0 - t = 0.0 - sx = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - t = jt-j - sx = sx + (t*t*t-t) - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - t = 0.0 - sy = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - t = jt - j - sy = sy + (t*t*t-t) - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - D = 0.0 - - for i in range(NN): - D += (X[i]-Y[i])*(X[i]-Y[i]) - - fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN)) - - return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN) - - -def calCorrelationRankText(dbdata,userdata,N): ### dcrowell = David Crowell, July 2008 - """Calculates correlation ranks with data formatted from the text file. - dbdata, userdata are lists of strings. N is an int. Returns a float. - Used by correlationPage""" - XX = [] - YY = [] - j = 0 - for i in range(N): - if (dbdata[i]!= None and userdata[i]!=None) and (dbdata[i]!= 'None' and userdata[i]!='None'): - XX.append((j,float(dbdata[i]))) - YY.append((j,float(userdata[i]))) - j += 1 - NN = len(XX) - if NN <6: - return (0.0,NN) - XX.sort(cmpOrder2) - YY.sort(cmpOrder2) - X = [0]*NN - Y = [0]*NN - - j = 1 - rank = 0.0 - t = 0.0 - sx = 0.0 - - while j < NN: - - if XX[j][1] != XX[j-1][1]: - X[XX[j-1][0]] = j - j = j+1 - - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (XX[jt][1] != XX[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - X[XX[ji][0]] = rank - t = jt-j - sx = sx + (t*t*t-t) - if (jt == NN-1): - if (XX[jt][1] == XX[j-1][1]): - X[XX[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if X[XX[NN-1][0]] == 0: - X[XX[NN-1][0]] = NN - - j = 1 - rank = 0.0 - t = 0.0 - sy = 0.0 - - while j < NN: - - if YY[j][1] != YY[j-1][1]: - Y[YY[j-1][0]] = j - j = j+1 - else: - jt = j+1 - ji = j - for jt in range(j+1, NN): - if (YY[jt][1] != YY[j-1][1]): - break - rank = 0.5*(j+jt) - for ji in range(j-1, jt): - Y[YY[ji][0]] = rank - t = jt - j - sy = sy + (t*t*t-t) - if (jt == NN-1): - if (YY[jt][1] == YY[j-1][1]): - Y[YY[NN-1][0]] = rank - j = jt+1 - - if j == NN: - if Y[YY[NN-1][0]] == 0: - Y[YY[NN-1][0]] = NN - - D = 0.0 - - for i in range(NN): - D += (X[i]-Y[i])*(X[i]-Y[i]) - - fac = (1.0 -sx/(NN*NN*NN-NN))*(1.0-sy/(NN*NN*NN-NN)) - - return ((1-(6.0/(NN*NN*NN-NN))*(D+(sx+sy)/12.0))/math.sqrt(fac),NN) - - - -def calCorrelation(dbdata,userdata,N): - X = [] - Y = [] - for i in range(N): - if dbdata[i]!= None and userdata[i]!= None: - X.append(dbdata[i]) - Y.append(userdata[i]) - NN = len(X) - if NN <6: - return (0.0,NN) - sx = reduce(lambda x,y:x+y,X,0.0) - sy = reduce(lambda x,y:x+y,Y,0.0) - meanx = sx/NN - meany = sy/NN - xyd = 0.0 - sxd = 0.0 - syd = 0.0 - for i in range(NN): - xyd += (X[i] - meanx)*(Y[i]-meany) - sxd += (X[i] - meanx)*(X[i] - meanx) - syd += (Y[i] - meany)*(Y[i] - meany) - try: - corr = xyd/(sqrt(sxd)*sqrt(syd)) - except: - corr = 0 - return (corr,NN) - -def calCorrelationText(dbdata,userdata,N): ### dcrowell July 2008 - """Calculates correlation coefficients with values formatted from text files. dbdata, userdata are lists of strings. N is an int. Returns a float - Used by correlationPage""" - X = [] - Y = [] - for i in range(N): - #if (dbdata[i]!= None and userdata[i]!= None) and (dbdata[i]!= 'None' and userdata[i]!= 'None'): - # X.append(float(dbdata[i])) - # Y.append(float(userdata[i])) - if dbdata[i] == None or dbdata[i] == 'None' or userdata[i] == None or userdata[i] == 'None': - continue - else: - X.append(float(dbdata[i])) - Y.append(float(userdata[i])) - NN = len(X) - if NN <6: - return (0.0,NN) - sx = sum(X) - sy = sum(Y) - meanx = sx/float(NN) - meany = sy/float(NN) - xyd = 0.0 - sxd = 0.0 - syd = 0.0 - for i in range(NN): - x1 = X[i]-meanx - y1 = Y[i]-meany - xyd += x1*y1 - sxd += x1**2 - syd += y1**2 - try: - corr = xyd/(sqrt(sxd)*sqrt(syd)) - except: - corr = 0 - return (corr,NN) - - def readLineCSV(line): ### dcrowell July 2008 """Parses a CSV string of text and returns a list containing each element as a string. Used by correlationPage""" @@ -757,45 +91,6 @@ def readLineCSV(line): ### dcrowell July 2008 returnList[0]=returnList[0][1:] return returnList - -def cmpCorr(A,B): - try: - if abs(A[1]) < abs(B[1]): - return 1 - elif abs(A[1]) == abs(B[1]): - return 0 - else: - return -1 - except: - return 0 - -def cmpLitCorr(A,B): - try: - if abs(A[3]) < abs(B[3]): return 1 - elif abs(A[3]) == abs(B[3]): - if abs(A[1]) < abs(B[1]): return 1 - elif abs(A[1]) == abs(B[1]): return 0 - else: return -1 - else: return -1 - except: - return 0 - -def cmpPValue(A,B): - try: - if A.corrPValue < B.corrPValue: - return -1 - elif A.corrPValue == B.corrPValue: - if abs(A.corr) > abs(B.corr): - return -1 - elif abs(A.corr) < abs(B.corr): - return 1 - else: - return 0 - else: - return 1 - except: - return 0 - def cmpEigenValue(A,B): try: if A[0] > B[0]: @@ -807,80 +102,6 @@ def cmpEigenValue(A,B): except: return 0 - -def cmpLRSFull(A,B): - try: - if A[0] < B[0]: - return -1 - elif A[0] == B[0]: - return 0 - else: - return 1 - except: - return 0 - -def cmpLRSInteract(A,B): - try: - if A[1] < B[1]: - return -1 - elif A[1] == B[1]: - return 0 - else: - return 1 - except: - return 0 - - -def cmpPos(A,B): - try: - try: - AChr = int(A.chr) - except: - AChr = 20 - try: - BChr = int(B.chr) - except: - BChr = 20 - if AChr > BChr: - return 1 - elif AChr == BChr: - if A.mb > B.mb: - return 1 - if A.mb == B.mb: - return 0 - else: - return -1 - else: - return -1 - except: - return 0 - -def cmpGenoPos(A,B): - try: - A1 = A.chr - B1 = B.chr - try: - A1 = int(A1) - except: - A1 = 25 - try: - B1 = int(B1) - except: - B1 = 25 - if A1 > B1: - return 1 - elif A1 == B1: - if A.mb > B.mb: - return 1 - if A.mb == B.mb: - return 0 - else: - return -1 - else: - return -1 - except: - return 0 - def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users): access_to_confidential_phenotype_trait = 0 if webqtlConfig.USERDICT[privilege] > webqtlConfig.USERDICT['user']: @@ -889,142 +110,4 @@ def hasAccessToConfidentialPhenotypeTrait(privilege, userName, authorized_users) AuthorizedUsersList=map(string.strip, string.split(authorized_users, ',')) if AuthorizedUsersList.__contains__(userName): access_to_confidential_phenotype_trait = 1 - return access_to_confidential_phenotype_trait - - -class VisualizeException(Exception): - def __init__(self, message): - self.message = message - def __str__(self): - return self.message - -# safeConvert : (string -> A) -> A -> A -# to convert a string to type A, using the supplied default value -# if the given conversion function doesn't work -def safeConvert(f, value, default): - try: - return f(value) - except: - return default - -# safeFloat : string -> float -> float -# to convert a string to a float safely -def safeFloat(value, default): - return safeConvert(float, value, default) - -# safeInt: string -> int -> int -# to convert a string to an int safely -def safeInt(value, default): - return safeConvert(int, value, default) - -# safeString : string -> (arrayof string) -> string -> string -# if a string is not in a list of strings to pick a default value -# for that string -def safeString(value, validChoices, default): - if value in validChoices: - return value - else: - return default - -# yesNoToInt: string -> int -# map "yes" -> 1 and "no" -> 0 -def yesNoToInt(value): - if value == "yes": - return 1 - elif value == "no": - return 0 - else: - return None - -# IntToYesNo: int -> string -# map 1 -> "yes" and 0 -> "no" -def intToYesNo(value): - if value == 1: - return "yes" - elif value == 0: - return "no" - else: - return None - -def formatField(name): - name = name.replace("_", " ") - name = name.title() - #name = name.replace("Mb Mm6", "Mb"); - return name.replace("Id", "ID") - -#XZ, 03/27/2009: This function is very specific. -#It is used by AJAX_table.py, correlationPage.py and dataPage.py - - -def genTableObj(tblobj=None, file="", sortby = ("", ""), tableID = "sortable", addIndex = "1", hiddenColumns=[]): - header = tblobj['header'] - body = tblobj['body'] - field, order = sortby - - #ZAS 9/12/2011 - The hiddenColumns array needs to be converted into a string so they can be placed into the javascript of each up/down button - hiddenColumnsString = ",".join(hiddenColumns) - - tbl = HT.TableLite(Class="collap b2", cellspacing=1, cellpadding=5) - - hiddenColumnIdx = [] #indices of columns to hide - idx = -1 - last_idx = 0 #ZS: This is the index of the last item in the regular table header (without any extra parameters). It is used to determine the index of each extra parameter. - for row in header: - hr = HT.TR() - for i, item in enumerate(row): - if (item.text == '') or (item.text not in hiddenColumns): - if item.sort and item.text: - down = HT.Href("javascript:xmlhttpPost('%smain.py?FormID=AJAX_table', '%s', 'sort=%s&order=down&file=%s&tableID=%s&addIndex=%s&hiddenColumns=%s')" % (webqtlConfig.CGIDIR, tableID, item.text, file, tableID, addIndex, hiddenColumnsString),IMGDESC) - up = HT.Href("javascript:xmlhttpPost('%smain.py?FormID=AJAX_table', '%s', 'sort=%s&order=up&file=%s&tableID=%s&addIndex=%s&hiddenColumns=%s')" % (webqtlConfig.CGIDIR, tableID, item.text, file, tableID, addIndex, hiddenColumnsString),IMGASC) - if item.text == field: - idx = item.idx - last_idx = idx - if order == 'up': - up = IMGASCON - elif order == 'down': - down = IMGDESCON - item.html.append(HT.Div(up, down, style="float: bottom;")) - hr.append(item.html) - else: - hiddenColumnIdx.append(i) - tbl.append(hr) - - for i, row in enumerate(body): - for j, item in enumerate(row): - if order == 'down': - if (item.val == '' or item.val == 'x' or item.val == 'None'): - item.val = 0 - if order == 'up': - if (item.val == '' or item.val == 'x' or item.val == 'None'): - item.val = 'zzzzz' - - if idx >= 0: - if order == 'down': - body.sort(lambda A, B: cmp(B[idx].val, A[idx].val), key=natsort_key) - elif order == 'up': - body.sort(lambda A, B: cmp(A[idx].val, B[idx].val), key=natsort_key) - else: - pass - - for i, row in enumerate(body): - hr = HT.TR(Id = row[0].text) - for j, item in enumerate(row): - if (j not in hiddenColumnIdx): - if j == 0: - if addIndex == "1": - item.html.contents = [i+1] + item.html.contents - hr.append(item.html) - tbl.append(hr) - - return tbl - -def natsort_key(string): - r = [] - for c in string: - try: - c = int(c) - try: r[-1] = r[-1] * 10 + c - except: r.append(c) - except: - r.append(c) - return r + return access_to_confidential_phenotype_trait
\ No newline at end of file |