From e6e3b12eeb3fc57b9652468304c1fd14a0a816d0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 10:08:29 +0300 Subject: Add callback handlers --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 38 ++++++++++++++++++++++++++ wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 41 ++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 wqflask/wqflask/my_pylmm/pyLMM/gn2.py create mode 100644 wqflask/wqflask/my_pylmm/pyLMM/standalone.py diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py new file mode 100644 index 00000000..e0c6c8a7 --- /dev/null +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -0,0 +1,38 @@ +# Genenetwork2 specific methods and callback handler +# +# Copyright (C) 2015 Pjotr Prins (pjotr.prins@thebird.nl) +# + +from __future__ import absolute_import, print_function, division + +import sys +import logging + +# logging.basicConfig(level=logging.DEBUG) + +def progress(location, count, total): + print("Progress: %s %i %i @%d%%" % (location,count,total,round(count*100.0/total))) + +def callbacks(): + return dict( + write = sys.stdout.write, + writeln = print, + debug = logging.debug, + info = logging.info, + warning = logging.warning, + error = logging.error, + critical = logging.critical, + progress = progress + ) + +# ----- Minor test cases: + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + logging.debug("Test %i" % (1)) + d = callbacks()['debug'] + d("TEST") + wrln = callbacks()['writeln'] + wrln("Hello %i" % 34) + progress = callbacks()['progress'] + progress("I am half way",50,100) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py new file mode 100644 index 00000000..a806729e --- /dev/null +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -0,0 +1,41 @@ +# Standalone specific methods and callback handler +# +# Copyright (C) 2015 Pjotr Prins (pjotr.prins@thebird.nl) +# +# Set the log level with +# +# logging.basicConfig(level=logging.DEBUG) + +from __future__ import absolute_import, print_function, division + +import sys +import logging + +logging.basicConfig(level=logging.DEBUG) + +def progress(location, count, total): + logging.info("Progress: %s %i %i @%d%%" % (location,count,total,round(count*100.0/total))) + +def callbacks(): + return dict( + write = sys.stdout.write, + writeln = print, + debug = logging.debug, + info = logging.info, + warning = logging.warning, + error = logging.error, + critical = logging.critical, + progress = progress + ) + +# ----- Minor test cases: + +if __name__ == '__main__': + # logging.basicConfig(level=logging.DEBUG) + logging.debug("Test %i" % (1)) + d = callbacks()['debug'] + d("TEST") + wrln = callbacks()['writeln'] + wrln("Hello %i" % 34) + progress = callbacks()['progress'] + progress("I am half way",50,100) -- cgit v1.2.3 From 178cdbbd1a52cfcab975ab27b36e148009cc3577 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 11:05:39 +0300 Subject: Introducing callbacks --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 17 ++++++++++++-- wqflask/wqflask/my_pylmm/pyLMM/kinship.py | 17 +++++++------- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 5 +++- wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 12 +++++++++- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 34 ++++++++++++++++++++++++++-- 5 files changed, 71 insertions(+), 14 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index e0c6c8a7..4702c670 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -5,13 +5,25 @@ from __future__ import absolute_import, print_function, division +import numpy as np import sys import logging # logging.basicConfig(level=logging.DEBUG) +# np.set_printoptions() def progress(location, count, total): - print("Progress: %s %i %i @%d%%" % (location,count,total,round(count*100.0/total))) + """ + Progress update + """ + logging.info("Progress: %s %d%%" % (location,round(count*100.0/total))) + +def mprint(msg,data): + """ + Array/matrix print function + """ + m = np.array(data) + print(msg,m.shape,"=\n",m) def callbacks(): return dict( @@ -22,7 +34,8 @@ def callbacks(): warning = logging.warning, error = logging.error, critical = logging.critical, - progress = progress + progress = progress, + mprint = mprint ) # ----- Minor test cases: diff --git a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py index 0c43587e..43e7fe36 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py @@ -155,20 +155,21 @@ def kinship(G,computeSize=1000,numThreads=None,useBLAS=False,verbose=True): # np.savetxt(outFile+".kve",Kve) return K -def kvakve(K, verbose=True): +def kvakve(K, callbacks): """ Obtain eigendecomposition for K and return Kva,Kve where Kva is cleaned of small values < 1e-6 (notably smaller than zero) """ - if verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) - + info = callbacks()['info'] + mprint = callbacks()['mprint'] + + info("Obtaining eigendecomposition for %dx%d matrix" % (K.shape[0],K.shape[1]) ) Kva,Kve = linalg.eigh(K) - if verbose: - print("Kva is: ", Kva.shape, Kva) - print("Kve is: ", Kve.shape, Kve) + mprint("Kva",Kva) + mprint("Kve",Kve) - if sum(Kva < 1e-6): - if verbose: sys.stderr.write("Cleaning %d eigen values (Kva<0)\n" % (sum(Kva < 0))) + if sum(Kva < 0): + info("Cleaning %d eigen values (Kva<0)" % (sum(Kva < 0))) Kva[Kva < 1e-6] = 1e-6 return Kva,Kve diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 8a24d98b..5ad644e2 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -54,11 +54,14 @@ import genotype import phenotype import gwas +# ---- A trick to decide on the environment: try: from wqflask.my_pylmm.pyLMM import chunks + from gn2 import callbacks except ImportError: print("WARNING: Standalone version missing the Genenetwork2 environment\n") has_gn2=False + from standalone import callbacks pass #np.seterr('raise') @@ -594,7 +597,7 @@ class LMM: # if self.verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) begin = time.time() # Kva,Kve = linalg.eigh(K) - Kva,Kve = kvakve(K) + Kva,Kve = kvakve(K,callbacks) end = time.time() if self.verbose: sys.stderr.write("Total time: %0.3f\n" % (end - begin)) print("sum(Kva),sum(Kve)=",sum(Kva),sum(Kve)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index d4b3ac82..6aefb9d3 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -24,6 +24,16 @@ from scipy import optimize from optmatrix import matrixMult import kinship +# A trick to decide on the environment: +try: + from wqflask.my_pylmm.pyLMM import chunks + from gn2 import callbacks +except ImportError: + print("WARNING: Standalone version missing the Genenetwork2 environment\n") + has_gn2=False + from standalone import callbacks + pass + def calculateKinship(W,center=False): """ W is an n x m matrix encoding SNP minor alleles. @@ -184,7 +194,7 @@ class LMM2: # if self.verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) begin = time.time() # Kva,Kve = linalg.eigh(K) - Kva,Kve = kinship.kvakve(K) + Kva,Kve = kinship.kvakve(K,callbacks) end = time.time() if self.verbose: sys.stderr.write("Total time: %0.3f\n" % (end - begin)) print("sum(Kva),sum(Kve)=",sum(Kva),sum(Kve)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index a806729e..bbee3cd7 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -8,13 +8,29 @@ from __future__ import absolute_import, print_function, division +import numpy as np import sys import logging logging.basicConfig(level=logging.DEBUG) +np.set_printoptions(precision=3,suppress=True) def progress(location, count, total): - logging.info("Progress: %s %i %i @%d%%" % (location,count,total,round(count*100.0/total))) + logging.info("Progress: %s %d%%" % (location,round(count*100.0/total))) + +def mprint(msg,data): + """ + Array/matrix print function + """ + m = np.array(data) + if m.ndim == 1: + print(msg,m.shape,"=\n",m[0:3]," ... ",m[-3:]) + if m.ndim == 2: + print(msg,m.shape,"=\n[", + m[0][0:3]," ... ",m[0][-3:],"\n ", + m[1][0:3]," ... ",m[1][-3:],"\n ...\n ", + m[-2][0:3]," ... ",m[-2][-3:],"\n ", + m[-1][0:3]," ... ",m[-1][-3:],"]") def callbacks(): return dict( @@ -25,7 +41,8 @@ def callbacks(): warning = logging.warning, error = logging.error, critical = logging.critical, - progress = progress + progress = progress, + mprint = mprint ) # ----- Minor test cases: @@ -39,3 +56,16 @@ if __name__ == '__main__': wrln("Hello %i" % 34) progress = callbacks()['progress'] progress("I am half way",50,100) + list = [0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15] + mprint("list",list) + matrix = [[1,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [2,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [3,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [4,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [5,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [6,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15]] + mprint("matrix",matrix) -- cgit v1.2.3 From 876e80148984274dfd3b8281677c7541504feb59 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 11:18:58 +0300 Subject: Added uses as syntax sugar for callbacks --- wqflask/wqflask/my_pylmm/pyLMM/kinship.py | 5 ++--- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 6 +++--- wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 8 ++++---- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 9 +++++++++ 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py index 43e7fe36..d3792570 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py @@ -155,13 +155,12 @@ def kinship(G,computeSize=1000,numThreads=None,useBLAS=False,verbose=True): # np.savetxt(outFile+".kve",Kve) return K -def kvakve(K, callbacks): +def kvakve(K, uses): """ Obtain eigendecomposition for K and return Kva,Kve where Kva is cleaned of small values < 1e-6 (notably smaller than zero) """ - info = callbacks()['info'] - mprint = callbacks()['mprint'] + info,mprint = uses('info','mprint') info("Obtaining eigendecomposition for %dx%d matrix" % (K.shape[0],K.shape[1]) ) Kva,Kve = linalg.eigh(K) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 5ad644e2..2076bc84 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -57,11 +57,11 @@ import gwas # ---- A trick to decide on the environment: try: from wqflask.my_pylmm.pyLMM import chunks - from gn2 import callbacks + from gn2 import uses except ImportError: print("WARNING: Standalone version missing the Genenetwork2 environment\n") has_gn2=False - from standalone import callbacks + from standalone import uses pass #np.seterr('raise') @@ -597,7 +597,7 @@ class LMM: # if self.verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) begin = time.time() # Kva,Kve = linalg.eigh(K) - Kva,Kve = kvakve(K,callbacks) + Kva,Kve = kvakve(K,uses) end = time.time() if self.verbose: sys.stderr.write("Total time: %0.3f\n" % (end - begin)) print("sum(Kva),sum(Kve)=",sum(Kva),sum(Kve)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index 6aefb9d3..5b93ae0d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -24,14 +24,14 @@ from scipy import optimize from optmatrix import matrixMult import kinship -# A trick to decide on the environment: +# ---- A trick to decide on the environment: try: from wqflask.my_pylmm.pyLMM import chunks - from gn2 import callbacks + from gn2 import uses except ImportError: print("WARNING: Standalone version missing the Genenetwork2 environment\n") has_gn2=False - from standalone import callbacks + from standalone import uses pass def calculateKinship(W,center=False): @@ -194,7 +194,7 @@ class LMM2: # if self.verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) begin = time.time() # Kva,Kve = linalg.eigh(K) - Kva,Kve = kinship.kvakve(K,callbacks) + Kva,Kve = kinship.kvakve(K,uses) end = time.time() if self.verbose: sys.stderr.write("Total time: %0.3f\n" % (end - begin)) print("sum(Kva),sum(Kve)=",sum(Kva),sum(Kve)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index bbee3cd7..705da21f 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -44,6 +44,12 @@ def callbacks(): progress = progress, mprint = mprint ) + +def uses(*funcs): + """ + Some sugar + """ + return [callbacks()[func] for func in funcs] # ----- Minor test cases: @@ -69,3 +75,6 @@ if __name__ == '__main__': [5,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], [6,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15]] mprint("matrix",matrix) + ix,dx = uses("info","debug") + ix("ix") + dx("dx") -- cgit v1.2.3 From 7f937ef3265f007c25ec2c386bc399a708bcdd5e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 11:46:06 +0300 Subject: Introduce sugar for callbacks --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 26 +++++++++++++-- wqflask/wqflask/my_pylmm/pyLMM/kinship.py | 49 +++++++++------------------- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 2 +- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 3 +- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 14 ++++---- 5 files changed, 50 insertions(+), 44 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index 4702c670..c71b9f22 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -37,11 +37,17 @@ def callbacks(): progress = progress, mprint = mprint ) - + +def uses(*funcs): + """ + Some sugar + """ + return [callbacks()[func] for func in funcs] + # ----- Minor test cases: if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) + # logging.basicConfig(level=logging.DEBUG) logging.debug("Test %i" % (1)) d = callbacks()['debug'] d("TEST") @@ -49,3 +55,19 @@ if __name__ == '__main__': wrln("Hello %i" % 34) progress = callbacks()['progress'] progress("I am half way",50,100) + list = [0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15, + 0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15] + mprint("list",list) + matrix = [[1,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [2,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [3,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [4,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [5,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15], + [6,0.5,0.6096595 , -0.31559815, -0.52793285, 1.16573418e-15]] + mprint("matrix",matrix) + ix,dx = uses("info","debug") + ix("ix") + dx("dx") diff --git a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py index d3792570..62f7be47 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py @@ -74,46 +74,39 @@ def f_init(q): # Calculate the kinship matrix from G (SNPs as rows!), returns K # -def kinship(G,computeSize=1000,numThreads=None,useBLAS=False,verbose=True): - numThreads = None - if numThreads: - numThreads = int(numThreads) +def kinship(G,uses,computeSize=1000,numThreads=None,useBLAS=False): + progress,debug,info,mprint = uses('progress','debug','info','mprint') + matrix_initialize(useBLAS) - - sys.stderr.write(str(G.shape)+"\n") + + mprint("G",G) n = G.shape[1] # inds inds = n m = G.shape[0] # snps snps = m - sys.stderr.write(str(m)+" SNPs\n") + info("%i SNPs" % (m)) assert snps>inds, "snps should be larger than inds (%i snps, %i inds)" % (snps,inds) q = mp.Queue() p = mp.Pool(numThreads, f_init, [q]) cpu_num = mp.cpu_count() - print "CPU cores:",cpu_num - print snps,computeSize + info("CPU cores: %i" % cpu_num) iterations = snps/computeSize+1 - # if testing: - # iterations = 8 - # jobs = range(0,8) # range(0,iterations) results = [] - K = np.zeros((n,n)) # The Kinship matrix has dimension individuals x individuals completed = 0 for job in range(iterations): - if verbose: - sys.stderr.write("Processing job %d first %d SNPs\n" % (job, ((job+1)*computeSize))) + info("Processing job %d first %d SNPs" % (job, ((job+1)*computeSize))) W = compute_W(job,G,n,snps,computeSize) if numThreads == 1: # Single-core compute_matrixMult(job,W,q) j,x = q.get() - if verbose: sys.stderr.write("Job "+str(j)+" finished\n") + debug("Job "+str(j)+" finished") + progress("kinship",j,iterations) K_j = x - # print j,K_j[:,0] K = K + K_j else: # Multi-core @@ -123,39 +116,27 @@ def kinship(G,computeSize=1000,numThreads=None,useBLAS=False,verbose=True): time.sleep(0.1) try: j,x = q.get_nowait() - if verbose: sys.stderr.write("Job "+str(j)+" finished\n") + debug("Job "+str(j)+" finished") K_j = x - # print j,K_j[:,0] K = K + K_j completed += 1 + progress("kinship",completed,iterations) except Queue.Empty: pass if numThreads == None or numThreads > 1: - # results contains the growing result set for job in range(len(results)-completed): j,x = q.get(True,15) - if verbose: sys.stderr.write("Job "+str(j)+" finished\n") + debug("Job "+str(j)+" finished") K_j = x - # print j,K_j[:,0] K = K + K_j completed += 1 + progress("kinship",completed,iterations) K = K / float(snps) - - # outFile = 'runtest.kin' - # if verbose: sys.stderr.write("Saving Kinship file to %s\n" % outFile) - # np.savetxt(outFile,K) - - # if saveKvaKve: - # if verbose: sys.stderr.write("Obtaining Eigendecomposition\n") - # Kva,Kve = linalg.eigh(K) - # if verbose: sys.stderr.write("Saving eigendecomposition to %s.[kva | kve]\n" % outFile) - # np.savetxt(outFile+".kva",Kva) - # np.savetxt(outFile+".kve",Kve) return K -def kvakve(K, uses): +def kvakve(K,uses): """ Obtain eigendecomposition for K and return Kva,Kve where Kva is cleaned of small values < 1e-6 (notably smaller than zero) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 2076bc84..5182e73c 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -395,7 +395,7 @@ def calculate_kinship_new(genotype_matrix, temp_data=None): print("call genotype.normalize") G = np.apply_along_axis( genotype.normalize, axis=0, arr=genotype_matrix) print("call calculate_kinship_new") - return kinship(G.T),G # G gets transposed, we'll turn this into an iterator (FIXME) + return kinship(G.T,uses),G # G gets transposed, we'll turn this into an iterator (FIXME) def calculate_kinship_old(genotype_matrix, temp_data=None): """ diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 324c4f2c..e3e8659c 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -25,6 +25,7 @@ from lmm import gn2_load_redis, calculate_kinship_old from kinship import kinship, kinship_full import genotype import phenotype +from standalone import uses usage = """ python runlmm.py [options] command @@ -193,7 +194,7 @@ elif cmd == 'kinship': k2 = round(K2[0][0],4) print "Genotype",G.shape, "\n", G - K3 = kinship(G.T) + K3 = kinship(G.T,uses) print "third Kinship method",K3.shape,"\n",K3 sys.stderr.write(options.geno+"\n") k3 = round(K3[0][0],4) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index 705da21f..538007f1 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -12,11 +12,13 @@ import numpy as np import sys import logging +# logger = logging.getLogger(__name__) +logger = logging.getLogger('lmm2') logging.basicConfig(level=logging.DEBUG) np.set_printoptions(precision=3,suppress=True) def progress(location, count, total): - logging.info("Progress: %s %d%%" % (location,round(count*100.0/total))) + logger.info("Progress: %s %d%%" % (location,round(count*100.0/total))) def mprint(msg,data): """ @@ -36,11 +38,11 @@ def callbacks(): return dict( write = sys.stdout.write, writeln = print, - debug = logging.debug, - info = logging.info, - warning = logging.warning, - error = logging.error, - critical = logging.critical, + debug = logger.debug, + info = logger.info, + warning = logger.warning, + error = logger.error, + critical = logger.critical, progress = progress, mprint = mprint ) -- cgit v1.2.3 From 204805157912aebb92967241850453f07729e2f6 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 12:00:01 +0300 Subject: Warning to stderr --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 2 +- wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 5182e73c..66c952aa 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -59,9 +59,9 @@ try: from wqflask.my_pylmm.pyLMM import chunks from gn2 import uses except ImportError: - print("WARNING: Standalone version missing the Genenetwork2 environment\n") has_gn2=False from standalone import uses + sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass #np.seterr('raise') diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index 5b93ae0d..aa6b473d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -29,7 +29,7 @@ try: from wqflask.my_pylmm.pyLMM import chunks from gn2 import uses except ImportError: - print("WARNING: Standalone version missing the Genenetwork2 environment\n") + sys.stderr.write("WARNING: LMM2 standalone version missing the Genenetwork2 environment\n") has_gn2=False from standalone import uses pass -- cgit v1.2.3 From f1056b9f4128fb91fbaf738914395697aa485b2e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 12:09:21 +0300 Subject: Warning to stderr --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 6 ++++-- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 5 +++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 66c952aa..95272818 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -57,10 +57,11 @@ import gwas # ---- A trick to decide on the environment: try: from wqflask.my_pylmm.pyLMM import chunks - from gn2 import uses + from gn2 import uses, set_progress_storage except ImportError: has_gn2=False - from standalone import uses + import standalone as handlers + from standalone import uses, set_progress_storage sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass @@ -816,6 +817,7 @@ def gn2_redis(key,species,new_code=True): params = json.loads(json_params) tempdata = temp_data.TempData(params['temp_uuid']) + set_progress_storage(tempdata) print('kinship', np.array(params['kinship_matrix'])) print('pheno', np.array(params['pheno_vector'])) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index 538007f1..e20d4092 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -17,7 +17,12 @@ logger = logging.getLogger('lmm2') logging.basicConfig(level=logging.DEBUG) np.set_printoptions(precision=3,suppress=True) +def set_progress_storage(location): + global storage + storage = location + def progress(location, count, total): + storage['percentage'] = round(count*100.0)/total) logger.info("Progress: %s %d%%" % (location,round(count*100.0/total))) def mprint(msg,data): -- cgit v1.2.3 From 6b8321d77e915dc5aec0c272c1cb84c2af3e6191 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 12:17:59 +0300 Subject: Replace progress meter --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 7 ++++++- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 17 ++++++----------- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index c71b9f22..f8033ac5 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -12,11 +12,16 @@ import logging # logging.basicConfig(level=logging.DEBUG) # np.set_printoptions() +def set_progress_storage(location): + global storage + storage = location + def progress(location, count, total): """ Progress update """ - logging.info("Progress: %s %d%%" % (location,round(count*100.0/total))) + storage.store("percent_complete",round(count*100.0)/total) + logger.info("Progress: %s %d%%" % (location,round(count*100.0/total))) def mprint(msg,data): """ diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 95272818..eab7d91d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -65,6 +65,8 @@ except ImportError: sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass +progress,info = uses('progress','info') + #np.seterr('raise') #def run_human(pheno_vector, @@ -171,10 +173,7 @@ def run_human(pheno_vector, #if count > 1000: # break count += 1 - - percent_complete = (float(count) / total_snps) * 100 - #print("percent_complete: ", percent_complete) - tempdata.store("percent_complete", percent_complete) + progress("human",count,total_snps) #with Bench("actual association"): ps, ts = human_association(snp, @@ -431,10 +430,7 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): continue keep.append(counter) genotype_matrix[:,counter] = (genotype_matrix[:,counter] - values_mean) / np.sqrt(vr) - - percent_complete = int(round((counter/m)*45)) - if temp_data != None: - temp_data.store("percent_complete", percent_complete) + progress('kinship_old',counter,m) genotype_matrix = genotype_matrix[:,keep] print("After kinship (old) genotype_matrix: ", pf(genotype_matrix)) @@ -539,9 +535,8 @@ def GWAS(pheno_vector, lmm_ob.fit(X=x) ts, ps, beta, betaVar = lmm_ob.association(x, REML=restricted_max_likelihood) - percent_complete = 45 + int(round((counter/m)*55)) - temp_data.store("percent_complete", percent_complete) - + progress("gwas_old",counter,m) + p_values.append(ps) t_statistics.append(ts) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index e20d4092..b3d480c3 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -22,7 +22,7 @@ def set_progress_storage(location): storage = location def progress(location, count, total): - storage['percentage'] = round(count*100.0)/total) + storage.store("percent_complete",round(count*100.0)/total) logger.info("Progress: %s %d%%" % (location,round(count*100.0/total))) def mprint(msg,data): -- cgit v1.2.3 From de84be30502af4be014fa4c0a2e7b54e51cff6f6 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 12:36:03 +0300 Subject: Progress handler --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 19 ++++++++++++++----- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 15 +++++++++++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index f8033ac5..b487ea25 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -12,17 +12,26 @@ import logging # logging.basicConfig(level=logging.DEBUG) # np.set_printoptions() +last_location = None +last_progress = 0 + def set_progress_storage(location): global storage storage = location def progress(location, count, total): - """ - Progress update - """ - storage.store("percent_complete",round(count*100.0)/total) - logger.info("Progress: %s %d%%" % (location,round(count*100.0/total))) + global last_location + global last_progress + + perc = round(count*100.0/total) + # print(last_progress,";",perc) + if perc != last_progress and (location != last_location or perc > 98 or perc > last_progress + 5): + storage.store("percent_complete",perc) + logger.info("Progress: %s %d%%" % (location,perc)) + last_location = location + last_progress = perc + def mprint(msg,data): """ Array/matrix print function diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index b3d480c3..7cc3e871 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -17,13 +17,24 @@ logger = logging.getLogger('lmm2') logging.basicConfig(level=logging.DEBUG) np.set_printoptions(precision=3,suppress=True) +last_location = None +last_progress = 0 + def set_progress_storage(location): global storage storage = location def progress(location, count, total): - storage.store("percent_complete",round(count*100.0)/total) - logger.info("Progress: %s %d%%" % (location,round(count*100.0/total))) + global last_location + global last_progress + + perc = round(count*100.0/total) + # print(last_progress,";",perc) + if perc != last_progress and (location != last_location or perc > 98 or perc > last_progress + 5): + storage.store("percent_complete",perc) + logger.info("Progress: %s %d%%" % (location,perc)) + last_location = location + last_progress = perc def mprint(msg,data): """ -- cgit v1.2.3 From f0653da318cac9736777495e40de6853227904ec Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 13:21:12 +0300 Subject: Cleaned up gwas.py to use uses and moved Redis call back into lmm.py --- wqflask/wqflask/my_pylmm/pyLMM/gwas.py | 70 +++++++++++----------------- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 10 ++-- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 31 +++++++----- 3 files changed, 52 insertions(+), 59 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gwas.py b/wqflask/wqflask/my_pylmm/pyLMM/gwas.py index b901c0e2..8b344a90 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gwas.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gwas.py @@ -19,7 +19,6 @@ import pdb import time -import sys # from utility import temp_data import lmm2 @@ -36,12 +35,10 @@ def formatResult(id,beta,betaSD,ts,ps): return "\t".join([str(x) for x in [id,beta,betaSD,ts,ps]]) + "\n" def compute_snp(j,n,snp_ids,lmm2,REML,q = None): - # print("COMPUTE SNP",j,snp_ids,"\n") result = [] for snp_id in snp_ids: snp,id = snp_id x = snp.reshape((n,1)) # all the SNPs - # print "X=",x # if refit: # L.fit(X=snp,REML=REML) ts,ps,beta,betaVar = lmm2.association(x,REML=REML,returnBeta=True) @@ -51,32 +48,28 @@ def compute_snp(j,n,snp_ids,lmm2,REML,q = None): q = compute_snp.q q.put([j,result]) return j - # PS.append(ps) - # TS.append(ts) - # return len(result) - # compute.q.put(result) - # return None def f_init(q): compute_snp.q = q -def gwas(Y,G,K,restricted_max_likelihood=True,refit=False,verbose=True): +def gwas(Y,G,K,uses,restricted_max_likelihood=True,refit=False,verbose=True): """ - Execute a GWAS. The G matrix should be n inds (cols) x m snps (rows) + GWAS. The G matrix should be n inds (cols) x m snps (rows) """ + progress,debug,info,mprint = uses('progress','debug','info','mprint') + matrix_initialize() cpu_num = mp.cpu_count() numThreads = None # for now use all available threads kfile2 = False reml = restricted_max_likelihood - sys.stderr.write(str(G.shape)+"\n") + mprint("G",G) n = G.shape[1] # inds inds = n m = G.shape[0] # snps snps = m - sys.stderr.write(str(m)+" SNPs\n") - # print "***** GWAS: G",G.shape,G + info("%s SNPs",snps) assert snps>inds, "snps should be larger than inds (snps=%d,inds=%d)" % (snps,inds) # CREATE LMM object for association @@ -85,19 +78,10 @@ def gwas(Y,G,K,restricted_max_likelihood=True,refit=False,verbose=True): lmm2 = LMM2(Y,K) # ,Kva,Kve,X0,verbose=verbose) if not refit: - if verbose: sys.stderr.write("Computing fit for null model\n") + info("Computing fit for null model") lmm2.fit() # follow GN model in run_other - if verbose: sys.stderr.write("\t heritability=%0.3f, sigma=%0.3f\n" % (lmm2.optH,lmm2.optSigma)) - - # outFile = "test.out" - # out = open(outFile,'w') - out = sys.stderr - - def outputResult(id,beta,betaSD,ts,ps): - out.write(formatResult(id,beta,betaSD,ts,ps)) - def printOutHead(): out.write("\t".join(["SNP_ID","BETA","BETA_SD","F_STAT","P_VALUE"]) + "\n") - - # printOutHead() + info("heritability=%0.3f, sigma=%0.3f" % (lmm2.optH,lmm2.optSigma)) + res = [] # Set up the pool @@ -106,26 +90,24 @@ def gwas(Y,G,K,restricted_max_likelihood=True,refit=False,verbose=True): p = mp.Pool(numThreads, f_init, [q]) collect = [] - # Buffers for pvalues and t-stats - # PS = [] - # TS = [] count = 0 job = 0 jobs_running = 0 + jobs_completed = 0 for snp in G: snp_id = (snp,'SNPID') count += 1 if count % 1000 == 0: job += 1 - if verbose: - sys.stderr.write("Job %d At SNP %d\n" % (job,count)) + debug("Job %d At SNP %d" % (job,count)) if numThreads == 1: - print "Running on 1 THREAD" + debug("Running on 1 THREAD") compute_snp(job,n,collect,lmm2,reml,q) collect = [] j,lst = q.get() - if verbose: - sys.stderr.write("Job "+str(j)+" finished\n") + debug("Job "+str(j)+" finished") + jobs_completed += 1 + progress("GWAS2",jobs_completed,snps/1000) res.append((j,lst)) else: p.apply_async(compute_snp,(job,n,collect,lmm2,reml)) @@ -134,8 +116,9 @@ def gwas(Y,G,K,restricted_max_likelihood=True,refit=False,verbose=True): while jobs_running > cpu_num: try: j,lst = q.get_nowait() - if verbose: - sys.stderr.write("Job "+str(j)+" finished\n") + debug("Job "+str(j)+" finished") + jobs_completed += 1 + progress("GWAS2",jobs_completed,snps/1000) res.append((j,lst)) jobs_running -= 1 except Queue.Empty: @@ -150,24 +133,23 @@ def gwas(Y,G,K,restricted_max_likelihood=True,refit=False,verbose=True): if numThreads==1 or count<1000 or len(collect)>0: job += 1 - print "Collect final batch size %i job %i @%i: " % (len(collect), job, count) + debug("Collect final batch size %i job %i @%i: " % (len(collect), job, count)) compute_snp(job,n,collect,lmm2,reml,q) collect = [] j,lst = q.get() res.append((j,lst)) - print "count=",count," running=",jobs_running," collect=",len(collect) + debug("count=%i running=%i collect=%i" % (count,jobs_running,len(collect))) for job in range(jobs_running): j,lst = q.get(True,15) # time out - if verbose: - sys.stderr.write("Job "+str(j)+" finished\n") + debug("Job "+str(j)+" finished") + jobs_completed += 1 + progress("GWAS2",jobs_completed,snps/1000) res.append((j,lst)) - print "Before sort",[res1[0] for res1 in res] + mprint("Before sort",[res1[0] for res1 in res]) res = sorted(res,key=lambda x: x[0]) - # if verbose: - # print "res=",res[0][0:10] - print "After sort",[res1[0] for res1 in res] - print [len(res1[1]) for res1 in res] + mprint("After sort",[res1[0] for res1 in res]) + info([len(res1[1]) for res1 in res]) ts = [item[0] for j,res1 in res for item in res1] ps = [item[1] for j,res1 in res for item in res1] return ts,ps diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index eab7d91d..1e00002a 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -57,11 +57,11 @@ import gwas # ---- A trick to decide on the environment: try: from wqflask.my_pylmm.pyLMM import chunks - from gn2 import uses, set_progress_storage + from gn2 import uses, progress_set_func except ImportError: has_gn2=False import standalone as handlers - from standalone import uses, set_progress_storage + from standalone import uses, progress_set_func sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass @@ -348,6 +348,7 @@ def run_other_new(pheno_vector, t_stats, p_values = gwas.gwas(Y, G.T, K, + uses, restricted_max_likelihood=True, refit=False,verbose=True) Bench().report() @@ -812,7 +813,10 @@ def gn2_redis(key,species,new_code=True): params = json.loads(json_params) tempdata = temp_data.TempData(params['temp_uuid']) - set_progress_storage(tempdata) + def update_tempdata(loc,i,total): + tempdata.store("percent_complete",round(i*100.0/total)) + debug("Updating REDIS percent_complete=%d" % (round(i*100.0/total))) + progress_set_func(update_tempdata) print('kinship', np.array(params['kinship_matrix'])) print('pheno', np.array(params['pheno_vector'])) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index 7cc3e871..36bf8fd5 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -17,24 +17,31 @@ logger = logging.getLogger('lmm2') logging.basicConfig(level=logging.DEBUG) np.set_printoptions(precision=3,suppress=True) -last_location = None -last_progress = 0 +progress_location = None +progress_current = None +progress_prev_perc = None -def set_progress_storage(location): - global storage - storage = location +def progress_default_func(location,count,total): + global progress_current + value = round(count*100.0/total) + progress_current = value + +progress_func = progress_default_func + +def progress_set_func(func): + global progress_func + progress_func = func def progress(location, count, total): - global last_location - global last_progress + global progress_location + global progress_prev_perc perc = round(count*100.0/total) - # print(last_progress,";",perc) - if perc != last_progress and (location != last_location or perc > 98 or perc > last_progress + 5): - storage.store("percent_complete",perc) + if perc != progress_prev_perc and (location != progress_location or perc > 98 or perc > progress_prev_perc + 5): + progress_func(location, count, total) logger.info("Progress: %s %d%%" % (location,perc)) - last_location = location - last_progress = perc + progress_location = location + progress_prev_perc = perc def mprint(msg,data): """ -- cgit v1.2.3 From 9b8a958494364fc6470cfe93f90d179e0bc7a787 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 13:23:06 +0300 Subject: Aligned gn2 handlers with standalone --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index b487ea25..f30cf1e6 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -12,25 +12,31 @@ import logging # logging.basicConfig(level=logging.DEBUG) # np.set_printoptions() -last_location = None -last_progress = 0 +progress_location = None +progress_current = None +progress_prev_perc = None -def set_progress_storage(location): - global storage - storage = location +def progress_default_func(location,count,total): + global progress_current + value = round(count*100.0/total) + progress_current = value + +progress_func = progress_default_func + +def progress_set_func(func): + global progress_func + progress_func = func def progress(location, count, total): - global last_location - global last_progress + global progress_location + global progress_prev_perc perc = round(count*100.0/total) - # print(last_progress,";",perc) - if perc != last_progress and (location != last_location or perc > 98 or perc > last_progress + 5): - storage.store("percent_complete",perc) + if perc != progress_prev_perc and (location != progress_location or perc > 98 or perc > progress_prev_perc + 5): + progress_func(location, count, total) logger.info("Progress: %s %d%%" % (location,perc)) - last_location = location - last_progress = perc - + progress_location = location + progress_prev_perc = perc def mprint(msg,data): """ -- cgit v1.2.3 From 130afd633fc50cbccaf2d12e5e643eb5f8b98c6f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 13:32:21 +0300 Subject: Add uses debug --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 1e00002a..e0fc8305 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -65,7 +65,7 @@ except ImportError: sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass -progress,info = uses('progress','info') +progress,debug,info = uses('progress','debug','info') #np.seterr('raise') -- cgit v1.2.3 From 803c3c56c37e448fd52fa102fdb6eef8431154cc Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 18 Mar 2015 13:36:35 +0300 Subject: Tagging 0.50-gn2-pre2 --- wqflask/wqflask/my_pylmm/README.md | 35 +++++++++++++++++------------- wqflask/wqflask/my_pylmm/pyLMM/__init__.py | 2 +- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/README.md b/wqflask/wqflask/my_pylmm/README.md index f6b0e72d..a84b5be2 100644 --- a/wqflask/wqflask/my_pylmm/README.md +++ b/wqflask/wqflask/my_pylmm/README.md @@ -1,21 +1,26 @@ -# RELEASE NOTES +# Genenetwork2/pylmm RELEASE NOTES -## 0.50-gn2-pre1 release +## 0.50-gn2-pre2 -This is the first test release of multi-core pylmm into GN2. Both -kinship calculation and GWAS have been made multi-threaded by -introducing the Python multiprocessing module. Note that only -run_other has been updated to use the new routines (so human is still -handled the old way). I have taken care that we can still run both -old-style and new-style LMM (through passing the 'new_code' -boolean). This could be an option in the web server for users to -select and test for any unexpected differences (of which there should -be none, naturally ;). +- Added abstractions for progress meter and info/debug statements; + Redis perc_complete is now updated through a lambda -The current version can handle missing phenotypes, but as they are -removed there is no way for GN2 to know what SNPs the P-values belong -to. A future version will pass a SNP index to allow for missing -phenotypes. +## 0.50-gn2-pre1 (release) + +- This is the first test release of multi-core pylmm into GN2. Both + kinship calculation and GWAS have been made multi-threaded by + introducing the Python multiprocessing module. Note that only + run_other has been updated to use the new routines (so human is + still handled the old way). I have taken care that we can still run + both old-style and new-style LMM (through passing the 'new_code' + boolean). This could be an option in the web server for users to + select and test for any unexpected differences (of which there + should be none, naturally ;). + +- The current version can handle missing phenotypes, but as they are + removed there is no way for GN2 to know what SNPs the P-values + belong to. A future version will pass a SNP index to allow for + missing phenotypes. \ No newline at end of file diff --git a/wqflask/wqflask/my_pylmm/pyLMM/__init__.py b/wqflask/wqflask/my_pylmm/pyLMM/__init__.py index c40c3221..6ab60d02 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/__init__.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/__init__.py @@ -1 +1 @@ -PYLMM_VERSION="0.50-gn2-pre1" +PYLMM_VERSION="0.50-gn2-pre2" -- cgit v1.2.3 From 8e9d7cde41800766fec835ca0c4a55c6327e05c8 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 20 Mar 2015 11:47:10 +0300 Subject: Trying to get kinship_old back in lmm1 --- wqflask/wqflask/my_pylmm/pyLMM/kinship.py | 14 +++++++----- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 35 ++++++++++++++--------------- wqflask/wqflask/my_pylmm/pyLMM/phenotype.py | 2 +- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 4 ++-- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py index 62f7be47..be12417e 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py @@ -28,17 +28,21 @@ import time from optmatrix import matrix_initialize, matrixMultT -def kinship_full(G): +def kinship_full(G,uses): """ Calculate the Kinship matrix using a full dot multiplication """ - print G.shape + info,mprint = uses('info','mprint') + + # mprint("kinship_full G",G) m = G.shape[0] # snps n = G.shape[1] # inds - sys.stderr.write(str(m)+" SNPs\n") - assert m>n, "n should be larger than m (snps>inds)" - m = np.dot(G.T,G) + info("%d SNPs",m) + assert m>n, "n should be larger than m (%d snps > %d inds)" % (m,n) + # m = np.dot(G.T,G) + m = matrixMultT(G.T) m = m/G.shape[0] + # mprint("kinship_full K",m) return m def compute_W(job,G,n,snps,compute_size): diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index e0fc8305..c040e3c2 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -65,7 +65,7 @@ except ImportError: sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass -progress,debug,info = uses('progress','debug','info') +progress,mprint,debug,info = uses('progress','mprint','debug','info') #np.seterr('raise') @@ -277,7 +277,7 @@ def run_other_old(pheno_vector, print("Running the original LMM engine in run_other (old)") print("REML=",restricted_max_likelihood," REFIT=",refit) with Bench("Calculate Kinship"): - kinship_matrix,genotype_matrix = calculate_kinship(genotype_matrix, tempdata) + kinship_matrix,genotype_matrix = calculate_kinship_old(genotype_matrix, tempdata) print("kinship_matrix: ", pf(kinship_matrix)) print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) @@ -331,7 +331,7 @@ def run_other_new(pheno_vector, # G = np.apply_along_axis( genotype.normalize, axis=1, arr=G) with Bench("Calculate Kinship"): - K,G = calculate_kinship(G, tempdata) + K,G = calculate_kinship_new(G, tempdata) print("kinship_matrix: ", pf(K)) print("kinship_matrix.shape: ", pf(K.shape)) @@ -393,9 +393,9 @@ def calculate_kinship_new(genotype_matrix, temp_data=None): Call the new kinship calculation where genotype_matrix contains inds (columns) by snps (rows). """ - print("call genotype.normalize") + info("call genotype.normalize") G = np.apply_along_axis( genotype.normalize, axis=0, arr=genotype_matrix) - print("call calculate_kinship_new") + info("call calculate_kinship_new") return kinship(G.T,uses),G # G gets transposed, we'll turn this into an iterator (FIXME) def calculate_kinship_old(genotype_matrix, temp_data=None): @@ -406,11 +406,11 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): normalizes the resulting vectors and returns the RRM matrix. """ - print("call calculate_kinship_old") + info("call calculate_kinship_old") n = genotype_matrix.shape[0] m = genotype_matrix.shape[1] - print("genotype 2D matrix n (inds) is:", n) - print("genotype 2D matrix m (snps) is:", m) + info("genotype 2D matrix n (inds) is: %d" % (n)) + info("genotype 2D matrix m (snps) is: %d" % (m)) assert m>n, "n should be larger than m (snps>inds)" keep = [] for counter in range(m): @@ -431,14 +431,13 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): continue keep.append(counter) genotype_matrix[:,counter] = (genotype_matrix[:,counter] - values_mean) / np.sqrt(vr) - progress('kinship_old',counter,m) + progress('kinship_old normalize genotype',counter,m) genotype_matrix = genotype_matrix[:,keep] - print("After kinship (old) genotype_matrix: ", pf(genotype_matrix)) - kinship_matrix = np.dot(genotype_matrix, genotype_matrix.T) * 1.0/float(m) - return kinship_matrix,genotype_matrix - -calculate_kinship = calculate_kinship_new # alias + mprint("After kinship (old) genotype_matrix: ", genotype_matrix) + # kinship_matrix = np.dot(genotype_matrix, genotype_matrix.T) * 1.0/float(m) + # return kinship_matrix,genotype_matrix + return kinship_full(genotype_matrix.T,uses),genotype_matrix def GWAS(pheno_vector, genotype_matrix, @@ -464,9 +463,9 @@ def GWAS(pheno_vector, refit - refit the variance component for each SNP """ - if kinship_eigen_vals == None: + if kinship_eigen_vals is None: kinship_eigen_vals = [] - if kinship_eigen_vectors == None: + if kinship_eigen_vectors is None: kinship_eigen_vectors = [] n = genotype_matrix.shape[0] @@ -570,7 +569,7 @@ class LMM: When this parameter is not provided, the constructor will set X0 to an n x 1 matrix of all ones to represent a mean effect. """ - if X0 == None: X0 = np.ones(len(Y)).reshape(len(Y),1) + if X0 is None: X0 = np.ones(len(Y)).reshape(len(Y),1) self.verbose = verbose #x = Y != -9 @@ -663,7 +662,7 @@ class LMM: REML is computed by adding additional terms to the standard LL and can be computed by setting REML=True. """ - if X == None: + if X is None: X = self.X0t elif stack: self.X0t_stack[:,(self.q)] = matrixMult(self.Kve.T,X)[:,0] diff --git a/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py b/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py index 682ba371..4c8175f7 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py @@ -24,7 +24,7 @@ def remove_missing(y,g,verbose=False): Remove missing data from matrices, make sure the genotype data has individuals as rows """ - assert(y!=None) + assert(y is not None) assert(y.shape[0] == g.shape[0]) y1 = y diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index e3e8659c..6a38da56 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -134,7 +134,7 @@ elif cmd == 'redis': # Emulating the redis setup of GN2 G = g print "Original G",G.shape, "\n", G - if y != None and options.remove_missing_phenotypes: + if y is not None and options.remove_missing_phenotypes: gnt = np.array(g).T Y,g,keep = phenotype.remove_missing(y,g.T,options.verbose) G = g.T @@ -165,7 +165,7 @@ elif cmd == 'redis': assert p1==0.0897, "p1=%f" % p1 assert p2==0.0405, "p2=%f" % p2 if options.geno == 'data/test8000.geno': - assert round(sum(ps)) == 4070 + assert int(sum(ps)) == 4070 assert len(ps) == 8000 elif cmd == 'kinship': G = g -- cgit v1.2.3 From 38594c7781b587a24be14b9631a73662ee3fdc2b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 20 Mar 2015 12:18:03 +0300 Subject: Fall back on calculate_kinship_new again --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 2 +- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index c040e3c2..a649029c 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -277,7 +277,7 @@ def run_other_old(pheno_vector, print("Running the original LMM engine in run_other (old)") print("REML=",restricted_max_likelihood," REFIT=",refit) with Bench("Calculate Kinship"): - kinship_matrix,genotype_matrix = calculate_kinship_old(genotype_matrix, tempdata) + kinship_matrix,genotype_matrix = calculate_kinship_new(genotype_matrix, tempdata) print("kinship_matrix: ", pf(kinship_matrix)) print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 6a38da56..88e2a033 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -184,7 +184,7 @@ elif cmd == 'kinship': gnt = None if options.test_kinship: - K = kinship_full(np.copy(G)) + K = kinship_full(np.copy(G),uses) print "Genotype",G.shape, "\n", G print "first Kinship method",K.shape,"\n",K k1 = round(K[0][0],4) -- cgit v1.2.3 From 490e0919b2757f6815a7e6c7f0cb08e55e1cd02e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 30 Mar 2015 10:32:11 +0200 Subject: Percentage complete: Add method description --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 8844118f..200424ba 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -814,6 +814,9 @@ def gn2_redis(key,species,new_code=True): tempdata = temp_data.TempData(params['temp_uuid']) def update_tempdata(loc,i,total): + """ + This is the single method that updates Redis for percentage complete! + """ tempdata.store("percent_complete",round(i*100.0/total)) debug("Updating REDIS percent_complete=%d" % (round(i*100.0/total))) progress_set_func(update_tempdata) -- cgit v1.2.3 From 6fc112431c0edb0ecae6cd5fa45716c349094a7f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 30 Mar 2015 11:49:43 +0200 Subject: Use of is vs == when testing None --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 4 ++-- wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 200424ba..f0473f99 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -278,7 +278,7 @@ def run_other_old(pheno_vector, print("Running the original LMM engine in run_other (old)") print("REML=",restricted_max_likelihood," REFIT=",refit) with Bench("Calculate Kinship"): - kinship_matrix,genotype_matrix = calculate_kinship_new(genotype_matrix, tempdata) + kinship_matrix,genotype_matrix = calculate_kinship_old(genotype_matrix, tempdata) print("kinship_matrix: ", pf(kinship_matrix)) print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) @@ -880,7 +880,7 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): k = kinship.tolist() params = dict(pheno_vector = pheno.tolist(), genotype_matrix = geno.tolist(), - kinship_matrix= k, + kinship_matrix = k, restricted_max_likelihood = True, refit = False, temp_uuid = "testrun_temp_uuid", diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index aa6b473d..d67e1205 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -85,7 +85,7 @@ def GWAS(Y, X, K, Kva=[], Kve=[], X0=None, REML=True, refit=False): print("genotype matrix n is:", n) print("genotype matrix m is:", m) - if X0 == None: + if X0 is None: X0 = np.ones((n,1)) # Remove missing values in Y and adjust associated parameters @@ -173,7 +173,7 @@ class LMM2: When this parameter is not provided, the constructor will set X0 to an n x 1 matrix of all ones to represent a mean effect. """ - if X0 == None: + if X0 is None: X0 = np.ones(len(Y)).reshape(len(Y),1) self.verbose = verbose @@ -260,7 +260,7 @@ class LMM2: REML is computed by adding additional terms to the standard LL and can be computed by setting REML=True. """ - if X == None: X = self.X0t + if X is None: X = self.X0t elif stack: self.X0t_stack[:,(self.q)] = matrixMult(self.Kve.T,X)[:,0] X = self.X0t_stack @@ -316,7 +316,7 @@ class LMM2: Given this optimum, the function computes the LL and associated ML solutions. """ - if X == None: X = self.X0t + if X is None: X = self.X0t else: #X = np.hstack([self.X0t,matrixMult(self.Kve.T, X)]) self.X0t_stack[:,(self.q)] = matrixMult(self.Kve.T,X)[:,0] @@ -340,7 +340,7 @@ class LMM2: def association(self,X,h=None,stack=True,REML=True,returnBeta=False): """ Calculates association statitics for the SNPs encoded in the vector X of size n. - If h == None, the optimal h stored in optH is used. + If h is None, the optimal h stored in optH is used. """ if False: @@ -358,7 +358,7 @@ class LMM2: self.X0t_stack[:,(self.q)] = m X = self.X0t_stack - if h == None: h = self.optH + if h is None: h = self.optH L,beta,sigma,betaVAR = self.LL(h,X,stack=False,REML=REML) q = len(beta) -- cgit v1.2.3 From 8b88be4f48baa6cd0cc3c37a851144d5b1dc24af Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 30 Mar 2015 13:01:22 +0200 Subject: Refactoring genotype normalization --- wqflask/wqflask/my_pylmm/pyLMM/genotype.py | 19 ++++++++++--------- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 9 +++++---- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 2 ++ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/genotype.py b/wqflask/wqflask/my_pylmm/pyLMM/genotype.py index 315fd824..49f32e3a 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/genotype.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/genotype.py @@ -37,14 +37,15 @@ def normalize(ind_g): Run for every SNP list (for one individual) and return normalized SNP genotype values with missing data filled in """ - g1 = np.copy(ind_g) # avoid side effects - x = True - np.isnan(ind_g) # Matrix of True/False - m = ind_g[x].mean() # Global mean value - s = np.sqrt(ind_g[x].var()) # Global stddev - g1[np.isnan(ind_g)] = m # Plug-in mean values for missing data - if s == 0: - g1 = g1 - m # Subtract the mean + g = np.copy(ind_g) # copy to avoid side effects + missing = np.isnan(g) + values = g[True - missing] + mean = values.mean() # Global mean value + stddev = np.sqrt(values.var()) # Global stddev + g[missing] = mean # Plug-in mean values for missing data + if stddev == 0: + g = g - mean # Subtract the mean else: - g1 = (g1 - m) / s # Normalize the deviation - return g1 + g = (g - mean) / stddev # Normalize the deviation + return g diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index f0473f99..035f31e8 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -414,6 +414,7 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): info("genotype 2D matrix m (snps) is: %d" % (m)) assert m>n, "n should be larger than m (snps>inds)" keep = [] + mprint("G (before old normalize)",genotype_matrix) for counter in range(m): #print("type of genotype_matrix[:,counter]:", pf(genotype_matrix[:,counter])) #Checks if any values in column are not numbers @@ -435,10 +436,10 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): progress('kinship_old normalize genotype',counter,m) genotype_matrix = genotype_matrix[:,keep] - mprint("After kinship (old) genotype_matrix: ", genotype_matrix) - # kinship_matrix = np.dot(genotype_matrix, genotype_matrix.T) * 1.0/float(m) - # return kinship_matrix,genotype_matrix - return kinship_full(genotype_matrix.T,uses),genotype_matrix + mprint("G (after old normalize)",genotype_matrix.T) + kinship_matrix = np.dot(genotype_matrix, genotype_matrix.T) * 1.0/float(m) + return kinship_matrix,genotype_matrix + # return kinship_full(genotype_matrix.T,uses),genotype_matrix def GWAS(pheno_vector, genotype_matrix, diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 88e2a033..fc7a4b9d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -106,6 +106,8 @@ if options.geno: if cmd == 'redis_new': # The main difference between redis_new and redis is that missing # phenotypes are handled by the first + if options.remove_missing_phenotypes: + raise Exception('Can not use --remove-missing-phenotypes with LMM2') Y = y G = g print "Original G",G.shape, "\n", G -- cgit v1.2.3 From 153317412a090d5b17bc176ff7da2e61e6ec4f2c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Apr 2015 09:55:42 +0200 Subject: Make the new version of genotype normalization default --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 15 ++++++++++----- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 5 +++-- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 4 ++-- wqflask/wqflask/my_pylmm/pyLMM/standalone.py | 5 +++++ 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index f30cf1e6..7bceb089 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -45,15 +45,20 @@ def mprint(msg,data): m = np.array(data) print(msg,m.shape,"=\n",m) +def fatal(msg): + logger.critical(msg) + raise Exception(msg) + def callbacks(): return dict( write = sys.stdout.write, writeln = print, - debug = logging.debug, - info = logging.info, - warning = logging.warning, - error = logging.error, - critical = logging.critical, + debug = logger.debug, + info = logger.info, + warning = logger.warning, + error = logger.error, + critical = logger.critical, + fatal = fatal, progress = progress, mprint = mprint ) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 035f31e8..8be3fc6f 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -66,7 +66,7 @@ except ImportError: sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") pass -progress,mprint,debug,info = uses('progress','mprint','debug','info') +progress,mprint,debug,info,fatal = uses('progress','mprint','debug','info','fatal') #np.seterr('raise') @@ -278,7 +278,7 @@ def run_other_old(pheno_vector, print("Running the original LMM engine in run_other (old)") print("REML=",restricted_max_likelihood," REFIT=",refit) with Bench("Calculate Kinship"): - kinship_matrix,genotype_matrix = calculate_kinship_old(genotype_matrix, tempdata) + kinship_matrix,genotype_matrix = calculate_kinship_new(genotype_matrix, tempdata) print("kinship_matrix: ", pf(kinship_matrix)) print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) @@ -408,6 +408,7 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): """ info("call calculate_kinship_old") + fatal("THE FUNCTION calculate_kinship_old IS OBSOLETE, use calculate_kinship_new instead - see Genotype Normalization Problem on Pjotr's blog") n = genotype_matrix.shape[0] m = genotype_matrix.shape[1] info("genotype 2D matrix n (inds) is: %d" % (n)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index fc7a4b9d..ef0bdd7e 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -21,7 +21,7 @@ from optparse import OptionParser import sys import tsvreader import numpy as np -from lmm import gn2_load_redis, calculate_kinship_old +from lmm import gn2_load_redis, calculate_kinship_new from kinship import kinship, kinship_full import genotype import phenotype @@ -190,7 +190,7 @@ elif cmd == 'kinship': print "Genotype",G.shape, "\n", G print "first Kinship method",K.shape,"\n",K k1 = round(K[0][0],4) - K2,G = calculate_kinship_old(np.copy(G).T,temp_data=None) + K2,G = calculate_kinship_new(np.copy(G).T,temp_data=None) print "Genotype",G.shape, "\n", G print "GN2 Kinship method",K2.shape,"\n",K2 k2 = round(K2[0][0],4) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py index 36bf8fd5..40b2021d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/standalone.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/standalone.py @@ -57,6 +57,10 @@ def mprint(msg,data): m[-2][0:3]," ... ",m[-2][-3:],"\n ", m[-1][0:3]," ... ",m[-1][-3:],"]") +def fatal(msg): + logger.critical(msg) + raise Exception(msg) + def callbacks(): return dict( write = sys.stdout.write, @@ -66,6 +70,7 @@ def callbacks(): warning = logger.warning, error = logger.error, critical = logger.critical, + fatal = fatal, progress = progress, mprint = mprint ) -- cgit v1.2.3 From 0f132d0cc4a77e69ab593fd9c8a2d5218d083ed7 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Apr 2015 10:15:49 +0200 Subject: Release 0.50-gn2 --- wqflask/wqflask/my_pylmm/README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/README.md b/wqflask/wqflask/my_pylmm/README.md index a84b5be2..4845ec03 100644 --- a/wqflask/wqflask/my_pylmm/README.md +++ b/wqflask/wqflask/my_pylmm/README.md @@ -1,11 +1,15 @@ # Genenetwork2/pylmm RELEASE NOTES -## 0.50-gn2-pre2 +## 0.50-gn2 (April 2nd, 2015) + +- Replaced the GN2 genotype normalization + +## 0.50-gn2-pre2 (March 18, 2015) - Added abstractions for progress meter and info/debug statements; Redis perc_complete is now updated through a lambda -## 0.50-gn2-pre1 (release) +## 0.50-gn2-pre1 (release, March 17, 2015) - This is the first test release of multi-core pylmm into GN2. Both kinship calculation and GWAS have been made multi-threaded by -- cgit v1.2.3 From 43295e57621e9a08ca4cb90e95cc14a87e0d8b5e Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Apr 2015 12:04:14 +0200 Subject: Create test geno iterator --- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 9 +++++++-- wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py | 25 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index ef0bdd7e..5a4bd268 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -99,11 +99,16 @@ if options.pheno: y = tsvreader.pheno(options.pheno) print y.shape -if options.geno: +if options.geno and cmd != 'iterator': g = tsvreader.geno(options.geno) print g.shape -if cmd == 'redis_new': +if cmd == 'iterator': + print "ITERATE over SNPs" + def pretty(snpid,values): + print snpid,values + print tsvreader.geno_iter(options.geno,pretty) +elif cmd == 'redis_new': # The main difference between redis_new and redis is that missing # phenotypes are handled by the first if options.remove_missing_phenotypes: diff --git a/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py b/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py index b4027fa3..7fe75e3f 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py @@ -74,3 +74,28 @@ def geno(fn): G = np.array(G1) return G +def geno(fn): + G1 = [] + def append(id,values): + G1.append(values) # <--- slow + geno_iter(fn,append) + G = np.array(G1) + return G + +def geno_iter(fn,func): + hab_mapper = {'A':0,'H':1,'B':2,'-':3} + pylmm_mapper = [ 0.0, 0.5, 1.0, float('nan') ] + + print fn + with open(fn,'r') as tsvin: + assert(tsvin.readline().strip() == "# Genotype format version 1.0") + tsvin.readline() + tsvin.readline() + tsvin.readline() + tsvin.readline() + tsv = csv.reader(tsvin, delimiter='\t') + for row in tsv: + id = row[0] + gs = list(row[1]) + gs2 = [pylmm_mapper[hab_mapper[g]] for g in gs] + func(id,gs2) -- cgit v1.2.3 From 5151bc389aa98415da9f4d49b3c279ed1380ea7d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Apr 2015 12:14:43 +0200 Subject: Prepare iterator --- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 5a4bd268..036bf7d5 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -104,10 +104,17 @@ if options.geno and cmd != 'iterator': print g.shape if cmd == 'iterator': - print "ITERATE over SNPs" - def pretty(snpid,values): - print snpid,values - print tsvreader.geno_iter(options.geno,pretty) + def snp_iterator(func): + tsvreader.geno_iter(options.geno,func) + + if options.remove_missing_phenotypes: + raise Exception('Can not use --remove-missing-phenotypes with LMM2') + ps, ts = gn2_iter_redis('testrun_iter','other',k,y,snp_iterator) + print np.array(ps) + print len(ps),sum(ps) + # Test results + p1 = round(ps[0],4) + p2 = round(ps[-1],4) elif cmd == 'redis_new': # The main difference between redis_new and redis is that missing # phenotypes are handled by the first -- cgit v1.2.3 From b9c79ef58ff6ec4da3e65290ea802c783bb17742 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 2 Apr 2015 13:40:42 +0200 Subject: Passing in an iterator --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 33 ++++++++++++++++++++++++++++- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 6 ++---- wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py | 26 ++++++++++++++++++++--- 3 files changed, 57 insertions(+), 8 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 8be3fc6f..07b55726 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -875,6 +875,9 @@ def gn2_main(): gn2_redis(key,species) def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): + """ + This function emulates current GN2 behaviour by pre-loading Redis + """ print("Loading Redis from parsed data") if kinship == None: k = None @@ -896,7 +899,35 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): Redis.expire(key, 60*60) return gn2_redis(key,species,new_code) - + +def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): + """ + This function emulates GN2 behaviour by pre-loading Redis with + a SNP iterator + """ + print("Loading Redis using a SNP iterator") + if kinship == None: + k = None + else: + k = kinship.tolist() + params = dict(pheno_vector = pheno.tolist(), + genotype_matrix = geno_iterator.tolist(), + kinship_matrix = k, + restricted_max_likelihood = True, + refit = False, + temp_uuid = "testrun_temp_uuid", + + # meta data + timestamp = datetime.datetime.now().isoformat(), + ) + + json_params = json.dumps(params) + Redis.set(key, json_params) + Redis.expire(key, 60*60) + + return gn2_redis(key,species,new_code) + + if __name__ == '__main__': print("WARNING: Calling pylmm from lmm.py will become OBSOLETE, use runlmm.py instead!") if has_gn2: diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 036bf7d5..3b0672b4 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -21,7 +21,7 @@ from optparse import OptionParser import sys import tsvreader import numpy as np -from lmm import gn2_load_redis, calculate_kinship_new +from lmm import gn2_load_redis, gn2_iter_redis, calculate_kinship_new from kinship import kinship, kinship_full import genotype import phenotype @@ -104,11 +104,9 @@ if options.geno and cmd != 'iterator': print g.shape if cmd == 'iterator': - def snp_iterator(func): - tsvreader.geno_iter(options.geno,func) - if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') + snp_iterator = tsvreader.geno_iter(options.geno) ps, ts = gn2_iter_redis('testrun_iter','other',k,y,snp_iterator) print np.array(ps) print len(ps),sum(ps) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py b/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py index 7fe75e3f..27daf43f 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py @@ -76,13 +76,12 @@ def geno(fn): def geno(fn): G1 = [] - def append(id,values): + for id,values in geno_iter(fn): G1.append(values) # <--- slow - geno_iter(fn,append) G = np.array(G1) return G -def geno_iter(fn,func): +def geno_callback(fn,func): hab_mapper = {'A':0,'H':1,'B':2,'-':3} pylmm_mapper = [ 0.0, 0.5, 1.0, float('nan') ] @@ -99,3 +98,24 @@ def geno_iter(fn,func): gs = list(row[1]) gs2 = [pylmm_mapper[hab_mapper[g]] for g in gs] func(id,gs2) + +def geno_iter(fn): + """ + Yield a tuple of snpid and values + """ + hab_mapper = {'A':0,'H':1,'B':2,'-':3} + pylmm_mapper = [ 0.0, 0.5, 1.0, float('nan') ] + + print fn + with open(fn,'r') as tsvin: + assert(tsvin.readline().strip() == "# Genotype format version 1.0") + tsvin.readline() + tsvin.readline() + tsvin.readline() + tsvin.readline() + tsv = csv.reader(tsvin, delimiter='\t') + for row in tsv: + id = row[0] + gs = list(row[1]) + gs2 = [pylmm_mapper[hab_mapper[g]] for g in gs] + yield (id,gs2) -- cgit v1.2.3 From 146b4a45c28b7d3ba4bf982cfaf93eda2e71d1ea Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 3 Apr 2015 10:58:53 +0200 Subject: Refactoring GN2 interface --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 07b55726..6e22e6c9 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -805,7 +805,7 @@ class LMM: pl.title(title) -def gn2_redis(key,species,new_code=True): +def gwas_using_redis(key,species,new_code=True): """ Invoke pylmm using Redis as a container. new_code runs the new version @@ -861,18 +861,6 @@ def gn2_redis(key,species,new_code=True): Redis.expire(results_key, 60*60) return ps, ts -# This is the main function used by Genenetwork2 (with environment) -def gn2_main(): - parser = argparse.ArgumentParser(description='Run pyLMM') - parser.add_argument('-k', '--key') - parser.add_argument('-s', '--species') - - opts = parser.parse_args() - - key = opts.key - species = opts.species - - gn2_redis(key,species) def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): """ @@ -898,7 +886,7 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): Redis.set(key, json_params) Redis.expire(key, 60*60) - return gn2_redis(key,species,new_code) + return gwas_using_redis(key,species,new_code) def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): """ @@ -925,7 +913,23 @@ def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): Redis.set(key, json_params) Redis.expire(key, 60*60) - return gn2_redis(key,species,new_code) + return gwas_using_redis(key,species,new_code) + +# This is the main function used by Genenetwork2 (with environment) +# +# Note that this calling route will become OBSOLETE (we should use runlmm.py +# instead) +def gn2_main(): + parser = argparse.ArgumentParser(description='Run pyLMM') + parser.add_argument('-k', '--key') + parser.add_argument('-s', '--species') + + opts = parser.parse_args() + + key = opts.key + species = opts.species + + gwas_using_redis(key,species) if __name__ == '__main__': -- cgit v1.2.3 From fabbcac393627badf0542377fc22325ae7e96f3d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 3 Apr 2015 11:15:29 +0200 Subject: Passing in an iterator --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 6e22e6c9..b8650938 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -891,15 +891,21 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): """ This function emulates GN2 behaviour by pre-loading Redis with - a SNP iterator + a SNP iterator, for this it sets a key for every genotype (SNP) """ print("Loading Redis using a SNP iterator") + for i,genotypes in enumerate(geno_iterator): + gkey = key+'_geno_'+str(i) + Redis.set(gkey, genotypes) + Redis.expire(gkey, 60*60) + if kinship == None: k = None else: k = kinship.tolist() params = dict(pheno_vector = pheno.tolist(), - genotype_matrix = geno_iterator.tolist(), + genotype_matrix = "iterator", + genotypes = i, kinship_matrix = k, restricted_max_likelihood = True, refit = False, @@ -913,7 +919,7 @@ def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): Redis.set(key, json_params) Redis.expire(key, 60*60) - return gwas_using_redis(key,species,new_code) + return gwas_using_redis(key,species) # This is the main function used by Genenetwork2 (with environment) # -- cgit v1.2.3 From 7d13eec7f67578aa75d8430bb5ed74a4dd825b51 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 3 Apr 2015 12:10:55 +0200 Subject: Refactoring Redis use to one function --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 65 ++++++++++++++++++-------------- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 4 +- 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index b8650938..88ca6a7f 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -805,6 +805,36 @@ class LMM: pl.title(title) +def gwas_without_redis(species,k,y,geno,cov,reml,refit,inputfn,new_code): + """ + Invoke pylmm using a genotype (SNP) iterator + """ + info("gwas_without_redis") + print('pheno', y) + + if species == "human" : + print('kinship', k ) + ps, ts = run_human(pheno_vector = y, + covariate_matrix = cov, + plink_input_file = inputfn, + kinship_matrix = k, + refit = refit, tempdata=tempdata) + else: + print('geno', geno.shape, geno) + + if new_code: + ps, ts = run_other_new(pheno_vector = y, + genotype_matrix = geno, + restricted_max_likelihood = reml, + refit = refit, + tempdata = tempdata) + else: + ps, ts = run_other_old(pheno_vector = y, + genotype_matrix = geno, + restricted_max_likelihood = reml, + refit = refit, + tempdata = tempdata) + def gwas_using_redis(key,species,new_code=True): """ Invoke pylmm using Redis as a container. new_code runs the new @@ -823,33 +853,7 @@ def gwas_using_redis(key,species,new_code=True): debug("Updating REDIS percent_complete=%d" % (round(i*100.0/total))) progress_set_func(update_tempdata) - - print('pheno', np.array(params['pheno_vector'])) - - if species == "human" : - print('kinship', np.array(params['kinship_matrix'])) - ps, ts = run_human(pheno_vector = np.array(params['pheno_vector']), - covariate_matrix = np.array(params['covariate_matrix']), - plink_input_file = params['input_file_name'], - kinship_matrix = np.array(params['kinship_matrix']), - refit = params['refit'], - tempdata = tempdata) - else: - geno = np.array(params['genotype_matrix']) - print('geno', geno.shape, geno) - - if new_code: - ps, ts = run_other_new(pheno_vector = np.array(params['pheno_vector']), - genotype_matrix = geno, - restricted_max_likelihood = params['restricted_max_likelihood'], - refit = params['refit'], - tempdata = tempdata) - else: - ps, ts = run_other_old(pheno_vector = np.array(params['pheno_vector']), - genotype_matrix = geno, - restricted_max_likelihood = params['restricted_max_likelihood'], - refit = params['refit'], - tempdata = tempdata) + ps,ts = gwas_without_redis(species,np.array(params['kinship_matrix']),np.array(params['pheno_vector']),np.array(params['genotype_matrix']),np.array(params['covariate_matrix']),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) results_key = "pylmm:results:" + params['temp_uuid'] @@ -874,6 +878,8 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): params = dict(pheno_vector = pheno.tolist(), genotype_matrix = geno.tolist(), kinship_matrix = k, + covariate_matrix = None, + input_file_name = None, restricted_max_likelihood = True, refit = False, temp_uuid = "testrun_temp_uuid", @@ -888,7 +894,7 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): return gwas_using_redis(key,species,new_code) -def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): +def gn2_load_redis_iter(key,species,kinship,pheno,geno_iterator): """ This function emulates GN2 behaviour by pre-loading Redis with a SNP iterator, for this it sets a key for every genotype (SNP) @@ -907,6 +913,8 @@ def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): genotype_matrix = "iterator", genotypes = i, kinship_matrix = k, + covariate_matrix = None, + input_file_name = None, restricted_max_likelihood = True, refit = False, temp_uuid = "testrun_temp_uuid", @@ -918,7 +926,6 @@ def gn2_iter_redis(key,species,kinship,pheno,geno_iterator): json_params = json.dumps(params) Redis.set(key, json_params) Redis.expire(key, 60*60) - return gwas_using_redis(key,species) # This is the main function used by Genenetwork2 (with environment) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 3b0672b4..ab698e41 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -21,7 +21,7 @@ from optparse import OptionParser import sys import tsvreader import numpy as np -from lmm import gn2_load_redis, gn2_iter_redis, calculate_kinship_new +from lmm import gn2_load_redis, gn2_load_redis_iter, calculate_kinship_new from kinship import kinship, kinship_full import genotype import phenotype @@ -107,7 +107,7 @@ if cmd == 'iterator': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') snp_iterator = tsvreader.geno_iter(options.geno) - ps, ts = gn2_iter_redis('testrun_iter','other',k,y,snp_iterator) + ps, ts = gn2_load_redis_iter('testrun_iter','other',k,y,snp_iterator) print np.array(ps) print len(ps),sum(ps) # Test results -- cgit v1.2.3 From fc6f0ef9fc8d2607e70c775c51ca55f50806cc7a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 3 Apr 2015 13:13:09 +0200 Subject: temp_data is no longer passed around --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 50 +++++++++++++++----------------- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 2 +- 2 files changed, 24 insertions(+), 28 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 88ca6a7f..9e25f56d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -81,8 +81,7 @@ def run_human(pheno_vector, covariate_matrix, plink_input_file, kinship_matrix, - refit=False, - tempdata=None): + refit=False): v = np.isnan(pheno_vector) keep = True - v @@ -262,23 +261,19 @@ def human_association(snp, def run_other_old(pheno_vector, genotype_matrix, restricted_max_likelihood=True, - refit=False, - tempdata=None # <---- can not be None - ): + refit=False): """Takes the phenotype vector and genotype matrix and returns a set of p-values and t-statistics restricted_max_likelihood -- whether to use restricted max likelihood; True or False refit -- whether to refit the variance component for each marker - temp_data -- TempData object that stores the progress for each major step of the - calculations ("calculate_kinship" and "GWAS" take the majority of time) """ print("Running the original LMM engine in run_other (old)") print("REML=",restricted_max_likelihood," REFIT=",refit) with Bench("Calculate Kinship"): - kinship_matrix,genotype_matrix = calculate_kinship_new(genotype_matrix, tempdata) + kinship_matrix,genotype_matrix = calculate_kinship_new(genotype_matrix) print("kinship_matrix: ", pf(kinship_matrix)) print("kinship_matrix.shape: ", pf(kinship_matrix.shape)) @@ -297,24 +292,19 @@ def run_other_old(pheno_vector, genotype_matrix, kinship_matrix, restricted_max_likelihood=True, - refit=False, - temp_data=tempdata) + refit=False) Bench().report() return p_values, t_stats def run_other_new(pheno_vector, genotype_matrix, restricted_max_likelihood=True, - refit=False, - tempdata=None # <---- can not be None - ): + refit=False): """Takes the phenotype vector and genotype matrix and returns a set of p-values and t-statistics restricted_max_likelihood -- whether to use restricted max likelihood; True or False refit -- whether to refit the variance component for each marker - temp_data -- TempData object that stores the progress for each major step of the - calculations ("calculate_kinship" and "GWAS" take the majority of time) """ @@ -332,7 +322,7 @@ def run_other_new(pheno_vector, # G = np.apply_along_axis( genotype.normalize, axis=1, arr=G) with Bench("Calculate Kinship"): - K,G = calculate_kinship_new(G, tempdata) + K,G = calculate_kinship_new(G) print("kinship_matrix: ", pf(K)) print("kinship_matrix.shape: ", pf(K.shape)) @@ -815,25 +805,24 @@ def gwas_without_redis(species,k,y,geno,cov,reml,refit,inputfn,new_code): if species == "human" : print('kinship', k ) ps, ts = run_human(pheno_vector = y, - covariate_matrix = cov, - plink_input_file = inputfn, - kinship_matrix = k, - refit = refit, tempdata=tempdata) + covariate_matrix = cov, + plink_input_file = inputfn, + kinship_matrix = k, + refit = refit) else: print('geno', geno.shape, geno) if new_code: ps, ts = run_other_new(pheno_vector = y, - genotype_matrix = geno, - restricted_max_likelihood = reml, - refit = refit, - tempdata = tempdata) + genotype_matrix = geno, + restricted_max_likelihood = reml, + refit = refit) else: ps, ts = run_other_old(pheno_vector = y, genotype_matrix = geno, restricted_max_likelihood = reml, - refit = refit, - tempdata = tempdata) + refit = refit) + return ps,ts def gwas_using_redis(key,species,new_code=True): """ @@ -853,7 +842,14 @@ def gwas_using_redis(key,species,new_code=True): debug("Updating REDIS percent_complete=%d" % (round(i*100.0/total))) progress_set_func(update_tempdata) - ps,ts = gwas_without_redis(species,np.array(params['kinship_matrix']),np.array(params['pheno_vector']),np.array(params['genotype_matrix']),np.array(params['covariate_matrix']),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) + def narray(key): + print(key) + v = params[key] + if v is not None: + v = np.array(v) + return v + + ps,ts = gwas_without_redis(species,narray('kinship_matrix'),narray('pheno_vector'),narray('genotype_matrix'),narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) results_key = "pylmm:results:" + params['temp_uuid'] diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index ab698e41..3801529e 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -200,7 +200,7 @@ elif cmd == 'kinship': print "Genotype",G.shape, "\n", G print "first Kinship method",K.shape,"\n",K k1 = round(K[0][0],4) - K2,G = calculate_kinship_new(np.copy(G).T,temp_data=None) + K2,G = calculate_kinship_new(np.copy(G).T) print "Genotype",G.shape, "\n", G print "GN2 Kinship method",K2.shape,"\n",K2 k2 = round(K2[0][0],4) -- cgit v1.2.3 From 3c738e6901ecc2ec0b4c1c667f20ebe3dc186f5c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 3 Apr 2015 13:17:56 +0200 Subject: Rename gwas_using_redis to gwas_with_redis --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 9e25f56d..ad6375e9 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -795,7 +795,7 @@ class LMM: pl.title(title) -def gwas_without_redis(species,k,y,geno,cov,reml,refit,inputfn,new_code): +def run_gwas(species,k,y,geno,cov,reml,refit,inputfn,new_code): """ Invoke pylmm using a genotype (SNP) iterator """ @@ -824,10 +824,10 @@ def gwas_without_redis(species,k,y,geno,cov,reml,refit,inputfn,new_code): refit = refit) return ps,ts -def gwas_using_redis(key,species,new_code=True): +def gwas_with_redis(key,species,new_code=True): """ Invoke pylmm using Redis as a container. new_code runs the new - version + version. All the Redis code goes here! """ json_params = Redis.get(key) @@ -849,7 +849,7 @@ def gwas_using_redis(key,species,new_code=True): v = np.array(v) return v - ps,ts = gwas_without_redis(species,narray('kinship_matrix'),narray('pheno_vector'),narray('genotype_matrix'),narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) + ps,ts = run_gwas(species,narray('kinship_matrix'),narray('pheno_vector'),narray('genotype_matrix'),narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) results_key = "pylmm:results:" + params['temp_uuid'] @@ -888,7 +888,7 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): Redis.set(key, json_params) Redis.expire(key, 60*60) - return gwas_using_redis(key,species,new_code) + return gwas_with_redis(key,species,new_code) def gn2_load_redis_iter(key,species,kinship,pheno,geno_iterator): """ @@ -922,7 +922,7 @@ def gn2_load_redis_iter(key,species,kinship,pheno,geno_iterator): json_params = json.dumps(params) Redis.set(key, json_params) Redis.expire(key, 60*60) - return gwas_using_redis(key,species) + return gwas_with_redis(key,species) # This is the main function used by Genenetwork2 (with environment) # @@ -938,7 +938,7 @@ def gn2_main(): key = opts.key species = opts.species - gwas_using_redis(key,species) + gwas_with_redis(key,species) if __name__ == '__main__': -- cgit v1.2.3 From e9865707ef447b8bc23eb8c872703f156936499d Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 3 Apr 2015 14:03:32 +0200 Subject: - Calculate n,m from the start - added test function to runlmm.py to run without Redis (25% faster) --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 17 ++++++++++------- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 17 +++++++++++++++-- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index ad6375e9..e51742c4 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -795,9 +795,9 @@ class LMM: pl.title(title) -def run_gwas(species,k,y,geno,cov,reml,refit,inputfn,new_code): +def run_gwas(species,n,m,k,y,geno,cov=None,reml=True,refit=False,inputfn=None,new_code=True): """ - Invoke pylmm using a genotype (SNP) iterator + Invoke pylmm using genotype as a matrix or as a (SNP) iterator. """ info("gwas_without_redis") print('pheno', y) @@ -848,8 +848,11 @@ def gwas_with_redis(key,species,new_code=True): if v is not None: v = np.array(v) return v - - ps,ts = run_gwas(species,narray('kinship_matrix'),narray('pheno_vector'),narray('genotype_matrix'),narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) + + y = narray('pheno_vector') + n = len(y) + m = params['num_genotypes'] + ps,ts = run_gwas(species,n,m,narray('kinship_matrix'),y,narray('genotype_matrix'),narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) results_key = "pylmm:results:" + params['temp_uuid'] @@ -873,6 +876,7 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): k = kinship.tolist() params = dict(pheno_vector = pheno.tolist(), genotype_matrix = geno.tolist(), + num_genotypes = geno.shape[1], kinship_matrix = k, covariate_matrix = None, input_file_name = None, @@ -881,8 +885,7 @@ def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): temp_uuid = "testrun_temp_uuid", # meta data - timestamp = datetime.datetime.now().isoformat(), - ) + timestamp = datetime.datetime.now().isoformat()) json_params = json.dumps(params) Redis.set(key, json_params) @@ -907,7 +910,7 @@ def gn2_load_redis_iter(key,species,kinship,pheno,geno_iterator): k = kinship.tolist() params = dict(pheno_vector = pheno.tolist(), genotype_matrix = "iterator", - genotypes = i, + num_genotypes = i, kinship_matrix = k, covariate_matrix = None, input_file_name = None, diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 3801529e..f095bb73 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -21,7 +21,7 @@ from optparse import OptionParser import sys import tsvreader import numpy as np -from lmm import gn2_load_redis, gn2_load_redis_iter, calculate_kinship_new +from lmm import gn2_load_redis, gn2_load_redis_iter, calculate_kinship_new, run_gwas from kinship import kinship, kinship_full import genotype import phenotype @@ -103,7 +103,20 @@ if options.geno and cmd != 'iterator': g = tsvreader.geno(options.geno) print g.shape -if cmd == 'iterator': +if cmd == 'run': + if options.remove_missing_phenotypes: + raise Exception('Can not use --remove-missing-phenotypes with LMM2') + snp_iterator = tsvreader.geno_iter(options.geno) + n = len(y) + m = g.shape[1] + ps, ts = run_gwas('other',n,m,k,y,g.T) + print np.array(ps) + print len(ps),sum(ps) + # Test results + p1 = round(ps[0],4) + p2 = round(ps[-1],4) + +elif cmd == 'iterator': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') snp_iterator = tsvreader.geno_iter(options.geno) -- cgit v1.2.3 From 163fe965bc1dcb807124c1c70c965d48bf2c2688 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 4 Apr 2015 09:52:24 +0200 Subject: Consolidate tests now they all agree for redis, redis_new and run --- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 74 +++++++++++++------------------- 1 file changed, 30 insertions(+), 44 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index f095bb73..2d02e195 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -27,6 +27,8 @@ import genotype import phenotype from standalone import uses +progress,mprint,debug,info,fatal = uses('progress','mprint','debug','info','fatal') + usage = """ python runlmm.py [options] command @@ -103,6 +105,29 @@ if options.geno and cmd != 'iterator': g = tsvreader.geno(options.geno) print g.shape +def check_results(ps,ts): + print np.array(ps) + print len(ps),sum(ps) + # Test results + p1 = round(ps[0],4) + p2 = round(ps[-1],4) + # sys.stderr.write(options.geno+"\n") + if options.geno == 'data/small.geno': + info("Validating results for "+options.geno) + assert p1==0.0708, "p1=%f" % p1 + assert p2==0.1417, "p2=%f" % p2 + if options.geno == 'data/small_na.geno': + info("Validating results for "+options.geno) + assert p1==0.0897, "p1=%f" % p1 + assert p2==0.0405, "p2=%f" % p2 + if options.geno == 'data/test8000.geno': + info("Validating results for "+options.geno) + # assert p1==0.8984, "p1=%f" % p1 + # assert p2==0.9621, "p2=%f" % p2 + assert round(sum(ps)) == 4070 + assert len(ps) == 8000 + info("Run completed") + if cmd == 'run': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') @@ -110,22 +135,13 @@ if cmd == 'run': n = len(y) m = g.shape[1] ps, ts = run_gwas('other',n,m,k,y,g.T) - print np.array(ps) - print len(ps),sum(ps) - # Test results - p1 = round(ps[0],4) - p2 = round(ps[-1],4) - + check_results(ps,ts) elif cmd == 'iterator': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') snp_iterator = tsvreader.geno_iter(options.geno) ps, ts = gn2_load_redis_iter('testrun_iter','other',k,y,snp_iterator) - print np.array(ps) - print len(ps),sum(ps) - # Test results - p1 = round(ps[0],4) - p2 = round(ps[-1],4) + check_results(ps,ts) elif cmd == 'redis_new': # The main difference between redis_new and redis is that missing # phenotypes are handled by the first @@ -138,23 +154,7 @@ elif cmd == 'redis_new': gt = G.T G = None ps, ts = gn2_load_redis('testrun','other',k,Y,gt,new_code=True) - print np.array(ps) - print len(ps),sum(ps) - # Test results - p1 = round(ps[0],4) - p2 = round(ps[-1],4) - sys.stderr.write(options.geno+"\n") - if options.geno == 'data/small.geno': - assert p1==0.0708, "p1=%f" % p1 - assert p2==0.1417, "p2=%f" % p2 - if options.geno == 'data/small_na.geno': - assert p1==0.0897, "p1=%f" % p1 - assert p2==0.0405, "p2=%f" % p2 - if options.geno == 'data/test8000.geno': - # assert p1==0.8984, "p1=%f" % p1 - # assert p2==0.9621, "p2=%f" % p2 - assert round(sum(ps)) == 4070 - assert len(ps) == 8000 + check_results(ps,ts) elif cmd == 'redis': # Emulating the redis setup of GN2 G = g @@ -177,21 +177,7 @@ elif cmd == 'redis': gt = G.T G = None ps, ts = gn2_load_redis('testrun','other',k,Y,gt, new_code=False) - print np.array(ps) - print len(ps),sum(ps) - # Test results 4070.02346579 - p1 = round(ps[0],4) - p2 = round(ps[-1],4) - sys.stderr.write(options.geno+"\n") - if options.geno == 'data/small.geno': - assert p1==0.0708, "p1=%f" % p1 - assert p2==0.1417, "p2=%f" % p2 - if options.geno == 'data/small_na.geno': - assert p1==0.0897, "p1=%f" % p1 - assert p2==0.0405, "p2=%f" % p2 - if options.geno == 'data/test8000.geno': - assert int(sum(ps)) == 4070 - assert len(ps) == 8000 + check_results(ps,ts) elif cmd == 'kinship': G = g print "Original G",G.shape, "\n", G @@ -235,4 +221,4 @@ elif cmd == 'kinship': assert k3==1.4352, "k3=%f" % k3 else: - print "Doing nothing" + fatal("Doing nothing") -- cgit v1.2.3 From 99fef2888f02551191cf6031c2c7222fce27e360 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 4 Apr 2015 12:33:07 +0200 Subject: Run works without transposes --- wqflask/wqflask/my_pylmm/pyLMM/gwas.py | 21 +++++++--- wqflask/wqflask/my_pylmm/pyLMM/kinship.py | 24 +++++++---- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 65 +++++++++++++++++++---------- wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 43 ++++++++++--------- wqflask/wqflask/my_pylmm/pyLMM/phenotype.py | 35 +++++++++++++--- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 21 +++++----- 6 files changed, 136 insertions(+), 73 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gwas.py b/wqflask/wqflask/my_pylmm/pyLMM/gwas.py index 8b344a90..ae3769d4 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gwas.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gwas.py @@ -19,7 +19,7 @@ import pdb import time -# from utility import temp_data +import sys import lmm2 import os @@ -31,6 +31,18 @@ from lmm2 import LMM2 import multiprocessing as mp # Multiprocessing is part of the Python stdlib import Queue +# ---- A trick to decide on the environment: +try: + from wqflask.my_pylmm.pyLMM import chunks + from gn2 import uses +except ImportError: + sys.stderr.write("WARNING: LMM2 standalone version missing the Genenetwork2 environment\n") + has_gn2=False + from standalone import uses + +progress,mprint,debug,info,fatal = uses('progress','mprint','debug','info','fatal') + + def formatResult(id,beta,betaSD,ts,ps): return "\t".join([str(x) for x in [id,beta,betaSD,ts,ps]]) + "\n" @@ -52,12 +64,11 @@ def compute_snp(j,n,snp_ids,lmm2,REML,q = None): def f_init(q): compute_snp.q = q -def gwas(Y,G,K,uses,restricted_max_likelihood=True,refit=False,verbose=True): +def gwas(Y,G,K,restricted_max_likelihood=True,refit=False,verbose=True): """ GWAS. The G matrix should be n inds (cols) x m snps (rows) """ - progress,debug,info,mprint = uses('progress','debug','info','mprint') - + info("In gwas.gwas") matrix_initialize() cpu_num = mp.cpu_count() numThreads = None # for now use all available threads @@ -70,7 +81,7 @@ def gwas(Y,G,K,uses,restricted_max_likelihood=True,refit=False,verbose=True): m = G.shape[0] # snps snps = m info("%s SNPs",snps) - assert snps>inds, "snps should be larger than inds (snps=%d,inds=%d)" % (snps,inds) + assert snps>=inds, "snps should be larger than inds (snps=%d,inds=%d)" % (snps,inds) # CREATE LMM object for association # if not kfile2: L = LMM(Y,K,Kva,Kve,X0,verbose=verbose) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py index be12417e..1c157fd8 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/kinship.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/kinship.py @@ -28,12 +28,21 @@ import time from optmatrix import matrix_initialize, matrixMultT -def kinship_full(G,uses): +# ---- A trick to decide on the environment: +try: + from wqflask.my_pylmm.pyLMM import chunks + from gn2 import uses, progress_set_func +except ImportError: + has_gn2=False + import standalone as handlers + from standalone import uses, progress_set_func + +progress,debug,info,mprint = uses('progress','debug','info','mprint') + +def kinship_full(G): """ Calculate the Kinship matrix using a full dot multiplication """ - info,mprint = uses('info','mprint') - # mprint("kinship_full G",G) m = G.shape[0] # snps n = G.shape[1] # inds @@ -78,8 +87,7 @@ def f_init(q): # Calculate the kinship matrix from G (SNPs as rows!), returns K # -def kinship(G,uses,computeSize=1000,numThreads=None,useBLAS=False): - progress,debug,info,mprint = uses('progress','debug','info','mprint') +def kinship(G,computeSize=1000,numThreads=None,useBLAS=False): matrix_initialize(useBLAS) @@ -89,7 +97,7 @@ def kinship(G,uses,computeSize=1000,numThreads=None,useBLAS=False): m = G.shape[0] # snps snps = m info("%i SNPs" % (m)) - assert snps>inds, "snps should be larger than inds (%i snps, %i inds)" % (snps,inds) + assert snps>=inds, "snps should be larger than inds (%i snps, %i inds)" % (snps,inds) q = mp.Queue() p = mp.Pool(numThreads, f_init, [q]) @@ -140,13 +148,11 @@ def kinship(G,uses,computeSize=1000,numThreads=None,useBLAS=False): K = K / float(snps) return K -def kvakve(K,uses): +def kvakve(K): """ Obtain eigendecomposition for K and return Kva,Kve where Kva is cleaned of small values < 1e-6 (notably smaller than zero) """ - info,mprint = uses('info','mprint') - info("Obtaining eigendecomposition for %dx%d matrix" % (K.shape[0],K.shape[1]) ) Kva,Kve = linalg.eigh(K) mprint("Kva",Kva) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index e51742c4..82bd7f0b 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -64,7 +64,6 @@ except ImportError: import standalone as handlers from standalone import uses, progress_set_func sys.stderr.write("WARNING: LMM standalone version missing the Genenetwork2 environment\n") - pass progress,mprint,debug,info,fatal = uses('progress','mprint','debug','info','fatal') @@ -296,8 +295,8 @@ def run_other_old(pheno_vector, Bench().report() return p_values, t_stats -def run_other_new(pheno_vector, - genotype_matrix, +def run_other_new(n,m,pheno_vector, + geno, restricted_max_likelihood=True, refit=False): @@ -312,8 +311,7 @@ def run_other_new(pheno_vector, print("REML=",restricted_max_likelihood," REFIT=",refit) # Adjust phenotypes - Y,G,keep = phenotype.remove_missing(pheno_vector,genotype_matrix,verbose=True) - print("Removed missing phenotypes",Y.shape) + n,Y,keep = phenotype.remove_missing_new(n,pheno_vector) # if options.maf_normalization: # G = np.apply_along_axis( genotype.replace_missing_with_MAF, axis=0, arr=g ) @@ -321,8 +319,9 @@ def run_other_new(pheno_vector, # if not options.skip_genotype_normalization: # G = np.apply_along_axis( genotype.normalize, axis=1, arr=G) + geno = geno[:,keep] with Bench("Calculate Kinship"): - K,G = calculate_kinship_new(G) + K,G = calculate_kinship_new(geno) print("kinship_matrix: ", pf(K)) print("kinship_matrix.shape: ", pf(K.shape)) @@ -337,9 +336,8 @@ def run_other_new(pheno_vector, with Bench("Doing GWAS"): t_stats, p_values = gwas.gwas(Y, - G.T, + G, K, - uses, restricted_max_likelihood=True, refit=False,verbose=True) Bench().report() @@ -378,18 +376,30 @@ def matrixMult(A,B): return linalg.fblas.dgemm(alpha=1.,a=AA,b=BB,trans_a=transA,trans_b=transB) +def calculate_kinship_new(genotype_matrix): + """ + Call the new kinship calculation where genotype_matrix contains + inds (columns) by snps (rows). + """ + assert type(genotype_matrix) is np.ndarray + info("call genotype.normalize") + G = np.apply_along_axis( genotype.normalize, axis=1, arr=genotype_matrix) + mprint("G",genotype_matrix) + info("call calculate_kinship_new") + return kinship(G),G # G gets transposed, we'll turn this into an iterator (FIXME) -def calculate_kinship_new(genotype_matrix, temp_data=None): +def calculate_kinship_iter(geno): """ Call the new kinship calculation where genotype_matrix contains inds (columns) by snps (rows). """ + assert type(genotype_matrix) is iter info("call genotype.normalize") G = np.apply_along_axis( genotype.normalize, axis=0, arr=genotype_matrix) info("call calculate_kinship_new") - return kinship(G.T,uses),G # G gets transposed, we'll turn this into an iterator (FIXME) + return kinship(G) -def calculate_kinship_old(genotype_matrix, temp_data=None): +def calculate_kinship_old(genotype_matrix): """ genotype_matrix is an n x m matrix encoding SNP minor alleles. @@ -430,7 +440,7 @@ def calculate_kinship_old(genotype_matrix, temp_data=None): mprint("G (after old normalize)",genotype_matrix.T) kinship_matrix = np.dot(genotype_matrix, genotype_matrix.T) * 1.0/float(m) return kinship_matrix,genotype_matrix - # return kinship_full(genotype_matrix.T,uses),genotype_matrix + # return kinship_full(genotype_matrix.T),genotype_matrix def GWAS(pheno_vector, genotype_matrix, @@ -586,7 +596,7 @@ class LMM: # if self.verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) begin = time.time() # Kva,Kve = linalg.eigh(K) - Kva,Kve = kvakve(K,uses) + Kva,Kve = kvakve(K) end = time.time() if self.verbose: sys.stderr.write("Total time: %0.3f\n" % (end - begin)) print("sum(Kva),sum(Kve)=",sum(Kva),sum(Kve)) @@ -794,12 +804,11 @@ class LMM: pl.ylabel("Probability of data") pl.title(title) - def run_gwas(species,n,m,k,y,geno,cov=None,reml=True,refit=False,inputfn=None,new_code=True): """ Invoke pylmm using genotype as a matrix or as a (SNP) iterator. """ - info("gwas_without_redis") + info("run_gwas") print('pheno', y) if species == "human" : @@ -813,8 +822,8 @@ def run_gwas(species,n,m,k,y,geno,cov=None,reml=True,refit=False,inputfn=None,ne print('geno', geno.shape, geno) if new_code: - ps, ts = run_other_new(pheno_vector = y, - genotype_matrix = geno, + ps, ts = run_other_new(n,m,pheno_vector = y, + geno = geno, restricted_max_likelihood = reml, refit = refit) else: @@ -849,10 +858,20 @@ def gwas_with_redis(key,species,new_code=True): v = np.array(v) return v + def narrayT(key): + m = narray(key) + if m is not None: + return m.T + return m + + # We are transposing before we enter run_gwas - this should happen on the webserver + # side (or when reading data from file) + k = narray('kinship_matrix') + g = narrayT('genotype_matrix') y = narray('pheno_vector') n = len(y) m = params['num_genotypes'] - ps,ts = run_gwas(species,n,m,narray('kinship_matrix'),y,narray('genotype_matrix'),narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) + ps,ts = run_gwas(species,n,m,k,y,g,narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) results_key = "pylmm:results:" + params['temp_uuid'] @@ -864,19 +883,19 @@ def gwas_with_redis(key,species,new_code=True): Redis.expire(results_key, 60*60) return ps, ts - def gn2_load_redis(key,species,kinship,pheno,geno,new_code=True): """ - This function emulates current GN2 behaviour by pre-loading Redis + This function emulates current GN2 behaviour by pre-loading Redis (note the input + genotype is transposed to emulate GN2 (FIXME!) """ - print("Loading Redis from parsed data") + info("Loading Redis from parsed data") if kinship == None: k = None else: k = kinship.tolist() params = dict(pheno_vector = pheno.tolist(), - genotype_matrix = geno.tolist(), - num_genotypes = geno.shape[1], + genotype_matrix = geno.T.tolist(), + num_genotypes = geno.shape[0], kinship_matrix = k, covariate_matrix = None, input_file_name = None, diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index d67e1205..358bf27e 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -32,7 +32,6 @@ except ImportError: sys.stderr.write("WARNING: LMM2 standalone version missing the Genenetwork2 environment\n") has_gn2=False from standalone import uses - pass def calculateKinship(W,center=False): """ @@ -149,28 +148,32 @@ def GWAS(Y, X, K, Kva=[], Kve=[], X0=None, REML=True, refit=False): class LMM2: - """ - This is a simple version of EMMA/fastLMM. - The main purpose of this module is to take a phenotype vector (Y), a set of covariates (X) and a kinship matrix (K) - and to optimize this model by finding the maximum-likelihood estimates for the model parameters. - There are three model parameters: heritability (h), covariate coefficients (beta) and the total - phenotypic variance (sigma). - Heritability as defined here is the proportion of the total variance (sigma) that is attributed to - the kinship matrix. - - For simplicity, we assume that everything being input is a numpy array. - If this is not the case, the module may throw an error as conversion from list to numpy array - is not done consistently. + """This is a simple version of EMMA/fastLMM. + + The main purpose of this module is to take a phenotype vector (Y), + a set of covariates (X) and a kinship matrix (K) and to optimize + this model by finding the maximum-likelihood estimates for the + model parameters. There are three model parameters: heritability + (h), covariate coefficients (beta) and the total phenotypic + variance (sigma). Heritability as defined here is the proportion + of the total variance (sigma) that is attributed to the kinship + matrix. + + For simplicity, we assume that everything being input is a numpy + array. If this is not the case, the module may throw an error as + conversion from list to numpy array is not done consistently. """ def __init__(self,Y,K,Kva=[],Kve=[],X0=None,verbose=False): - """ - The constructor takes a phenotype vector or array Y of size n. - It takes a kinship matrix K of size n x n. Kva and Kve can be computed as Kva,Kve = linalg.eigh(K) and cached. - If they are not provided, the constructor will calculate them. - X0 is an optional covariate matrix of size n x q, where there are q covariates. - When this parameter is not provided, the constructor will set X0 to an n x 1 matrix of all ones to represent a mean effect. + """The constructor takes a phenotype vector or array Y of size n. It + takes a kinship matrix K of size n x n. Kva and Kve can be + computed as Kva,Kve = linalg.eigh(K) and cached. If they are + not provided, the constructor will calculate them. X0 is an + optional covariate matrix of size n x q, where there are q + covariates. When this parameter is not provided, the + constructor will set X0 to an n x 1 matrix of all ones to + represent a mean effect. """ if X0 is None: @@ -194,7 +197,7 @@ class LMM2: # if self.verbose: sys.stderr.write("Obtaining eigendecomposition for %dx%d matrix\n" % (K.shape[0],K.shape[1]) ) begin = time.time() # Kva,Kve = linalg.eigh(K) - Kva,Kve = kinship.kvakve(K,uses) + Kva,Kve = kinship.kvakve(K) end = time.time() if self.verbose: sys.stderr.write("Total time: %0.3f\n" % (end - begin)) print("sum(Kva),sum(Kve)=",sum(Kva),sum(Kve)) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py b/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py index 4c8175f7..7b652515 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/phenotype.py @@ -19,22 +19,47 @@ import sys import numpy as np -def remove_missing(y,g,verbose=False): +# ---- A trick to decide on the environment: +try: + from wqflask.my_pylmm.pyLMM import chunks + from gn2 import uses, progress_set_func +except ImportError: + has_gn2=False + import standalone as handlers + from standalone import uses, progress_set_func + +progress,debug,info,mprint = uses('progress','debug','info','mprint') + +def remove_missing(n,y,g): """ Remove missing data from matrices, make sure the genotype data has individuals as rows """ assert(y is not None) - assert(y.shape[0] == g.shape[0]) + assert y.shape[0] == g.shape[0],"y (n) %d, g (n,m) %s" % (y.shape[0],g.shape) y1 = y g1 = g v = np.isnan(y) keep = True - v if v.sum(): - if verbose: - sys.stderr.write("runlmm.py: Cleaning the phenotype vector and genotype matrix by removing %d individuals...\n" % (v.sum())) + info("runlmm.py: Cleaning the phenotype vector and genotype matrix by removing %d individuals...\n" % (v.sum())) y1 = y[keep] g1 = g[keep,:] - return y1,g1,keep + n = y1.shape[0] + return n,y1,g1,keep + +def remove_missing_new(n,y): + """ + Remove missing data. Returns new n,y,keep + """ + assert(y is not None) + y1 = y + v = np.isnan(y) + keep = True - v + if v.sum(): + info("runlmm.py: Cleaning the phenotype vector by removing %d individuals" % (v.sum())) + y1 = y[keep] + n = y1.shape[0] + return n,y1,keep diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 2d02e195..d248dee2 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -131,16 +131,15 @@ def check_results(ps,ts): if cmd == 'run': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') - snp_iterator = tsvreader.geno_iter(options.geno) n = len(y) m = g.shape[1] - ps, ts = run_gwas('other',n,m,k,y,g.T) + ps, ts = run_gwas('other',n,m,k,y,g) # <--- pass in geno by SNP check_results(ps,ts) elif cmd == 'iterator': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') - snp_iterator = tsvreader.geno_iter(options.geno) - ps, ts = gn2_load_redis_iter('testrun_iter','other',k,y,snp_iterator) + geno_iterator = tsvreader.geno_iter(options.geno) + ps, ts = gn2_load_redis_iter('testrun_iter','other',k,y,geno_iterator) check_results(ps,ts) elif cmd == 'redis_new': # The main difference between redis_new and redis is that missing @@ -150,10 +149,9 @@ elif cmd == 'redis_new': Y = y G = g print "Original G",G.shape, "\n", G - - gt = G.T - G = None - ps, ts = gn2_load_redis('testrun','other',k,Y,gt,new_code=True) + # gt = G.T + # G = None + ps, ts = gn2_load_redis('testrun','other',k,Y,G,new_code=True) check_results(ps,ts) elif cmd == 'redis': # Emulating the redis setup of GN2 @@ -174,9 +172,10 @@ elif cmd == 'redis': g = None gnt = None - gt = G.T - G = None - ps, ts = gn2_load_redis('testrun','other',k,Y,gt, new_code=False) + # gt = G.T + # G = None + mprint("G",G) + ps, ts = gn2_load_redis('testrun','other',k,Y,G, new_code=False) check_results(ps,ts) elif cmd == 'kinship': G = g -- cgit v1.2.3 From 49f5eb3e825c953bc7f6da87460ccfe9b891d493 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 4 Apr 2015 13:01:44 +0200 Subject: Fixing transpose issues --- wqflask/wqflask/my_pylmm/pyLMM/gwas.py | 1 - wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 2 +- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 26 ++++++++++++-------------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gwas.py b/wqflask/wqflask/my_pylmm/pyLMM/gwas.py index ae3769d4..247a8729 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gwas.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gwas.py @@ -36,7 +36,6 @@ try: from wqflask.my_pylmm.pyLMM import chunks from gn2 import uses except ImportError: - sys.stderr.write("WARNING: LMM2 standalone version missing the Genenetwork2 environment\n") has_gn2=False from standalone import uses diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 82bd7f0b..6f03eaf7 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -288,7 +288,7 @@ def run_other_old(pheno_vector, with Bench("Doing GWAS"): t_stats, p_values = GWAS(pheno_vector, - genotype_matrix, + genotype_matrix.T, kinship_matrix, restricted_max_likelihood=True, refit=False) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index d248dee2..44d5c0f4 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -108,26 +108,25 @@ if options.geno and cmd != 'iterator': def check_results(ps,ts): print np.array(ps) print len(ps),sum(ps) - # Test results p1 = round(ps[0],4) p2 = round(ps[-1],4) - # sys.stderr.write(options.geno+"\n") if options.geno == 'data/small.geno': info("Validating results for "+options.geno) - assert p1==0.0708, "p1=%f" % p1 - assert p2==0.1417, "p2=%f" % p2 + assert p1==0.7387, "p1=%f" % p1 + assert p2==0.7387, "p2=%f" % p2 if options.geno == 'data/small_na.geno': info("Validating results for "+options.geno) - assert p1==0.0897, "p1=%f" % p1 - assert p2==0.0405, "p2=%f" % p2 + assert p1==0.062, "p1=%f" % p1 + assert p2==0.062, "p2=%f" % p2 if options.geno == 'data/test8000.geno': info("Validating results for "+options.geno) - # assert p1==0.8984, "p1=%f" % p1 - # assert p2==0.9621, "p2=%f" % p2 assert round(sum(ps)) == 4070 assert len(ps) == 8000 info("Run completed") - + +if y is not None: + n = y.shape[0] + if cmd == 'run': if options.remove_missing_phenotypes: raise Exception('Can not use --remove-missing-phenotypes with LMM2') @@ -159,7 +158,7 @@ elif cmd == 'redis': print "Original G",G.shape, "\n", G if y is not None and options.remove_missing_phenotypes: gnt = np.array(g).T - Y,g,keep = phenotype.remove_missing(y,g.T,options.verbose) + n,Y,g,keep = phenotype.remove_missing(n,y,gnt) G = g.T print "Removed missing phenotypes",G.shape, "\n", G else: @@ -174,7 +173,6 @@ elif cmd == 'redis': # gt = G.T # G = None - mprint("G",G) ps, ts = gn2_load_redis('testrun','other',k,Y,G, new_code=False) check_results(ps,ts) elif cmd == 'kinship': @@ -182,7 +180,7 @@ elif cmd == 'kinship': print "Original G",G.shape, "\n", G if y != None and options.remove_missing_phenotypes: gnt = np.array(g).T - Y,g = phenotype.remove_missing(y,g.T,options.verbose) + n,Y,g,keep = phenotype.remove_missing(n,y,g.T) G = g.T print "Removed missing phenotypes",G.shape, "\n", G if options.maf_normalization: @@ -194,7 +192,7 @@ elif cmd == 'kinship': gnt = None if options.test_kinship: - K = kinship_full(np.copy(G),uses) + K = kinship_full(np.copy(G)) print "Genotype",G.shape, "\n", G print "first Kinship method",K.shape,"\n",K k1 = round(K[0][0],4) @@ -204,7 +202,7 @@ elif cmd == 'kinship': k2 = round(K2[0][0],4) print "Genotype",G.shape, "\n", G - K3 = kinship(G.T,uses) + K3 = kinship(G.T) print "third Kinship method",K3.shape,"\n",K3 sys.stderr.write(options.geno+"\n") k3 = round(K3[0][0],4) -- cgit v1.2.3 From 17f453e50ebac657d9f3096811d92bedc9bfc064 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 4 Apr 2015 13:15:48 +0200 Subject: Regression tests --- wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 44d5c0f4..52c3c80a 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -196,20 +196,20 @@ elif cmd == 'kinship': print "Genotype",G.shape, "\n", G print "first Kinship method",K.shape,"\n",K k1 = round(K[0][0],4) - K2,G = calculate_kinship_new(np.copy(G).T) + K2,G = calculate_kinship_new(np.copy(G)) print "Genotype",G.shape, "\n", G print "GN2 Kinship method",K2.shape,"\n",K2 k2 = round(K2[0][0],4) print "Genotype",G.shape, "\n", G - K3 = kinship(G.T) + K3 = kinship(G) print "third Kinship method",K3.shape,"\n",K3 sys.stderr.write(options.geno+"\n") k3 = round(K3[0][0],4) if options.geno == 'data/small.geno': - assert k1==0.8, "k1=%f" % k1 - assert k2==0.7939, "k2=%f" % k2 - assert k3==0.7939, "k3=%f" % k3 + assert k1==0.8333, "k1=%f" % k1 + assert k2==0.9375, "k2=%f" % k2 + assert k3==0.9375, "k3=%f" % k3 if options.geno == 'data/small_na.geno': assert k1==0.8333, "k1=%f" % k1 assert k2==0.7172, "k2=%f" % k2 -- cgit v1.2.3 From 102523493e2f8a7660c63f117f1d8dfd009eff02 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 13 Apr 2015 08:14:43 +0000 Subject: Improved assertion message --- wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py b/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py index b4027fa3..b24ffe8f 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/tsvreader.py @@ -56,7 +56,8 @@ def geno(fn): print fn with open(fn,'r') as tsvin: - assert(tsvin.readline().strip() == "# Genotype format version 1.0") + line = tsvin.readline().strip() + assert line == "# Genotype format version 1.0", line tsvin.readline() tsvin.readline() tsvin.readline() -- cgit v1.2.3 From 6cef9c3b27d92383f89a432f6fa0e9fd16107f66 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 13 Apr 2015 10:15:59 +0200 Subject: Added examples for convertlmm.py --- wqflask/wqflask/my_pylmm/pyLMM/convertlmm.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/convertlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/convertlmm.py index 3b6b5d70..4312fed0 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/convertlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/convertlmm.py @@ -1,5 +1,5 @@ -# This is a converter for common LMM formats, so as to keep complexity -# outside the main routines. +# This is a converter for common LMM formats, so as to keep file +# reader complexity outside the main routines. # Copyright (C) 2015 Pjotr Prins (pjotr.prins@thebird.nl) # @@ -31,6 +31,12 @@ python convertlmm.py [--plink] [--prefix out_basename] [--kinship kfile] [--phen Convert files for runlmm.py processing. Writes to stdout by default. try --help for more information + +Examples: + + python ./my_pylmm/pyLMM/convertlmm.py --plink --pheno data/test_snps.132k.clean.noX.fake.phenos > test.pheno + + python ./my_pylmm/pyLMM/convertlmm.py --plink --pheno data/test_snps.132k.clean.noX.fake.phenos --geno data/test_snps.132k.clean.noX > test.geno """ # if len(args) == 0: -- cgit v1.2.3 From 85ccb971687fda00538b248722454ea2aa514e27 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Mon, 13 Apr 2015 10:16:11 +0200 Subject: Started on INSTALL information --- INSTALL.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 INSTALL.md diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 00000000..26eacc3a --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,26 @@ +# INSTALL Genenetwork2 (GN2) + +## Fetch GN2 from github + +Clone the repository (currently ~800Mb) to local + + git clone git@github.com:genenetwork2/genenetwork2.git + +## Dependencies + +GN2 requires + +* redis +* mysql + +## Required python modules + +Install the following python modules: + +* Flask +* pyyaml +* redis +* qtlreaper +* numarray +* pp +* Flask-SQLAlchemy -- cgit v1.2.3 From 25bb886b733362edea657c72e7d29172b7e22755 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 17 Apr 2015 14:33:26 +0200 Subject: INSTALL --- INSTALL.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 26eacc3a..84b3d37c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -24,3 +24,41 @@ Install the following python modules: * numarray * pp * Flask-SQLAlchemy + +## Set up local file settings.py + +```python +LOGFILE = """/tmp/flask_gn_log""" + +#This is needed because Flask turns key errors into a +#400 bad request response with no exception/log +TRAP_BAD_REQUEST_ERRORS = True + +DB_URI = """mysql://gn2:password@localhost/db_webqtl""" +SQLALCHEMY_DATABASE_URI = 'mysql://gn2:password@localhost/db_webqtl' + +# http://pythonhosted.org/Flask-Security/configuration.html +SECURITY_CONFIRMABLE = True +SECURITY_TRACKABLE = True +SECURITY_REGISTERABLE = True +SECURITY_RECOVERABLE = True + +SECURITY_EMAIL_SENDER = "no-reply@genenetwork.org" +SECURITY_POST_LOGIN_VIEW = "/thank_you" +SQLALCHEMY_POOL_RECYCLE = 3600 + +SERVER_PORT = 5051 + +SECRET_HMAC_CODE = '*' +``` + +```sh + export WQFLASK_SETTINGS=$HOME/settings.py + source /home/pjotr/ve27/bin/activate + cd genenetwork2/wqflask + python ./runserver.py + + or + + python ./secure_server.py +``` -- cgit v1.2.3 From bb8e466e00c622f7b28209378c1871a1d8469572 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 17 Apr 2015 14:41:40 +0200 Subject: spacing --- INSTALL.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 84b3d37c..38d15090 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -53,12 +53,12 @@ SECRET_HMAC_CODE = '*' ``` ```sh - export WQFLASK_SETTINGS=$HOME/settings.py - source /home/pjotr/ve27/bin/activate - cd genenetwork2/wqflask - python ./runserver.py +export WQFLASK_SETTINGS=$HOME/settings.py +source /home/pjotr/ve27/bin/activate +cd genenetwork2/wqflask +python ./runserver.py - or +or - python ./secure_server.py +python ./secure_server.py ``` -- cgit v1.2.3 From 0929b16a5183538811260aef5c37f7406c302026 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 18 Apr 2015 08:54:07 +0200 Subject: Use reduced outputter for GN2 logs too --- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index 7bceb089..40b2021d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -1,7 +1,10 @@ -# Genenetwork2 specific methods and callback handler +# Standalone specific methods and callback handler # # Copyright (C) 2015 Pjotr Prins (pjotr.prins@thebird.nl) # +# Set the log level with +# +# logging.basicConfig(level=logging.DEBUG) from __future__ import absolute_import, print_function, division @@ -9,8 +12,10 @@ import numpy as np import sys import logging -# logging.basicConfig(level=logging.DEBUG) -# np.set_printoptions() +# logger = logging.getLogger(__name__) +logger = logging.getLogger('lmm2') +logging.basicConfig(level=logging.DEBUG) +np.set_printoptions(precision=3,suppress=True) progress_location = None progress_current = None @@ -37,13 +42,20 @@ def progress(location, count, total): logger.info("Progress: %s %d%%" % (location,perc)) progress_location = location progress_prev_perc = perc - + def mprint(msg,data): """ Array/matrix print function """ m = np.array(data) - print(msg,m.shape,"=\n",m) + if m.ndim == 1: + print(msg,m.shape,"=\n",m[0:3]," ... ",m[-3:]) + if m.ndim == 2: + print(msg,m.shape,"=\n[", + m[0][0:3]," ... ",m[0][-3:],"\n ", + m[1][0:3]," ... ",m[1][-3:],"\n ...\n ", + m[-2][0:3]," ... ",m[-2][-3:],"\n ", + m[-1][0:3]," ... ",m[-1][-3:],"]") def fatal(msg): logger.critical(msg) @@ -68,7 +80,7 @@ def uses(*funcs): Some sugar """ return [callbacks()[func] for func in funcs] - + # ----- Minor test cases: if __name__ == '__main__': -- cgit v1.2.3 From a1d8f68d5428a4ceec9a2d9a771b000ecabec5e6 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 18 Apr 2015 09:36:00 +0000 Subject: pylmm: fix integration problems --- wqflask/runserver.py | 6 ++--- .../wqflask/marker_regression/marker_regression.py | 14 +++++----- wqflask/wqflask/my_pylmm/pyLMM/gn2.py | 30 +++++++++++++++------- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 19 ++++++++------ wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 18 ++++++++++--- 5 files changed, 56 insertions(+), 31 deletions(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 9d5686a9..fadae6bf 100755 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -20,9 +20,9 @@ from wqflask import app import logging #from themodule import TheHandlerYouWant -file_handler = logging.FileHandler("/tmp/flask_gn_log_danny_unsecure") -file_handler.setLevel(logging.DEBUG) -app.logger.addHandler(file_handler) +# file_handler = logging.FileHandler("/tmp/flask_gn_log_danny_unsecure") +# file_handler.setLevel(logging.DEBUG) +# app.logger.addHandler(file_handler) import logging_tree logging_tree.printout() diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 7708356b..ae3e062f 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -40,6 +40,7 @@ from utility import temp_data from utility.benchmark import Bench +PYLMM_COMMAND= 'python /home/pjotr/izip/git/opensource/python/gn2/wqflask/wqflask/my_pylmm/pyLMM/lmm.py' class MarkerRegression(object): @@ -272,7 +273,7 @@ class MarkerRegression(object): """) def run_rqtl_geno(self): - print("Calling R/qtl from python") + print("Calling R/qtl") self.geno_to_rqtl_function() @@ -655,8 +656,7 @@ class MarkerRegression(object): Redis.set(key, json_params) Redis.expire(key, 60*60) - command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, - "other") + command = PYLMM_COMMAND+' --key {} --species {}'.format(key,"other") os.system(command) @@ -713,8 +713,8 @@ class MarkerRegression(object): # "refit": False, # "temp_data": tempdata} - print("genotype_matrix:", str(genotype_matrix.tolist())) - print("pheno_vector:", str(pheno_vector.tolist())) + # print("genotype_matrix:", str(genotype_matrix.tolist())) + # print("pheno_vector:", str(pheno_vector.tolist())) params = dict(pheno_vector = pheno_vector.tolist(), genotype_matrix = genotype_matrix.tolist(), @@ -732,7 +732,7 @@ class MarkerRegression(object): Redis.expire(key, 60*60) print("before printing command") - command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + command = PYLMM_COMMAND + ' --key {} --species {}'.format(key, "other") print("command is:", command) print("after printing command") @@ -806,7 +806,7 @@ class MarkerRegression(object): print("Before creating the command") - command = 'python /home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/lmm.py --key {} --species {}'.format(key, + command = PYLMM_COMMAND+' --key {} --species {}'.format(key, "human") print("command is:", command) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py index 7bceb089..b128bfab 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/gn2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/gn2.py @@ -1,7 +1,10 @@ -# Genenetwork2 specific methods and callback handler +# Standalone specific methods and callback handler # # Copyright (C) 2015 Pjotr Prins (pjotr.prins@thebird.nl) # +# Set the log level with +# +# logging.basicConfig(level=logging.DEBUG) from __future__ import absolute_import, print_function, division @@ -9,10 +12,12 @@ import numpy as np import sys import logging -# logging.basicConfig(level=logging.DEBUG) -# np.set_printoptions() +# logger = logging.getLogger(__name__) +logger = logging.getLogger('lmm2') +logging.basicConfig(level=logging.DEBUG) +np.set_printoptions(precision=3,suppress=True) -progress_location = None +progress_location = None progress_current = None progress_prev_perc = None @@ -20,30 +25,37 @@ def progress_default_func(location,count,total): global progress_current value = round(count*100.0/total) progress_current = value - + progress_func = progress_default_func def progress_set_func(func): global progress_func progress_func = func - + def progress(location, count, total): global progress_location global progress_prev_perc - + perc = round(count*100.0/total) if perc != progress_prev_perc and (location != progress_location or perc > 98 or perc > progress_prev_perc + 5): progress_func(location, count, total) logger.info("Progress: %s %d%%" % (location,perc)) progress_location = location progress_prev_perc = perc - + def mprint(msg,data): """ Array/matrix print function """ m = np.array(data) - print(msg,m.shape,"=\n",m) + if m.ndim == 1: + print(msg,m.shape,"=\n",m[0:3]," ... ",m[-3:]) + if m.ndim == 2: + print(msg,m.shape,"=\n[", + m[0][0:3]," ... ",m[0][-3:],"\n ", + m[1][0:3]," ... ",m[1][-3:],"\n ...\n ", + m[-2][0:3]," ... ",m[-2][-3:],"\n ", + m[-1][0:3]," ... ",m[-1][-3:],"]") def fatal(msg): logger.critical(msg) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index b2067b27..6fff5f1d 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -42,25 +42,27 @@ from redis import Redis Redis = Redis() import sys -sys.path.append("/home/zas1024/gene/wqflask/") - -has_gn2=True from utility.benchmark import Bench from utility import temp_data -sys.path.append("/home/zas1024/gene/wqflask/wqflask/my_pylmm/pyLMM/") - from kinship import kinship, kinship_full, kvakve import genotype import phenotype import gwas +has_gn2=True +sys.stderr.write("INFO: pylmm system path is "+":".join(sys.path)+"\n") +sys.stderr.write("INFO: pylmm file is "+__file__+"\n") + # ---- A trick to decide on the environment: try: - from wqflask.my_pylmm.pyLMM import chunks + sys.stderr.write("INFO: trying loading module\n") + import utility.formatting # this is never used, just to check the environment + sys.stderr.write("INFO: This is a genenetwork2 environment\n") from gn2 import uses, progress_set_func except ImportError: + # Failed to load gn2 has_gn2=False import standalone as handlers from standalone import uses, progress_set_func @@ -856,7 +858,8 @@ def gwas_with_redis(key,species,new_code=True): print(key) v = params[key] if v is not None: - v = np.array(v) + v = np.array(v).astype(np.float) + print(v) return v def narrayT(key): @@ -969,6 +972,6 @@ if __name__ == '__main__': if has_gn2: gn2_main() else: - print("Run from runlmm.py instead") + fatal("Run from runlmm.py instead") diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index 358bf27e..c65843ec 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -24,14 +24,24 @@ from scipy import optimize from optmatrix import matrixMult import kinship +sys.stderr.write("INFO: pylmm (lmm2) system path is "+":".join(sys.path)+"\n") +sys.stderr.write("INFO: pylmm (lmm2) file is "+__file__+"\n") + # ---- A trick to decide on the environment: try: - from wqflask.my_pylmm.pyLMM import chunks - from gn2 import uses + sys.stderr.write("INFO: trying loading module\n") + import utility.formatting # this is never used, just to check the environment + sys.stderr.write("INFO: This is a genenetwork2 environment (lmm2)\n") + from gn2 import uses, progress_set_func except ImportError: - sys.stderr.write("WARNING: LMM2 standalone version missing the Genenetwork2 environment\n") + # Failed to load gn2 has_gn2=False - from standalone import uses + import standalone as handlers + from standalone import uses, progress_set_func + sys.stderr.write("WARNING: LMM2 standalone version missing the Genenetwork2 environment\n") + +progress,mprint,debug,info,fatal = uses('progress','mprint','debug','info','fatal') + def calculateKinship(W,center=False): """ -- cgit v1.2.3 From 02660b9406a97943d4c33946250fc3f08b80c556 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 18 Apr 2015 09:40:57 +0000 Subject: pylmm: fix integration problems --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 6fff5f1d..618f8332 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -854,22 +854,23 @@ def gwas_with_redis(key,species,new_code=True): debug("Updating REDIS percent_complete=%d" % (round(i*100.0/total))) progress_set_func(update_tempdata) - def narray(key): - print(key) - v = params[key] + def narray(t): + info("Type is "+t) + v = params[t] if v is not None: v = np.array(v).astype(np.float) print(v) return v - def narrayT(key): - m = narray(key) + def narrayT(t): + m = narray(t) if m is not None: return m.T return m # We are transposing before we enter run_gwas - this should happen on the webserver # side (or when reading data from file) + print(params) k = narray('kinship_matrix') g = narrayT('genotype_matrix') y = narray('pheno_vector') -- cgit v1.2.3 From 8706319923b3830a4d8cd63fd9a3f6b9a2b04563 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 18 Apr 2015 10:58:26 +0000 Subject: Fix NA to float tranforms --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 37 +++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 618f8332..5b06c9ae 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -856,30 +856,47 @@ def gwas_with_redis(key,species,new_code=True): def narray(t): info("Type is "+t) - v = params[t] + v = params.get(t) if v is not None: - v = np.array(v).astype(np.float) - print(v) + # Note input values can be array of string or float + v1 = [x if x != 'NA' else 'nan' for x in v] + v = np.array(v1).astype(np.float) return v - def narrayT(t): - m = narray(t) + def marray(t): + info("Type is "+t) + v = params.get(t) + if v is not None: + m = [] + for r in v: + # Note input values can be array of string or float + r1 = [x if x != 'NA' else 'nan' for x in r] + m.append(np.array(r1).astype(np.float)) + return np.array(m) + return np.array(v) + + def marrayT(t): + m = marray(t) if m is not None: return m.T return m # We are transposing before we enter run_gwas - this should happen on the webserver # side (or when reading data from file) - print(params) - k = narray('kinship_matrix') - g = narrayT('genotype_matrix') + k = marray('kinship_matrix') + g = marrayT('genotype_matrix') + mprint("geno",g) y = narray('pheno_vector') n = len(y) - m = params['num_genotypes'] - ps,ts = run_gwas(species,n,m,k,y,g,narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params['input_file_name'],new_code) + m = params.get('num_genotypes') + if m is None: + m = g.shape[0] + info("m=%d,n=%d" % (m,n)) + ps,ts = run_gwas(species,n,m,k,y,g,narray('covariate_matrix'),params['restricted_max_likelihood'],params['refit'],params.get('input_file_name'),new_code) results_key = "pylmm:results:" + params['temp_uuid'] + # fatal(results_key) json_results = json.dumps(dict(p_values = ps, t_stats = ts)) -- cgit v1.2.3 From ced6f0c49c155a2ab47adfe93578d4718504566b Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 18 Apr 2015 11:09:20 +0000 Subject: Disable some print statements - will introduce debug levels soon --- wqflask/wqflask/marker_regression/marker_regression.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index ae3e062f..c80bba8e 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -128,7 +128,7 @@ class MarkerRegression(object): #Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary self.qtl_results = [] for qtl in self.filtered_markers: - print("lod score is:", qtl['lod_score']) + # print("lod score is:", qtl['lod_score']) if qtl['chr'] == highest_chr and highest_chr != "X" and highest_chr != "X/Y": print("changing to X") self.json_data['chr'].append("X") @@ -145,7 +145,7 @@ class MarkerRegression(object): self.json_data['chrnames'].append([self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length]) chromosome_mb_lengths[key] = self.species.chromosomes.chromosomes[key].mb_length - print("json_data:", self.json_data) + # print("json_data:", self.json_data) self.js_data = dict( @@ -745,7 +745,7 @@ class MarkerRegression(object): json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60) results = json.loads(json_results[1]) p_values = [float(result) for result in results['p_values']] - print("p_values:", p_values) + print("p_values:", p_values[:10]) #p_values = self.trim_results(p_values) t_stats = results['t_stats'] -- cgit v1.2.3 From 1f6386cbddfd02d8abbd4e9bcb502c06be6864d1 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sat, 18 Apr 2015 12:41:07 +0000 Subject: Show first 40 LOD scores --- wqflask/wqflask/marker_regression/marker_regression.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index c80bba8e..fba34b99 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -127,8 +127,9 @@ class MarkerRegression(object): #Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary self.qtl_results = [] - for qtl in self.filtered_markers: - # print("lod score is:", qtl['lod_score']) + for index,qtl in enumerate(self.filtered_markers): + if index<40: + print("lod score is:", qtl['lod_score']) if qtl['chr'] == highest_chr and highest_chr != "X" and highest_chr != "X/Y": print("changing to X") self.json_data['chr'].append("X") -- cgit v1.2.3 From 3736bd0044ddee68180a06809847af7542951743 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 09:46:15 +0200 Subject: INSTALL info --- INSTALL.md | 15 ++++++++++++--- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 38d15090..afe22678 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -1,5 +1,10 @@ # INSTALL Genenetwork2 (GN2) +## Use a Docker image + +A Docker image can be generated from +[here](https://github.com/lomereiter/gn2-docker). + ## Fetch GN2 from github Clone the repository (currently ~800Mb) to local @@ -10,12 +15,14 @@ Clone the repository (currently ~800Mb) to local GN2 requires +* python * redis * mysql ## Required python modules -Install the following python modules: +Install the following python modules (it is probably wise to use a local +Python with environment for this) * Flask * pyyaml @@ -53,12 +60,14 @@ SECRET_HMAC_CODE = '*' ``` ```sh +# Use a working copy of python +export python=$HOME/ve27/bin/python export WQFLASK_SETTINGS=$HOME/settings.py source /home/pjotr/ve27/bin/activate cd genenetwork2/wqflask -python ./runserver.py +$python ./runserver.py or -python ./secure_server.py +$python ./secure_server.py ``` diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index b2067b27..98bbead8 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -856,7 +856,7 @@ def gwas_with_redis(key,species,new_code=True): print(key) v = params[key] if v is not None: - v = np.array(v) + v = np.array(v).astype(np.float) return v def narrayT(key): @@ -969,6 +969,6 @@ if __name__ == '__main__': if has_gn2: gn2_main() else: - print("Run from runlmm.py instead") + fatal("Run from runlmm.py instead") -- cgit v1.2.3 From 5dad3ceb4acd652dd28d183f784005479089aa8a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 10:55:42 +0200 Subject: Restore logger in runserver --- wqflask/runserver.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index fadae6bf..20d79218 100755 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -19,10 +19,9 @@ from wqflask import app #_log.addHandler(_ch) import logging -#from themodule import TheHandlerYouWant -# file_handler = logging.FileHandler("/tmp/flask_gn_log_danny_unsecure") -# file_handler.setLevel(logging.DEBUG) -# app.logger.addHandler(file_handler) +file_handler = logging.FileHandler(app.config['LOGFILE']) +file_handler.setLevel(logging.DEBUG) +app.logger.addHandler(file_handler) import logging_tree logging_tree.printout() -- cgit v1.2.3 From 78dde9ccb4c24ea900b7a6d64ef392ec30ac89ea Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 10:58:43 +0200 Subject: INSTALL: refer to information in ./misc --- INSTALL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index afe22678..a971ff78 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -71,3 +71,7 @@ or $python ./secure_server.py ``` + +## Other information + +Check also the ./misc/ directory for settings \ No newline at end of file -- cgit v1.2.3 From 93f663a2e865484cb6f476fb7a0fa2415410e4fd Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 11:01:56 +0200 Subject: runserver: set port --- wqflask/runserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 20d79218..4ae91e64 100755 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -27,7 +27,7 @@ import logging_tree logging_tree.printout() app.run(host='0.0.0.0', - port=5003, + port=app.config['SERVER_PORT'], use_debugger=False, threaded=True, use_reloader=True) -- cgit v1.2.3 From 240c2db33b70b7d10a6bdd18e043fc0aa6766715 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 11:04:31 +0200 Subject: Output configi --- wqflask/runserver.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wqflask/runserver.py b/wqflask/runserver.py index 4ae91e64..5a76d1e2 100755 --- a/wqflask/runserver.py +++ b/wqflask/runserver.py @@ -18,6 +18,8 @@ from wqflask import app #_ch = logging.StreamHandler() #_log.addHandler(_ch) +print app.config + import logging file_handler = logging.FileHandler(app.config['LOGFILE']) file_handler.setLevel(logging.DEBUG) -- cgit v1.2.3 From e7cbe10d754e1e334746fc43a01e8b9fa3a666c0 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 11:37:32 +0200 Subject: pylmm: Copied benchmark (preparing for a module) --- wqflask/wqflask/my_pylmm/pyLMM/benchmark.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100755 wqflask/wqflask/my_pylmm/pyLMM/benchmark.py diff --git a/wqflask/wqflask/my_pylmm/pyLMM/benchmark.py b/wqflask/wqflask/my_pylmm/pyLMM/benchmark.py new file mode 100755 index 00000000..6c6c9f88 --- /dev/null +++ b/wqflask/wqflask/my_pylmm/pyLMM/benchmark.py @@ -0,0 +1,44 @@ +from __future__ import print_function, division, absolute_import + +import collections +import inspect +import time + +class Bench(object): + entries = collections.OrderedDict() + + def __init__(self, name=None): + self.name = name + + def __enter__(self): + if self.name: + print("Starting benchmark: %s" % (self.name)) + else: + print("Starting benchmark at: %s [%i]" % (inspect.stack()[1][3], inspect.stack()[1][2])) + self.start_time = time.time() + + def __exit__(self, type, value, traceback): + if self.name: + name = self.name + else: + name = "That" + + time_taken = time.time() - self.start_time + print(" %s took: %f seconds" % (name, (time_taken))) + + if self.name: + Bench.entries[self.name] = Bench.entries.get(self.name, 0) + time_taken + + + @classmethod + def report(cls): + total_time = sum((time_taken for time_taken in cls.entries.itervalues())) + print("\nTiming report\n") + for name, time_taken in cls.entries.iteritems(): + percent = int(round((time_taken/total_time) * 100)) + print("[{}%] {}: {}".format(percent, name, time_taken)) + print() + + def reset(cls): + """Reset the entries""" + cls.entries = collections.OrderedDict() -- cgit v1.2.3 From ac57e839c8ffb52f65e74a9064e4adeb15c76b49 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 11:54:42 +0200 Subject: pylmm: module loading --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 38 ++++++++++++++++------------------ wqflask/wqflask/my_pylmm/pyLMM/lmm2.py | 2 +- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 5b06c9ae..135ba1f4 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -19,48 +19,46 @@ from __future__ import absolute_import, print_function, division import sys import time -import argparse +# import argparse import uuid import numpy as np from scipy import linalg from scipy import optimize from scipy import stats -import pdb +# import pdb -import simplejson as json - -import gzip -import zlib +# import gzip +# import zlib import datetime -import cPickle as pickle -import simplejson as json - +# import cPickle as pickle from pprint import pformat as pf -from redis import Redis -Redis = Redis() - -import sys - -from utility.benchmark import Bench -from utility import temp_data - +# pylmm imports from kinship import kinship, kinship_full, kvakve import genotype import phenotype import gwas +from benchmark import Bench + +# The following imports are for exchanging data with the webserver +import simplejson as json +from redis import Redis +Redis = Redis() +from utility import temp_data + +has_gn2=None -has_gn2=True -sys.stderr.write("INFO: pylmm system path is "+":".join(sys.path)+"\n") +# sys.stderr.write("INFO: pylmm system path is "+":".join(sys.path)+"\n") sys.stderr.write("INFO: pylmm file is "+__file__+"\n") # ---- A trick to decide on the environment: try: - sys.stderr.write("INFO: trying loading module\n") + sys.stderr.write("INFO: lmm try loading module\n") import utility.formatting # this is never used, just to check the environment sys.stderr.write("INFO: This is a genenetwork2 environment\n") from gn2 import uses, progress_set_func + has_gn2=True except ImportError: # Failed to load gn2 has_gn2=False diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py index c65843ec..d871d8d2 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm2.py @@ -29,7 +29,7 @@ sys.stderr.write("INFO: pylmm (lmm2) file is "+__file__+"\n") # ---- A trick to decide on the environment: try: - sys.stderr.write("INFO: trying loading module\n") + sys.stderr.write("INFO: lmm2 try loading module\n") import utility.formatting # this is never used, just to check the environment sys.stderr.write("INFO: This is a genenetwork2 environment (lmm2)\n") from gn2 import uses, progress_set_func -- cgit v1.2.3 From e3adbf898dd537688339c8af1b59ac440aef3848 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 12:05:38 +0200 Subject: pylmm: move temp_data local --- wqflask/wqflask/my_pylmm/pyLMM/benchmark.py | 0 wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 2 +- wqflask/wqflask/my_pylmm/pyLMM/temp_data.py | 25 +++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) mode change 100755 => 100644 wqflask/wqflask/my_pylmm/pyLMM/benchmark.py create mode 100644 wqflask/wqflask/my_pylmm/pyLMM/temp_data.py diff --git a/wqflask/wqflask/my_pylmm/pyLMM/benchmark.py b/wqflask/wqflask/my_pylmm/pyLMM/benchmark.py old mode 100755 new mode 100644 diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 135ba1f4..2d9ca812 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -45,7 +45,7 @@ from benchmark import Bench import simplejson as json from redis import Redis Redis = Redis() -from utility import temp_data +import temp_data has_gn2=None diff --git a/wqflask/wqflask/my_pylmm/pyLMM/temp_data.py b/wqflask/wqflask/my_pylmm/pyLMM/temp_data.py new file mode 100644 index 00000000..004d45c6 --- /dev/null +++ b/wqflask/wqflask/my_pylmm/pyLMM/temp_data.py @@ -0,0 +1,25 @@ +from __future__ import print_function, division, absolute_import +from redis import Redis + +import simplejson as json + +class TempData(object): + + def __init__(self, temp_uuid): + self.temp_uuid = temp_uuid + self.redis = Redis() + self.key = "tempdata:{}".format(self.temp_uuid) + + def store(self, field, value): + self.redis.hset(self.key, field, value) + self.redis.expire(self.key, 60*15) # Expire in 15 minutes + + def get_all(self): + return self.redis.hgetall(self.key) + + +if __name__ == "__main__": + redis = Redis() + for key in redis.keys(): + for field in redis.hkeys(key): + print("{}.{}={}".format(key, field, redis.hget(key, field))) -- cgit v1.2.3 From 3c1e043dd63fe2d65a0bd44764867254b13aba32 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 12:28:58 +0200 Subject: pylmm: auto add to pythonpath --- INSTALL.md | 4 ++-- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 6 ++++++ wqflask/wqflask/my_pylmm/pyLMM/runlmm.py | 8 ++++++++ 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index a971ff78..9f28ac28 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -16,8 +16,8 @@ Clone the repository (currently ~800Mb) to local GN2 requires * python -* redis -* mysql +* redis-server +* mysql-server ## Required python modules diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 2d9ca812..4e35a4ac 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -34,6 +34,12 @@ import datetime # import cPickle as pickle from pprint import pformat as pf +# Add local dir to PYTHONPATH +import os +cwd = os.path.dirname(__file__) +if sys.path[0] != cwd: + sys.path.insert(1,cwd) + # pylmm imports from kinship import kinship, kinship_full, kvakve import genotype diff --git a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py index 52c3c80a..6b241cd6 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/runlmm.py @@ -21,6 +21,14 @@ from optparse import OptionParser import sys import tsvreader import numpy as np + +# Add local dir to PYTHONPATH +import os +cwd = os.path.dirname(__file__) +if sys.path[0] != cwd: + sys.path.insert(1,cwd) + +# pylmm modules from lmm import gn2_load_redis, gn2_load_redis_iter, calculate_kinship_new, run_gwas from kinship import kinship, kinship_full import genotype -- cgit v1.2.3 From 6e4a01fbdf2ac72230346d6f474edaa56a623bfe Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 10:40:15 +0000 Subject: pylmm: can now be called from GN2 without path set --- wqflask/wqflask/my_pylmm/pyLMM/lmm.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py index 4e35a4ac..2a0c7fdc 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py @@ -19,7 +19,6 @@ from __future__ import absolute_import, print_function, division import sys import time -# import argparse import uuid import numpy as np @@ -977,6 +976,7 @@ def gn2_load_redis_iter(key,species,kinship,pheno,geno_iterator): # Note that this calling route will become OBSOLETE (we should use runlmm.py # instead) def gn2_main(): + import argparse parser = argparse.ArgumentParser(description='Run pyLMM') parser.add_argument('-k', '--key') parser.add_argument('-s', '--species') @@ -991,9 +991,5 @@ def gn2_main(): if __name__ == '__main__': print("WARNING: Calling pylmm from lmm.py will become OBSOLETE, use runlmm.py instead!") - if has_gn2: - gn2_main() - else: - fatal("Run from runlmm.py instead") - + gn2_main() -- cgit v1.2.3 From 1b71fc64719ba5c5b56d23aad9d9dc45bc2898a9 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 12:57:10 +0200 Subject: pylmm: Release --- wqflask/wqflask/my_pylmm/README.md | 7 +++++++ wqflask/wqflask/my_pylmm/pyLMM/__init__.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/wqflask/wqflask/my_pylmm/README.md b/wqflask/wqflask/my_pylmm/README.md index 4845ec03..b844c845 100644 --- a/wqflask/wqflask/my_pylmm/README.md +++ b/wqflask/wqflask/my_pylmm/README.md @@ -1,5 +1,12 @@ # Genenetwork2/pylmm RELEASE NOTES +## 0.51-gn2 (April 19, 2015) + +- Improved GN2 integration +- Less matrix transposes +- Able to run pylmm standalone without Redis again (still requires + the modules) + ## 0.50-gn2 (April 2nd, 2015) - Replaced the GN2 genotype normalization diff --git a/wqflask/wqflask/my_pylmm/pyLMM/__init__.py b/wqflask/wqflask/my_pylmm/pyLMM/__init__.py index 6ab60d02..f33c4e74 100644 --- a/wqflask/wqflask/my_pylmm/pyLMM/__init__.py +++ b/wqflask/wqflask/my_pylmm/pyLMM/__init__.py @@ -1 +1 @@ -PYLMM_VERSION="0.50-gn2-pre2" +PYLMM_VERSION="0.51-gn2" -- cgit v1.2.3 From 561ad00c82f440aefd5dbaf741d927cb63a37e0f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 18:57:56 +0200 Subject: Pick up PYLMM_PATH from environment or setting.py --- wqflask/wqflask/marker_regression/marker_regression.py | 10 +++++++++- wqflask/wqflask/model.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index fba34b99..8d3eba48 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -25,6 +25,7 @@ from redis import Redis Redis = Redis() from flask import Flask, g +from wqflask import app from base.trait import GeneralTrait from base import data_set @@ -40,7 +41,14 @@ from utility import temp_data from utility.benchmark import Bench -PYLMM_COMMAND= 'python /home/pjotr/izip/git/opensource/python/gn2/wqflask/wqflask/my_pylmm/pyLMM/lmm.py' +import os +if os.environ['PYLMM_PATH'] is None: + PYLMM_PATH=app.config['PYLMM_PATH'] + if PYLMM_PATH is None: + PYLMM_PATH=os.environ['HOME']+'/gene/wqflask/wqflask/my_pylmm/pyLMM' +if !os.path.isfile(PYLMM_PATH+'lmm.py'): + raise 'PYLMM_PATH unknown or faulty' +PYLMM_COMMAND= 'python '+PYLMM_PATH+'/lmm.py' class MarkerRegression(object): diff --git a/wqflask/wqflask/model.py b/wqflask/wqflask/model.py index fa8c1aab..042cb8df 100755 --- a/wqflask/wqflask/model.py +++ b/wqflask/wqflask/model.py @@ -194,4 +194,4 @@ def display_collapsible(number): def user_uuid(): """Unique cookie for a user""" user_uuid = request.cookies.get('user_uuid') - \ No newline at end of file + -- cgit v1.2.3 From 4f4ceddafa1d172515b2ef24658e5cf39730e6c6 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 19 Apr 2015 17:03:10 +0000 Subject: Pick up PYLMM_PATH from environment or setting.py --- wqflask/wqflask/marker_regression/marker_regression.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py index 8d3eba48..67e1df0d 100755 --- a/wqflask/wqflask/marker_regression/marker_regression.py +++ b/wqflask/wqflask/marker_regression/marker_regression.py @@ -42,11 +42,11 @@ from utility import temp_data from utility.benchmark import Bench import os -if os.environ['PYLMM_PATH'] is None: - PYLMM_PATH=app.config['PYLMM_PATH'] +if os.environ.get('PYLMM_PATH') is None: + PYLMM_PATH=app.config.get('PYLMM_PATH') if PYLMM_PATH is None: PYLMM_PATH=os.environ['HOME']+'/gene/wqflask/wqflask/my_pylmm/pyLMM' -if !os.path.isfile(PYLMM_PATH+'lmm.py'): +if not os.path.isfile(PYLMM_PATH+'/lmm.py'): raise 'PYLMM_PATH unknown or faulty' PYLMM_COMMAND= 'python '+PYLMM_PATH+'/lmm.py' -- cgit v1.2.3 From f8dee92b54a3b5f727ec574158c0faa7c5774241 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Wed, 6 May 2015 14:05:21 +0300 Subject: email and pass validation --- .../wqflask/templates/new_security/login_user.html | 43 +++++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/wqflask/wqflask/templates/new_security/login_user.html b/wqflask/wqflask/templates/new_security/login_user.html index 81e8b53a..a66a85d6 100755 --- a/wqflask/wqflask/templates/new_security/login_user.html +++ b/wqflask/wqflask/templates/new_security/login_user.html @@ -24,18 +24,18 @@

Already have an account? Sign in here.

-
+
-
+
-
+

Forgot your password?
@@ -45,14 +45,14 @@
-
+
Remember me
-
+
@@ -66,9 +66,42 @@ {% endblock %} +{% block css %} + +{% endblock %} + {% block js %} + + + {% include "new_security/_scripts.html" %} {% endblock %} -- cgit v1.2.3 From 92a47463b73c25a9a5cffc6caaea696d628cca3b Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Sat, 18 Apr 2015 18:43:54 +0300 Subject: allow for attribute table to not exist --- wqflask/wqflask/show_trait/SampleList.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py index d196e722..060cb519 100755 --- a/wqflask/wqflask/show_trait/SampleList.py +++ b/wqflask/wqflask/show_trait/SampleList.py @@ -25,7 +25,11 @@ class SampleList(object): self.sample_list = [] # The actual list - self.get_attributes() + try: + self.get_attributes() + except Exception: + print("failed to get attributes") + self.attributes = {} print("camera: attributes are:", pf(self.attributes)) @@ -243,4 +247,4 @@ def natural_sort_key(x): x = int(x) except ValueError: pass - return x \ No newline at end of file + return x -- cgit v1.2.3 From 53d8a8bff6418a203e3d2565aaeca4fcb5dc0752 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Thu, 7 May 2015 17:45:07 +0300 Subject: fix CSV export of trait data --- wqflask/wqflask/show_trait/show_trait.py | 1 + .../static/new/javascript/show_trait.coffee | 17 +- .../wqflask/static/new/javascript/show_trait.js | 888 ++++++++++----------- wqflask/wqflask/views.py | 3 +- 4 files changed, 454 insertions(+), 455 deletions(-) diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py index 836a0be5..b28c9b76 100755 --- a/wqflask/wqflask/show_trait/show_trait.py +++ b/wqflask/wqflask/show_trait/show_trait.py @@ -123,6 +123,7 @@ class ShowTrait(object): hddn['control_marker'] = "" hddn['maf'] = 0.01 hddn['compare_traits'] = [] + hddn['export_data'] = "" # We'll need access to this_trait and hddn in the Jinja2 Template, so we put it inside self self.hddn = hddn diff --git a/wqflask/wqflask/static/new/javascript/show_trait.coffee b/wqflask/wqflask/static/new/javascript/show_trait.coffee index d05ccbf7..ee57831f 100755 --- a/wqflask/wqflask/static/new/javascript/show_trait.coffee +++ b/wqflask/wqflask/static/new/javascript/show_trait.coffee @@ -377,11 +377,9 @@ $ -> ##Get Sample Data From Table Code - get_sample_table_data = -> - samples = {} - primary_samples = [] - other_samples = [] - $('#sortable1').find('.value_se').each (_index, element) => + get_sample_table_data = (table_name) -> + samples = [] + $('#' + table_name).find('.value_se').each (_index, element) => row_data = {} row_data.name = $.trim($(element).find('.column_name-Sample').text()) row_data.value = $(element).find('.edit_sample_value').val() @@ -391,10 +389,7 @@ $ -> row_data[attribute_info.name] = $.trim($(element).find( '.column_name-'+attribute_info.name.replace(" ", "_")).text()) console.log("row_data is:", row_data) - primary_samples.push(row_data) - console.log("primary_samples is:", primary_samples) - samples.primary_samples = primary_samples - samples.other_samples = other_samples + samples.push(row_data) return samples ##End Get Sample Data from Table Code @@ -402,7 +397,9 @@ $ -> ##Export Sample Table Data Code export_sample_table_data = -> - sample_data = get_sample_table_data() + sample_data = {} + sample_data.primary_samples = get_sample_table_data('samples_primary') + sample_data.other_samples = get_sample_table_data('samples_other') console.log("sample_data is:", sample_data) json_sample_data = JSON.stringify(sample_data) console.log("json_sample_data is:", json_sample_data) diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index 159dafcb..c2dc07ee 100755 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -1,469 +1,469 @@ -// Generated by CoffeeScript 1.8.0 -var Stat_Table_Rows, is_number, - __hasProp = {}.hasOwnProperty, - __slice = [].slice; +// Generated by CoffeeScript 1.9.2 +(function() { + var Stat_Table_Rows, is_number, + hasProp = {}.hasOwnProperty, + slice = [].slice; -console.log("start_b"); + console.log("start_b"); -is_number = function(o) { - return !isNaN((o - 0) && o !== null); -}; - -Stat_Table_Rows = [ - { - vn: "n_of_samples", - pretty: "N of Samples", - digits: 0 - }, { - vn: "mean", - pretty: "Mean", - digits: 2 - }, { - vn: "median", - pretty: "Median", - digits: 2 - }, { - vn: "std_error", - pretty: "Standard Error (SE)", - digits: 2 - }, { - vn: "std_dev", - pretty: "Standard Deviation (SD)", - digits: 2 - }, { - vn: "min", - pretty: "Minimum", - digits: 2 - }, { - vn: "max", - pretty: "Maximum", - digits: 2 - }, { - vn: "range", - pretty: "Range (log2)", - digits: 2 - }, { - vn: "range_fold", - pretty: "Range (fold)", - digits: 2 - }, { - vn: "interquartile", - pretty: "Interquartile Range", - url: "/glossary.html#Interquartile", - digits: 2 - } -]; + is_number = function(o) { + return !isNaN((o - 0) && o !== null); + }; -$(function() { - var block_by_attribute_value, block_by_index, block_outliers, change_stats_value, create_value_dropdown, edit_data_change, export_sample_table_data, get_sample_table_data, hide_no_value, hide_tabs, make_table, on_corr_method_change, open_trait_selection, populate_sample_attributes_values_dropdown, process_id, reset_samples_table, sample_group_types, sample_lists, show_hide_outliers, stats_mdp_change, update_stat_values; - sample_lists = js_data.sample_lists; - sample_group_types = js_data.sample_group_types; - root.bar_chart = new Bar_Chart(sample_lists[0]); - root.histogram = new Histogram(sample_lists[0]); - new Box_Plot(sample_lists[0]); - $('.bar_chart_samples_group').change(function() { - var all_samples, group; - $('#bar_chart').remove(); - $('#bar_chart_container').append('
'); - group = $(this).val(); - if (group === "samples_primary") { - return root.bar_chart = new Bar_Chart(sample_lists[0]); - } else if (group === "samples_other") { - return root.bar_chart = new Bar_Chart(sample_lists[1]); - } else if (group === "samples_all") { - all_samples = sample_lists[0].concat(sample_lists[1]); - return root.bar_chart = new Bar_Chart(all_samples); - } - }); - $('.box_plot_samples_group').change(function() { - var all_samples, group; - $('#box_plot').remove(); - $('#box_plot_container').append('
'); - group = $(this).val(); - if (group === "samples_primary") { - return new Box_Plot(sample_lists[0]); - } else if (group === "samples_other") { - return new Box_Plot(sample_lists[1]); - } else if (group === "samples_all") { - all_samples = sample_lists[0].concat(sample_lists[1]); - return new Box_Plot(all_samples); + Stat_Table_Rows = [ + { + vn: "n_of_samples", + pretty: "N of Samples", + digits: 0 + }, { + vn: "mean", + pretty: "Mean", + digits: 2 + }, { + vn: "median", + pretty: "Median", + digits: 2 + }, { + vn: "std_error", + pretty: "Standard Error (SE)", + digits: 2 + }, { + vn: "std_dev", + pretty: "Standard Deviation (SD)", + digits: 2 + }, { + vn: "min", + pretty: "Minimum", + digits: 2 + }, { + vn: "max", + pretty: "Maximum", + digits: 2 + }, { + vn: "range", + pretty: "Range (log2)", + digits: 2 + }, { + vn: "range_fold", + pretty: "Range (fold)", + digits: 2 + }, { + vn: "interquartile", + pretty: "Interquartile Range", + url: "/glossary.html#Interquartile", + digits: 2 } - }); - d3.select("#select_compare_trait").on("click", (function(_this) { - return function() { - $('.scatter-matrix-container').remove(); - return open_trait_selection(); - }; - })(this)); - d3.select("#clear_compare_trait").on("click", (function(_this) { - return function() { - return $('.scatter-matrix-container').remove(); - }; - })(this)); - open_trait_selection = function() { - return $('#collections_holder').load('/collections/list?color_by_trait #collections_list', (function(_this) { + ]; + + $(function() { + var block_by_attribute_value, block_by_index, block_outliers, change_stats_value, create_value_dropdown, edit_data_change, export_sample_table_data, get_sample_table_data, hide_no_value, hide_tabs, make_table, on_corr_method_change, open_trait_selection, populate_sample_attributes_values_dropdown, process_id, reset_samples_table, sample_group_types, sample_lists, show_hide_outliers, stats_mdp_change, update_stat_values; + sample_lists = js_data.sample_lists; + sample_group_types = js_data.sample_group_types; + root.bar_chart = new Bar_Chart(sample_lists[0]); + root.histogram = new Histogram(sample_lists[0]); + new Box_Plot(sample_lists[0]); + $('.bar_chart_samples_group').change(function() { + var all_samples, group; + $('#bar_chart').remove(); + $('#bar_chart_container').append('
'); + group = $(this).val(); + if (group === "samples_primary") { + return root.bar_chart = new Bar_Chart(sample_lists[0]); + } else if (group === "samples_other") { + return root.bar_chart = new Bar_Chart(sample_lists[1]); + } else if (group === "samples_all") { + all_samples = sample_lists[0].concat(sample_lists[1]); + return root.bar_chart = new Bar_Chart(all_samples); + } + }); + $('.box_plot_samples_group').change(function() { + var all_samples, group; + $('#box_plot').remove(); + $('#box_plot_container').append('
'); + group = $(this).val(); + if (group === "samples_primary") { + return new Box_Plot(sample_lists[0]); + } else if (group === "samples_other") { + return new Box_Plot(sample_lists[1]); + } else if (group === "samples_all") { + all_samples = sample_lists[0].concat(sample_lists[1]); + return new Box_Plot(all_samples); + } + }); + d3.select("#select_compare_trait").on("click", (function(_this) { return function() { - $.colorbox({ - inline: true, - href: "#collections_holder" - }); - return $('a.collection_name').attr('onClick', 'return false'); + $('.scatter-matrix-container').remove(); + return open_trait_selection(); }; })(this)); - }; - hide_tabs = function(start) { - var x, _i, _results; - _results = []; - for (x = _i = start; start <= 10 ? _i <= 10 : _i >= 10; x = start <= 10 ? ++_i : --_i) { - _results.push($("#stats_tabs" + x).hide()); - } - return _results; - }; - stats_mdp_change = function() { - var selected; - selected = $(this).val(); - hide_tabs(0); - return $("#stats_tabs" + selected).show(); - }; - change_stats_value = function(sample_sets, category, value_type, decimal_places) { - var current_value, id, in_box, the_value, title_value; - id = "#" + process_id(category, value_type); - console.log("the_id:", id); - in_box = $(id).html; - current_value = parseFloat($(in_box)).toFixed(decimal_places); - the_value = sample_sets[category][value_type](); - console.log("After running sample_sets, the_value is:", the_value); - if (decimal_places > 0) { - title_value = the_value.toFixed(decimal_places * 2); - the_value = the_value.toFixed(decimal_places); - } else { - title_value = null; - } - console.log("*-* the_value:", the_value); - console.log("*-* current_value:", current_value); - if (the_value !== current_value) { - console.log("object:", $(id).html(the_value)); - $(id).html(the_value).effect("highlight"); - } - if (title_value) { - return $(id).attr('title', title_value); - } - }; - update_stat_values = function(sample_sets) { - var category, row, _i, _len, _ref, _results; - _ref = ['samples_primary', 'samples_other', 'samples_all']; - _results = []; - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - category = _ref[_i]; - _results.push((function() { - var _j, _len1, _results1; - _results1 = []; - for (_j = 0, _len1 = Stat_Table_Rows.length; _j < _len1; _j++) { - row = Stat_Table_Rows[_j]; - console.log("Calling change_stats_value"); - _results1.push(change_stats_value(sample_sets, category, row.vn, row.digits)); - } - return _results1; - })()); - } - return _results; - }; - make_table = function() { - var header, key, row, row_line, table, the_id, the_rows, value, _i, _len, _ref, _ref1; - header = " "; - console.log("js_data.sample_group_types:", js_data.sample_group_types); - _ref = js_data.sample_group_types; - for (key in _ref) { - if (!__hasProp.call(_ref, key)) continue; - value = _ref[key]; - console.log("aa key:", key); - console.log("aa value:", value); - the_id = process_id("column", key); - header += "" + value + ""; - } - header += ""; - console.log("windex header is:", header); - the_rows = ""; - for (_i = 0, _len = Stat_Table_Rows.length; _i < _len; _i++) { - row = Stat_Table_Rows[_i]; - console.log("rowing"); - row_line = ""; - if (row.url != null) { - row_line += "" + row.pretty + ""; + d3.select("#clear_compare_trait").on("click", (function(_this) { + return function() { + return $('.scatter-matrix-container').remove(); + }; + })(this)); + open_trait_selection = function() { + return $('#collections_holder').load('/collections/list?color_by_trait #collections_list', (function(_this) { + return function() { + $.colorbox({ + inline: true, + href: "#collections_holder" + }); + return $('a.collection_name').attr('onClick', 'return false'); + }; + })(this)); + }; + hide_tabs = function(start) { + var i, ref, results, x; + results = []; + for (x = i = ref = start; ref <= 10 ? i <= 10 : i >= 10; x = ref <= 10 ? ++i : --i) { + results.push($("#stats_tabs" + x).hide()); + } + return results; + }; + stats_mdp_change = function() { + var selected; + selected = $(this).val(); + hide_tabs(0); + return $("#stats_tabs" + selected).show(); + }; + change_stats_value = function(sample_sets, category, value_type, decimal_places) { + var current_value, id, in_box, the_value, title_value; + id = "#" + process_id(category, value_type); + console.log("the_id:", id); + in_box = $(id).html; + current_value = parseFloat($(in_box)).toFixed(decimal_places); + the_value = sample_sets[category][value_type](); + console.log("After running sample_sets, the_value is:", the_value); + if (decimal_places > 0) { + title_value = the_value.toFixed(decimal_places * 2); + the_value = the_value.toFixed(decimal_places); } else { - row_line += "" + row.pretty + ""; + title_value = null; } - console.log("box - js_data.sample_group_types:", js_data.sample_group_types); - _ref1 = js_data.sample_group_types; - for (key in _ref1) { - if (!__hasProp.call(_ref1, key)) continue; - value = _ref1[key]; - console.log("apple key:", key); - the_id = process_id(key, row.vn); - console.log("the_id:", the_id); - row_line += "foo"; + console.log("*-* the_value:", the_value); + console.log("*-* current_value:", current_value); + if (the_value !== current_value) { + console.log("object:", $(id).html(the_value)); + $(id).html(the_value).effect("highlight"); } - row_line += ""; - console.log("row line:", row_line); - the_rows += row_line; - } - the_rows += ""; - table = header + the_rows; - console.log("table is:", table); - return $("#stats_table").append(table); - }; - process_id = function() { - var processed, value, values, _i, _len; - values = 1 <= arguments.length ? __slice.call(arguments, 0) : []; - - /* Make an id or a class valid javascript by, for example, eliminating spaces */ - processed = ""; - for (_i = 0, _len = values.length; _i < _len; _i++) { - value = values[_i]; - console.log("value:", value); - value = value.replace(" ", "_"); - if (processed.length) { - processed += "-"; + if (title_value) { + return $(id).attr('title', title_value); } - processed += value; - } - return processed; - }; - edit_data_change = function() { - var already_seen, checkbox, checked, name, real_value, row, rows, sample_sets, table, tables, _i, _j, _len, _len1; - already_seen = {}; - sample_sets = { - samples_primary: new Stats([]), - samples_other: new Stats([]), - samples_all: new Stats([]) }; - console.log("at beginning:", sample_sets); - tables = ['samples_primary', 'samples_other']; - for (_i = 0, _len = tables.length; _i < _len; _i++) { - table = tables[_i]; - rows = $("#" + table).find('tr'); - for (_j = 0, _len1 = rows.length; _j < _len1; _j++) { - row = rows[_j]; - name = $(row).find('.edit_sample_sample_name').html(); - name = $.trim(name); - real_value = $(row).find('.edit_sample_value').val(); - console.log("real_value:", real_value); - checkbox = $(row).find(".edit_sample_checkbox"); - checked = $(checkbox).attr('checked'); - if (checked && is_number(real_value) && real_value !== "") { - console.log("in the iffy if"); - real_value = parseFloat(real_value); - sample_sets[table].add_value(real_value); - console.log("checking name of:", name); - if (!(name in already_seen)) { - console.log("haven't seen"); - sample_sets['samples_all'].add_value(real_value); - already_seen[name] = true; + update_stat_values = function(sample_sets) { + var category, i, len, ref, results, row; + ref = ['samples_primary', 'samples_other', 'samples_all']; + results = []; + for (i = 0, len = ref.length; i < len; i++) { + category = ref[i]; + results.push((function() { + var j, len1, results1; + results1 = []; + for (j = 0, len1 = Stat_Table_Rows.length; j < len1; j++) { + row = Stat_Table_Rows[j]; + console.log("Calling change_stats_value"); + results1.push(change_stats_value(sample_sets, category, row.vn, row.digits)); } + return results1; + })()); + } + return results; + }; + make_table = function() { + var header, i, key, len, ref, ref1, row, row_line, table, the_id, the_rows, value; + header = " "; + console.log("js_data.sample_group_types:", js_data.sample_group_types); + ref = js_data.sample_group_types; + for (key in ref) { + if (!hasProp.call(ref, key)) continue; + value = ref[key]; + console.log("aa key:", key); + console.log("aa value:", value); + the_id = process_id("column", key); + header += "" + value + ""; + } + header += ""; + console.log("windex header is:", header); + the_rows = ""; + for (i = 0, len = Stat_Table_Rows.length; i < len; i++) { + row = Stat_Table_Rows[i]; + console.log("rowing"); + row_line = ""; + if (row.url != null) { + row_line += "" + row.pretty + ""; + } else { + row_line += "" + row.pretty + ""; } + console.log("box - js_data.sample_group_types:", js_data.sample_group_types); + ref1 = js_data.sample_group_types; + for (key in ref1) { + if (!hasProp.call(ref1, key)) continue; + value = ref1[key]; + console.log("apple key:", key); + the_id = process_id(key, row.vn); + console.log("the_id:", the_id); + row_line += "foo"; + } + row_line += ""; + console.log("row line:", row_line); + the_rows += row_line; } - } - console.log("towards end:", sample_sets); - return update_stat_values(sample_sets); - }; - show_hide_outliers = function() { - var label; - console.log("FOOBAR in beginning of show_hide_outliers"); - label = $('#show_hide_outliers').val(); - console.log("lable is:", label); - if (label === "Hide Outliers") { - return $('#show_hide_outliers').val("Show Outliers"); - } else if (label === "Show Outliers") { - console.log("Found Show Outliers"); - $('#show_hide_outliers').val("Hide Outliers"); - return console.log("Should be now Hide Outliers"); - } - }; - on_corr_method_change = function() { - var corr_method; - console.log("in beginning of on_corr_method_change"); - corr_method = $('select[name=corr_method]').val(); - console.log("corr_method is:", corr_method); - $('.correlation_desc').hide(); - $('#' + corr_method + "_r_desc").show().effect("highlight"); - if (corr_method === "lit") { - return $("#corr_sample_method_options").hide(); - } else { - return $("#corr_sample_method_options").show(); - } - }; - $('select[name=corr_method]').change(on_corr_method_change); - create_value_dropdown = function(value) { - return ""; - }; - populate_sample_attributes_values_dropdown = function() { - var attribute_info, key, sample_attributes, selected_attribute, value, _i, _len, _ref, _ref1, _results; - console.log("in beginning of psavd"); - $('#attribute_values').empty(); - sample_attributes = {}; - _ref = js_data.attribute_names; - for (key in _ref) { - if (!__hasProp.call(_ref, key)) continue; - attribute_info = _ref[key]; - sample_attributes[attribute_info.name] = attribute_info.distinct_values; - } - console.log("[visa] attributes is:", sample_attributes); - selected_attribute = $('#exclude_menu').val().replace("_", " "); - console.log("selected_attribute is:", selected_attribute); - _ref1 = sample_attributes[selected_attribute]; - _results = []; - for (_i = 0, _len = _ref1.length; _i < _len; _i++) { - value = _ref1[_i]; - _results.push($(create_value_dropdown(value)).appendTo($('#attribute_values'))); - } - return _results; - }; - if (js_data.attribute_names.length > 0) { - populate_sample_attributes_values_dropdown(); - } - $('#exclude_menu').change(populate_sample_attributes_values_dropdown); - block_by_attribute_value = function() { - var attribute_name, cell_class, exclude_by_value; - attribute_name = $('#exclude_menu').val(); - exclude_by_value = $('#attribute_values').val(); - cell_class = ".column_name-" + attribute_name; - return $(cell_class).each((function(_this) { - return function(index, element) { - var row; - if ($.trim($(element).text()) === exclude_by_value) { - row = $(element).parent('tr'); - return $(row).find(".trait_value_input").val("x"); + the_rows += ""; + table = header + the_rows; + console.log("table is:", table); + return $("#stats_table").append(table); + }; + process_id = function() { + var i, len, processed, value, values; + values = 1 <= arguments.length ? slice.call(arguments, 0) : []; + + /* Make an id or a class valid javascript by, for example, eliminating spaces */ + processed = ""; + for (i = 0, len = values.length; i < len; i++) { + value = values[i]; + console.log("value:", value); + value = value.replace(" ", "_"); + if (processed.length) { + processed += "-"; } + processed += value; + } + return processed; + }; + edit_data_change = function() { + var already_seen, checkbox, checked, i, j, len, len1, name, real_value, row, rows, sample_sets, table, tables; + already_seen = {}; + sample_sets = { + samples_primary: new Stats([]), + samples_other: new Stats([]), + samples_all: new Stats([]) }; - })(this)); - }; - $('#exclude_group').click(block_by_attribute_value); - block_by_index = function() { - var end_index, error, index, index_list, index_set, index_string, start_index, _i, _j, _k, _len, _len1, _ref, _results; - index_string = $('#remove_samples_field').val(); - index_list = []; - _ref = index_string.split(","); - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - index_set = _ref[_i]; - if (index_set.indexOf('-') !== -1) { - try { - start_index = parseInt(index_set.split("-")[0]); - end_index = parseInt(index_set.split("-")[1]); - for (index = _j = start_index; start_index <= end_index ? _j <= end_index : _j >= end_index; index = start_index <= end_index ? ++_j : --_j) { - index_list.push(index); + console.log("at beginning:", sample_sets); + tables = ['samples_primary', 'samples_other']; + for (i = 0, len = tables.length; i < len; i++) { + table = tables[i]; + rows = $("#" + table).find('tr'); + for (j = 0, len1 = rows.length; j < len1; j++) { + row = rows[j]; + name = $(row).find('.edit_sample_sample_name').html(); + name = $.trim(name); + real_value = $(row).find('.edit_sample_value').val(); + console.log("real_value:", real_value); + checkbox = $(row).find(".edit_sample_checkbox"); + checked = $(checkbox).attr('checked'); + if (checked && is_number(real_value) && real_value !== "") { + console.log("in the iffy if"); + real_value = parseFloat(real_value); + sample_sets[table].add_value(real_value); + console.log("checking name of:", name); + if (!(name in already_seen)) { + console.log("haven't seen"); + sample_sets['samples_all'].add_value(real_value); + already_seen[name] = true; + } } - } catch (_error) { - error = _error; - alert("Syntax error"); } - } else { - index = parseInt(index_set); - console.log("index:", index); - index_list.push(index); } - } - console.log("index_list:", index_list); - _results = []; - for (_k = 0, _len1 = index_list.length; _k < _len1; _k++) { - index = index_list[_k]; - if ($('#block_group').val() === "primary") { - console.log("block_group:", $('#block_group').val()); - console.log("row:", $('#Primary_' + index.toString())); - _results.push($('#Primary_' + index.toString()).find('.trait_value_input').val("x")); - } else if ($('#block_group').val() === "other") { - console.log("block_group:", $('#block_group').val()); - console.log("row:", $('#Other_' + index.toString())); - _results.push($('#Other_' + index.toString()).find('.trait_value_input').val("x")); + console.log("towards end:", sample_sets); + return update_stat_values(sample_sets); + }; + show_hide_outliers = function() { + var label; + console.log("FOOBAR in beginning of show_hide_outliers"); + label = $('#show_hide_outliers').val(); + console.log("lable is:", label); + if (label === "Hide Outliers") { + return $('#show_hide_outliers').val("Show Outliers"); + } else if (label === "Show Outliers") { + console.log("Found Show Outliers"); + $('#show_hide_outliers').val("Hide Outliers"); + return console.log("Should be now Hide Outliers"); + } + }; + on_corr_method_change = function() { + var corr_method; + console.log("in beginning of on_corr_method_change"); + corr_method = $('select[name=corr_method]').val(); + console.log("corr_method is:", corr_method); + $('.correlation_desc').hide(); + $('#' + corr_method + "_r_desc").show().effect("highlight"); + if (corr_method === "lit") { + return $("#corr_sample_method_options").hide(); } else { - _results.push(void 0); + return $("#corr_sample_method_options").show(); + } + }; + $('select[name=corr_method]').change(on_corr_method_change); + create_value_dropdown = function(value) { + return ""; + }; + populate_sample_attributes_values_dropdown = function() { + var attribute_info, i, key, len, ref, ref1, results, sample_attributes, selected_attribute, value; + console.log("in beginning of psavd"); + $('#attribute_values').empty(); + sample_attributes = {}; + ref = js_data.attribute_names; + for (key in ref) { + if (!hasProp.call(ref, key)) continue; + attribute_info = ref[key]; + sample_attributes[attribute_info.name] = attribute_info.distinct_values; } + console.log("[visa] attributes is:", sample_attributes); + selected_attribute = $('#exclude_menu').val().replace("_", " "); + console.log("selected_attribute is:", selected_attribute); + ref1 = sample_attributes[selected_attribute]; + results = []; + for (i = 0, len = ref1.length; i < len; i++) { + value = ref1[i]; + results.push($(create_value_dropdown(value)).appendTo($('#attribute_values'))); + } + return results; + }; + if (js_data.attribute_names.length > 0) { + populate_sample_attributes_values_dropdown(); } - return _results; - }; - $('#block_by_index').click(block_by_index); - hide_no_value = function() { - return $('.value_se').each((function(_this) { - return function(_index, element) { - if ($(element).find('.trait_value_input').val() === 'x') { - return $(element).hide(); - } - }; - })(this)); - }; - $('#hide_no_value').click(hide_no_value); - block_outliers = function() { - return $('.outlier').each((function(_this) { - return function(_index, element) { - return $(element).find('.trait_value_input').val('x'); - }; - })(this)); - }; - $('#block_outliers').click(block_outliers); - reset_samples_table = function() { - return $('.trait_value_input').each((function(_this) { - return function(_index, element) { - console.log("value is:", $(element).val()); - $(element).val($(element).data('value')); - console.log("data-value is:", $(element).data('value')); - return $(element).parents('.value_se').show(); - }; - })(this)); - }; - $('#reset').click(reset_samples_table); - get_sample_table_data = function() { - var other_samples, primary_samples, samples; - samples = {}; - primary_samples = []; - other_samples = []; - $('#sortable1').find('.value_se').each((function(_this) { - return function(_index, element) { - var attribute_info, key, row_data, _ref; - row_data = {}; - row_data.name = $.trim($(element).find('.column_name-Sample').text()); - row_data.value = $(element).find('.edit_sample_value').val(); - if ($(element).find('.edit_sample_se').length !== -1) { - row_data.se = $(element).find('.edit_sample_se').val(); + $('#exclude_menu').change(populate_sample_attributes_values_dropdown); + block_by_attribute_value = function() { + var attribute_name, cell_class, exclude_by_value; + attribute_name = $('#exclude_menu').val(); + exclude_by_value = $('#attribute_values').val(); + cell_class = ".column_name-" + attribute_name; + return $(cell_class).each((function(_this) { + return function(index, element) { + var row; + if ($.trim($(element).text()) === exclude_by_value) { + row = $(element).parent('tr'); + return $(row).find(".trait_value_input").val("x"); + } + }; + })(this)); + }; + $('#exclude_group').click(block_by_attribute_value); + block_by_index = function() { + var end_index, error, i, index, index_list, index_set, index_string, j, k, len, len1, ref, ref1, ref2, results, start_index; + index_string = $('#remove_samples_field').val(); + index_list = []; + ref = index_string.split(","); + for (i = 0, len = ref.length; i < len; i++) { + index_set = ref[i]; + if (index_set.indexOf('-') !== -1) { + try { + start_index = parseInt(index_set.split("-")[0]); + end_index = parseInt(index_set.split("-")[1]); + for (index = j = ref1 = start_index, ref2 = end_index; ref1 <= ref2 ? j <= ref2 : j >= ref2; index = ref1 <= ref2 ? ++j : --j) { + index_list.push(index); + } + } catch (_error) { + error = _error; + alert("Syntax error"); + } + } else { + index = parseInt(index_set); + console.log("index:", index); + index_list.push(index); } - _ref = js_data.attribute_names; - for (key in _ref) { - if (!__hasProp.call(_ref, key)) continue; - attribute_info = _ref[key]; - row_data[attribute_info.name] = $.trim($(element).find('.column_name-' + attribute_info.name.replace(" ", "_")).text()); + } + console.log("index_list:", index_list); + results = []; + for (k = 0, len1 = index_list.length; k < len1; k++) { + index = index_list[k]; + if ($('#block_group').val() === "primary") { + console.log("block_group:", $('#block_group').val()); + console.log("row:", $('#Primary_' + index.toString())); + results.push($('#Primary_' + index.toString()).find('.trait_value_input').val("x")); + } else if ($('#block_group').val() === "other") { + console.log("block_group:", $('#block_group').val()); + console.log("row:", $('#Other_' + index.toString())); + results.push($('#Other_' + index.toString()).find('.trait_value_input').val("x")); + } else { + results.push(void 0); } - console.log("row_data is:", row_data); - return primary_samples.push(row_data); - }; - })(this)); - console.log("primary_samples is:", primary_samples); - samples.primary_samples = primary_samples; - samples.other_samples = other_samples; - return samples; - }; - export_sample_table_data = function() { - var format, json_sample_data, sample_data; - sample_data = get_sample_table_data(); - console.log("sample_data is:", sample_data); - json_sample_data = JSON.stringify(sample_data); - console.log("json_sample_data is:", json_sample_data); - $('input[name=export_data]').val(json_sample_data); - console.log("export_data is", $('input[name=export_data]').val()); - format = $('#export_format').val(); - if (format === "excel") { - $('#trait_data_form').attr('action', '/export_trait_excel'); - } else { - $('#trait_data_form').attr('action', '/export_trait_csv'); - } - console.log("action is:", $('#trait_data_form').attr('action')); - return $('#trait_data_form').submit(); - }; - $('#export').click(export_sample_table_data); - console.log("before registering block_outliers"); - $('#block_outliers').click(block_outliers); - console.log("after registering block_outliers"); - _.mixin(_.str.exports()); - $('#edit_sample_lists').change(edit_data_change); - console.log("loaded"); - make_table(); - edit_data_change(); - return console.log("end"); -}); + } + return results; + }; + $('#block_by_index').click(block_by_index); + hide_no_value = function() { + return $('.value_se').each((function(_this) { + return function(_index, element) { + if ($(element).find('.trait_value_input').val() === 'x') { + return $(element).hide(); + } + }; + })(this)); + }; + $('#hide_no_value').click(hide_no_value); + block_outliers = function() { + return $('.outlier').each((function(_this) { + return function(_index, element) { + return $(element).find('.trait_value_input').val('x'); + }; + })(this)); + }; + $('#block_outliers').click(block_outliers); + reset_samples_table = function() { + return $('.trait_value_input').each((function(_this) { + return function(_index, element) { + console.log("value is:", $(element).val()); + $(element).val($(element).data('value')); + console.log("data-value is:", $(element).data('value')); + return $(element).parents('.value_se').show(); + }; + })(this)); + }; + $('#reset').click(reset_samples_table); + get_sample_table_data = function(table_name) { + var samples; + samples = []; + $('#' + table_name).find('.value_se').each((function(_this) { + return function(_index, element) { + var attribute_info, key, ref, row_data; + row_data = {}; + row_data.name = $.trim($(element).find('.column_name-Sample').text()); + row_data.value = $(element).find('.edit_sample_value').val(); + if ($(element).find('.edit_sample_se').length !== -1) { + row_data.se = $(element).find('.edit_sample_se').val(); + } + ref = js_data.attribute_names; + for (key in ref) { + if (!hasProp.call(ref, key)) continue; + attribute_info = ref[key]; + row_data[attribute_info.name] = $.trim($(element).find('.column_name-' + attribute_info.name.replace(" ", "_")).text()); + } + console.log("row_data is:", row_data); + return samples.push(row_data); + }; + })(this)); + return samples; + }; + export_sample_table_data = function() { + var format, json_sample_data, sample_data; + sample_data = {}; + sample_data.primary_samples = get_sample_table_data('samples_primary'); + sample_data.other_samples = get_sample_table_data('samples_other'); + console.log("sample_data is:", sample_data); + json_sample_data = JSON.stringify(sample_data); + console.log("json_sample_data is:", json_sample_data); + $('input[name=export_data]').val(json_sample_data); + console.log("export_data is", $('input[name=export_data]').val()); + format = $('#export_format').val(); + if (format === "excel") { + $('#trait_data_form').attr('action', '/export_trait_excel'); + } else { + $('#trait_data_form').attr('action', '/export_trait_csv'); + } + console.log("action is:", $('#trait_data_form').attr('action')); + return $('#trait_data_form').submit(); + }; + $('#export').click(export_sample_table_data); + console.log("before registering block_outliers"); + $('#block_outliers').click(block_outliers); + console.log("after registering block_outliers"); + _.mixin(_.str.exports()); + $('#edit_sample_lists').change(edit_data_change); + console.log("loaded"); + make_table(); + edit_data_change(); + return console.log("end"); + }); + +}).call(this); diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 39004f07..abd8f05b 100755 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -186,7 +186,8 @@ def environments(): doc = docs.Docs("environments") return render_template("docs.html", **doc.__dict__) -@app.route('/export_trait_csv', methods=('POST',)) +# TODO +@app.route('/export_trait_excel', methods=('POST',)) def export_trait_excel(): """Excel file consisting of the sample data from the trait data and analysis page""" print("In export_trait_excel") -- cgit v1.2.3 From cd9e120fa77a68f570d615a17071928d0c91aac2 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Thu, 7 May 2015 18:05:57 +0300 Subject: Excel export of trait data --- misc/requirements.txt | 1 + wqflask/wqflask/views.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/misc/requirements.txt b/misc/requirements.txt index ec02a2c2..3487aa09 100644 --- a/misc/requirements.txt +++ b/misc/requirements.txt @@ -32,3 +32,4 @@ scipy==0.11.0 simplejson==3.0.7 wsgiref==0.1.2 yolk==0.4.3 +XlsxWriter=0.7.2 diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index abd8f05b..79c1c967 100755 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -5,6 +5,7 @@ import sys print("sys.path is:", sys.path) import csv +import xlsxwriter import StringIO # Todo: Use cStringIO? import gc @@ -186,7 +187,6 @@ def environments(): doc = docs.Docs("environments") return render_template("docs.html", **doc.__dict__) -# TODO @app.route('/export_trait_excel', methods=('POST',)) def export_trait_excel(): """Excel file consisting of the sample data from the trait data and analysis page""" @@ -197,15 +197,18 @@ def export_trait_excel(): print("sample_data - type: %s -- size: %s" % (type(sample_data), len(sample_data))) buff = StringIO.StringIO() - writer = csv.writer(buff) - for row in sample_data: - writer.writerow(row) - csv_data = buff.getvalue() + workbook = xlsxwriter.Workbook(buff, {'in_memory': True}) + worksheet = workbook.add_worksheet() + for i, row in enumerate(sample_data): + worksheet.write(i, 0, row[0]) + worksheet.write(i, 1, row[1]) + workbook.close() + excel_data = buff.getvalue() buff.close() - return Response(csv_data, - mimetype='text/csv', - headers={"Content-Disposition":"attachment;filename=test.csv"}) + return Response(excel_data, + mimetype='application/vnd.ms-excel', + headers={"Content-Disposition":"attachment;filename=test.xlsx"}) @app.route('/export_trait_csv', methods=('POST',)) def export_trait_csv(): -- cgit v1.2.3 From bf860dd3c6e2c8d32e9d40ae129e33613151fab5 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Fri, 8 May 2015 12:59:38 +0300 Subject: Fix #29 The number of bins was set to 20 for all datasets. Changed the number to be sqrt(#samples) as suggested on Wikipedia. --- .../wqflask/static/new/javascript/histogram.coffee | 5 +- wqflask/wqflask/static/new/javascript/histogram.js | 223 +++++++++++---------- 2 files changed, 117 insertions(+), 111 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/histogram.coffee b/wqflask/wqflask/static/new/javascript/histogram.coffee index 98f89ac6..7adb53f9 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.coffee +++ b/wqflask/wqflask/static/new/javascript/histogram.coffee @@ -53,8 +53,9 @@ class Histogram get_histogram_data: () -> console.log("sample_vals:", @sample_vals) + n_bins = sqrt(@sample_vals.length) @histogram_data = d3.layout.histogram() - .bins(@x_scale.ticks(20))(@sample_vals) + .bins(@x_scale.ticks(n_bins))(@sample_vals) console.log("histogram_data:", @histogram_data[0]) create_y_scale: () -> @@ -131,4 +132,4 @@ class Histogram return @format_count(d.y) ) -root.Histogram = Histogram \ No newline at end of file +root.Histogram = Histogram diff --git a/wqflask/wqflask/static/new/javascript/histogram.js b/wqflask/wqflask/static/new/javascript/histogram.js index 561a3068..3ab5eefa 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.js +++ b/wqflask/wqflask/static/new/javascript/histogram.js @@ -1,124 +1,129 @@ -// Generated by CoffeeScript 1.8.0 -var Histogram, root; +// Generated by CoffeeScript 1.9.2 +(function() { + var Histogram, root; -root = typeof exports !== "undefined" && exports !== null ? exports : this; + root = typeof exports !== "undefined" && exports !== null ? exports : this; -Histogram = (function() { - function Histogram(sample_list, sample_group) { - this.sample_list = sample_list; - this.sample_group = sample_group; - this.sort_by = "name"; - this.format_count = d3.format(",.0f"); - this.get_sample_vals(); - this.margin = { - top: 10, - right: 30, - bottom: 30, - left: 30 - }; - this.plot_width = 960 - this.margin.left - this.margin.right; - this.plot_height = 500 - this.margin.top - this.margin.bottom; - this.x_buffer = this.plot_width / 20; - this.y_buffer = this.plot_height / 20; - this.y_min = d3.min(this.sample_vals); - this.y_max = d3.max(this.sample_vals) * 1.1; - this.plot_height -= this.y_buffer; - this.create_x_scale(); - this.get_histogram_data(); - this.create_y_scale(); - this.svg = this.create_svg(); - this.create_graph(); - } + Histogram = (function() { + function Histogram(sample_list, sample_group) { + this.sample_list = sample_list; + this.sample_group = sample_group; + this.sort_by = "name"; + this.format_count = d3.format(",.0f"); + this.get_sample_vals(); + this.margin = { + top: 10, + right: 30, + bottom: 30, + left: 30 + }; + this.plot_width = 960 - this.margin.left - this.margin.right; + this.plot_height = 500 - this.margin.top - this.margin.bottom; + this.x_buffer = this.plot_width / 20; + this.y_buffer = this.plot_height / 20; + this.y_min = d3.min(this.sample_vals); + this.y_max = d3.max(this.sample_vals) * 1.1; + this.plot_height -= this.y_buffer; + this.create_x_scale(); + this.get_histogram_data(); + this.create_y_scale(); + this.svg = this.create_svg(); + this.create_graph(); + } - Histogram.prototype.get_sample_vals = function() { - var sample; - return this.sample_vals = (function() { - var _i, _len, _ref, _results; - _ref = this.sample_list; - _results = []; - for (_i = 0, _len = _ref.length; _i < _len; _i++) { - sample = _ref[_i]; - if (sample.value !== null) { - _results.push(sample.value); + Histogram.prototype.get_sample_vals = function() { + var sample; + return this.sample_vals = (function() { + var i, len, ref, results; + ref = this.sample_list; + results = []; + for (i = 0, len = ref.length; i < len; i++) { + sample = ref[i]; + if (sample.value !== null) { + results.push(sample.value); + } } - } - return _results; - }).call(this); - }; + return results; + }).call(this); + }; - Histogram.prototype.create_svg = function() { - var svg; - svg = d3.select("#histogram").append("svg").attr("class", "histogram").attr("width", this.plot_width + this.margin.left + this.margin.right).attr("height", this.plot_height + this.margin.top + this.margin.bottom).append("g").attr("transform", "translate(" + this.margin.left + "," + this.margin.top + ")"); - return svg; - }; + Histogram.prototype.create_svg = function() { + var svg; + svg = d3.select("#histogram").append("svg").attr("class", "histogram").attr("width", this.plot_width + this.margin.left + this.margin.right).attr("height", this.plot_height + this.margin.top + this.margin.bottom).append("g").attr("transform", "translate(" + this.margin.left + "," + this.margin.top + ")"); + return svg; + }; + + Histogram.prototype.create_x_scale = function() { + var x0; + console.log("min/max:", d3.min(this.sample_vals) + "," + d3.max(this.sample_vals)); + x0 = Math.max(-d3.min(this.sample_vals), d3.max(this.sample_vals)); + return this.x_scale = d3.scale.linear().domain([d3.min(this.sample_vals), d3.max(this.sample_vals)]).range([0, this.plot_width]).nice(); + }; - Histogram.prototype.create_x_scale = function() { - var x0; - console.log("min/max:", d3.min(this.sample_vals) + "," + d3.max(this.sample_vals)); - x0 = Math.max(-d3.min(this.sample_vals), d3.max(this.sample_vals)); - return this.x_scale = d3.scale.linear().domain([d3.min(this.sample_vals), d3.max(this.sample_vals)]).range([0, this.plot_width]).nice(); - }; + Histogram.prototype.get_histogram_data = function() { + var n_bins; + console.log("sample_vals:", this.sample_vals); + n_bins = Math.sqrt(this.sample_vals.length); + this.histogram_data = d3.layout.histogram().bins(this.x_scale.ticks(n_bins))(this.sample_vals); + return console.log("histogram_data:", this.histogram_data[0]); + }; - Histogram.prototype.get_histogram_data = function() { - console.log("sample_vals:", this.sample_vals); - this.histogram_data = d3.layout.histogram().bins(this.x_scale.ticks(20))(this.sample_vals); - return console.log("histogram_data:", this.histogram_data[0]); - }; + Histogram.prototype.create_y_scale = function() { + return this.y_scale = d3.scale.linear().domain([ + 0, d3.max(this.histogram_data, (function(_this) { + return function(d) { + return d.y; + }; + })(this)) + ]).range([this.plot_height, 0]); + }; - Histogram.prototype.create_y_scale = function() { - return this.y_scale = d3.scale.linear().domain([ - 0, d3.max(this.histogram_data, (function(_this) { - return function(d) { - return d.y; - }; - })(this)) - ]).range([this.plot_height, 0]); - }; + Histogram.prototype.create_graph = function() { + this.add_x_axis(); + this.add_y_axis(); + return this.add_bars(); + }; - Histogram.prototype.create_graph = function() { - this.add_x_axis(); - this.add_y_axis(); - return this.add_bars(); - }; + Histogram.prototype.add_x_axis = function() { + var x_axis; + x_axis = d3.svg.axis().scale(this.x_scale).orient("bottom"); + return this.svg.append("g").attr("class", "x axis").attr("transform", "translate(0," + this.plot_height + ")").call(x_axis); + }; - Histogram.prototype.add_x_axis = function() { - var x_axis; - x_axis = d3.svg.axis().scale(this.x_scale).orient("bottom"); - return this.svg.append("g").attr("class", "x axis").attr("transform", "translate(0," + this.plot_height + ")").call(x_axis); - }; + Histogram.prototype.add_y_axis = function() { + var yAxis; + yAxis = d3.svg.axis().scale(this.y_scale).orient("left").ticks(5); + return this.svg.append("g").attr("class", "y axis").call(yAxis).append("text").attr("transform", "rotate(-90)").attr("y", 6).attr("dy", ".71em").style("text-anchor", "end"); + }; - Histogram.prototype.add_y_axis = function() { - var yAxis; - yAxis = d3.svg.axis().scale(this.y_scale).orient("left").ticks(5); - return this.svg.append("g").attr("class", "y axis").call(yAxis).append("text").attr("transform", "rotate(-90)").attr("y", 6).attr("dy", ".71em").style("text-anchor", "end"); - }; + Histogram.prototype.add_bars = function() { + var bar; + console.log("bar_width:", this.x_scale(this.histogram_data[0].dx)); + bar = this.svg.selectAll(".bar").data(this.histogram_data).enter().append("g").attr("class", "bar").attr("transform", (function(_this) { + return function(d) { + return "translate(" + _this.x_scale(d.x) + "," + _this.y_scale(d.y) + ")"; + }; + })(this)); + bar.append("rect").attr("x", 1).attr("width", this.x_scale(this.histogram_data[0].x + this.histogram_data[0].dx) - 1).attr("height", (function(_this) { + return function(d) { + return _this.plot_height - _this.y_scale(d.y); + }; + })(this)); + return bar.append("text").attr("dy", ".75em").attr("y", 6).attr("x", this.x_scale(this.histogram_data[0].dx) / 2).attr("text-anchor", "middle").style("fill", "#fff").text((function(_this) { + return function(d) { + var bar_height; + bar_height = _this.plot_height - _this.y_scale(d.y); + if (bar_height > 20) { + return _this.format_count(d.y); + } + }; + })(this)); + }; - Histogram.prototype.add_bars = function() { - var bar; - console.log("bar_width:", this.x_scale(this.histogram_data[0].dx)); - bar = this.svg.selectAll(".bar").data(this.histogram_data).enter().append("g").attr("class", "bar").attr("transform", (function(_this) { - return function(d) { - return "translate(" + _this.x_scale(d.x) + "," + _this.y_scale(d.y) + ")"; - }; - })(this)); - bar.append("rect").attr("x", 1).attr("width", this.x_scale(this.histogram_data[0].x + this.histogram_data[0].dx) - 1).attr("height", (function(_this) { - return function(d) { - return _this.plot_height - _this.y_scale(d.y); - }; - })(this)); - return bar.append("text").attr("dy", ".75em").attr("y", 6).attr("x", this.x_scale(this.histogram_data[0].dx) / 2).attr("text-anchor", "middle").style("fill", "#fff").text((function(_this) { - return function(d) { - var bar_height; - bar_height = _this.plot_height - _this.y_scale(d.y); - if (bar_height > 20) { - return _this.format_count(d.y); - } - }; - })(this)); - }; + return Histogram; - return Histogram; + })(); -})(); + root.Histogram = Histogram; -root.Histogram = Histogram; +}).call(this); -- cgit v1.2.3 From 22903fc679e4dbbd820b2849703d6f41950483b3 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Fri, 8 May 2015 13:37:09 +0300 Subject: fix --- wqflask/wqflask/static/new/javascript/histogram.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wqflask/wqflask/static/new/javascript/histogram.coffee b/wqflask/wqflask/static/new/javascript/histogram.coffee index 7adb53f9..879c0fb8 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.coffee +++ b/wqflask/wqflask/static/new/javascript/histogram.coffee @@ -53,7 +53,7 @@ class Histogram get_histogram_data: () -> console.log("sample_vals:", @sample_vals) - n_bins = sqrt(@sample_vals.length) + n_bins = Math.sqrt(@sample_vals.length) @histogram_data = d3.layout.histogram() .bins(@x_scale.ticks(n_bins))(@sample_vals) console.log("histogram_data:", @histogram_data[0]) -- cgit v1.2.3 From 4e6c08c51dbd2c4ec3cdf92e10f24c3987675bf8 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Fri, 8 May 2015 14:15:50 +0300 Subject: fix histogram bin width It was calculated wrongly when bins began from a non-zero value --- .../wqflask/static/new/javascript/histogram.coffee | 22 +++++++++----- wqflask/wqflask/static/new/javascript/histogram.js | 35 ++++++++++++---------- 2 files changed, 34 insertions(+), 23 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/histogram.coffee b/wqflask/wqflask/static/new/javascript/histogram.coffee index 879c0fb8..b27a389e 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.coffee +++ b/wqflask/wqflask/static/new/javascript/histogram.coffee @@ -4,7 +4,6 @@ class Histogram constructor: (@sample_list, @sample_group) -> @sort_by = "name" @format_count = d3.format(",.0f") #a formatter for counts - @get_sample_vals() @margin = {top: 10, right: 30, bottom: 30, left: 30} @plot_width = 960 - @margin.left - @margin.right @@ -12,21 +11,25 @@ class Histogram @x_buffer = @plot_width/20 @y_buffer = @plot_height/20 + @plot_height -= @y_buffer + + @redraw(@sample_list) + redraw: (sample_list) -> + @get_sample_vals(sample_list) @y_min = d3.min(@sample_vals) @y_max = d3.max(@sample_vals) * 1.1 - - @plot_height -= @y_buffer + @create_x_scale() @get_histogram_data() @create_y_scale() + $("#histogram").empty() @svg = @create_svg() - @create_graph() - get_sample_vals: () -> - @sample_vals = (sample.value for sample in @sample_list when sample.value != null) + get_sample_vals: (sample_list) -> + @sample_vals = (sample.value for sample in sample_list when sample.value != null) create_svg: () -> svg = d3.select("#histogram") @@ -113,17 +116,20 @@ class Histogram .attr("class", "bar") .attr("transform", (d) => return "translate(" + @x_scale(d.x) + "," + @y_scale(d.y) + ")") + + rect_width = @x_scale(@histogram_data[0].x + @histogram_data[0].dx) - + @x_scale(@histogram_data[0].x) bar.append("rect") .attr("x", 1) - .attr("width", @x_scale(@histogram_data[0].x + @histogram_data[0].dx) - 1) + .attr("width", rect_width - 1) .attr("height", (d) => return @plot_height - @y_scale(d.y) ) bar.append("text") .attr("dy", ".75em") .attr("y", 6) - .attr("x", @x_scale(@histogram_data[0].dx)/2) + .attr("x", rect_width / 2) .attr("text-anchor", "middle") .style("fill", "#fff") .text((d) => diff --git a/wqflask/wqflask/static/new/javascript/histogram.js b/wqflask/wqflask/static/new/javascript/histogram.js index 3ab5eefa..94144661 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.js +++ b/wqflask/wqflask/static/new/javascript/histogram.js @@ -5,12 +5,11 @@ root = typeof exports !== "undefined" && exports !== null ? exports : this; Histogram = (function() { - function Histogram(sample_list, sample_group) { - this.sample_list = sample_list; + function Histogram(sample_list1, sample_group) { + this.sample_list = sample_list1; this.sample_group = sample_group; this.sort_by = "name"; this.format_count = d3.format(",.0f"); - this.get_sample_vals(); this.margin = { top: 10, right: 30, @@ -21,30 +20,35 @@ this.plot_height = 500 - this.margin.top - this.margin.bottom; this.x_buffer = this.plot_width / 20; this.y_buffer = this.plot_height / 20; + this.plot_height -= this.y_buffer; + this.redraw(this.sample_list); + } + + Histogram.prototype.redraw = function(sample_list) { + this.get_sample_vals(sample_list); this.y_min = d3.min(this.sample_vals); this.y_max = d3.max(this.sample_vals) * 1.1; - this.plot_height -= this.y_buffer; this.create_x_scale(); this.get_histogram_data(); this.create_y_scale(); + $("#histogram").empty(); this.svg = this.create_svg(); - this.create_graph(); - } + return this.create_graph(); + }; - Histogram.prototype.get_sample_vals = function() { + Histogram.prototype.get_sample_vals = function(sample_list) { var sample; return this.sample_vals = (function() { - var i, len, ref, results; - ref = this.sample_list; + var i, len, results; results = []; - for (i = 0, len = ref.length; i < len; i++) { - sample = ref[i]; + for (i = 0, len = sample_list.length; i < len; i++) { + sample = sample_list[i]; if (sample.value !== null) { results.push(sample.value); } } return results; - }).call(this); + })(); }; Histogram.prototype.create_svg = function() { @@ -97,19 +101,20 @@ }; Histogram.prototype.add_bars = function() { - var bar; + var bar, rect_width; console.log("bar_width:", this.x_scale(this.histogram_data[0].dx)); bar = this.svg.selectAll(".bar").data(this.histogram_data).enter().append("g").attr("class", "bar").attr("transform", (function(_this) { return function(d) { return "translate(" + _this.x_scale(d.x) + "," + _this.y_scale(d.y) + ")"; }; })(this)); - bar.append("rect").attr("x", 1).attr("width", this.x_scale(this.histogram_data[0].x + this.histogram_data[0].dx) - 1).attr("height", (function(_this) { + rect_width = this.x_scale(this.histogram_data[0].x + this.histogram_data[0].dx) - this.x_scale(this.histogram_data[0].x); + bar.append("rect").attr("x", 1).attr("width", rect_width - 1).attr("height", (function(_this) { return function(d) { return _this.plot_height - _this.y_scale(d.y); }; })(this)); - return bar.append("text").attr("dy", ".75em").attr("y", 6).attr("x", this.x_scale(this.histogram_data[0].dx) / 2).attr("text-anchor", "middle").style("fill", "#fff").text((function(_this) { + return bar.append("text").attr("dy", ".75em").attr("y", 6).attr("x", rect_width / 2).attr("text-anchor", "middle").style("fill", "#fff").text((function(_this) { return function(d) { var bar_height; bar_height = _this.plot_height - _this.y_scale(d.y); -- cgit v1.2.3 From 3f3469d007c3e5ff229425911b28f24211c84301 Mon Sep 17 00:00:00 2001 From: Artem Tarasov Date: Fri, 8 May 2015 14:42:19 +0300 Subject: updating histogram on values' change (#28) --- wqflask/wqflask/static/new/javascript/histogram.coffee | 6 +++--- wqflask/wqflask/static/new/javascript/histogram.js | 7 ++++--- wqflask/wqflask/static/new/javascript/show_trait.coffee | 1 + wqflask/wqflask/static/new/javascript/show_trait.js | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/wqflask/wqflask/static/new/javascript/histogram.coffee b/wqflask/wqflask/static/new/javascript/histogram.coffee index b27a389e..68d9b5a2 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.coffee +++ b/wqflask/wqflask/static/new/javascript/histogram.coffee @@ -13,10 +13,10 @@ class Histogram @y_buffer = @plot_height/20 @plot_height -= @y_buffer - @redraw(@sample_list) + @get_sample_vals(@sample_list) + @redraw(@sample_vals) - redraw: (sample_list) -> - @get_sample_vals(sample_list) + redraw: (@sample_vals) -> @y_min = d3.min(@sample_vals) @y_max = d3.max(@sample_vals) * 1.1 diff --git a/wqflask/wqflask/static/new/javascript/histogram.js b/wqflask/wqflask/static/new/javascript/histogram.js index 94144661..d872a3ba 100755 --- a/wqflask/wqflask/static/new/javascript/histogram.js +++ b/wqflask/wqflask/static/new/javascript/histogram.js @@ -21,11 +21,12 @@ this.x_buffer = this.plot_width / 20; this.y_buffer = this.plot_height / 20; this.plot_height -= this.y_buffer; - this.redraw(this.sample_list); + this.get_sample_vals(this.sample_list); + this.redraw(this.sample_vals); } - Histogram.prototype.redraw = function(sample_list) { - this.get_sample_vals(sample_list); + Histogram.prototype.redraw = function(sample_vals) { + this.sample_vals = sample_vals; this.y_min = d3.min(this.sample_vals); this.y_max = d3.max(this.sample_vals) * 1.1; this.create_x_scale(); diff --git a/wqflask/wqflask/static/new/javascript/show_trait.coffee b/wqflask/wqflask/static/new/javascript/show_trait.coffee index ee57831f..1d3123ba 100755 --- a/wqflask/wqflask/static/new/javascript/show_trait.coffee +++ b/wqflask/wqflask/static/new/javascript/show_trait.coffee @@ -238,6 +238,7 @@ $ -> sample_sets['samples_all'].add_value(real_value) already_seen[name] = true console.log("towards end:", sample_sets) + root.histogram.redraw(sample_sets['samples_primary'].the_values) update_stat_values(sample_sets) show_hide_outliers = -> diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js index c2dc07ee..9323862a 100755 --- a/wqflask/wqflask/static/new/javascript/show_trait.js +++ b/wqflask/wqflask/static/new/javascript/show_trait.js @@ -265,6 +265,7 @@ } } console.log("towards end:", sample_sets); + root.histogram.redraw(sample_sets['samples_primary'].the_values); return update_stat_values(sample_sets); }; show_hide_outliers = function() { -- cgit v1.2.3