From 18cf0075003b70b2d97f9fe20794b8ed3f6e1233 Mon Sep 17 00:00:00 2001 From: zsloan Date: Thu, 15 Sep 2016 17:10:51 +0000 Subject: Conflict in wqflask/wqflask/templates/show_trait_mapping_tools.html --- wqflask/wqflask/templates/show_trait_mapping_tools.html | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) (limited to 'wqflask') diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html index 3d9c2521..9c1101ca 100644 --- a/wqflask/wqflask/templates/show_trait_mapping_tools.html +++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html @@ -32,6 +32,18 @@ {% if use_pylmm_rqtl and not use_plink_gemma and dataset.group.species != "human" %}
+ {% if genofiles and genofiles|length>0 %} +
+ +
+ +
+
+ {% endif %}
@@ -90,8 +102,8 @@
- -
+ + 
+    {{ stack[0] }}
     {{ message }} (error)
     {{ stack[-3] }}
     {{ stack[-2] }}
@@ -45,8 +46,7 @@
   Toggle full stack trace
   

     
-      {% for line in stack %}
-        {{ line }}
+      {% for line in stack %} {{ line }}
       {% endfor %}
     

   

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 40a77df3..12d43052 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -91,10 +91,11 @@ def shutdown_session(exception=None):
 def handle_bad_request(e):
     err_msg = str(e)
     logger.error(err_msg)
+    logger.error(request.url)
     # get the stack trace and send it to the logger
     exc_type, exc_value, exc_traceback = sys.exc_info()
     logger.error(traceback.format_exc())
-    formatted_lines = traceback.format_exc().splitlines()
+    formatted_lines = [request.url]+traceback.format_exc().splitlines()
 
     # Handle random animations
     # Use a cookie to have one animation on refresh
-- 
cgit 1.4.1


From 1107259dc258d0cfcb447519cc36e59e78facdef Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sun, 2 Oct 2016 11:52:47 +0000
Subject: Errors: added time stamps in UTC

---
 wqflask/utility/logger.py | 6 +++++-
 wqflask/wqflask/views.py  | 6 +++++-
 2 files changed, 10 insertions(+), 2 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/utility/logger.py b/wqflask/utility/logger.py
index ddc0ea82..b873e16f 100644
--- a/wqflask/utility/logger.py
+++ b/wqflask/utility/logger.py
@@ -31,6 +31,7 @@ import string
 from inspect import isfunction
 from pprint import pformat as pf
 from inspect import stack
+import datetime
 
 from utility.tools import LOG_LEVEL, LOG_LEVEL_DEBUG, LOG_SQL, LOG_FORMAT
 
@@ -66,7 +67,10 @@ LOG_LEVEL_DEBUG (NYI).
 
     def error(self,*args):
         """Call logging.error for multiple args"""
-        self.collect(self.logger.error,*args)
+        now = datetime.datetime.utcnow()
+        time_str = now.strftime('%H:%M:%S UTC %Y%m%d')
+        l = [time_str]+list(args)
+        self.collect(self.logger.error,*l)
 
     def infof(self,*args):
         """Call logging.info for multiple args lazily"""
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 12d43052..7abcc17d 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -7,6 +7,8 @@ from __future__ import absolute_import, division, print_function
 import traceback # for error page
 import os        # for error gifs
 import random    # for random error gif
+import datetime  # for errors
+import time      # for errors
 import sys
 import csv
 import xlsxwriter
@@ -95,7 +97,9 @@ def handle_bad_request(e):
     # get the stack trace and send it to the logger
     exc_type, exc_value, exc_traceback = sys.exc_info()
     logger.error(traceback.format_exc())
-    formatted_lines = [request.url]+traceback.format_exc().splitlines()
+    now = datetime.datetime.utcnow()
+    time_str = now.strftime('%l:%M%p UTC %b %d, %Y')
+    formatted_lines = [request.url + " ("+time_str+")"]+traceback.format_exc().splitlines()
 
     # Handle random animations
     # Use a cookie to have one animation on refresh
-- 
cgit 1.4.1


From 2115fcfde74375252f556153d48f52f4d15fb298 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sun, 2 Oct 2016 12:02:46 +0000
Subject: Error page: trim error messages

---
 wqflask/wqflask/templates/error.html | 6 +++++-
 wqflask/wqflask/views.py             | 4 ++--
 2 files changed, 7 insertions(+), 3 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/templates/error.html b/wqflask/wqflask/templates/error.html
index 95485f1e..7ab2bf2f 100644
--- a/wqflask/wqflask/templates/error.html
+++ b/wqflask/wqflask/templates/error.html
@@ -25,11 +25,15 @@
       to reproduce this ERROR. Next to those steps, copy-paste below
       stack trace, either as
       a new
-      issue (GeneNetwork error: {{message}}) or E-mail this full page to one of the developers
+      issue or E-mail this full page to one of the developers
       directly.
+

+ (GeneNetwork error: {{message[:128]}}) +

+ 
     {{ stack[0] }}
     {{ message }} (error)
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 7abcc17d..cf316bee 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -103,7 +103,7 @@ def handle_bad_request(e):
 
     # Handle random animations
     # Use a cookie to have one animation on refresh
-    animation = request.cookies.get(err_msg)
+    animation = request.cookies.get(err_msg[:64])
     if not animation:
         list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ]
         animation = random.choice(list)
@@ -111,7 +111,7 @@ def handle_bad_request(e):
     resp = make_response(render_template("error.html",message=err_msg,stack=formatted_lines,error_image=animation))
 
     # logger.error("Set cookie %s with %s" % (err_msg, animation))
-    resp.set_cookie(err_msg,animation)
+    resp.set_cookie(err_msg[:64],animation)
     return resp
 
 @app.route("/")
-- 
cgit 1.4.1


From e7e1fbfda99c47bbe547b11c926831e523a37147 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 3 Oct 2016 06:59:08 +0000
Subject: Error page: reduce cookie size

---
 wqflask/wqflask/views.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index cf316bee..33fab84d 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -103,7 +103,7 @@ def handle_bad_request(e):
 
     # Handle random animations
     # Use a cookie to have one animation on refresh
-    animation = request.cookies.get(err_msg[:64])
+    animation = request.cookies.get(err_msg[:32])
     if not animation:
         list = [fn for fn in os.listdir("./wqflask/static/gif/error") if fn.endswith(".gif") ]
         animation = random.choice(list)
@@ -111,7 +111,7 @@ def handle_bad_request(e):
     resp = make_response(render_template("error.html",message=err_msg,stack=formatted_lines,error_image=animation))
 
     # logger.error("Set cookie %s with %s" % (err_msg, animation))
-    resp.set_cookie(err_msg[:64],animation)
+    resp.set_cookie(err_msg[:32],animation)
     return resp
 
 @app.route("/")
-- 
cgit 1.4.1


From e1ba11b55fe849737a649939156c56fe90b0d3e9 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 3 Oct 2016 08:07:00 +0000
Subject: Config: use server port setting

---
 wqflask/runserver.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 12ec904e..5a8766b0 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -16,34 +16,38 @@ GREEN = '\033[92m'
 BOLD  = '\033[1m'
 ENDC  = '\033[0m'
 
-logger.info("GN2 is running. Visit %shttp://localhost:5003/%s" % (BLUE,ENDC))
 
 import os
 app.config['SECRET_KEY'] = os.urandom(24)
 
-from utility.tools import WEBSERVER_MODE
+from utility.tools import WEBSERVER_MODE,get_setting_int
+
+port = get_setting_int("SERVER_PORT")
+
+logger.info("GN2 is running. Visit %shttp://localhost:%s/%s" % (BLUE,port,ENDC))
 
 werkzeug_logger = logging.getLogger('werkzeug')
 
 if WEBSERVER_MODE == 'DEBUG':
     app.run(host='0.0.0.0',
-            port=app.config['SERVER_PORT'],
+            port=port,
             debug=True,
-            use_debugger=True,
+            use_debugger=False,
             threaded=False,
+            processes=0,
             use_reloader=True)
 elif WEBSERVER_MODE == 'DEV':
     werkzeug_logger.setLevel(logging.WARNING)
     app.run(host='0.0.0.0',
-            port=app.config['SERVER_PORT'],
+            port=port,
             debug=False,
             use_debugger=False,
             threaded=False,
             processes=0,
             use_reloader=True)
-else: #production mode
+else: # staging/production modes
     app.run(host='0.0.0.0',
-            port=app.config['SERVER_PORT'],
+            port=port,
             debug=False,
             use_debugger=False,
             threaded=True,
-- 
cgit 1.4.1


From 2acd8aa41580aaccc4119a6f504bdfb5e986e0c2 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 3 Oct 2016 08:29:30 +0000
Subject: Debug: sanitize some output

---
 wqflask/base/webqtlCaseData.py | 2 --
 wqflask/runserver.py           | 1 -
 wqflask/utility/tools.py       | 9 ++++-----
 wqflask/wqflask/__init__.py    | 8 ++------
 4 files changed, 6 insertions(+), 14 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/base/webqtlCaseData.py b/wqflask/base/webqtlCaseData.py
index 8df9939e..2f88f778 100644
--- a/wqflask/base/webqtlCaseData.py
+++ b/wqflask/base/webqtlCaseData.py
@@ -16,8 +16,6 @@
 # Contact Drs. Robert W. Williams and Xiaodong Zhou (2010)
 # at rwilliams@uthsc.edu and xzhou15@uthsc.edu
 #
-#
-#
 # This module is used by GeneNetwork project (www.genenetwork.org)
 #
 # Created by GeneNetwork Core Team 2010/08/10
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 5a8766b0..0342b7ad 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -16,7 +16,6 @@ GREEN = '\033[92m'
 BOLD  = '\033[1m'
 ENDC  = '\033[0m'
 
-
 import os
 app.config['SECRET_KEY'] = os.urandom(24)
 
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 22779739..545c0427 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -50,7 +50,7 @@ def get_setting(command_id,guess=None):
         if command is None:
             command = value(guess)
             if command is None or command == "":
-                print command
+                # print command
                 raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).')
     logger.debug("Set "+command_id+"="+str(command))
     return command
@@ -170,15 +170,14 @@ def show_settings():
     logging.basicConfig(level=log_level)
 
     logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC)
-    print "runserver.py: ****** The webserver has the following configuration ******"
+    print "runserver.py: ****** Webserver configuration ******"
     keylist = app.config.keys()
     keylist.sort()
     for k in keylist:
         try:
-            print("%s %s%s%s%s" % (k,BLUE,BOLD,get_setting(k),ENDC))
+            print("%s: %s%s%s%s" % (k,BLUE,BOLD,get_setting(k),ENDC))
         except:
-            print("%s %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC))
-
+            print("%s: %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC))
 
 # Cached values
 WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py
index af271d02..602246d9 100644
--- a/wqflask/wqflask/__init__.py
+++ b/wqflask/wqflask/__init__.py
@@ -8,17 +8,13 @@ from utility import formatting
 
 import logging
 logger = logging.getLogger(__name__ )
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(level=logging.WARN)
 
 app = Flask(__name__)
 
 app.config.from_object('cfg.default_settings')   # Get the defaults from cfg.default_settings
 app.config.from_envvar('WQFLASK_SETTINGS')       # See http://flask.pocoo.org/docs/config/#configuring-from-files
-
-logger.debug("System path is")
-logger.debug(sys.path)
-logger.debug("App.config is")
-logger.debug(app.config)
+# Note we also use WQFLASK_OVERRIDES
 
 app.jinja_env.globals.update(
     undefined = jinja2.StrictUndefined,
-- 
cgit 1.4.1


From 598fbead85b0ecb9127989a9c5a88a2d7bc91a5d Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 3 Oct 2016 09:23:57 +0000
Subject: Config: supporting JSON OVERRIDES

---
 bin/genenetwork2            |  2 +-
 wqflask/utility/tools.py    | 35 +++++++++++++++++++++++++++++------
 wqflask/wqflask/__init__.py |  2 +-
 3 files changed, 31 insertions(+), 8 deletions(-)

(limited to 'wqflask')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 872e9326..30a4dc95 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -56,7 +56,7 @@ if [ -z $TEMPDIR ]; then
 fi
 
 # Now handle command parameter -c
-if [ $1 = '-c' ] ; then
+if [ "$1" = '-c' ] ; then
     cd $GN2_BASE_PATH/wqflask
     cmd=${2#wqflask/}
     echo PYTHONPATH=$PYTHONPATH
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 545c0427..23d6fb62 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -3,6 +3,7 @@
 
 import os
 import sys
+import json
 
 from wqflask import app
 
@@ -10,6 +11,8 @@ from wqflask import app
 import logging
 logger = logging.getLogger(__name__ )
 
+OVERRIDES = {}
+
 def get_setting(command_id,guess=None):
     """Resolve a setting from the environment or the global settings in
     app.config, with valid_path is a function checking whether the
@@ -45,13 +48,15 @@ def get_setting(command_id,guess=None):
     logger.debug("Looking for "+command_id+"\n")
     command = value(os.environ.get(command_id))
     if command is None or command == "":
-        # ---- Check whether setting exists in app
-        command = value(app.config.get(command_id))
+        command = OVERRIDES.get(command_id)
         if command is None:
-            command = value(guess)
-            if command is None or command == "":
-                # print command
-                raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).')
+            # ---- Check whether setting exists in app
+            command = value(app.config.get(command_id))
+            if command is None:
+                command = value(guess)
+                if command is None or command == "":
+                    # print command
+                    raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).')
     logger.debug("Set "+command_id+"="+str(command))
     return command
 
@@ -169,6 +174,7 @@ def show_settings():
     log_level = getattr(logging, LOG_LEVEL.upper())
     logging.basicConfig(level=log_level)
 
+    logger.info(OVERRIDES)
     logger.info(BLUE+"Mr. Mojo Risin 2"+ENDC)
     print "runserver.py: ****** Webserver configuration ******"
     keylist = app.config.keys()
@@ -179,7 +185,9 @@ def show_settings():
         except:
             print("%s: %s%s%s%s" % (k,GREEN,BOLD,app.config[k],ENDC))
 
+
 # Cached values
+HOME               = get_setting('HOME')
 WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
 GN_SERVER_URL      = get_setting('GN_SERVER_URL')
 SQL_URI            = get_setting('SQL_URI')
@@ -198,3 +206,18 @@ PYLMM_COMMAND      = pylmm_command()
 GEMMA_COMMAND      = gemma_command()
 PLINK_COMMAND      = plink_command()
 TEMPDIR            = tempdir()
+
+from six import string_types
+
+if os.environ.get('WQFLASK_OVERRIDES'):
+    jsonfn = get_setting('WQFLASK_OVERRIDES')
+    logger.error("WQFLASK_OVERRIDES: %s" % jsonfn)
+    with open(jsonfn) as data_file:
+        overrides = json.load(data_file)
+        for k in overrides:
+            cmd = overrides[k]
+            if isinstance(cmd, string_types):
+                OVERRIDES[k] = eval(cmd)
+            else:
+                OVERRIDES[k] = cmd
+            logger.debug(OVERRIDES)
diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py
index 602246d9..2188ce17 100644
--- a/wqflask/wqflask/__init__.py
+++ b/wqflask/wqflask/__init__.py
@@ -8,7 +8,7 @@ from utility import formatting
 
 import logging
 logger = logging.getLogger(__name__ )
-logging.basicConfig(level=logging.WARN)
+logging.basicConfig(level=logging.INFO)
 
 app = Flask(__name__)
 
-- 
cgit 1.4.1


From 5fd49a1e036b02eaa82bdb8ce747a3c2cbc8c8de Mon Sep 17 00:00:00 2001
From: zsloan
Date: Mon, 3 Oct 2016 16:46:47 +0000
Subject: Moved rqtl mapping code out of marker_regression and into its own
 file.

Later we will want to run all mapping methods outside of the web server, though
---
 .../wqflask/marker_regression/marker_regression.py | 191 +--------------------
 wqflask/wqflask/marker_regression/rqtl_mapping.py  | 190 ++++++++++++++++++++
 2 files changed, 192 insertions(+), 189 deletions(-)
 create mode 100644 wqflask/wqflask/marker_regression/rqtl_mapping.py

(limited to 'wqflask')

diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index d2b27991..bb964961 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -35,7 +35,7 @@ from utility import helper_functions
 from utility import Plot, Bunch
 from utility import temp_data
 from utility.benchmark import Bench
-from wqflask.marker_regression import gemma_mapping
+from wqflask.marker_regression import gemma_mapping, rqtl_mapping
 
 from utility.tools import locate, locate_ignore_error, PYLMM_COMMAND, GEMMA_COMMAND, PLINK_COMMAND, TEMPDIR
 from utility.external import shell
@@ -166,7 +166,7 @@ class MarkerRegression(object):
             self.model = start_vars['mapmodel_rqtl_geno']
             if start_vars['pair_scan'] == "true":
                 self.pair_scan = True
-            results = self.run_rqtl_geno()
+            results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan)
         elif self.mapping_method == "reaper":
             if "startMb" in start_vars: #ZS: Check if first time page loaded, so it can default to ON
                 if "additiveCheck" in start_vars:
@@ -349,193 +349,6 @@ class MarkerRegression(object):
 
         count, p_values = self.parse_rqtl_output(plink_output_filename)
 
-    def geno_to_rqtl_function(self):        # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly
-
-        ro.r("""
-           trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) }
-
-           getGenoCode <- function(header, name = 'unk'){
-             mat = which(unlist(lapply(header,function(x){ length(grep(paste('@',name,sep=''), x)) })) == 1)
-             return(trim(strsplit(header[mat],':')[[1]][2]))
-           }
-
-           GENOtoCSVR <- function(genotypes = '%s', out = 'cross.csvr', phenotype = NULL, sex = NULL, verbose = FALSE){
-             header = readLines(genotypes, 40)                                                                                 # Assume a geno header is not longer than 40 lines
-             toskip = which(unlist(lapply(header, function(x){ length(grep("Chr\t", x)) })) == 1)-1                            # Major hack to skip the geno headers
-
-             genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat'))                # Get the genotype codes
-             type <- getGenoCode(header, 'type')
-             genodata <- read.csv(genotypes, sep='\t', skip=toskip, header=TRUE, na.strings=getGenoCode(header,'unk'), colClasses='character', comment.char = '#')
-             cat('Genodata:', toskip, " ", dim(genodata), genocodes, '\n')
-             if(is.null(phenotype)) phenotype <- runif((ncol(genodata)-4))                                                     # If there isn't a phenotype, generate a random one
-             if(is.null(sex)) sex <- rep('m', (ncol(genodata)-4))                                                              # If there isn't a sex phenotype, treat all as males
-             outCSVR <- rbind(c('Pheno', '', '', phenotype),                                                                   # Phenotype
-                              c('sex', '', '', sex),                                                                           # Sex phenotype for the mice
-                              cbind(genodata[,c('Locus','Chr', 'cM')], genodata[, 5:ncol(genodata)]))                          # Genotypes
-             write.table(outCSVR, file = out, row.names=FALSE, col.names=FALSE,quote=FALSE, sep=',')                           # Save it to a file
-             require(qtl)
-             cross = read.cross(file=out, 'csvr', genotypes=genocodes)                                                         # Load the created cross file using R/qtl read.cross
-             if(type == 'riset') cross <- convert2riself(cross)                                                                # If its a RIL, convert to a RIL in R/qtl
-             return(cross)
-          }
-        """ % (self.dataset.group.name + ".geno"))
-
-    def run_rqtl_geno(self):
-        self.geno_to_rqtl_function()
-
-        ## Get pointers to some common R functions
-        r_library     = ro.r["library"]                 # Map the library function
-        r_c           = ro.r["c"]                       # Map the c function
-        r_sum         = ro.r["sum"]                     # Map the sum function
-        plot          = ro.r["plot"]                    # Map the plot function
-        postscript    = ro.r["postscript"]              # Map the postscript function
-        png           = ro.r["png"]                     # Map the png function
-        dev_off       = ro.r["dev.off"]                 # Map the device off function
-
-        print(r_library("qtl"))                         # Load R/qtl
-
-        ## Get pointers to some R/qtl functions
-        scanone         = ro.r["scanone"]               # Map the scanone function
-        scantwo         = ro.r["scantwo"]               # Map the scantwo function
-        calc_genoprob   = ro.r["calc.genoprob"]         # Map the calc.genoprob function
-        read_cross      = ro.r["read.cross"]            # Map the read.cross function
-        write_cross     = ro.r["write.cross"]           # Map the write.cross function
-        GENOtoCSVR      = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
-
-        crossname = self.dataset.group.name
-        genofilelocation  = locate(crossname + ".geno", "genotype")
-        crossfilelocation = TMPDIR + crossname + ".cross"
-
-        #print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation)
-
-        cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)                                  # TODO: Add the SEX if that is available
-
-        if self.manhattan_plot:
-            cross_object = calc_genoprob(cross_object)
-        else:
-            cross_object = calc_genoprob(cross_object, step=1, stepwidth="max")
-
-        cross_object = self.add_phenotype(cross_object, self.sanitize_rqtl_phenotype())                 # Add the phenotype
-
-        # for debug: write_cross(cross_object, "csvr", "test.csvr")
-
-        # Scan for QTLs
-        covar = self.create_covariates(cross_object)                                                    # Create the additive covariate matrix
-
-        if self.pair_scan:
-            if self.do_control == "true":                                                # If sum(covar) > 0 we have a covariate matrix
-                print("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covar, model=self.model, method=self.method, n_cluster = 16)
-            else:
-                print("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=self.model, method=self.method, n_cluster = 16)
-
-            #print("Pair scan results:", result_data_frame)
-
-            self.pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png"
-            png(file=TEMPDIR+self.pair_scan_filename)
-            plot(result_data_frame)
-            dev_off()
-
-            return self.process_pair_scan_results(result_data_frame)
-
-        else:
-            if self.do_control == "true":
-                print("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covar, model=self.model, method=self.method)
-            else:
-                print("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=self.model, method=self.method)
-
-            if self.num_perm > 0 and self.permCheck == "ON":                                                                   # Do permutation (if requested by user)
-                if self.do_control == "true":
-                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covar, n_perm = self.num_perm, model=self.model, method=self.method)
-                else:
-                    perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = self.num_perm, model=self.model, method=self.method)
-
-                self.process_rqtl_perm_results(perm_data_frame)                                          # Functions that sets the thresholds for the webinterface
-
-            return self.process_rqtl_results(result_data_frame)
-
-    def add_phenotype(self, cross, pheno_as_string):
-        ro.globalenv["the_cross"] = cross
-        ro.r('the_cross$pheno <- cbind(pull.pheno(the_cross), the_pheno = '+ pheno_as_string +')')
-        return ro.r["the_cross"]
-
-    def create_covariates(self, cross):
-        ro.globalenv["the_cross"] = cross
-        ro.r('genotypes <- pull.geno(the_cross)')                             # Get the genotype matrix
-        userinputS = self.control_marker.replace(" ", "").split(",")                 # TODO: sanitize user input, Never Ever trust a user
-        covariate_names = ', '.join('"{0}"'.format(w) for w in userinputS)
-        #print("Marker names of selected covariates:", covariate_names)
-        ro.r('covnames <- c(' + covariate_names + ')')
-        ro.r('covInGeno <- which(covnames %in% colnames(genotypes))')
-        ro.r('covnames <- covnames[covInGeno]')
-        ro.r("cat('covnames (purged): ', covnames,'\n')")
-        ro.r('covariates <- genotypes[,covnames]')                            # Get the covariate matrix by using the marker name as index to the genotype file
-        #print("R/qtl matrix of covariates:", ro.r["covariates"])
-        return ro.r["covariates"]
-
-    def sanitize_rqtl_phenotype(self):
-        pheno_as_string = "c("
-        for i, val in enumerate(self.vals):
-            if val == "x":
-                if i < (len(self.vals) - 1):
-                    pheno_as_string +=  "NA,"
-                else:
-                    pheno_as_string += "NA"
-            else:
-                if i < (len(self.vals) - 1):
-                    pheno_as_string += str(val) + ","
-                else:
-                    pheno_as_string += str(val)
-        pheno_as_string += ")"
-        return pheno_as_string
-
-    def process_pair_scan_results(self, result):
-        pair_scan_results = []
-
-        result = result[1]
-        output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)]
-        #print("R/qtl scantwo output:", output)
-
-        for i, line in enumerate(result.iter_row()):
-            marker = {}
-            marker['name'] = result.rownames[i]
-            marker['chr1'] = output[i][0]
-            marker['Mb'] = output[i][1]
-            marker['chr2'] = int(output[i][2])
-            pair_scan_results.append(marker)
-
-        #print("pair_scan_results:", pair_scan_results)
-
-        return pair_scan_results
-
-    def process_rqtl_results(self, result):        # TODO: how to make this a one liner and not copy the stuff in a loop
-        qtl_results = []
-
-        output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)]
-        #print("R/qtl scanone output:", output)
-
-        for i, line in enumerate(result.iter_row()):
-            marker = {}
-            marker['name'] = result.rownames[i]
-            marker['chr'] = output[i][0]
-            marker['Mb'] = output[i][1]
-            marker['lod_score'] = output[i][2]
-            qtl_results.append(marker)
-
-        return qtl_results
-
-    def process_rqtl_perm_results(self, results):
-        perm_vals = []
-        for line in str(results).split("\n")[1:(self.num_perm+1)]:
-            #print("R/qtl permutation line:", line.split())
-            perm_vals.append(float(line.split()[1]))
-
-        self.perm_output = perm_vals
-        self.suggestive = np.percentile(np.array(perm_vals), 67)
-        self.significant = np.percentile(np.array(perm_vals), 95)
-
-        return self.suggestive, self.significant
-
-
     def run_plink(self):
         plink_output_filename = webqtlUtil.genRandStr("%s_%s_"%(self.dataset.group.name, self.this_trait.name))
 
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
new file mode 100644
index 00000000..53ea6053
--- /dev/null
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -0,0 +1,190 @@
+import rpy2.robjects as ro
+
+from base.webqtlConfig import TMPDIR
+from utility import webqtlUtil
+from utility.tools import locate, TEMPDIR
+
+def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control, control_marker, manhattan_plot, pair_scan):
+    geno_to_rqtl_function(dataset)
+
+    ## Get pointers to some common R functions
+    r_library     = ro.r["library"]                 # Map the library function
+    r_c           = ro.r["c"]                       # Map the c function
+    r_sum         = ro.r["sum"]                     # Map the sum function
+    plot          = ro.r["plot"]                    # Map the plot function
+    postscript    = ro.r["postscript"]              # Map the postscript function
+    png           = ro.r["png"]                     # Map the png function
+    dev_off       = ro.r["dev.off"]                 # Map the device off function
+
+    print(r_library("qtl"))                         # Load R/qtl
+
+    ## Get pointers to some R/qtl functions
+    scanone         = ro.r["scanone"]               # Map the scanone function
+    scantwo         = ro.r["scantwo"]               # Map the scantwo function
+    calc_genoprob   = ro.r["calc.genoprob"]         # Map the calc.genoprob function
+    read_cross      = ro.r["read.cross"]            # Map the read.cross function
+    write_cross     = ro.r["write.cross"]           # Map the write.cross function
+    GENOtoCSVR      = ro.r["GENOtoCSVR"]            # Map the local GENOtoCSVR function
+
+    crossname = dataset.group.name
+    genofilelocation  = locate(crossname + ".geno", "genotype")
+    crossfilelocation = TMPDIR + crossname + ".cross"
+
+    #print("Conversion of geno to cross at location:", genofilelocation, " to ", crossfilelocation)
+
+    cross_object = GENOtoCSVR(genofilelocation, crossfilelocation)                                  # TODO: Add the SEX if that is available
+
+    if manhattan_plot:
+        cross_object = calc_genoprob(cross_object)
+    else:
+        cross_object = calc_genoprob(cross_object, step=1, stepwidth="max")
+
+    cross_object = add_phenotype(cross_object, sanitize_rqtl_phenotype(vals))                 # Add the phenotype
+
+    # for debug: write_cross(cross_object, "csvr", "test.csvr")
+
+    # Scan for QTLs
+    covar = create_covariates(control_marker, cross_object)                                                    # Create the additive covariate matrix
+
+    if pair_scan:
+        if do_control == "true":                                                # If sum(covar) > 0 we have a covariate matrix
+            print("Using covariate"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method, n_cluster = 16)
+        else:
+            print("No covariates"); result_data_frame = scantwo(cross_object, pheno = "the_pheno", model=model, method=method, n_cluster = 16)
+
+        #print("Pair scan results:", result_data_frame)
+
+        pair_scan_filename = webqtlUtil.genRandStr("scantwo_") + ".png"
+        png(file=TEMPDIR+pair_scan_filename)
+        plot(result_data_frame)
+        dev_off()
+
+        return process_pair_scan_results(result_data_frame)
+    else:
+        if do_control == "true":
+            print("Using covariate"); result_data_frame = scanone(cross_object, pheno = "the_pheno", addcovar = covar, model=model, method=method)
+        else:
+            print("No covariates"); result_data_frame = scanone(cross_object, pheno = "the_pheno", model=model, method=method)
+
+        if num_perm > 0 and permCheck == "ON":                                                                   # Do permutation (if requested by user)
+            if do_control == "true":
+                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", addcovar = covar, n_perm = num_perm, model=model, method=method)
+            else:
+                perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
+
+            process_rqtl_perm_results(num_perm, perm_data_frame)                                          # Functions that sets the thresholds for the webinterface
+
+        return process_rqtl_results(result_data_frame)
+
+def geno_to_rqtl_function(dataset):        # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly
+
+    ro.r("""
+       trim <- function( x ) { gsub("(^[[:space:]]+|[[:space:]]+$)", "", x) }
+
+       getGenoCode <- function(header, name = 'unk'){
+         mat = which(unlist(lapply(header,function(x){ length(grep(paste('@',name,sep=''), x)) })) == 1)
+         return(trim(strsplit(header[mat],':')[[1]][2]))
+       }
+
+       GENOtoCSVR <- function(genotypes = '%s', out = 'cross.csvr', phenotype = NULL, sex = NULL, verbose = FALSE){
+         header = readLines(genotypes, 40)                                                                                 # Assume a geno header is not longer than 40 lines
+         toskip = which(unlist(lapply(header, function(x){ length(grep("Chr\t", x)) })) == 1)-1                            # Major hack to skip the geno headers
+
+         genocodes <- c(getGenoCode(header, 'mat'), getGenoCode(header, 'het'), getGenoCode(header, 'pat'))                # Get the genotype codes
+         type <- getGenoCode(header, 'type')
+         genodata <- read.csv(genotypes, sep='\t', skip=toskip, header=TRUE, na.strings=getGenoCode(header,'unk'), colClasses='character', comment.char = '#')
+         cat('Genodata:', toskip, " ", dim(genodata), genocodes, '\n')
+         if(is.null(phenotype)) phenotype <- runif((ncol(genodata)-4))                                                     # If there isn't a phenotype, generate a random one
+         if(is.null(sex)) sex <- rep('m', (ncol(genodata)-4))                                                              # If there isn't a sex phenotype, treat all as males
+         outCSVR <- rbind(c('Pheno', '', '', phenotype),                                                                   # Phenotype
+                          c('sex', '', '', sex),                                                                           # Sex phenotype for the mice
+                          cbind(genodata[,c('Locus','Chr', 'cM')], genodata[, 5:ncol(genodata)]))                          # Genotypes
+         write.table(outCSVR, file = out, row.names=FALSE, col.names=FALSE,quote=FALSE, sep=',')                           # Save it to a file
+         require(qtl)
+         cross = read.cross(file=out, 'csvr', genotypes=genocodes)                                                         # Load the created cross file using R/qtl read.cross
+         if(type == 'riset') cross <- convert2riself(cross)                                                                # If its a RIL, convert to a RIL in R/qtl
+         return(cross)
+      }
+    """ % (dataset.group.name + ".geno"))
+        
+def add_phenotype(cross, pheno_as_string):
+    ro.globalenv["the_cross"] = cross
+    ro.r('the_cross$pheno <- cbind(pull.pheno(the_cross), the_pheno = '+ pheno_as_string +')')
+    return ro.r["the_cross"]
+
+def create_covariates(control_marker, cross):
+    ro.globalenv["the_cross"] = cross
+    ro.r('genotypes <- pull.geno(the_cross)')                             # Get the genotype matrix
+    userinputS = control_marker.replace(" ", "").split(",")                 # TODO: sanitize user input, Never Ever trust a user
+    covariate_names = ', '.join('"{0}"'.format(w) for w in userinputS)
+    #print("Marker names of selected covariates:", covariate_names)
+    ro.r('covnames <- c(' + covariate_names + ')')
+    ro.r('covInGeno <- which(covnames %in% colnames(genotypes))')
+    ro.r('covnames <- covnames[covInGeno]')
+    ro.r("cat('covnames (purged): ', covnames,'\n')")
+    ro.r('covariates <- genotypes[,covnames]')                            # Get the covariate matrix by using the marker name as index to the genotype file
+    #print("R/qtl matrix of covariates:", ro.r["covariates"])
+    return ro.r["covariates"]
+
+def sanitize_rqtl_phenotype(vals):
+    pheno_as_string = "c("
+    for i, val in enumerate(vals):
+        if val == "x":
+            if i < (len(vals) - 1):
+                pheno_as_string +=  "NA,"
+            else:
+                pheno_as_string += "NA"
+        else:
+            if i < (len(vals) - 1):
+                pheno_as_string += str(val) + ","
+            else:
+                pheno_as_string += str(val)
+    pheno_as_string += ")"
+    return pheno_as_string
+
+def process_pair_scan_results(result):
+    pair_scan_results = []
+
+    result = result[1]
+    output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)]
+    #print("R/qtl scantwo output:", output)
+
+    for i, line in enumerate(result.iter_row()):
+        marker = {}
+        marker['name'] = result.rownames[i]
+        marker['chr1'] = output[i][0]
+        marker['Mb'] = output[i][1]
+        marker['chr2'] = int(output[i][2])
+        pair_scan_results.append(marker)
+
+    #print("pair_scan_results:", pair_scan_results)
+
+    return pair_scan_results
+
+def process_rqtl_results(result):        # TODO: how to make this a one liner and not copy the stuff in a loop
+    qtl_results = []
+
+    output = [tuple([result[j][i] for j in range(result.ncol)]) for i in range(result.nrow)]
+    #print("R/qtl scanone output:", output)
+
+    for i, line in enumerate(result.iter_row()):
+        marker = {}
+        marker['name'] = result.rownames[i]
+        marker['chr'] = output[i][0]
+        marker['Mb'] = output[i][1]
+        marker['lod_score'] = output[i][2]
+        qtl_results.append(marker)
+
+    return qtl_results
+
+def process_rqtl_perm_results(num_perm, results):
+    perm_vals = []
+    for line in str(results).split("\n")[1:(num_perm+1)]:
+        #print("R/qtl permutation line:", line.split())
+        perm_vals.append(float(line.split()[1]))
+
+    perm_output = perm_vals
+    suggestive = np.percentile(np.array(perm_vals), 67)
+    significant = np.percentile(np.array(perm_vals), 95)
+
+    return perm_output, suggestive, significant 
\ No newline at end of file
-- 
cgit 1.4.1


From 6a0a3626baad96deb1e8dc7d27fe1fa15e8c5b98 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Mon, 3 Oct 2016 19:16:56 +0000
Subject: Fixed problem that caused R/qtl with permutations to not work
 correctly

---
 .../wqflask/marker_regression/marker_regression.py |  5 +++-
 wqflask/wqflask/marker_regression/rqtl_mapping.py  | 35 ++++++++++++----------
 2 files changed, 23 insertions(+), 17 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index bb964961..f2a2eb8c 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -166,7 +166,10 @@ class MarkerRegression(object):
             self.model = start_vars['mapmodel_rqtl_geno']
             if start_vars['pair_scan'] == "true":
                 self.pair_scan = True
-            results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan)
+            if self.permCheck and self.num_perm > 0:
+                perm_output, self.suggestive, self.significant, results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan)
+            else:
+                results = rqtl_mapping.run_rqtl_geno(self.vals, self.dataset, self.method, self.model, self.permCheck, self.num_perm, self.do_control, self.control_marker, self.manhattan_plot, self.pair_scan)
         elif self.mapping_method == "reaper":
             if "startMb" in start_vars: #ZS: Check if first time page loaded, so it can default to ON
                 if "additiveCheck" in start_vars:
diff --git a/wqflask/wqflask/marker_regression/rqtl_mapping.py b/wqflask/wqflask/marker_regression/rqtl_mapping.py
index 53ea6053..93bf717c 100644
--- a/wqflask/wqflask/marker_regression/rqtl_mapping.py
+++ b/wqflask/wqflask/marker_regression/rqtl_mapping.py
@@ -1,4 +1,5 @@
 import rpy2.robjects as ro
+import numpy as np
 
 from base.webqtlConfig import TMPDIR
 from utility import webqtlUtil
@@ -72,9 +73,10 @@ def run_rqtl_geno(vals, dataset, method, model, permCheck, num_perm, do_control,
             else:
                 perm_data_frame = scanone(cross_object, pheno_col = "the_pheno", n_perm = num_perm, model=model, method=method)
 
-            process_rqtl_perm_results(num_perm, perm_data_frame)                                          # Functions that sets the thresholds for the webinterface
-
-        return process_rqtl_results(result_data_frame)
+            perm_output, suggestive, significant = process_rqtl_perm_results(num_perm, perm_data_frame)          # Functions that sets the thresholds for the webinterface
+            return perm_output, suggestive, significant, process_rqtl_results(result_data_frame)
+        else:
+            return process_rqtl_results(result_data_frame)
 
 def geno_to_rqtl_function(dataset):        # TODO: Need to figure out why some genofiles have the wrong format and don't convert properly
 
@@ -161,6 +163,19 @@ def process_pair_scan_results(result):
 
     return pair_scan_results
 
+def process_rqtl_perm_results(num_perm, results):
+    perm_vals = []
+    for line in str(results).split("\n")[1:(num_perm+1)]:
+        #print("R/qtl permutation line:", line.split())
+        perm_vals.append(float(line.split()[1]))
+
+    perm_output = perm_vals
+    suggestive = np.percentile(np.array(perm_vals), 67)
+    significant = np.percentile(np.array(perm_vals), 95)
+    print("SIGNIFICANT:", significant)
+
+    return perm_output, suggestive, significant
+    
 def process_rqtl_results(result):        # TODO: how to make this a one liner and not copy the stuff in a loop
     qtl_results = []
 
@@ -175,16 +190,4 @@ def process_rqtl_results(result):        # TODO: how to make this a one liner an
         marker['lod_score'] = output[i][2]
         qtl_results.append(marker)
 
-    return qtl_results
-
-def process_rqtl_perm_results(num_perm, results):
-    perm_vals = []
-    for line in str(results).split("\n")[1:(num_perm+1)]:
-        #print("R/qtl permutation line:", line.split())
-        perm_vals.append(float(line.split()[1]))
-
-    perm_output = perm_vals
-    suggestive = np.percentile(np.array(perm_vals), 67)
-    significant = np.percentile(np.array(perm_vals), 95)
-
-    return perm_output, suggestive, significant 
\ No newline at end of file
+    return qtl_results 
\ No newline at end of file
-- 
cgit 1.4.1


From da127da677a8976675fb95d75213dd3e5326fb36 Mon Sep 17 00:00:00 2001
From: zsloan
Date: Tue, 4 Oct 2016 19:39:17 +0000
Subject: Fixed issue that caused adding to existing collections when not
 logged in to not work correctly.

The problem was that it would get the collection name from the "new_collection" field instead of the "existing_collections" dropdown.
---
 wqflask/wqflask/collect.py                     | 17 +++--------------
 wqflask/wqflask/templates/collections/add.html |  2 +-
 2 files changed, 4 insertions(+), 15 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/collect.py b/wqflask/wqflask/collect.py
index 7e7aba89..81d03d6c 100644
--- a/wqflask/wqflask/collect.py
+++ b/wqflask/wqflask/collect.py
@@ -164,7 +164,7 @@ class UserCollection(object):
             if not uc:
                 return create_new("Default")
         else:
-            uc = model.UserCollection.query.get(params['existing_collection'])
+            uc = model.UserCollection.query.get(params['existing_collection'].split(":")[0])
         members =  list(uc.members_as_set()) #set(json.loads(uc.members))
         len_before = len(members)
 
@@ -218,7 +218,6 @@ def collections_add():
         anon_collections = user_manager.AnonUser().get_collections()
         collection_names = []
         for collection in anon_collections:
-            print("COLLECTION:", collection)
             collection_names.append({'id':collection['id'], 'name':collection['name']})
         return render_template("collections/add.html",
                                 traits = traits,
@@ -229,21 +228,17 @@ def collections_add():
 @app.route("/collections/new")
 def collections_new():
     params = request.args
-    #print("request.args in collections_new are:", params)
-
-    collection_name = params['new_collection']
     
-    if "anonymous_add" in params:
-        AnonCollection(name=collection_name).add_traits(params, "Default")
-        return redirect(url_for('view_collection'))
     if "sign_in" in params:
         return redirect(url_for('login'))
 
     if "create_new" in params:
         print("in create_new")
+        collection_name = params['new_collection']
         return create_new(collection_name)
     elif "add_to_existing" in params:
         print("in add to existing")
+        collection_name = params['existing_collection'].split(":")[1]
         if g.user_session.logged_in:
             return UserCollection().add_traits(params, collection_name)
         else:
@@ -251,7 +246,6 @@ def collections_new():
             ac.add_traits(params)
             return redirect(url_for('view_collection', collection_id=ac.id))
     else:
-        print("ELSE")
         CauseAnError
 
 
@@ -288,7 +282,6 @@ def create_new(collection_name):
         ac = AnonCollection(collection_name)
         ac.changed_timestamp = datetime.datetime.utcnow().strftime('%b %d %Y %I:%M%p')       
         ac.add_traits(params)
-        print("Collection ID:", ac.id)
         return redirect(url_for('view_collection', collection_id=ac.id))
 
 @app.route("/collections/list")
@@ -367,7 +360,6 @@ def view_collection():
         uc_id = params['uc_id']
         uc = model.UserCollection.query.get(uc_id)
         traits = json.loads(uc.members)
-        print("traits are:", traits)
     else:
         user_collections = json.loads(Redis.get(user_manager.AnonUser().key))
         this_collection = {}
@@ -384,7 +376,6 @@ def view_collection():
     json_version = []
 
     for atrait in traits:
-        print("atrait is:", atrait)
         name, dataset_name = atrait.split(':')                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        
                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          
         trait_ob = trait.GeneralTrait(name=name, dataset_name=dataset_name)
@@ -393,8 +384,6 @@ def view_collection():
 
         json_version.append(trait_ob.jsonable())
 
-    print("trait_obs:", trait_obs)
-
     if "uc_id" in params:
         collection_info = dict(trait_obs=trait_obs,
                                uc = uc)
diff --git a/wqflask/wqflask/templates/collections/add.html b/wqflask/wqflask/templates/collections/add.html
index 47b87d73..d45aa015 100644
--- a/wqflask/wqflask/templates/collections/add.html
+++ b/wqflask/wqflask/templates/collections/add.html
@@ -25,7 +25,7 @@
 
                 
                 

-- 
cgit 1.4.1


From ff48bdbce4b4f5cb4c561b06268fd91d8fe546a4 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 5 Oct 2016 06:55:50 +0000
Subject: print statements should be logger

---
 wqflask/utility/helper_functions.py                    | 12 +++++++-----
 wqflask/wqflask/marker_regression/marker_regression.py |  2 +-
 2 files changed, 8 insertions(+), 6 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/utility/helper_functions.py b/wqflask/utility/helper_functions.py
index 149ee553..377f6b26 100644
--- a/wqflask/utility/helper_functions.py
+++ b/wqflask/utility/helper_functions.py
@@ -5,19 +5,21 @@ from base import data_set
 from base.species import TheSpecies
 
 from wqflask import user_manager
+import logging
+logger = logging.getLogger(__name__ )
 
 
 def get_species_dataset_trait(self, start_vars):
     #assert type(read_genotype) == type(bool()), "Expecting boolean value for read_genotype"
     self.dataset = data_set.create_dataset(start_vars['dataset'])
-    print("After creating dataset")
+    logger.debug("After creating dataset")
     self.species = TheSpecies(dataset=self.dataset)
-    print("After creating species")
+    logger.debug("After creating species")
     self.this_trait = GeneralTrait(dataset=self.dataset,
                                    name=start_vars['trait_id'],
                                    cellid=None,
                                    get_qtl_info=True)
-    print("After creating trait")
+    logger.debug("After creating trait")
 
     #if read_genotype:
     #self.dataset.group.read_genotype_file()
@@ -27,7 +29,7 @@ def get_species_dataset_trait(self, start_vars):
 def get_trait_db_obs(self, trait_db_list):
     if isinstance(trait_db_list, basestring):
         trait_db_list = trait_db_list.split(",")
-        
+
     self.trait_list = []
     for trait in trait_db_list:
         data, _separator, hmac = trait.rpartition(':')
@@ -38,4 +40,4 @@ def get_trait_db_obs(self, trait_db_list):
         trait_ob = GeneralTrait(dataset=dataset_ob,
                                name=trait_name,
                                cellid=None)
-        self.trait_list.append((trait_ob, dataset_ob))
\ No newline at end of file
+        self.trait_list.append((trait_ob, dataset_ob))
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index f28407b3..324f128c 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -814,7 +814,7 @@ class MarkerRegression(object):
                     timestamp = datetime.datetime.now().isoformat(),
                     )
 
-        print("After creating params")
+        logger.debug("After creating params")
 
         json_params = json.dumps(params)
         Redis.set(key, json_params)
-- 
cgit 1.4.1


From d99da978111aa90f154be84bf40f09953239ac48 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 5 Oct 2016 07:02:35 +0000
Subject: print statements should be logger

---
 .../wqflask/marker_regression/marker_regression.py | 64 +++++++++++-----------
 1 file changed, 32 insertions(+), 32 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index 324f128c..19e2d50a 100644
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -205,12 +205,12 @@ class MarkerRegression(object):
         elif self.mapping_method == "plink":
             results = self.run_plink()
         elif self.mapping_method == "pylmm":
-            print("RUNNING PYLMM")
+            logger.debug("RUNNING PYLMM")
             if self.num_perm > 0:
                 self.run_permutations(str(temp_uuid))
             results = self.gen_data(str(temp_uuid))
         else:
-            print("RUNNING NOTHING")
+            logger.debug("RUNNING NOTHING")
 
         if self.pair_scan == True:
             self.qtl_results = []
@@ -264,9 +264,9 @@ class MarkerRegression(object):
             #Need to convert the QTL objects that qtl reaper returns into a json serializable dictionary
             for index, qtl in enumerate(self.qtl_results):
                 #if index<40:
-                #    print("lod score is:", qtl['lod_score'])
+                #    logger.debug("lod score is:", qtl['lod_score'])
                 if qtl['chr'] == highest_chr and highest_chr != "X" and highest_chr != "X/Y":
-                    #print("changing to X")
+                    #logger.debug("changing to X")
                     self.json_data['chr'].append("X")
                 else:
                     self.json_data['chr'].append(str(qtl['chr']))
@@ -284,7 +284,7 @@ class MarkerRegression(object):
                 self.json_data['chrnames'].append([self.species.chromosomes.chromosomes[key].name, self.species.chromosomes.chromosomes[key].mb_length])
                 chromosome_mb_lengths[key] = self.species.chromosomes.chromosomes[key].mb_length
 
-            # print("json_data:", self.json_data)
+            # logger.debug("json_data:", self.json_data)
 
             self.js_data = dict(
                 result_score_type = self.score_type,
@@ -312,7 +312,7 @@ class MarkerRegression(object):
                                                                                                  self.dataset.group.name,
                                                                                                  self.dataset.group.name,
                                                                                                  self.dataset.group.name)
-        #print("gemma_command:" + gemma_command)
+        #logger.debug("gemma_command:" + gemma_command)
 
         os.system(gemma_command)
 
@@ -334,7 +334,7 @@ class MarkerRegression(object):
                     included_markers.append(line.split("\t")[1])
                     p_values.append(float(line.split("\t")[10]))
                     #p_values[line.split("\t")[1]] = float(line.split("\t")[10])
-        #print("p_values: ", p_values)
+        #logger.debug("p_values: ", p_values)
         return included_markers, p_values
 
     def gen_pheno_txt_file(self):
@@ -362,7 +362,7 @@ class MarkerRegression(object):
         self.gen_pheno_txt_file_plink(pheno_filename = plink_output_filename)
 
         plink_command = PLINK_COMMAND + ' --noweb --ped %s/%s.ped --no-fid --no-parents --no-sex --no-pheno --map %s/%s.map --pheno %s%s.txt --pheno-name %s --maf %s --missing-phenotype -9999 --out %s%s --assoc ' % (PLINK_PATH, self.dataset.group.name, PLINK_PATH, self.dataset.group.name, TMPDIR, plink_output_filename, self.this_trait.name, self.maf, TMPDIR, plink_output_filename)
-        print("plink_command:", plink_command)
+        logger.debug("plink_command:", plink_command)
 
         os.system(plink_command)
 
@@ -370,11 +370,11 @@ class MarkerRegression(object):
 
         #for marker in self.dataset.group.markers.markers:
         #    if marker['name'] not in included_markers:
-        #        print("marker:", marker)
+        #        logger.debug("marker:", marker)
         #        self.dataset.group.markers.markers.remove(marker)
         #        #del self.dataset.group.markers.markers[marker]
 
-        print("p_values:", pf(p_values))
+        logger.debug("p_values:", pf(p_values))
 
         self.dataset.group.markers.add_pvalues(p_values)
 
@@ -641,7 +641,7 @@ class MarkerRegression(object):
 
         top_lod_scores = []
 
-        #print("self.num_perm:", self.num_perm)
+        #logger.debug("self.num_perm:", self.num_perm)
 
         for permutation in range(self.num_perm):
 
@@ -686,10 +686,10 @@ class MarkerRegression(object):
                     if p_value < lowest_p_value:
                         lowest_p_value = p_value
 
-                #print("lowest_p_value:", lowest_p_value)
+                #logger.debug("lowest_p_value:", lowest_p_value)
                 top_lod_scores.append(-math.log10(lowest_p_value))
 
-        #print("top_lod_scores:", top_lod_scores)
+        #logger.debug("top_lod_scores:", top_lod_scores)
 
         self.suggestive = np.percentile(top_lod_scores, 67)
         self.significant = np.percentile(top_lod_scores, 95)
@@ -698,13 +698,13 @@ class MarkerRegression(object):
         """Generates p-values for each marker"""
 
 
-        print("self.vals is:", self.vals)
+        logger.debug("self.vals is:", self.vals)
         pheno_vector = np.array([(val == "x" or val == "") and np.nan or float(val) for val in self.vals])
 
         #lmm_uuid = str(uuid.uuid4())
 
         key = "pylmm:input:" + temp_uuid
-        print("key is:", pf(key))
+        logger.debug("key is:", pf(key))
         #with Bench("Loading cache"):
         #    result = Redis.get(key)
 
@@ -713,7 +713,7 @@ class MarkerRegression(object):
             #p_values = self.trim_results(p_values)
 
         else:
-            print("NOW CWD IS:", os.getcwd())
+            logger.debug("NOW CWD IS:", os.getcwd())
             genotype_data = [marker['genotypes'] for marker in self.dataset.group.markers.markers]
 
             no_val_samples = self.identify_empty_samples()
@@ -721,9 +721,9 @@ class MarkerRegression(object):
 
             genotype_matrix = np.array(genotype_data).T
 
-            #print("pheno_vector: ", pf(pheno_vector))
-            #print("genotype_matrix: ", pf(genotype_matrix))
-            #print("genotype_matrix.shape: ", pf(genotype_matrix.shape))
+            #logger.debug("pheno_vector: ", pf(pheno_vector))
+            #logger.debug("genotype_matrix: ", pf(genotype_matrix))
+            #logger.debug("genotype_matrix.shape: ", pf(genotype_matrix.shape))
 
             #params = {"pheno_vector": pheno_vector,
             #            "genotype_matrix": genotype_matrix,
@@ -731,8 +731,8 @@ class MarkerRegression(object):
             #            "refit": False,
             #            "temp_data": tempdata}
 
-            # print("genotype_matrix:", str(genotype_matrix.tolist()))
-            # print("pheno_vector:", str(pheno_vector.tolist()))
+            # logger.debug("genotype_matrix:", str(genotype_matrix.tolist()))
+            # logger.debug("pheno_vector:", str(pheno_vector.tolist()))
 
             params = dict(pheno_vector = pheno_vector.tolist(),
                         genotype_matrix = genotype_matrix.tolist(),
@@ -745,14 +745,14 @@ class MarkerRegression(object):
                         )
 
             json_params = json.dumps(params)
-            #print("json_params:", json_params)
+            #logger.debug("json_params:", json_params)
             Redis.set(key, json_params)
             Redis.expire(key, 60*60)
-            print("before printing command")
+            logger.debug("before printing command")
 
             command = PYLMM_COMMAND + ' --key {} --species {}'.format(key, "other")
-            print("command is:", command)
-            print("after printing command")
+            logger.debug("command is:", command)
+            logger.debug("after printing command")
 
             shell(command)
 
@@ -762,7 +762,7 @@ class MarkerRegression(object):
             json_results = Redis.blpop("pylmm:results:" + temp_uuid, 45*60)
             results = json.loads(json_results[1])
             p_values = [float(result) for result in results['p_values']]
-            #print("p_values:", p_values[:10])
+            #logger.debug("p_values:", p_values[:10])
             #p_values = self.trim_results(p_values)
             t_stats = results['t_stats']
 
@@ -773,7 +773,7 @@ class MarkerRegression(object):
             #    refit=False,
             #    temp_data=tempdata
             #)
-            #print("p_values:", p_values)
+            #logger.debug("p_values:", p_values)
 
         self.dataset.group.markers.add_pvalues(p_values)
 
@@ -782,7 +782,7 @@ class MarkerRegression(object):
         return self.dataset.group.markers.markers
 
     def trim_results(self, p_values):
-        print("len_p_values:", len(p_values))
+        logger.debug("len_p_values:", len(p_values))
         if len(p_values) > 500:
             p_values.sort(reverse=True)
             trimmed_values = p_values[:500]
@@ -801,7 +801,7 @@ class MarkerRegression(object):
         kinship_matrix = np.fromfile(open(file_base + '.kin','r'),sep=" ")
         kinship_matrix.resize((len(plink_input.indivs),len(plink_input.indivs)))
 
-        print("Before creating params")
+        logger.debug("Before creating params")
 
         params = dict(pheno_vector = pheno_vector.tolist(),
                     covariate_matrix = covariate_matrix.tolist(),
@@ -820,12 +820,12 @@ class MarkerRegression(object):
         Redis.set(key, json_params)
         Redis.expire(key, 60*60)
 
-        print("Before creating the command")
+        logger.debug("Before creating the command")
 
         command = PYLMM_COMMAND+' --key {} --species {}'.format(key,
                                                                                                                 "human")
 
-        print("command is:", command)
+        logger.debug("command is:", command)
 
         os.system(command)
 
@@ -848,7 +848,7 @@ class MarkerRegression(object):
         return p_values, t_stats
 
     def get_lod_score_cutoff(self):
-        print("INSIDE GET LOD CUTOFF")
+        logger.debug("INSIDE GET LOD CUTOFF")
         high_qtl_count = 0
         for marker in self.dataset.group.markers.markers:
             if marker['lod_score'] > 1:
-- 
cgit 1.4.1


From ec0d6b9ee8ff1ec3baafb3967dcf5c28e4a823b6 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 5 Oct 2016 08:07:05 +0000
Subject: Reverted on genotype selector on page show_trait_mapping_tools.html -
 pending the full PR

---
 .../templates/show_trait_mapping_tools.html        | 38 ++++++++--------------
 1 file changed, 13 insertions(+), 25 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 9c1101ca..0f293942 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -2,7 +2,7 @@
     {% if (use_pylmm_rqtl and dataset.group.species != "human") or use_plink_gemma %}
     

         
 
-        
+
             
    
                 {% if use_pylmm_rqtl and not use_plink_gemma and dataset.group.species != "human" %}
                 
  • 
@@ -32,18 +32,6 @@
                 {% if use_pylmm_rqtl and not use_plink_gemma and dataset.group.species != "human" %}
                 
    
                     
    
-                        {% if genofiles and genofiles|length>0 %}
-                        
    
-                            
-                            
    
-                                
-                            
    
-                        
    
-                        {% endif %}
                         
    
                             
                             
    
@@ -73,7 +61,7 @@
                                     No
                                 
                             
    
-                        
     
+                        
    
 
 
 
    
+  {% if request.form['trait_list'].split(",")|length < 4 %}
+  
+  {% else %}
   
    CTL analysis parameters
    
   {{(request.form['trait_list'].split(',')|length -1)}} phenotypes as input
 
-
    
-	
+  
+    
+    
    
+      
+      
    
+        
+      
    
+    
    
 
-	
    
-		
-		
    
-		
-		
    
-	
    
+    
    
+      
+      
    
+        
+      
    
+    
    
 
-	
    
-		
-		
    
-		
-		
    
-	
    
-
-	
    
-		
-		
    
-		
-		
    
-	
    
-
-
-	
    
-		
-		
    
-		
-		
    
-	
    
-  
    
-	
    
-		
    
-			
-		
    
-	
    
+    
    
+      
+      
    
+        
+      
    
+    
    
 
+    
    
+      
+      
    
+        
+      
    
+    
    
+    
    
+    
    
+      
    
+        
+      
    
+    
    
 
    
-
+{% endif %}
 
    
 {% endblock %}
diff --git a/wqflask/wqflask/templates/wgcna_setup.html b/wqflask/wqflask/templates/wgcna_setup.html
index b4a5730d..c5461497 100644
--- a/wqflask/wqflask/templates/wgcna_setup.html
+++ b/wqflask/wqflask/templates/wgcna_setup.html
@@ -4,49 +4,46 @@
 {% block content %} 
 
     WGCNA analysis parameters
    
 
    
-		{% if request.form['trait_list'].split(",")|length <= 4 %}
-
-		
-    {% else %}
- 		
    
-				
-				
    
-					
-					
    
-						
-					
    
-				
    
-				
    
-					
-					
    
-						
-					
    
-				
    
-				
    
-					
-					
    
-						
-					
    
-				
    
-				
    
-					
-					
    
-						
-					
    
-				
    
-				
    
-					
    
-						
-					
    
-				
    
-		
    
-		{% endif %}
+  {% if request.form['trait_list'].split(",")|length < 4 %}
+  
+  {% else %}
+  
    
+    
+    
    
+      
+      
    
+        
+      
    
+    
    
+    
    
+      
+      
    
+        
+      
    
+    
    
+    
    
+      
+      
    
+        
+      
    
+    
    
+    
    
+      
+      
    
+        
+      
    
+    
    
+    
    
+      
    
+        
+      
    
+    
    
+  
    
+  {% endif %}
 
    
 {% endblock %}
-- 
cgit 1.4.1


From 416363b3f8940d145d8aec076865b167e78705d9 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Thu, 6 Oct 2016 14:24:36 +0000
Subject: CTL requires 2 traits minimally. Also use traits/phenotypes
 consistently

---
 wqflask/wqflask/templates/ctl_setup.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

(limited to 'wqflask')

diff --git a/wqflask/wqflask/templates/ctl_setup.html b/wqflask/wqflask/templates/ctl_setup.html
index 592e4b36..51553322 100644
--- a/wqflask/wqflask/templates/ctl_setup.html
+++ b/wqflask/wqflask/templates/ctl_setup.html
@@ -3,16 +3,16 @@
 
 {% block content %} 
 
    
-  {% if request.form['trait_list'].split(",")|length < 4 %}
+  {% if request.form['trait_list'].split(",")|length < 2 %}
   
   {% else %}
   
    CTL analysis parameters
    
-  {{(request.form['trait_list'].split(',')|length -1)}} phenotypes as input
+  {{(request.form['trait_list'].split(',')|length -1)}} traits as input
 
   
    
     
-- 
cgit 1.4.1