From c38cf0af74c3927f5b7d1da5416671f14c789f72 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 31 Jan 2018 11:22:42 +0000
Subject: Startup: added -cli option

---
 bin/genenetwork2 | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index cbaba76f..65330ec8 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,8 +1,11 @@
 #! /bin/bash
 #
+# This is the startup script for GN2. It sets the environment variables to pick
+# up a Guix profile and allows for overriding parameters.
+#
 # Typical usage
 #
-#   env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2
+#   env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py
 #
 # Where GN2_PROFILE points to the GNU Guix profile used for deployment.
 #
@@ -41,6 +44,7 @@
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi"
 
 SCRIPT=$(realpath "$0")
+echo SCRIPT=$SCRIPT
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
 GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
@@ -96,6 +100,8 @@ if [ -z $GN2_PROFILE ] ; then
     if [ -d $GN2_PROFILE ]; then
         echo "Best guess is $GN2_PROFILE"
     fi
+    echo "ERROR: always set GN2_PROFILE"
+    exit 1
 fi
 if [ -z $GN2_PROFILE ]; then
     read -p "PRESS [ENTER] TO CONTINUE..."
@@ -143,7 +149,7 @@ set|grep guix
 set|grep $GN2_PROFILE
 set|grep TMPDIR
 
-# Now handle command parameter -c
+# Now handle command parameter -c which runs python
 if [ "$1" = '-c' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=${2#wqflask/}
@@ -152,6 +158,16 @@ if [ "$1" = '-c' ] ; then
     python $cmd
     exit $?
 fi
+# Now handle command parameter -cli which runs in bash
+if [ "$1" = "-cli" ] ; then
+    echo "HERE"
+    cd $GN2_BASE_DIR/wqflask
+    cmd=$2
+    echo PYTHONPATH=$PYTHONPATH
+    echo RUNNING COMMAND $cmd
+    $cmd
+    exit $?
+fi
 if [ "$1" = '-gunicorn' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=$2
-- 
cgit v1.2.3


From 8e79b13e9126a1a978aff64cabfd374760e0b5cc Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 31 Jan 2018 11:48:19 +0000
Subject: Startup: longer timeout for gunicorn-dev mode

---
 bin/genenetwork2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 65330ec8..f64576d5 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -180,7 +180,7 @@ if [ "$1" = '-gunicorn-dev' ] ; then
     cd $GN2_BASE_DIR/wqflask
     echo PYTHONPATH=$PYTHONPATH
     if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
-    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --reload wsgi"
+    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi"
     echo RUNNING gunicorn $cmd
     gunicorn $cmd
     exit $?
-- 
cgit v1.2.3


From 97ccc478543a7bb08fe0cf178e320dd5811d8bfa Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sat, 10 Feb 2018 07:53:53 +0000
Subject: Check for bimbam path and bail out if it is missing

---
 bin/genenetwork2             | 1 +
 wqflask/base/webqtlConfig.py | 1 +
 wqflask/utility/tools.py     | 4 ----
 3 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index f64576d5..7148c022 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -45,6 +45,7 @@
 
 SCRIPT=$(realpath "$0")
 echo SCRIPT=$SCRIPT
+echo GN2_PROFILE=$GN2_PROFILE
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
 GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index 1ef2bc26..1e66e957 100644
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -82,6 +82,7 @@ assert_writable_dir(GENERATED_TEXT_DIR)
 # Flat file directories
 GENODIR              = flat_files('genotype')+'/'
 assert_dir(GENODIR)
+assert_dir(GENODIR+'bimbam') # for gemma
 
 # JSON genotypes are OBSOLETE
 JSON_GENODIR         = flat_files('genotype/json')+'/'
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 57f97a81..e3fcd7c7 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -278,8 +278,4 @@ if os.environ.get('WQFLASK_OVERRIDES'):
             logger.debug(OVERRIDES)
 
 # assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData")
-# assert_dir(get_setting("JS_BIODALLIANCE"))
-# assert_file(get_setting("JS_BIODALLIANCE")+"/build/dalliance-all.js")
-# assert_file(get_setting("JS_BIODALLIANCE")+"/build/worker-all.js")
-# assert_dir(get_setting("JS_TWITTER_POST_FETCHER"))
 assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js")
-- 
cgit v1.2.3


From b3d9c748008d401ca5b5fa5d60a6a1eff505ee3e Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sat, 10 Feb 2018 08:22:41 +0000
Subject: Fixed finding PATHs - one return statement!

---
 wqflask/utility/tools.py                           | 9 +++++----
 wqflask/wqflask/marker_regression/gemma_mapping.py | 7 ++++---
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index e3fcd7c7..d3113302 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -16,7 +16,7 @@ OVERRIDES = {}
 def app_set(command_id, value):
     """Set application wide value"""
     app.config.setdefault(command_id,value)
-    value
+    return value
 
 def get_setting(command_id,guess=None):
     """Resolve a setting from the environment or the global settings in
@@ -51,7 +51,7 @@ def get_setting(command_id,guess=None):
             return None
 
     # ---- Check whether environment exists
-    logger.debug("Looking for "+command_id+"\n")
+    # print("Looking for "+command_id+"\n")
     command = value(os.environ.get(command_id))
     if command is None or command == "":
         command = OVERRIDES.get(command_id)
@@ -63,7 +63,7 @@ def get_setting(command_id,guess=None):
                 if command is None or command == "":
                     # print command
                     raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).')
-    logger.debug("Set "+command_id+"="+str(command))
+    # print("Set "+command_id+"="+str(command))
     return command
 
 def get_setting_bool(id):
@@ -253,6 +253,7 @@ JS_GN_PATH         = get_setting('JS_GN_PATH')
 
 PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
 GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
+assert(GEMMA_COMMAND is not None)
 PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
 GEMMA_WRAPPER_COMMAND = gemma_wrapper_command()
 TEMPDIR            = tempdir() # defaults to UNIX TMPDIR
@@ -266,7 +267,7 @@ from six import string_types
 
 if os.environ.get('WQFLASK_OVERRIDES'):
     jsonfn = get_setting('WQFLASK_OVERRIDES')
-    logger.error("WQFLASK_OVERRIDES: %s" % jsonfn)
+    logger.info("WQFLASK_OVERRIDES: %s" % jsonfn)
     with open(jsonfn) as data_file:
         overrides = json.load(data_file)
         for k in overrides:
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index a24e43d4..c004983e 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -3,7 +3,7 @@ import os, math, string, random, json
 from base import webqtlConfig
 from base.trait import GeneralTrait
 from base.data_set import create_dataset
-from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR
+from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin
 
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
@@ -11,6 +11,7 @@ logger = utility.logger.getLogger(__name__ )
 def run_gemma(this_dataset, samples, vals, covariates, method, use_loco):
     """Generates p-values for each marker using GEMMA"""
 
+    assert_bin(GEMMA_COMMAND);
     if this_dataset.group.genofile != None:
         genofile_name = this_dataset.group.genofile[:-5]
     else:
@@ -27,7 +28,7 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco):
         if i < (len(this_chromosomes) - 1):
             chr_list_string += this_chromosomes[i+1].name + ","
         else:
-            chr_list_string += this_chromosomes[i+1].name  
+            chr_list_string += this_chromosomes[i+1].name
 
     if covariates != "":
         gen_covariates_file(this_dataset, covariates)
@@ -247,4 +248,4 @@ def parse_loco_output(this_dataset, gwa_output_filename):
                     included_markers.append(line.split("\t")[1])
                     p_values.append(float(line.split("\t")[10]))
 
-    return marker_obs
\ No newline at end of file
+    return marker_obs
-- 
cgit v1.2.3


From 91365baf9ba657f56c19201874cdf60aede75dd1 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sat, 10 Feb 2018 08:53:27 +0000
Subject: Fixing GEMMA LOCO

---
 bin/genenetwork2                                   |  4 +++-
 wqflask/wqflask/marker_regression/gemma_mapping.py | 11 ++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 7148c022..52144f4d 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -121,7 +121,9 @@ else
     export PLINK_COMMAND="$GN2_PROFILE/bin/plink2"
     export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis"
     export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
-    export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
+    if [ -z GEMMA_WRAPPER_COMMAND ]; then
+        export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
+    fi
     if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi
     if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
     if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index c004983e..68920130 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -3,7 +3,7 @@ import os, math, string, random, json
 from base import webqtlConfig
 from base.trait import GeneralTrait
 from base.data_set import create_dataset
-from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin
+from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin, assert_file
 
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
@@ -210,8 +210,13 @@ def parse_gemma_output(genofile_name):
 def parse_loco_output(this_dataset, gwa_output_filename):
 
     output_filelist = []
-    with open("{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json") as data_file:
-       data = json.load(data_file)
+    jsonfn = "{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json"
+    assert_file(jsonfn)
+    try:
+        with open(jsonfn) as data_file:
+            data = json.load(data_file)
+    except:
+        logger.error("Can not parse "+jsonfn)
 
     files = data['files']
     for file in files:
-- 
cgit v1.2.3


From 7009d382c4515340cc7080192c58cc3fc0c1ced2 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sat, 10 Feb 2018 09:38:22 +0000
Subject: Fix env variable $GEMMA_WRAPPER_COMMAND

---
 bin/genenetwork2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 52144f4d..d1b8d25f 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -121,7 +121,7 @@ else
     export PLINK_COMMAND="$GN2_PROFILE/bin/plink2"
     export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis"
     export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
-    if [ -z GEMMA_WRAPPER_COMMAND ]; then
+    if [ -z $GEMMA_WRAPPER_COMMAND ]; then
         export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
     fi
     if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi
-- 
cgit v1.2.3


From 7f3a493db99998992f86af9a5e333103f255be6d Mon Sep 17 00:00:00 2001
From: zsloan
Date: Sun, 11 Feb 2018 11:26:01 +0100
Subject: Using eventlet workers

---
 bin/genenetwork2 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index d1b8d25f..2f0b841e 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -192,7 +192,8 @@ if [ "$1" = '-gunicorn-prod' ] ; then
     cd $GN2_BASE_DIR/wqflask
     echo PYTHONPATH=$PYTHONPATH
     if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
-    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=32 --max-requests 1000 --timeout 1200 wsgi"
+    cmd="--bind 0.0.0.0:$SERVER_PORT -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi"
+    # cmd="--bind 0.0.0.0:$SERVER_PORT --workers 20 --keep-alive 75 --max-requests 1000 --timeout 1200 wsgi"
     echo RUNNING gunicorn $cmd
     gunicorn $cmd
     exit $?
-- 
cgit v1.2.3


From 018c9d6c50e90d783bc7bd1b2a0d85b080ebf394 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Sun, 11 Feb 2018 11:37:36 +0000
Subject: Added gunicorn pid so we can not run twice

---
 bin/genenetwork2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 2f0b841e..8886e4bc 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -192,8 +192,8 @@ if [ "$1" = '-gunicorn-prod' ] ; then
     cd $GN2_BASE_DIR/wqflask
     echo PYTHONPATH=$PYTHONPATH
     if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
-    cmd="--bind 0.0.0.0:$SERVER_PORT -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi"
-    # cmd="--bind 0.0.0.0:$SERVER_PORT --workers 20 --keep-alive 75 --max-requests 1000 --timeout 1200 wsgi"
+    PID=$TMPDIR/gunicorn.$USER.pid
+    cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi"
     echo RUNNING gunicorn $cmd
     gunicorn $cmd
     exit $?
-- 
cgit v1.2.3