From 06b203e722c0061aae6c16f877619dc7af03638e Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 29 Jan 2018 11:30:28 +0000
Subject: Added checks in startup script, removed r-auwerx path

---
 bin/genenetwork2 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index a7edb1c2..c6b08771 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -86,6 +86,9 @@ else
     export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis"
     export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
     export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
+    if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi
+    if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
+    if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi
 fi
 if [ -z $PYTHONPATH ] ; then
     echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!"
@@ -127,7 +130,7 @@ dbfilename gn2.rdb
 " | redis-server - &
 
 # Overrides for packages that are not yet public (currently r-auwerx)
-export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib
+# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib
 
 # Start the flask server running GN2
 cd $GN2_BASE_DIR/wqflask
-- 
cgit v1.2.3


From b3724d60d2e50dd19e61140406aa85dff981ac16 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 29 Jan 2018 14:15:49 +0000
Subject: startup script: adapting for gunicorn

---
 bin/genenetwork2                 | 16 ++++++++++++++++
 wqflask/__init__.py              |  1 -
 wqflask/gunicorn/__init__.py     |  0
 wqflask/gunicorn/genenetwork2.py | 10 ++++++++++
 wqflask/gunicorn/wsgi.py         |  5 +++++
 wqflask/runserver.py             |  2 ++
 6 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 wqflask/gunicorn/__init__.py
 create mode 100644 wqflask/gunicorn/genenetwork2.py
 create mode 100644 wqflask/gunicorn/wsgi.py

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index c6b08771..f32c76eb 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -22,6 +22,14 @@
 # a -c switch, e.g.
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
+#
+# For development you may want to run
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
+#
+# For production we use gunicorn. Run with
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=2 --chdir wqflask/gunicorn/ wsgi"
 
 SCRIPT=$(readlink -f "$0")
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
@@ -123,6 +131,14 @@ if [ "$1" = '-c' ] ; then
     python $cmd
     exit 0
 fi
+if [ "$1" = '-gunicorn' ] ; then
+    cd $GN2_BASE_DIR
+    cmd=${2}
+    echo PYTHONPATH=$PYTHONPATH
+    echo RUNNING COMMAND gunicorn $cmd
+    gunicorn $cmd
+    exit 0
+fi
 
 echo "Starting the redis server:"
 echo -n "dir $TMPDIR
diff --git a/wqflask/__init__.py b/wqflask/__init__.py
index 315b709e..e69de29b 100644
--- a/wqflask/__init__.py
+++ b/wqflask/__init__.py
@@ -1 +0,0 @@
-from wqflask import app
diff --git a/wqflask/gunicorn/__init__.py b/wqflask/gunicorn/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/wqflask/gunicorn/genenetwork2.py b/wqflask/gunicorn/genenetwork2.py
new file mode 100644
index 00000000..afbdc934
--- /dev/null
+++ b/wqflask/gunicorn/genenetwork2.py
@@ -0,0 +1,10 @@
+from flask import Flask
+application = Flask(__name__)
+
+@application.route("/")
+def hello():
+    return "<h1 style='color:blue'>Hello There!</h1>"
+
+if __name__ == "__main__":
+    application.run(host='0.0.0.0')
+
diff --git a/wqflask/gunicorn/wsgi.py b/wqflask/gunicorn/wsgi.py
new file mode 100644
index 00000000..8c14ee6b
--- /dev/null
+++ b/wqflask/gunicorn/wsgi.py
@@ -0,0 +1,5 @@
+from genenetwork2 import application
+
+if __name__ == "__main__":
+    application.run()
+
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index 50f134db..e62e34f2 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -1,5 +1,7 @@
 # Starts the webserver with the ./bin/genenetwork2 command
 #
+# This uses Werkzeug WSGI, see gunicorn.py for the alternative
+#
 # Please note, running with host set externally below combined with
 # debug mode is a security risk unless you have a firewall setup, e.g.
 #
-- 
cgit v1.2.3


From 85819dbeaf922456dc205e482e7e338952945d9b Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 29 Jan 2018 15:10:09 +0000
Subject: Gunicorn works

---
 bin/genenetwork2                 | 28 +++++++++++++++++++---------
 etc/default_settings.py          |  2 +-
 wqflask/gunicorn/__init__.py     |  0
 wqflask/gunicorn/genenetwork2.py | 10 ----------
 wqflask/gunicorn/wsgi.py         |  5 -----
 wqflask/run_gunicorn.py          | 19 +++++++++++++++++++
 wqflask/runserver.py             |  2 +-
 wqflask/wsgi.py                  |  4 ++++
 8 files changed, 44 insertions(+), 26 deletions(-)
 delete mode 100644 wqflask/gunicorn/__init__.py
 delete mode 100644 wqflask/gunicorn/genenetwork2.py
 delete mode 100644 wqflask/gunicorn/wsgi.py
 create mode 100644 wqflask/run_gunicorn.py
 create mode 100644 wqflask/wsgi.py

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index f32c76eb..f6ae3807 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -31,7 +31,7 @@
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=2 --chdir wqflask/gunicorn/ wsgi"
 
-SCRIPT=$(readlink -f "$0")
+SCRIPT=$(realpath "$0")
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
 GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
@@ -50,15 +50,25 @@ else
 fi
 echo GN_VERSION=$GN_VERSION
 
+if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then
+    echo "Can not use $1 switch without default settings file"
+    exit 1
+fi
 # Handle settings parameter (can be .py or .json)
-settings=$1
-ext="${settings##*.}"
-if [ -z "$settings" -o "$ext" = "json" -o "$ext" = "JSON" ]; then
-    overrides=$settings
+settings=$(realpath "$1")
+if [ ! -e $settings ]; then
     settings=$GN2_BASE_DIR/etc/default_settings.py
 else
     shift
 fi
+
+ext="${settings##*.}"
+if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
+    overrides=$settings
+else
+    echo $settings
+fi
+
 if [ ! -e $settings ]; then
     echo "ERROR: can not locate settings file - pass it in the command line"
     exit 1
@@ -132,12 +142,12 @@ if [ "$1" = '-c' ] ; then
     exit 0
 fi
 if [ "$1" = '-gunicorn' ] ; then
-    cd $GN2_BASE_DIR
-    cmd=${2}
+    cd $GN2_BASE_DIR/wqflask
+    cmd=$2
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND gunicorn $cmd
+    echo RUNNING gunicorn $cmd
     gunicorn $cmd
-    exit 0
+    exit $?
 fi
 
 echo "Starting the redis server:"
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 59e22f1a..699d21f1 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -27,7 +27,7 @@ import sys
 GN_VERSION = open("../etc/VERSION","r").read()
 SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s"
 SQL_ALCHEMY_POOL_RECYCLE = 3600
-GN_SERVER_URL = "http://localhost:8880/"
+GN_SERVER_URL = "http://localhost:8880/" # REST API server
 
 # ---- Flask configuration (see website)
 TRAP_BAD_REQUEST_ERRORS = True
diff --git a/wqflask/gunicorn/__init__.py b/wqflask/gunicorn/__init__.py
deleted file mode 100644
index e69de29b..00000000
diff --git a/wqflask/gunicorn/genenetwork2.py b/wqflask/gunicorn/genenetwork2.py
deleted file mode 100644
index afbdc934..00000000
--- a/wqflask/gunicorn/genenetwork2.py
+++ /dev/null
@@ -1,10 +0,0 @@
-from flask import Flask
-application = Flask(__name__)
-
-@application.route("/")
-def hello():
-    return "<h1 style='color:blue'>Hello There!</h1>"
-
-if __name__ == "__main__":
-    application.run(host='0.0.0.0')
-
diff --git a/wqflask/gunicorn/wsgi.py b/wqflask/gunicorn/wsgi.py
deleted file mode 100644
index 8c14ee6b..00000000
--- a/wqflask/gunicorn/wsgi.py
+++ /dev/null
@@ -1,5 +0,0 @@
-from genenetwork2 import application
-
-if __name__ == "__main__":
-    application.run()
-
diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py
new file mode 100644
index 00000000..14a2d689
--- /dev/null
+++ b/wqflask/run_gunicorn.py
@@ -0,0 +1,19 @@
+# Run with gunicorn, see ./bin/genenetwork2 for an example
+#
+# Run standalone with
+#
+#   ./bin/genenetwork2 ./etc/default_settings.py -c run_gunicorn.py
+
+# from flask import Flask
+# application = Flask(__name__)
+
+print "Starting up Gunicorn process"
+
+from wqflask import app
+
+@app.route("/gunicorn")
+def hello():
+    return "<h1 style='color:blue'>Hello There!</h1>"
+
+if __name__ == "__main__":
+    app.run(host='0.0.0.0')
diff --git a/wqflask/runserver.py b/wqflask/runserver.py
index e62e34f2..a0c76e51 100644
--- a/wqflask/runserver.py
+++ b/wqflask/runserver.py
@@ -1,6 +1,6 @@
 # Starts the webserver with the ./bin/genenetwork2 command
 #
-# This uses Werkzeug WSGI, see gunicorn.py for the alternative
+# This uses Werkzeug WSGI, see ./run_gunicorn.py for the alternative
 #
 # Please note, running with host set externally below combined with
 # debug mode is a security risk unless you have a firewall setup, e.g.
diff --git a/wqflask/wsgi.py b/wqflask/wsgi.py
new file mode 100644
index 00000000..be9c7b37
--- /dev/null
+++ b/wqflask/wsgi.py
@@ -0,0 +1,4 @@
+from run_gunicorn import app as application # expect application as a name
+
+if __name__ == "__main__":
+    application.run()
-- 
cgit v1.2.3


From 84d3b397bd5b51c4cde0b48f4a133e1a5dd2a3e9 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 29 Jan 2018 15:12:18 +0000
Subject: Fixed gunicorn example

---
 bin/genenetwork2 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index f6ae3807..6878643d 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -27,9 +27,9 @@
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
 #
-# For production we use gunicorn. Run with
+# For production we use gunicorn. Run with something like
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=2 --chdir wqflask/gunicorn/ wsgi"
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=1 wsgi"
 
 SCRIPT=$(realpath "$0")
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
-- 
cgit v1.2.3


From f60d49527bedfc1ecda401f49fae9c9a5bbcf5f9 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Tue, 30 Jan 2018 11:09:51 +0000
Subject: gunicorn dev and prod modes added

---
 bin/genenetwork2         | 45 +++++++++++++++++++++++++++++++++++++--------
 wqflask/wqflask/views.py |  2 +-
 2 files changed, 38 insertions(+), 9 deletions(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 6878643d..cbaba76f 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -27,9 +27,18 @@
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
 #
-# For production we use gunicorn. Run with something like
+# For staging and production we use gunicorn. Run with something like
+# (note you have to provide the server port). Provide a settings file!
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:8000 --workers=1 wsgi"
+#   env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod
+#
+# For development use
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev
+#
+# For extra flexibility you can also provide gunicorn parameters yourself with something like
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi"
 
 SCRIPT=$(realpath "$0")
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
@@ -55,11 +64,13 @@ if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then
     exit 1
 fi
 # Handle settings parameter (can be .py or .json)
-settings=$(realpath "$1")
-if [ ! -e $settings ]; then
-    settings=$GN2_BASE_DIR/etc/default_settings.py
-else
-    shift
+if [ ! -z $1 ]; then
+    settings=$(realpath "$1")
+    if [ ! -e $settings ]; then
+        settings=$GN2_BASE_DIR/etc/default_settings.py
+    else
+        shift
+    fi
 fi
 
 ext="${settings##*.}"
@@ -139,7 +150,7 @@ if [ "$1" = '-c' ] ; then
     echo PYTHONPATH=$PYTHONPATH
     echo RUNNING COMMAND $cmd
     python $cmd
-    exit 0
+    exit $?
 fi
 if [ "$1" = '-gunicorn' ] ; then
     cd $GN2_BASE_DIR/wqflask
@@ -149,6 +160,24 @@ if [ "$1" = '-gunicorn' ] ; then
     gunicorn $cmd
     exit $?
 fi
+if [ "$1" = '-gunicorn-dev' ] ; then
+    cd $GN2_BASE_DIR/wqflask
+    echo PYTHONPATH=$PYTHONPATH
+    if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
+    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --reload wsgi"
+    echo RUNNING gunicorn $cmd
+    gunicorn $cmd
+    exit $?
+fi
+if [ "$1" = '-gunicorn-prod' ] ; then
+    cd $GN2_BASE_DIR/wqflask
+    echo PYTHONPATH=$PYTHONPATH
+    if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
+    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=32 --max-requests 1000 --timeout 1200 wsgi"
+    echo RUNNING gunicorn $cmd
+    gunicorn $cmd
+    exit $?
+fi
 
 echo "Starting the redis server:"
 echo -n "dir $TMPDIR
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index d8f5616d..4e81c29c 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -766,7 +766,7 @@ def corr_scatter_plot_page():
 def submit_bnw():
     logger.error(request.url)
     template_vars = get_bnw_input(request.form)
-    return render_template("empty_collection.html", **{'tool':'Correlation Matrix'}) 
+    return render_template("empty_collection.html", **{'tool':'Correlation Matrix'})
 
 # Todo: Can we simplify this? -Sam
 def sharing_info_page():
-- 
cgit v1.2.3


From c38cf0af74c3927f5b7d1da5416671f14c789f72 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 31 Jan 2018 11:22:42 +0000
Subject: Startup: added -cli option

---
 bin/genenetwork2 | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index cbaba76f..65330ec8 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,8 +1,11 @@
 #! /bin/bash
 #
+# This is the startup script for GN2. It sets the environment variables to pick
+# up a Guix profile and allows for overriding parameters.
+#
 # Typical usage
 #
-#   env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2
+#   env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py
 #
 # Where GN2_PROFILE points to the GNU Guix profile used for deployment.
 #
@@ -41,6 +44,7 @@
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi"
 
 SCRIPT=$(realpath "$0")
+echo SCRIPT=$SCRIPT
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
 GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
@@ -96,6 +100,8 @@ if [ -z $GN2_PROFILE ] ; then
     if [ -d $GN2_PROFILE ]; then
         echo "Best guess is $GN2_PROFILE"
     fi
+    echo "ERROR: always set GN2_PROFILE"
+    exit 1
 fi
 if [ -z $GN2_PROFILE ]; then
     read -p "PRESS [ENTER] TO CONTINUE..."
@@ -143,7 +149,7 @@ set|grep guix
 set|grep $GN2_PROFILE
 set|grep TMPDIR
 
-# Now handle command parameter -c
+# Now handle command parameter -c which runs python
 if [ "$1" = '-c' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=${2#wqflask/}
@@ -152,6 +158,16 @@ if [ "$1" = '-c' ] ; then
     python $cmd
     exit $?
 fi
+# Now handle command parameter -cli which runs in bash
+if [ "$1" = "-cli" ] ; then
+    echo "HERE"
+    cd $GN2_BASE_DIR/wqflask
+    cmd=$2
+    echo PYTHONPATH=$PYTHONPATH
+    echo RUNNING COMMAND $cmd
+    $cmd
+    exit $?
+fi
 if [ "$1" = '-gunicorn' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=$2
-- 
cgit v1.2.3


From 8e79b13e9126a1a978aff64cabfd374760e0b5cc Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 31 Jan 2018 11:48:19 +0000
Subject: Startup: longer timeout for gunicorn-dev mode

---
 bin/genenetwork2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 65330ec8..f64576d5 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -180,7 +180,7 @@ if [ "$1" = '-gunicorn-dev' ] ; then
     cd $GN2_BASE_DIR/wqflask
     echo PYTHONPATH=$PYTHONPATH
     if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
-    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --reload wsgi"
+    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi"
     echo RUNNING gunicorn $cmd
     gunicorn $cmd
     exit $?
-- 
cgit v1.2.3