Merge pull request #1 from genenetwork/testing

Updating my testing branch

2 files changed, 180 insertions, 167 deletions

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 52d3155c..1c538e86 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -1,83 +1,156 @@
 #! /bin/bash
 #
+# This is the startup script for GN2. It sets the environment variables to pick
+# up a Guix profile and allows for overriding parameters.
+#
+# Typical usage
+#
+#   env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py
+#
+# Where GN2_PROFILE points to the GNU Guix profile used for deployment.
+#
 # This will run the GN2 server (with default settings if none
-# supplied). Pass in your own settings file, e.g.
+# supplied). Typically you need a GNU Guix profile which is set with
+# an environment variable (this profile is dictated by the
+# installation path of genenetwork). Say your profile is in
+# ~/opt/gn-latest-guix
+#
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2
+#
+# You can pass in your own settings file, e.g.
+#
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py
+#
+# To run a maintenance python script with settings (instead of the
+# webserver) run from the base-dir with settings file and add that
+# script with a -c switch, e.g.
+#
+#   env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py
+#
+# To run any script in the environment
+#
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD"
 #
-#   ./bin/genenetwork2 ~/my_settings.py
+# To get a python REPL(!)
 #
-# But better is to retain the defaults and override them with
-# JSON file (see the deploy docs).
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python
 #
-#   ./bin/genenetwork2 ~/my_overrides.json
+# For development you may want to run
 #
-# To run a maintenance script with settings (instead of the webserver) add that with
-# a -c switch, e.g.
+#   env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
 #
-#   ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
+# For staging and production we use gunicorn. Run with something like
+# (note you have to provide the server port). Provide a settings file!
 #
-# Environment settings can be used to preconfigure as well as a
-# settings.py file.
+#   env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod
 #
-# GN2_BASE_PATH is base directory of wqflask source code
+# For development use
 #
+#   env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev
+#
+# For extra flexibility you can also provide gunicorn parameters yourself with something like
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi"
 
-SCRIPT=$(readlink -f "$0")
+SCRIPT=$(realpath "$0")
+echo SCRIPT=$SCRIPT
+echo GN2_PROFILE=$GN2_PROFILE
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
+GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
 echo GN2_BASE_DIR=$GN2_BASE_DIR
 
-GIT_HASH=$(git rev-parse HEAD)
-GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
-export GN_VERSION=$(cat $GN2_BASE_DIR/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9}
+GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages
+if [ -d $GUIX_SITE ]; then
+    echo INFO: GN2 is running from GNU Guix
+    GN2_BASE_DIR=$GUIX_SITE
+    export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)
+else
+    echo INFO: GN2 is running from a source tree
+    GIT_HASH=$(git rev-parse HEAD)
+    GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+    export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9}
+fi
 echo GN_VERSION=$GN_VERSION
 
-# Handle settings parameter (can be .py or .json)
+if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then
+    echo "Can not use $1 switch without default settings file"
+    exit 1
+fi
+
 settings=$1
-ext="${settings##*.}"
-if [ -z "$settings" -o "$ext" = "json" -o "$ext" = "JSON" ]; then
-    overrides=$settings
+if [ -z $settings ]; then
     settings=$GN2_BASE_DIR/etc/default_settings.py
 else
     shift
 fi
+settings=$(realpath $settings)
+
+# ext="${settings##*.}"
+# if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
+#     overrides=$settings
+# else
+#     echo $settings
+# fi
+
 if [ ! -e $settings ]; then
-    GUIX_ETC=$GN2_BASE_DIR/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg
-    if [ -d $GUIX_ETC ]; then
-        echo INFO: GN2 is actually running from GNU Guix
-    else
-        echo "ERROR: can not locate settings file - pass it in the command line"
-        exit 1
-    fi
+    echo "ERROR: can not locate settings file - pass it in the command line"
+    exit 1
 fi
-export WQFLASK_SETTINGS=$settings     # Python
-export WQFLASK_OVERRIDES=$overrides   # JSON
-
-echo WQFLASK_SETTINGS=$settings
-echo WQFLASK_OVERRIDES=$overrides
-
-# ---- Checks and balances
-if [ -z $GUIX_ETC ]; then
-    if [ -z $GN2_PROFILE ] ; then
-        echo "WARNING: GN2_PROFILE has not been set - I hope you know what you are doing!"
-    else
-        export PATH=$GN2_PROFILE/bin:$PATH
-        export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages
-        export R_LIBS_SITE=$GN2_PROFILE/site-library
-        export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0"
-        export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0"
-        export XDG_DATA_DIRS="$GN2_PROFILE/share"
-        export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules"
-        export LC_ALL=C
-    fi
-    if [ -z $PYTHONPATH ] ; then
-        echo "WARNING PYTHONPATH has not been set - use GN2_PROFILE!"
+
+export GN2_SETTINGS=$settings     # Python
+echo GN2_SETTINGS=$settings
+
+# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead
+# if [ -z $ELASTICSEARCH_PROFILE ]; then
+#     echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE";
+# else
+#     PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
+# fi
+
+if [ -z $GN2_PROFILE ] ; then
+    echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
+    export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
+    if [ -d $GN2_PROFILE ]; then
+        echo "Best guess is $GN2_PROFILE"
     fi
-    if [ ! -d $R_LIBS_SITE ] ; then
-        echo "ERROR: R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!"
-        echo "Paste into your shell the output of (for example)"
-        echo "guix package -p \$GN2_PROFILE --search-paths"
-        exit 1
+    echo "ERROR: always set GN2_PROFILE"
+    exit 1
+fi
+if [ -z $GN2_PROFILE ]; then
+    read -p "PRESS [ENTER] TO CONTINUE..."
+else
+    export PATH=$GN2_PROFILE/bin:$PATH
+    export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!!
+    export R_LIBS_SITE=$GN2_PROFILE/site-library
+    export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
+    export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0"
+    export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0"
+    export XDG_DATA_DIRS="$GN2_PROFILE/share"
+    export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules"
+    export LC_ALL=C # FIXME
+    export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2"
+    export PLINK_COMMAND="$GN2_PROFILE/bin/plink2"
+    export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
+    if [ -z $GEMMA_WRAPPER_COMMAND ]; then
+        export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
     fi
+    while IFS=":" read -ra PPATH; do
+	for PPART in "${PPATH[@]}"; do
+	    if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi
+	done
+    done <<< "$PYTHONPATH"
+    if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
+fi
+if [ -z $PYTHONPATH ] ; then
+    echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!"
+    exit 1
+fi
+if [ ! -d $R_LIBS_SITE ] ; then
+    echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!"
+    echo "Paste into your shell the output of (for example)"
+    echo "guix package -p \$GN2_PROFILE --search-paths"
+    exit 1
 fi
 
 # We may change this one:
@@ -88,17 +161,58 @@ if [ -z $TMPDIR ]; then
     TMPDIR="/tmp"
 fi
 
+# Show environment settings
+set|grep guix
 set|grep $GN2_PROFILE
 set|grep TMPDIR
 
-# Now handle command parameter -c
+# Now handle command parameter -c which runs python
 if [ "$1" = '-c' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=${2#wqflask/}
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND $cmd
-    /usr/bin/env python $cmd
-    exit 0
+    shift ; shift
+    echo RUNNING COMMAND $cmd $*
+    python $cmd $*
+    exit $?
+fi
+
+# Now handle command parameter -cli which runs in bash
+if [ "$1" = "-cli" ] ; then
+    cd $GN2_BASE_DIR/wqflask
+    cmd=$2
+    echo PYTHONPATH=$PYTHONPATH
+    shift ; shift
+    echo RUNNING COMMAND $cmd $*
+    $cmd $*
+    exit $?
+fi
+if [ "$1" = '-gunicorn' ] ; then
+    cd $GN2_BASE_DIR/wqflask
+    cmd=$2
+    echo PYTHONPATH=$PYTHONPATH
+    echo RUNNING gunicorn $cmd
+    gunicorn $cmd
+    exit $?
+fi
+if [ "$1" = '-gunicorn-dev' ] ; then
+    cd $GN2_BASE_DIR/wqflask
+    echo PYTHONPATH=$PYTHONPATH
+    if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
+    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi"
+    echo RUNNING gunicorn $cmd
+    gunicorn $cmd
+    exit $?
+fi
+if [ "$1" = '-gunicorn-prod' ] ; then
+    cd $GN2_BASE_DIR/wqflask
+    echo PYTHONPATH=$PYTHONPATH
+    if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
+    PID=$TMPDIR/gunicorn.$USER.pid
+    cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi"
+    echo RUNNING gunicorn $cmd
+    gunicorn $cmd
+    exit $?
 fi
 
 echo "Starting the redis server:"
@@ -106,7 +220,10 @@ echo -n "dir $TMPDIR
 dbfilename gn2.rdb
 " | redis-server - &
 
+# Overrides for packages that are not yet public (currently r-auwerx)
+# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib
+
 # Start the flask server running GN2
 cd $GN2_BASE_DIR/wqflask
 echo "Starting with $settings"
-/usr/bin/env python runserver.py
+python runserver.py
diff --git a/bin/test-website b/bin/test-website
index be223d94..7fbcfd2f 100755
--- a/bin/test-website
+++ b/bin/test-website
@@ -1,111 +1,7 @@
-#!/usr/bin/env ruby
+#! /bin/bash
 
-
-USAGE = <<EOT
-This is Mechanical-Rob - an automated web server tester for
-                         Genenetwork.org that uses the brilliant
-                         mechanize gem with minitest.
-
-To use this software you need to install mechanize. Run it from
-the root of the genenetwork2 source tree as, for example,
-
-  ./bin/test-website http://localhost:5003/ (default)
-
-If you are using the small deployment database you can use
-
-  ./bin/test-website --skip -n
-
-To run all tests
-
-  ./bin/test-website --all
-
-To run individual tests on localhost you can do
-
-  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
-
-For more information see http://genenetwork.org/
-
-EOT
-$stderr.print USAGE
-
-require 'optparse'
-
-options = { database: :small, link_checker: false}
-opts = OptionParser.new do |o|
-  o.banner = "Usage: #{File.basename($0)} [options] URL"
-
-  o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s|
-    options[:database] =
-      case s
-      when 'xx'
-        :unknown
-      else
-        :small
-      end
-  end
-
-  o.on('--all', 'Run all tests') do
-    options[:all] = true
-  end
-
-  o.on('-l','--link-checker', 'Check for dead links') do
-    options[:link_checker] = true
-  end
-
-  o.on('--navigation', 'Check for navigation') do
-    options[:navigation] = true
-  end
-
-  o.on('--mapping', 'Check for mapping') do
-    options[:mapping] = true
-  end
-
-  o.on('--skip-broken', 'Skip tests that are known to be broken') do
-    options[:skip_broken] = true
-  end
-
-  o.separator ""
-  o.on_tail('-h', '--help', 'display this help and exit') do
-    options[:show_help] = true
-  end
-end
-
-opts.parse!(ARGV)
-
-if options[:show_help]
-  print opts
+if [ -z $GN2_PROFILE ]; then
+  echo "Run request tests with something like"
+  echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003
   exit 1
-end
-
-$options = options  # we are using a global here
-$host =
-  if ARGV.size>0
-    ARGV.shift
-  else
-    "http://localhost:5003"
-  end
-
-$stderr.print "Testing <",$host,">\n"
-
-require 'mechanize'
-require 'minitest/spec'
-require 'minitest/autorun'
-
-# These are the actual testing modules
-
-libpath = File.dirname(File.dirname(__FILE__))
-$: << File.join(libpath,'test/lib')
-
-require 'main_web_functionality'
-
-if options[:all] or options[:mapping]
-  require 'mapping'
-end
-
-if options[:all] or options[:link_checker]
-  require 'link_checker'
-end
-
-if options[:all] or options[:navigation]
-  require 'navigation'
-end
+fi