diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/genenetwork2 | 233 | ||||
| -rwxr-xr-x | bin/test-website | 114 |
2 files changed, 180 insertions, 167 deletions
diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 52d3155c..1c538e86 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,83 +1,156 @@ #! /bin/bash # +# This is the startup script for GN2. It sets the environment variables to pick +# up a Guix profile and allows for overriding parameters. +# +# Typical usage +# +# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py +# +# Where GN2_PROFILE points to the GNU Guix profile used for deployment. +# # This will run the GN2 server (with default settings if none -# supplied). Pass in your own settings file, e.g. +# supplied). Typically you need a GNU Guix profile which is set with +# an environment variable (this profile is dictated by the +# installation path of genenetwork). Say your profile is in +# ~/opt/gn-latest-guix +# +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 +# +# You can pass in your own settings file, e.g. +# +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py +# +# To run a maintenance python script with settings (instead of the +# webserver) run from the base-dir with settings file and add that +# script with a -c switch, e.g. +# +# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py +# +# To run any script in the environment +# +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" # -# ./bin/genenetwork2 ~/my_settings.py +# To get a python REPL(!) # -# But better is to retain the defaults and override them with -# JSON file (see the deploy docs). +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python # -# ./bin/genenetwork2 ~/my_overrides.json +# For development you may want to run # -# To run a maintenance script with settings (instead of the webserver) add that with -# a -c switch, e.g. +# env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 # -# ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py +# For staging and production we use gunicorn. Run with something like +# (note you have to provide the server port). Provide a settings file! # -# Environment settings can be used to preconfigure as well as a -# settings.py file. +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod # -# GN2_BASE_PATH is base directory of wqflask source code +# For development use # +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# +# For extra flexibility you can also provide gunicorn parameters yourself with something like +# +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" -SCRIPT=$(readlink -f "$0") +SCRIPT=$(realpath "$0") +echo SCRIPT=$SCRIPT +echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) +GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) echo GN2_BASE_DIR=$GN2_BASE_DIR -GIT_HASH=$(git rev-parse HEAD) -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -export GN_VERSION=$(cat $GN2_BASE_DIR/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} +GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages +if [ -d $GUIX_SITE ]; then + echo INFO: GN2 is running from GNU Guix + GN2_BASE_DIR=$GUIX_SITE + export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION) +else + echo INFO: GN2 is running from a source tree + GIT_HASH=$(git rev-parse HEAD) + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} +fi echo GN_VERSION=$GN_VERSION -# Handle settings parameter (can be .py or .json) +if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then + echo "Can not use $1 switch without default settings file" + exit 1 +fi + settings=$1 -ext="${settings##*.}" -if [ -z "$settings" -o "$ext" = "json" -o "$ext" = "JSON" ]; then - overrides=$settings +if [ -z $settings ]; then settings=$GN2_BASE_DIR/etc/default_settings.py else shift fi +settings=$(realpath $settings) + +# ext="${settings##*.}" +# if [ "$ext" = "json" -o "$ext" = "JSON" ]; then +# overrides=$settings +# else +# echo $settings +# fi + if [ ! -e $settings ]; then - GUIX_ETC=$GN2_BASE_DIR/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg - if [ -d $GUIX_ETC ]; then - echo INFO: GN2 is actually running from GNU Guix - else - echo "ERROR: can not locate settings file - pass it in the command line" - exit 1 - fi + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 fi -export WQFLASK_SETTINGS=$settings # Python -export WQFLASK_OVERRIDES=$overrides # JSON - -echo WQFLASK_SETTINGS=$settings -echo WQFLASK_OVERRIDES=$overrides - -# ---- Checks and balances -if [ -z $GUIX_ETC ]; then - if [ -z $GN2_PROFILE ] ; then - echo "WARNING: GN2_PROFILE has not been set - I hope you know what you are doing!" - else - export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages - export R_LIBS_SITE=$GN2_PROFILE/site-library - export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" - export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" - export XDG_DATA_DIRS="$GN2_PROFILE/share" - export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" - export LC_ALL=C - fi - if [ -z $PYTHONPATH ] ; then - echo "WARNING PYTHONPATH has not been set - use GN2_PROFILE!" + +export GN2_SETTINGS=$settings # Python +echo GN2_SETTINGS=$settings + +# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead +# if [ -z $ELASTICSEARCH_PROFILE ]; then +# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; +# else +# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +# fi + +if [ -z $GN2_PROFILE ] ; then + echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" + export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) + if [ -d $GN2_PROFILE ]; then + echo "Best guess is $GN2_PROFILE" fi - if [ ! -d $R_LIBS_SITE ] ; then - echo "ERROR: R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" - echo "Paste into your shell the output of (for example)" - echo "guix package -p \$GN2_PROFILE --search-paths" - exit 1 + echo "ERROR: always set GN2_PROFILE" + exit 1 +fi +if [ -z $GN2_PROFILE ]; then + read -p "PRESS [ENTER] TO CONTINUE..." +else + export PATH=$GN2_PROFILE/bin:$PATH + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!! + export R_LIBS_SITE=$GN2_PROFILE/site-library + export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript + export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" + export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" + export XDG_DATA_DIRS="$GN2_PROFILE/share" + export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" + export LC_ALL=C # FIXME + export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" + export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" + export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" + if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi + while IFS=":" read -ra PPATH; do + for PPART in "${PPATH[@]}"; do + if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + done + done <<< "$PYTHONPATH" + if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi +fi +if [ -z $PYTHONPATH ] ; then + echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" + exit 1 +fi +if [ ! -d $R_LIBS_SITE ] ; then + echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" + echo "Paste into your shell the output of (for example)" + echo "guix package -p \$GN2_PROFILE --search-paths" + exit 1 fi # We may change this one: @@ -88,17 +161,58 @@ if [ -z $TMPDIR ]; then TMPDIR="/tmp" fi +# Show environment settings +set|grep guix set|grep $GN2_PROFILE set|grep TMPDIR -# Now handle command parameter -c +# Now handle command parameter -c which runs python if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - /usr/bin/env python $cmd - exit 0 + shift ; shift + echo RUNNING COMMAND $cmd $* + python $cmd $* + exit $? +fi + +# Now handle command parameter -cli which runs in bash +if [ "$1" = "-cli" ] ; then + cd $GN2_BASE_DIR/wqflask + cmd=$2 + echo PYTHONPATH=$PYTHONPATH + shift ; shift + echo RUNNING COMMAND $cmd $* + $cmd $* + exit $? +fi +if [ "$1" = '-gunicorn' ] ; then + cd $GN2_BASE_DIR/wqflask + cmd=$2 + echo PYTHONPATH=$PYTHONPATH + echo RUNNING gunicorn $cmd + gunicorn $cmd + exit $? +fi +if [ "$1" = '-gunicorn-dev' ] ; then + cd $GN2_BASE_DIR/wqflask + echo PYTHONPATH=$PYTHONPATH + if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi + cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" + echo RUNNING gunicorn $cmd + gunicorn $cmd + exit $? +fi +if [ "$1" = '-gunicorn-prod' ] ; then + cd $GN2_BASE_DIR/wqflask + echo PYTHONPATH=$PYTHONPATH + if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi + PID=$TMPDIR/gunicorn.$USER.pid + cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi" + echo RUNNING gunicorn $cmd + gunicorn $cmd + exit $? fi echo "Starting the redis server:" @@ -106,7 +220,10 @@ echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & +# Overrides for packages that are not yet public (currently r-auwerx) +# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib + # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" -/usr/bin/env python runserver.py +python runserver.py diff --git a/bin/test-website b/bin/test-website index be223d94..7fbcfd2f 100755 --- a/bin/test-website +++ b/bin/test-website @@ -1,111 +1,7 @@ -#!/usr/bin/env ruby +#! /bin/bash - -USAGE = <<EOT -This is Mechanical-Rob - an automated web server tester for - Genenetwork.org that uses the brilliant - mechanize gem with minitest. - -To use this software you need to install mechanize. Run it from -the root of the genenetwork2 source tree as, for example, - - ./bin/test-website http://localhost:5003/ (default) - -If you are using the small deployment database you can use - - ./bin/test-website --skip -n - -To run all tests - - ./bin/test-website --all - -To run individual tests on localhost you can do - - ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/" - -For more information see http://genenetwork.org/ - -EOT -$stderr.print USAGE - -require 'optparse' - -options = { database: :small, link_checker: false} -opts = OptionParser.new do |o| - o.banner = "Usage: #{File.basename($0)} [options] URL" - - o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s| - options[:database] = - case s - when 'xx' - :unknown - else - :small - end - end - - o.on('--all', 'Run all tests') do - options[:all] = true - end - - o.on('-l','--link-checker', 'Check for dead links') do - options[:link_checker] = true - end - - o.on('--navigation', 'Check for navigation') do - options[:navigation] = true - end - - o.on('--mapping', 'Check for mapping') do - options[:mapping] = true - end - - o.on('--skip-broken', 'Skip tests that are known to be broken') do - options[:skip_broken] = true - end - - o.separator "" - o.on_tail('-h', '--help', 'display this help and exit') do - options[:show_help] = true - end -end - -opts.parse!(ARGV) - -if options[:show_help] - print opts +if [ -z $GN2_PROFILE ]; then + echo "Run request tests with something like" + echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py -a http://localhost:5003 exit 1 -end - -$options = options # we are using a global here -$host = - if ARGV.size>0 - ARGV.shift - else - "http://localhost:5003" - end - -$stderr.print "Testing <",$host,">\n" - -require 'mechanize' -require 'minitest/spec' -require 'minitest/autorun' - -# These are the actual testing modules - -libpath = File.dirname(File.dirname(__FILE__)) -$: << File.join(libpath,'test/lib') - -require 'main_web_functionality' - -if options[:all] or options[:mapping] - require 'mapping' -end - -if options[:all] or options[:link_checker] - require 'link_checker' -end - -if options[:all] or options[:navigation] - require 'navigation' -end +fi |