From 03caa57ad209f3bdd135be9d6516b94261c9b8de Mon Sep 17 00:00:00 2001 From: BonfaceKilz Date: Thu, 28 Oct 2021 11:05:19 +0300 Subject: Remove all elasticsearch references in gn2 --- bin/genenetwork2 | 7 ------- 1 file changed, 7 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 2b94b2a2..5f714d2e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -101,13 +101,6 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead -# if [ -z $ELASTICSEARCH_PROFILE ]; then -# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; -# else -# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python3.8/site-packages" -# fi - if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) -- cgit v1.2.3 From d9843928fe5140c953078b5238c2ddb8fb12a96c Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 1 Dec 2021 14:30:00 +0000 Subject: Introduce GN3_PYTHONPATH environment variable --- README.md | 6 ++++++ bin/genenetwork2 | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/README.md b/README.md index 6921d299..62ead0bd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,12 @@ startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/ Also mariadb and redis need to be running, see [INSTALL](./doc/README.org). +## Development + +It may be useful to pull in the GN3 python modules locally. For this +use `GN3_PYTHONPATH` environment that gets injected in +the ./bin/genenetwork2 startup. + ## Testing To have tests pass, the redis and mariadb instance should be running, because of diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5f714d2e..b5c940a8 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -147,7 +147,7 @@ if [ ! -d $R_LIBS_SITE ] ; then fi # We may change this one: -export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$PYTHONPATH +export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then -- cgit v1.2.3 From f4e336eb1ea526156e112cff97a3ec8137a2bc90 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Mon, 3 Jan 2022 17:45:14 +0530 Subject: bin: Set GN2_PROFILE from GUIX_ENVIRONMENT. `guix shell' sets the profile path in an environment variable---GUIX_ENVIRONMENT. There is no need to pass it explicity as an input in GN2_PROFILE. * README.md: Remove all references to GN2_PROFILE. * bin/genenetwork2: Set GN2_PROFILE from GUIX_ENVIRONMENT. Do not check the value of GN2_PROFILE now that it is set automatically. Remove all references to GN2_PROFILE in example invocations. --- README.md | 17 +++++------- bin/genenetwork2 | 83 ++++++++++++++++++++------------------------------------ 2 files changed, 37 insertions(+), 63 deletions(-) (limited to 'bin') diff --git a/README.md b/README.md index 972d5c50..d5539a9f 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ genenetwork2 A quick example is ```sh -env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 \ +env SERVER_PORT=5300 \ GENENETWORK_FILES=~/data/gn2_data/ \ GN_PROXY_URL="http://localhost:8080"\ GN3_LOCAL_URL="http://localhost:8081"\ @@ -69,19 +69,16 @@ We are building 'Mechanical Rob' automated testing using Python which can be run with: ```sh -env GN2_PROFILE=~/opt/gn-latest \ - ./bin/genenetwork2 \ +env ./bin/genenetwork2 \ GN_PROXY_URL="http://localhost:8080" \ GN3_LOCAL_URL="http://localhost:8081 "\ ./etc/default_settings.py -c \ ../test/requests/test-website.py -a http://localhost:5003 ``` -The GN2_PROFILE is the Guix profile that contains all -dependencies. The ./bin/genenetwork2 script sets up the environment -and executes test-website.py in a Python interpreter. The -a switch -says to run all tests and the URL points to the running GN2 http -server. +The ./bin/genenetwork2 script sets up the environment and executes +test-website.py in a Python interpreter. The -a switch says to run all +tests and the URL points to the running GN2 http server. #### Unit tests @@ -102,9 +99,9 @@ runcmd coverage html The `runcmd` and `runpython` are shell aliases defined in the following way: ```sh -alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 +alias runpython="env TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 -alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" +alias runcmd="time env TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" ``` Replace some of the env variables as per your use case. diff --git a/bin/genenetwork2 b/bin/genenetwork2 index b5c940a8..93ab02f8 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -5,55 +5,45 @@ # # Typical usage # -# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py -# -# Where GN2_PROFILE points to the GNU Guix profile used for deployment. +# ./bin/genenetwork2 ~/my_settings.py # # This will run the GN2 server (with default settings if none -# supplied). Typically you need a GNU Guix profile which is set with -# an environment variable (this profile is dictated by the -# installation path of genenetwork). Say your profile is in -# ~/opt/gn-latest-guix -# -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 -# -# You can pass in your own settings file, e.g. -# -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py +# supplied). # # To run a maintenance python script with settings (instead of the # webserver) run from the base-dir with settings file and add that # script with a -c switch, e.g. # -# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py +# env TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py # # To run any script in the environment # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" +# ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" # # To get a python REPL(!) # -# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python +# ./bin/genenetwork2 ./etc/default_settings.py -cli python # # For development you may want to run # -# env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 +# env WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 # # For staging and production we use gunicorn. Run with something like # (note you have to provide the server port). Provide a settings file! # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod +# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod # # For development use # -# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev # # For extra flexibility you can also provide gunicorn parameters yourself with something like # -# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" +# ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") echo SCRIPT=$SCRIPT +export GN2_PROFILE=$GUIX_ENVIRONMENT echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) @@ -101,40 +91,27 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -if [ -z $GN2_PROFILE ] ; then - echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" - export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) - if [ -d $GN2_PROFILE ]; then - echo "Best guess is $GN2_PROFILE" - fi - echo "ERROR: always set GN2_PROFILE" - exit 1 -fi -if [ -z $GN2_PROFILE ]; then - read -p "PRESS [ENTER] TO CONTINUE..." -else - export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! - export R_LIBS_SITE=$GN2_PROFILE/site-library - export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript - export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" - export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" - export XDG_DATA_DIRS="$GN2_PROFILE/share" - export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" - export LC_ALL=C # FIXME - export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" - export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" - export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" - if [ -z $GEMMA_WRAPPER_COMMAND ]; then - export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" - fi - while IFS=":" read -ra PPATH; do - for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi - done - done <<< "$PYTHONPATH" - if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi +export PATH=$GN2_PROFILE/bin:$PATH +export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! +export R_LIBS_SITE=$GN2_PROFILE/site-library +export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript +export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" +export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" +export XDG_DATA_DIRS="$GN2_PROFILE/share" +export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" +export LC_ALL=C # FIXME +export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" +export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" +export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" +if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi +while IFS=":" read -ra PPATH; do + for PPART in "${PPATH[@]}"; do + if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + done +done <<< "$PYTHONPATH" +if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ -z $PYTHONPATH ] ; then echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" exit 1 -- cgit v1.2.3 From 344e428126b60932bff4c62c5ded8c36519155e8 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Mon, 3 Jan 2022 17:51:04 +0530 Subject: bin: Do not set environment variables set by guix shell. The Guix python package accepts GUIX_PYTHONPATH instead of PYTHONPATH. `guix shell' sets GI_TYPELIB_PATH, GIO_EXTRA_MODULES, GUIX_GTK3_PATH, GUIX_PYTHONPATH, PATH, R_LIB_SITE and XDG_DATA_DIRS when necessary. There is no need to set these environment variables explicitly. * bin/genenetwork2: Do not set GI_TYPELIB_PATH, GIO_EXTRA_MODULES, GUIX_GTK3_PATH, PATH, PYTHONPATH, R_LIBS_SITE and XDG_DATA_DIRS. * README.md [Development]: Remove paragraph on injecting python modules locally. --- README.md | 4 ---- bin/genenetwork2 | 34 ---------------------------------- 2 files changed, 38 deletions(-) (limited to 'bin') diff --git a/README.md b/README.md index d5539a9f..8dcbac9e 100644 --- a/README.md +++ b/README.md @@ -46,10 +46,6 @@ Also mariadb and redis need to be running, see ## Development -It may be useful to pull in the GN3 python modules locally. For this -use `GN3_PYTHONPATH` environment that gets injected in -the ./bin/genenetwork2 startup. - ## Testing To have tests pass, the redis and mariadb instance should be running, because of diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 93ab02f8..934a10d8 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -91,14 +91,7 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -export PATH=$GN2_PROFILE/bin:$PATH -export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! -export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" -export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" -export XDG_DATA_DIRS="$GN2_PROFILE/share" -export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" export LC_ALL=C # FIXME export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" @@ -106,25 +99,6 @@ export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" if [ -z $GEMMA_WRAPPER_COMMAND ]; then export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi -while IFS=":" read -ra PPATH; do - for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi - done -done <<< "$PYTHONPATH" -if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi -if [ -z $PYTHONPATH ] ; then - echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" - exit 1 -fi -if [ ! -d $R_LIBS_SITE ] ; then - echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" - echo "Paste into your shell the output of (for example)" - echo "guix package -p \$GN2_PROFILE --search-paths" - exit 1 -fi - -# We may change this one: -export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then @@ -140,7 +114,6 @@ set|grep TMPDIR if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} - echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* python $cmd $* @@ -151,7 +124,6 @@ fi if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 - echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* $cmd $* @@ -160,14 +132,12 @@ fi if [ "$1" = '-gunicorn' ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 - echo PYTHONPATH=$PYTHONPATH echo RUNNING gunicorn $cmd gunicorn $cmd exit $? fi if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" echo RUNNING gunicorn $cmd @@ -176,7 +146,6 @@ if [ "$1" = '-gunicorn-dev' ] ; then fi if [ "$1" = '-gunicorn-prod' ] ; then cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" @@ -190,9 +159,6 @@ echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & -# Overrides for packages that are not yet public (currently r-auwerx) -# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib - # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" -- cgit v1.2.3 From 80003d3f892846c4475a1365c5c5129424970def Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 11:29:27 +0530 Subject: bin: Set shebang to sh instead of bash. /bin/sh is present on Guix System and other distros while /bin/bash is present only on other distros. * bin/genenetwork2: Set shebang to sh instead of bash. --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 934a10d8..eaf3f6ca 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh # # This is the startup script for GN2. It sets the environment variables to pick # up a Guix profile and allows for overriding parameters. -- cgit v1.2.3 From 339e4d084bc8400f86b0fb3cf83bb3049f6b39ed Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Tue, 4 Jan 2022 11:32:40 +0530 Subject: bin: Stop execution on error. * bin/genenetwork2: Pass -e flag to sh. --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index eaf3f6ca..ce3678e4 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,4 +1,4 @@ -#! /bin/sh +#! /bin/sh -e # # This is the startup script for GN2. It sets the environment variables to pick # up a Guix profile and allows for overriding parameters. -- cgit v1.2.3