From 627f86392f082b9a42bd9efeb59e7ee9374932b3 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Thu, 27 Jan 2022 10:37:03 +0300 Subject: Fix 'Bad substitution' issue In posix shell, string indexing is undefined. This commit replaces the string indexing with a more portable implementation. --- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index ce3678e4..7669a89a 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -59,7 +59,7 @@ else echo INFO: GN2 is running from a source tree GIT_HASH=$(git rev-parse HEAD) GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} + export GN_VERSION="${GN2_ID}:$(cat ${GN2_BASE_DIR}/etc/VERSION)-${GIT_BRANCH}-$(echo ${GIT_HASH} | cut -c1-9)" fi echo GN_VERSION=$GN_VERSION -- cgit v1.2.3 From 2b629077ba1ec52233c118707d44dca2fb5e6afb Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Fri, 28 Jan 2022 13:38:20 +0530 Subject: Revert "bin: Do not set environment variables set by guix shell." This reverts commit 344e428126b60932bff4c62c5ded8c36519155e8. --- README.md | 4 ++++ bin/genenetwork2 | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) (limited to 'bin') diff --git a/README.md b/README.md index 2b41a415..7b4fc01f 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,10 @@ Also mariadb and redis need to be running, see ## Development +It may be useful to pull in the GN3 python modules locally. For this +use `GN3_PYTHONPATH` environment that gets injected in +the ./bin/genenetwork2 startup. + ## Testing To have tests pass, the redis and mariadb instance should be running, because of diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 7669a89a..f1e04eef 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -91,7 +91,14 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings +export PATH=$GN2_PROFILE/bin:$PATH +export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! +export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript +export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" +export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" +export XDG_DATA_DIRS="$GN2_PROFILE/share" +export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" export LC_ALL=C # FIXME export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" @@ -99,6 +106,25 @@ export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" if [ -z $GEMMA_WRAPPER_COMMAND ]; then export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi +while IFS=":" read -ra PPATH; do + for PPART in "${PPATH[@]}"; do + if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + done +done <<< "$PYTHONPATH" +if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi +if [ -z $PYTHONPATH ] ; then + echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" + exit 1 +fi +if [ ! -d $R_LIBS_SITE ] ; then + echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" + echo "Paste into your shell the output of (for example)" + echo "guix package -p \$GN2_PROFILE --search-paths" + exit 1 +fi + +# We may change this one: +export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONPATH # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then @@ -114,6 +140,7 @@ set|grep TMPDIR if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} + echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* python $cmd $* @@ -124,6 +151,7 @@ fi if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 + echo PYTHONPATH=$PYTHONPATH shift ; shift echo RUNNING COMMAND $cmd $* $cmd $* @@ -132,12 +160,14 @@ fi if [ "$1" = '-gunicorn' ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 + echo PYTHONPATH=$PYTHONPATH echo RUNNING gunicorn $cmd gunicorn $cmd exit $? fi if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask + echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" echo RUNNING gunicorn $cmd @@ -146,6 +176,7 @@ if [ "$1" = '-gunicorn-dev' ] ; then fi if [ "$1" = '-gunicorn-prod' ] ; then cd $GN2_BASE_DIR/wqflask + echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" @@ -159,6 +190,9 @@ echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & +# Overrides for packages that are not yet public (currently r-auwerx) +# export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib + # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" -- cgit v1.2.3 From fadf0b836d512c9839f91df77f5f957e7aca6a1c Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Fri, 28 Jan 2022 13:38:24 +0530 Subject: Revert "bin: Set GN2_PROFILE from GUIX_ENVIRONMENT." This reverts commit f4e336eb1ea526156e112cff97a3ec8137a2bc90. --- README.md | 17 +++++++----- bin/genenetwork2 | 83 ++++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 63 insertions(+), 37 deletions(-) (limited to 'bin') diff --git a/README.md b/README.md index 7b4fc01f..52382c98 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ genenetwork2 A quick example is ```sh -env SERVER_PORT=5300 \ +env GN2_PROFILE=~/opt/gn-latest SERVER_PORT=5300 \ GENENETWORK_FILES=~/data/gn2_data/ \ GN_PROXY_URL="http://localhost:8080"\ GN3_LOCAL_URL="http://localhost:8081"\ @@ -69,16 +69,19 @@ We are building 'Mechanical Rob' automated testing using Python which can be run with: ```sh -env ./bin/genenetwork2 \ +env GN2_PROFILE=~/opt/gn-latest \ + ./bin/genenetwork2 \ GN_PROXY_URL="http://localhost:8080" \ GN3_LOCAL_URL="http://localhost:8081 "\ ./etc/default_settings.py -c \ ../test/requests/test-website.py -a http://localhost:5003 ``` -The ./bin/genenetwork2 script sets up the environment and executes -test-website.py in a Python interpreter. The -a switch says to run all -tests and the URL points to the running GN2 http server. +The GN2_PROFILE is the Guix profile that contains all +dependencies. The ./bin/genenetwork2 script sets up the environment +and executes test-website.py in a Python interpreter. The -a switch +says to run all tests and the URL points to the running GN2 http +server. #### Unit tests @@ -99,9 +102,9 @@ runcmd coverage html The `runcmd` and `runpython` are shell aliases defined in the following way: ```sh -alias runpython="env TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 +alias runpython="env GN2_PROFILE=~/opt/gn-latest TMPDIR=/tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 -alias runcmd="time env TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" +alias runcmd="time env GN2_PROFILE=~/opt/gn-latest TMPDIR=//tmp SERVER_PORT=5004 GENENETWORK_FILES=/gnu/data/gn2_data/ GN_PROXY_URL="http://localhost:8080" GN3_LOCAL_URL="http://localhost:8081" ./bin/genenetwork2 ./etc/default_settings.py -cli" ``` Replace some of the env variables as per your use case. diff --git a/bin/genenetwork2 b/bin/genenetwork2 index f1e04eef..ea5a4f50 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -5,45 +5,55 @@ # # Typical usage # -# ./bin/genenetwork2 ~/my_settings.py +# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py +# +# Where GN2_PROFILE points to the GNU Guix profile used for deployment. # # This will run the GN2 server (with default settings if none -# supplied). +# supplied). Typically you need a GNU Guix profile which is set with +# an environment variable (this profile is dictated by the +# installation path of genenetwork). Say your profile is in +# ~/opt/gn-latest-guix +# +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 +# +# You can pass in your own settings file, e.g. +# +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py # # To run a maintenance python script with settings (instead of the # webserver) run from the base-dir with settings file and add that # script with a -c switch, e.g. # -# env TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py +# env GN2_PROFILE=/usr/local/guix-profiles/gn-latest-20190905 TMPDIR=/export/local/home/zas1024/gn2-zach/tmp WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG SERVER_PORT=5002 GENENETWORK_FILES=/export/local/home/zas1024/gn2-zach/genotype_files SQL_URI=mysql://webqtlout:webqtlout@lily.uthsc.edu/db_webqtl ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py # # To run any script in the environment # -# ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD" # # To get a python REPL(!) # -# ./bin/genenetwork2 ./etc/default_settings.py -cli python +# env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python # # For development you may want to run # -# env WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 +# env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2 # # For staging and production we use gunicorn. Run with something like # (note you have to provide the server port). Provide a settings file! # -# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-prod # # For development use # -# env SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev +# env GN2_PROFILE=~/opt/gn-latest-guix SERVER_PORT=5003 ./bin/genenetwork2 ./etc/default_settings.py -gunicorn-dev # # For extra flexibility you can also provide gunicorn parameters yourself with something like # -# ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") echo SCRIPT=$SCRIPT -export GN2_PROFILE=$GUIX_ENVIRONMENT echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) @@ -91,27 +101,40 @@ fi export GN2_SETTINGS=$settings # Python echo GN2_SETTINGS=$settings -export PATH=$GN2_PROFILE/bin:$PATH -export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! -export R_LIBS_SITE=$GN2_PROFILE/site-library -export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" -export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" -export XDG_DATA_DIRS="$GN2_PROFILE/share" -export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" -export LC_ALL=C # FIXME -export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" -export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" -export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" -if [ -z $GEMMA_WRAPPER_COMMAND ]; then - export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" +if [ -z $GN2_PROFILE ] ; then + echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" + export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) + if [ -d $GN2_PROFILE ]; then + echo "Best guess is $GN2_PROFILE" + fi + echo "ERROR: always set GN2_PROFILE" + exit 1 +fi +if [ -z $GN2_PROFILE ]; then + read -p "PRESS [ENTER] TO CONTINUE..." +else + export PATH=$GN2_PROFILE/bin:$PATH + export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! + export R_LIBS_SITE=$GN2_PROFILE/site-library + export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript + export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" + export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" + export XDG_DATA_DIRS="$GN2_PROFILE/share" + export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" + export LC_ALL=C # FIXME + export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" + export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" + export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" + if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" + fi + while IFS=":" read -ra PPATH; do + for PPART in "${PPATH[@]}"; do + if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + done + done <<< "$PYTHONPATH" + if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi fi -while IFS=":" read -ra PPATH; do - for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi - done -done <<< "$PYTHONPATH" -if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ -z $PYTHONPATH ] ; then echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" exit 1 -- cgit v1.2.3 From 631504680ce31b1a181cf6617e87ef0cc2cefa26 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 28 Jan 2022 13:40:10 -0600 Subject: Revert 'bin: Set shebang to sh instead of bash' It seems that this also needed to be reverted, because I was getting the error below (which was prevented by reverting to #! /bin/bash): ./bin/genenetwork2: 135: ./bin/genenetwork2: Syntax error: redirection unexpected--- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index ea5a4f50..ea70835b 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,4 +1,4 @@ -#! /bin/sh -e +#! /bin/bash # # This is the startup script for GN2. It sets the environment variables to pick # up a Guix profile and allows for overriding parameters. -- cgit v1.2.3 From b9f16a3b28c4d996ac91befa5a919a9e0a7ac981 Mon Sep 17 00:00:00 2001 From: zsloan Date: Fri, 28 Jan 2022 14:53:53 -0600 Subject: Update PYTHONPATH The PYTHONPATH apparently needs to be updated to 3.9, because 3.8 throws an error (at least on production).--- bin/genenetwork2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index ea70835b..1d645c23 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -114,7 +114,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH="$GN2_PROFILE/lib/python3.8/site-packages" # never inject another PYTHONPATH!! + export PYTHONPATH="$GN2_PROFILE/lib/python3.9/site-packages" # never inject another PYTHONPATH!! export R_LIBS_SITE=$GN2_PROFILE/site-library export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" -- cgit v1.2.3 From a01033438a85e3097e4ade30e9a208fd5109d4e1 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 6 Dec 2021 17:58:28 +0300 Subject: Add "Partial Correlations" UI elements on collections page Issue: https://github.com/genenetwork/gn-gemtext-threads/blob/main/topics/gn1-migration-to-gn2/partial-correlations.gmi * Add UI elements to trigger the partial correlations feature * Connect partial correlation view to the application * Point to correct application element for gunicorn --- bin/genenetwork2 | 2 +- wqflask/wqflask/__init__.py | 1 + wqflask/wqflask/partial_correlations_views.py | 263 +++++++++++++++++++++ wqflask/wqflask/templates/collections/view.html | 17 ++ .../wqflask/templates/partial_correlations.html | 199 ++++++++++++++++ 5 files changed, 481 insertions(+), 1 deletion(-) create mode 100644 wqflask/wqflask/partial_correlations_views.py create mode 100644 wqflask/wqflask/templates/partial_correlations.html (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 1d645c23..0a2c18cf 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -192,7 +192,7 @@ if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi - cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" + cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload run_gunicorn:app" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py index 05e040ed..ab8b9e66 100644 --- a/wqflask/wqflask/__init__.py +++ b/wqflask/wqflask/__init__.py @@ -98,3 +98,4 @@ from wqflask import user_login from wqflask import user_session import wqflask.views +import wqflask.partial_correlations_views diff --git a/wqflask/wqflask/partial_correlations_views.py b/wqflask/wqflask/partial_correlations_views.py new file mode 100644 index 00000000..bee0a033 --- /dev/null +++ b/wqflask/wqflask/partial_correlations_views.py @@ -0,0 +1,263 @@ +from typing import Union, Tuple + +import MySQLdb +from gn3.db.traits import retrieve_trait_info +from flask import flash, request, current_app, render_template +from gn3.computations.partial_correlations import partial_correlations_entry + +from wqflask import app + +def parse_trait(trait_str: str) -> Union[dict, None]: + keys = ("name", "dataset", "symbol", "description", "data_hmac") + parts = tuple(part.strip() for part in trait_str.split(":::")) + if len(parts) == len(keys): + return dict(zip(keys, parts)) + return None + +def process_step_select_primary( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + if primary_trait is None: + flash("You must select a primary trait", "alert-danger") + return ( + "select-primary", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + return ( + "select-controls", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list + if trait["data_hmac"] != primary_trait["data_hmac"]), + corr_method) + +def process_step_select_controls( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + if len(control_traits) == 0 or len(control_traits) > 3: + flash( + ("You must select a minimum of one control trait, up to a maximum " + "of three control traits."), + "alert-danger") + return ( + "select-controls", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + hmacs =(primary_trait["data_hmac"],) + tuple( + trait["data_hmac"] for trait in control_traits) + return ( + "select-targets", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list if trait["data_hmac"] not in hmacs), + corr_method) + +def process_step_select_targets( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + if len(target_traits) == 0: + flash( + "You must select at least one target trait.", "alert-danger") + return ( + "select-targets", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + hmacs =(primary_trait["data_hmac"],) + tuple( + trait["data_hmac"] for trait in (control_traits + target_traits)) + return ( + "select-corr-method", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list if trait["data_hmac"] not in hmacs), + corr_method) + +def process_step_select_corr_method( + primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], + traits_list: Tuple[dict, ...], corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + methods = ( + "genetic correlation, pearson's r", + "genetic correlation, spearman's rho", + "sgo literature correlation", + "tissue correlation, pearson's r", + "tissue correlation, spearman's rho") + if corr_method.lower() not in methods: + flash( + "Selected method is unknown.", "alert-danger") + return ( + "select-corr-method", primary_trait, control_traits, target_traits, + traits_list, corr_method) + + hmacs =(primary_trait["data_hmac"],) + tuple( + trait["data_hmac"] for trait in (control_traits + target_traits)) + return ( + "run-correlation", primary_trait, control_traits, target_traits, + tuple( + trait for trait in traits_list if trait["data_hmac"] not in hmacs), + corr_method) + +def process_step( + step: str, primary_trait: dict, control_traits: Tuple[dict, ...], + target_traits: Tuple[dict, ...], traits_list: Tuple[dict, ...], + corr_method: str) -> Tuple[ + str, dict, Tuple[dict, ...], Tuple[dict, ...], Tuple[dict, ...], + str]: + processor_functions = { + # "select-traits": lambda arg: arg, + "select-primary": process_step_select_primary, + "select-controls": process_step_select_controls, + "select-targets": process_step_select_targets, + "select-corr-method": process_step_select_corr_method + } + return processor_functions[(step or "select-primary")]( + primary_trait, control_traits, target_traits, traits_list, corr_method) + +def sequence_of_traits(trait_strs) -> Tuple[dict, ...]: + return tuple(filter( + lambda trt: trt is not None, + (parse_trait(tstr.strip()) for tstr in trait_strs))) + +def publish_target_dabases(conn, group, threshold): + query = ( + "SELECT PublishFreeze.FullName,PublishFreeze.Name " + "FROM PublishFreeze, InbredSet " + "WHERE PublishFreeze.InbredSetId = InbredSet.Id " + "AND InbredSet.Name = %(group)s " + "AND PublishFreeze.public > %(threshold)s") + with conn.cursor() as cursor: + cursor.execute(query, {"group": group, "threshold": threshold}) + res = cursor.fetchall() + if res: + return tuple( + dict(zip(("description", "value"), row)) for row in res) + + return tuple() + +def geno_target_databases(conn, group, threshold): + query = ( + "SELECT GenoFreeze.FullName,GenoFreeze.Name " + "FROM GenoFreeze, InbredSet " + "WHERE GenoFreeze.InbredSetId = InbredSet.Id " + "AND InbredSet.Name = %(group)s " + "AND GenoFreeze.public > %(threshold)s") + with conn.cursor() as cursor: + cursor.execute(query, {"group": group, "threshold": threshold}) + res = cursor.fetchall() + if res: + return tuple( + dict(zip(("description", "value"), row)) for row in res) + + return tuple() + +def probeset_target_databases(conn, group, threshold): + query1 = "SELECT Id, Name FROM Tissue order by Name" + query2 = ( + "SELECT ProbeFreeze.TissueId, ProbeSetFreeze.FullName, ProbeSetFreeze.Name " + "FROM ProbeSetFreeze, ProbeFreeze, InbredSet " + "WHERE ProbeSetFreeze.ProbeFreezeId = ProbeFreeze.Id " + "AND ProbeFreeze.TissueId IN %(tissue_ids)s " + "AND ProbeSetFreeze.public > %(threshold)s " + "AND ProbeFreeze.InbredSetId = InbredSet.Id " + "AND InbredSet.Name like %(group)s " + "ORDER BY ProbeSetFreeze.CreateTime desc, ProbeSetFreeze.AvgId") + with conn.cursor() as cursor: + cursor.execute(query1) + tissue_res = cursor.fetchall() + if tissue_res: + tissue_ids = tuple(row[0] for row in tissue_res) + cursor.execute( + query2,{ + "tissue_ids": tissue_ids, "group": f"{group}%%", + "threshold": threshold + }) + db_res = cursor.fetchall() + if db_res: + databases = tuple( + dict(zip(("tissue_id", "description", "value"), row)) + for row in db_res) + return tuple( + {tissue_name: tuple( + { + "value": item["value"], + "description": item["description"] + } for item in databases + if item["tissue_id"] == tissue_id)} + for tissue_id, tissue_name in tissue_res) + + return tuple() + +def target_databases(conn, step, trait, threshold): + """ + Retrieves the names of possible target databases from the database. + """ + if step != "select-corr-method": + return None + + trait_info = retrieve_trait_info( + threshold, f"{trait['dataset']}::{trait['name']}", conn) + group = trait_info["group"] + return ( + publish_target_dabases(conn, group, threshold) + + geno_target_databases(conn, group, threshold) + + probeset_target_databases(conn, group, threshold)) + +def pcorrelations(conn, values): + if values["step"] != "run-correlation": + return None + + def trait_fullname(trait): + return f"{trait['dataset']}::{trait['name']}" + + return partial_correlations_entry( + conn, trait_fullname(values["primary_trait"]), + tuple(trait_fullname(trait) for trait in values["control_traits"]), + values["method"], values["criteria"], values["target_db"]) + +@app.route("/partial_correlations", methods=("POST",)) +def partial_correlations(): + form = request.form + traits_list = tuple(filter( + lambda trt: trt is not None, + (parse_trait(tstr) for tstr in form.get("traits_list", "").split("|||")))) + + args_dict = dict(zip( + ("step", "primary_trait", "control_traits", "target_traits", + "traits_list", "method"), + process_step( + form.get("step", None), + parse_trait(form.get("primary_trait", "")), + sequence_of_traits( + form.getlist("control_traits[]") or + form.get("control_traits", "").split("|||")), + sequence_of_traits( + form.getlist("target_traits[]") or + form.get("target_traits", "").split("|||")), + sequence_of_traits(form.get("traits_list", "").split("|||")), + form.get("method")))) + + conn = MySQLdb.Connect( + db=current_app.config.get("DB_NAME"), + user=current_app.config.get("DB_USER"), + passwd=current_app.config.get("DB_PASS"), + host=current_app.config.get("DB_HOST")) + target_dbs = target_databases( + conn, args_dict["step"], args_dict["primary_trait"], 0) + + if args_dict["step"] == "run-correlation": + args_dict = { + **args_dict, "target_db": form.get("target_db"), + "criteria": int(form.get("criteria", 500))} + + corr_results = pcorrelations(conn, args_dict) + + return render_template( + "partial_correlations.html", **args_dict, target_dbs=target_dbs, + corr_results=corr_results) diff --git a/wqflask/wqflask/templates/collections/view.html b/wqflask/wqflask/templates/collections/view.html index 0ded66a6..f4270b67 100644 --- a/wqflask/wqflask/templates/collections/view.html +++ b/wqflask/wqflask/templates/collections/view.html @@ -34,6 +34,23 @@ +
+ + + +
+

diff --git a/wqflask/wqflask/templates/partial_correlations.html b/wqflask/wqflask/templates/partial_correlations.html new file mode 100644 index 00000000..b61f7fc4 --- /dev/null +++ b/wqflask/wqflask/templates/partial_correlations.html @@ -0,0 +1,199 @@ +{%extends "base.html"%} + +{%block content%} +
+ + {%with messages = get_flashed_messages(with_categories=true)%} + {%if messages:%} +
    + {%for category, message in messages:%} +
  • {{message}}
    • + {%endfor%} +
+ {%endif%} + {%endwith%} + + + + + {%if primary_trait:%} + +

+ Primary Trait: {{primary_trait["name"]}} -  + {{primary_trait["symbol"]}} - {{primary_trait["description"]}} +

+ {%endif%} + + {%if control_traits:%} + +

+ Control Traits: +

    + {%for trait in control_traits:%} +
  • + {{trait["name"]}} - {{trait["symbol"]}} -  + {{trait["description"]}}
    • + {%endfor%} +
+

+ {%endif%} + + {%if target_traits:%} + +

+ Target Traits: +

    + {%for trait in target_traits:%} +
  • + {{trait["name"]}} - {{trait["symbol"]}} -  + {{trait["description"]}}
    • + {%endfor%} +
+

+ {%endif%} + + + + {%if step == "select-primary":%} +

Please select the primary trait (X)

+ {%for trait in traits_list:%} +
+ + +
+ {%endfor%} + + {%endif%} + + + + {%if step == "select-controls":%} + +

Select a maximum of three (3) control traits (Z)

+ {%for trait in traits_list:%} +
+ + +
+ {%endfor%} + + {%endif%} + + + + {%if step == "select-targets":%} +

Select at least one target trait (Y)

+ {%for trait in traits_list:%} +
+ + +
+ {%endfor%} + + {%endif%} + + + + {%if step == "select-corr-method":%} +
+ + +
+ +
+ + +
+ +
+ + +
+ + + {%endif%} + + {%if step == "run-correlation":%} + + + + + {{corr_results}} + {%endif%} + +
+{%endblock%} -- cgit v1.2.3 From c0d4de52e28135b002ab18ff59ac0fb6fd827a4f Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Fri, 10 Dec 2021 09:52:38 +0300 Subject: Automate setting of the development GN3 repo * When developing the GN2 code, there are times we need to use the latest code in GN3, and rather than manually editing the `bin/genenetwork` file every time we need that, this commit enables the use of an environment variable. In this case, launching the application with something like: $ env GN3_DEV_REPO_PATH="$HOME/genenetwork3" ...\ ./bin/genenetwork2 ./the_settings.py -gunicorn-dev will add the path to the development version to the PYTHONPATH. If the GN3_DEV_REPO_PATH variable is not set, then the PYTHONPATH is not altered, and would be as if there is no development version of GN3 in use. --- bin/genenetwork2 | 2 ++ 1 file changed, 2 insertions(+) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 0a2c18cf..1b933e80 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -149,6 +149,8 @@ fi # We may change this one: export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONPATH +export PYTHONPATH="${GN3_DEV_REPO_PATH:+$GN3_DEV_REPO_PATH:}:${PYTHONPATH}" + # Our UNIX TMPDIR defaults to /tmp - change this on a shared server if [ -z $TMPDIR ]; then TMPDIR="/tmp" -- cgit v1.2.3 From 04203f785e5a8c86509089bc68340e1e01162ffd Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 31 Jan 2022 06:46:50 +0300 Subject: Double quote shell variables to prevent globbing and word splitting --- bin/genenetwork2 | 87 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 45 insertions(+), 42 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 1b933e80..a6ba7241 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -53,25 +53,27 @@ # env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") -echo SCRIPT=$SCRIPT -echo GN2_PROFILE=$GN2_PROFILE -GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) -GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) +echo SCRIPT="${SCRIPT}" +echo GN2_PROFILE="${GN2_PROFILE}" +GN2_BASE_DIR=$(dirname "$(dirname "${SCRIPT}")") +GN2_ID=$(cat /etc/hostname):$(basename "${GN2_BASE_DIR}") -echo GN2_BASE_DIR=$GN2_BASE_DIR +echo GN2_BASE_DIR="${GN2_BASE_DIR}" GUIX_SITE=$GN2_BASE_DIR/lib/python3.8/site-packages -if [ -d $GUIX_SITE ]; then +if [ -d "${GUIX_SITE}" ]; then echo INFO: GN2 is running from GNU Guix GN2_BASE_DIR=$GUIX_SITE - export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION) + GN_VERSION="${GN2_ID}:$(cat "${GN2_BASE_DIR}"/etc/VERSION)" + export GN_VERSION else echo INFO: GN2 is running from a source tree GIT_HASH=$(git rev-parse HEAD) GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - export GN_VERSION="${GN2_ID}:$(cat ${GN2_BASE_DIR}/etc/VERSION)-${GIT_BRANCH}-$(echo ${GIT_HASH} | cut -c1-9)" + GN_VERSION="${GN2_ID}:$(cat "${GN2_BASE_DIR}"/etc/VERSION)-${GIT_BRANCH}-$(echo "${GIT_HASH}" | cut -c1-9)" + export GN_VERSION fi -echo GN_VERSION=$GN_VERSION +echo GN_VERSION="${GN_VERSION}" if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then echo "Can not use $1 switch without default settings file" @@ -79,12 +81,12 @@ if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then fi settings=$1 -if [ -z $settings ]; then +if [ -z "${settings}" ]; then settings=$GN2_BASE_DIR/etc/default_settings.py else shift fi -settings=$(realpath $settings) +settings=$(realpath "${settings}") # ext="${settings##*.}" # if [ "$ext" = "json" -o "$ext" = "JSON" ]; then @@ -93,24 +95,25 @@ settings=$(realpath $settings) # echo $settings # fi -if [ ! -e $settings ]; then +if [ ! -e "${settings}" ]; then echo "ERROR: can not locate settings file - pass it in the command line" exit 1 fi export GN2_SETTINGS=$settings # Python -echo GN2_SETTINGS=$settings +echo GN2_SETTINGS="${settings}" -if [ -z $GN2_PROFILE ] ; then +if [ -z "${GN2_PROFILE}" ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" - export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) - if [ -d $GN2_PROFILE ]; then + GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) + export GN2_PROFILE + if [ -d "${GN2_PROFILE}" ]; then echo "Best guess is $GN2_PROFILE" fi echo "ERROR: always set GN2_PROFILE" exit 1 fi -if [ -z $GN2_PROFILE ]; then +if [ -z "${GN2_PROFILE}" ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH @@ -125,21 +128,21 @@ else export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" - if [ -z $GEMMA_WRAPPER_COMMAND ]; then + if [ -z "${GEMMA_WRAPPER_COMMAND}" ]; then export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" fi while IFS=":" read -ra PPATH; do for PPART in "${PPATH[@]}"; do - if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi + if [ ! -d "${PPART}" ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi done done <<< "$PYTHONPATH" - if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi + if [ ! -d "${R_LIBS_SITE}" ] ; then echo "R_LIBS_SITE not valid ${R_LIBS_SITE}" ; exit 1 ; fi fi -if [ -z $PYTHONPATH ] ; then +if [ -z "${PYTHONPATH}" ] ; then echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" exit 1 fi -if [ ! -d $R_LIBS_SITE ] ; then +if [ ! -d "${R_LIBS_SITE}" ] ; then echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" echo "Paste into your shell the output of (for example)" echo "guix package -p \$GN2_PROFILE --search-paths" @@ -152,60 +155,60 @@ export PYTHONPATH=$PYTHON_GN_PATH:$GN2_BASE_DIR/wqflask:$GN3_PYTHONPATH:$PYTHONP export PYTHONPATH="${GN3_DEV_REPO_PATH:+$GN3_DEV_REPO_PATH:}:${PYTHONPATH}" # Our UNIX TMPDIR defaults to /tmp - change this on a shared server -if [ -z $TMPDIR ]; then +if [ -z "${TMPDIR}" ]; then TMPDIR="/tmp" fi # Show environment settings set|grep guix -set|grep $GN2_PROFILE +set|grep GN2_PROFILE set|grep TMPDIR # Now handle command parameter -c which runs python if [ "$1" = '-c' ] ; then - cd $GN2_BASE_DIR/wqflask + cd "${GN2_BASE_DIR}/wqflask" cmd=${2#wqflask/} - echo PYTHONPATH=$PYTHONPATH + echo PYTHONPATH="${PYTHONPATH}" shift ; shift - echo RUNNING COMMAND $cmd $* + echo "RUNNING COMMAND ${cmd} ${*}" python $cmd $* exit $? fi # Now handle command parameter -cli which runs in bash if [ "$1" = "-cli" ] ; then - cd $GN2_BASE_DIR/wqflask + cd "${GN2_BASE_DIR}/wqflask" cmd=$2 - echo PYTHONPATH=$PYTHONPATH + echo PYTHONPATH="${PYTHONPATH}" shift ; shift - echo RUNNING COMMAND $cmd $* + echo "RUNNING COMMAND ${cmd} ${*}" $cmd $* exit $? fi if [ "$1" = '-gunicorn' ] ; then - cd $GN2_BASE_DIR/wqflask + cd "${GN2_BASE_DIR}/wqflask" cmd=$2 - echo PYTHONPATH=$PYTHONPATH - echo RUNNING gunicorn $cmd + echo PYTHONPATH="${PYTHONPATH}" + echo "RUNNING gunicorn ${cmd}" gunicorn $cmd exit $? fi if [ "$1" = '-gunicorn-dev' ] ; then - cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH - if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi + cd "${GN2_BASE_DIR}/wqflask" + echo PYTHONPATH="${PYTHONPATH}" + if [ -z "${SERVER_PORT}" ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload run_gunicorn:app" - echo RUNNING gunicorn $cmd + echo "RUNNING gunicorn ${cmd}" gunicorn $cmd exit $? fi if [ "$1" = '-gunicorn-prod' ] ; then - cd $GN2_BASE_DIR/wqflask - echo PYTHONPATH=$PYTHONPATH - if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi + cd "${GN2_BASE_DIR}/wqflask" + echo PYTHONPATH="${PYTHONPATH}" + if [ -z "${SERVER_PORT}" ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi PID=$TMPDIR/gunicorn.$USER.pid cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID --workers 20 --keep-alive 6000 --max-requests 100 --max-requests-jitter 30 --timeout 1200 wsgi" - echo RUNNING gunicorn $cmd + echo "RUNNING gunicorn ${cmd}" gunicorn $cmd exit $? fi @@ -219,6 +222,6 @@ dbfilename gn2.rdb # export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib # Start the flask server running GN2 -cd $GN2_BASE_DIR/wqflask +cd "${GN2_BASE_DIR}/wqflask" echo "Starting with $settings" python runserver.py -- cgit v1.2.3 From 84a1f972cab8d7976919a65c7bc4e14a76d78d02 Mon Sep 17 00:00:00 2001 From: Arun Isaac Date: Thu, 17 Feb 2022 22:50:20 +0530 Subject: bin: Allow for not starting redis. redis needs to be started and managed as a separate service. We create a flag NO_REDIS so that it can be so. If NO_REDIS is not defined, the present behavior of starting redis remains. Thus current use cases are not broken. * bin/genenetwork2: Allow for not starting redis with the NO_REDIS flag. --- bin/genenetwork2 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a6ba7241..36bf11a6 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -213,10 +213,12 @@ if [ "$1" = '-gunicorn-prod' ] ; then exit $? fi -echo "Starting the redis server:" -echo -n "dir $TMPDIR +if [ -z "$NO_REDIS" ] ; then + echo "Starting the redis server:" + echo -n "dir $TMPDIR dbfilename gn2.rdb " | redis-server - & +fi # Overrides for packages that are not yet public (currently r-auwerx) # export R_LIBS_SITE=$R_LIBS_SITE:$HOME/.Rlibs/das1i1pm54dj6lbdcsw5w0sdwhccyj1a-r-3.3.2/lib/R/lib -- cgit v1.2.3