diff options
| -rwxr-xr-x | bin/genenetwork2 | 30 | ||||
| -rw-r--r-- | wqflask/base/webqtlConfig.py | 1 | ||||
| -rw-r--r-- | wqflask/utility/tools.py | 13 | ||||
| -rw-r--r-- | wqflask/wqflask/marker_regression/gemma_mapping.py | 16 |
4 files changed, 42 insertions, 18 deletions
diff --git a/bin/genenetwork2 b/bin/genenetwork2 index cbaba76f..8886e4bc 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,8 +1,11 @@ #! /bin/bash # +# This is the startup script for GN2. It sets the environment variables to pick +# up a Guix profile and allows for overriding parameters. +# # Typical usage # -# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 +# env GN2_PROFILE=~/opt/genenetwork2-phewas ./bin/genenetwork2 ~/my_settings.py # # Where GN2_PROFILE points to the GNU Guix profile used for deployment. # @@ -41,6 +44,8 @@ # env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -gunicorn "--bind 0.0.0.0:5003 --workers=1 wsgi" SCRIPT=$(realpath "$0") +echo SCRIPT=$SCRIPT +echo GN2_PROFILE=$GN2_PROFILE GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) @@ -96,6 +101,8 @@ if [ -z $GN2_PROFILE ] ; then if [ -d $GN2_PROFILE ]; then echo "Best guess is $GN2_PROFILE" fi + echo "ERROR: always set GN2_PROFILE" + exit 1 fi if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." @@ -114,7 +121,9 @@ else export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" - export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" + if [ -z $GEMMA_WRAPPER_COMMAND ]; then + export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" + fi if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi @@ -143,7 +152,7 @@ set|grep guix set|grep $GN2_PROFILE set|grep TMPDIR -# Now handle command parameter -c +# Now handle command parameter -c which runs python if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} @@ -152,6 +161,16 @@ if [ "$1" = '-c' ] ; then python $cmd exit $? fi +# Now handle command parameter -cli which runs in bash +if [ "$1" = "-cli" ] ; then + echo "HERE" + cd $GN2_BASE_DIR/wqflask + cmd=$2 + echo PYTHONPATH=$PYTHONPATH + echo RUNNING COMMAND $cmd + $cmd + exit $? +fi if [ "$1" = '-gunicorn' ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 @@ -164,7 +183,7 @@ if [ "$1" = '-gunicorn-dev' ] ; then cd $GN2_BASE_DIR/wqflask echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi - cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --reload wsgi" + cmd="--bind 0.0.0.0:$SERVER_PORT --workers=1 --timeout 180 --reload wsgi" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? @@ -173,7 +192,8 @@ if [ "$1" = '-gunicorn-prod' ] ; then cd $GN2_BASE_DIR/wqflask echo PYTHONPATH=$PYTHONPATH if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi - cmd="--bind 0.0.0.0:$SERVER_PORT --workers=32 --max-requests 1000 --timeout 1200 wsgi" + PID=$TMPDIR/gunicorn.$USER.pid + cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi" echo RUNNING gunicorn $cmd gunicorn $cmd exit $? diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py index 1ef2bc26..1e66e957 100644 --- a/wqflask/base/webqtlConfig.py +++ b/wqflask/base/webqtlConfig.py @@ -82,6 +82,7 @@ assert_writable_dir(GENERATED_TEXT_DIR) # Flat file directories GENODIR = flat_files('genotype')+'/' assert_dir(GENODIR) +assert_dir(GENODIR+'bimbam') # for gemma # JSON genotypes are OBSOLETE JSON_GENODIR = flat_files('genotype/json')+'/' diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 57f97a81..d3113302 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -16,7 +16,7 @@ OVERRIDES = {} def app_set(command_id, value): """Set application wide value""" app.config.setdefault(command_id,value) - value + return value def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in @@ -51,7 +51,7 @@ def get_setting(command_id,guess=None): return None # ---- Check whether environment exists - logger.debug("Looking for "+command_id+"\n") + # print("Looking for "+command_id+"\n") command = value(os.environ.get(command_id)) if command is None or command == "": command = OVERRIDES.get(command_id) @@ -63,7 +63,7 @@ def get_setting(command_id,guess=None): if command is None or command == "": # print command raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).') - logger.debug("Set "+command_id+"="+str(command)) + # print("Set "+command_id+"="+str(command)) return command def get_setting_bool(id): @@ -253,6 +253,7 @@ JS_GN_PATH = get_setting('JS_GN_PATH') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +assert(GEMMA_COMMAND is not None) PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) GEMMA_WRAPPER_COMMAND = gemma_wrapper_command() TEMPDIR = tempdir() # defaults to UNIX TMPDIR @@ -266,7 +267,7 @@ from six import string_types if os.environ.get('WQFLASK_OVERRIDES'): jsonfn = get_setting('WQFLASK_OVERRIDES') - logger.error("WQFLASK_OVERRIDES: %s" % jsonfn) + logger.info("WQFLASK_OVERRIDES: %s" % jsonfn) with open(jsonfn) as data_file: overrides = json.load(data_file) for k in overrides: @@ -278,8 +279,4 @@ if os.environ.get('WQFLASK_OVERRIDES'): logger.debug(OVERRIDES) # assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData") -# assert_dir(get_setting("JS_BIODALLIANCE")) -# assert_file(get_setting("JS_BIODALLIANCE")+"/build/dalliance-all.js") -# assert_file(get_setting("JS_BIODALLIANCE")+"/build/worker-all.js") -# assert_dir(get_setting("JS_TWITTER_POST_FETCHER")) assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js") diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py index a24e43d4..68920130 100644 --- a/wqflask/wqflask/marker_regression/gemma_mapping.py +++ b/wqflask/wqflask/marker_regression/gemma_mapping.py @@ -3,7 +3,7 @@ import os, math, string, random, json from base import webqtlConfig from base.trait import GeneralTrait from base.data_set import create_dataset -from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR +from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin, assert_file import utility.logger logger = utility.logger.getLogger(__name__ ) @@ -11,6 +11,7 @@ logger = utility.logger.getLogger(__name__ ) def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): """Generates p-values for each marker using GEMMA""" + assert_bin(GEMMA_COMMAND); if this_dataset.group.genofile != None: genofile_name = this_dataset.group.genofile[:-5] else: @@ -27,7 +28,7 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco): if i < (len(this_chromosomes) - 1): chr_list_string += this_chromosomes[i+1].name + "," else: - chr_list_string += this_chromosomes[i+1].name + chr_list_string += this_chromosomes[i+1].name if covariates != "": gen_covariates_file(this_dataset, covariates) @@ -209,8 +210,13 @@ def parse_gemma_output(genofile_name): def parse_loco_output(this_dataset, gwa_output_filename): output_filelist = [] - with open("{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json") as data_file: - data = json.load(data_file) + jsonfn = "{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json" + assert_file(jsonfn) + try: + with open(jsonfn) as data_file: + data = json.load(data_file) + except: + logger.error("Can not parse "+jsonfn) files = data['files'] for file in files: @@ -247,4 +253,4 @@ def parse_loco_output(this_dataset, gwa_output_filename): included_markers.append(line.split("\t")[1]) p_values.append(float(line.split("\t")[10])) - return marker_obs
\ No newline at end of file + return marker_obs |