From 8677d1b7bd5b32791c440bf0de2fc4c61c98299f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 21 May 2017 07:21:44 +0000 Subject: Merge from the diet branch. * refactored ./bin/genenetwork2 and /etc/default_settings - better detection of Guix - removed unused parameters, such as GEMMA_RESULT_PATH and TEMP_TRAITS - removing some default settings, such as PYLMM_COMMAND - rename some settings, such as SQL_URI and LOG_SQL_ALCHEMY - added GUIX_JS_PATH for JS modules * Added documentation on installing binary GN2 * Updated Twitter board to latest version in Guix - it had broken * Updated many tools in the latest Guix profile --- bin/genenetwork2 | 104 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 58 insertions(+), 46 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 52d3155c..df688989 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -1,34 +1,39 @@ #! /bin/bash # # This will run the GN2 server (with default settings if none -# supplied). Pass in your own settings file, e.g. +# supplied). Typically you need a GNU Guix profile which is set with +# an environment variable (this profile is dictated by the +# installation path of genenetwork). Say your profile is in +# ~/opt/gn-latest-guix # -# ./bin/genenetwork2 ~/my_settings.py +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 # -# But better is to retain the defaults and override them with -# JSON file (see the deploy docs). +# You can pass in your own settings file, e.g. # -# ./bin/genenetwork2 ~/my_overrides.json +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_settings.py # # To run a maintenance script with settings (instead of the webserver) add that with # a -c switch, e.g. # -# ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py -# -# Environment settings can be used to preconfigure as well as a -# settings.py file. -# -# GN2_BASE_PATH is base directory of wqflask source code -# +# env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py SCRIPT=$(readlink -f "$0") GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT")) +GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR) echo GN2_BASE_DIR=$GN2_BASE_DIR -GIT_HASH=$(git rev-parse HEAD) -GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -export GN_VERSION=$(cat $GN2_BASE_DIR/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} +GUIX_SITE=$GN2_BASE_DIR/lib/python2.7/site-packages +if [ -d $GUIX_SITE ]; then + echo INFO: GN2 is running from GNU Guix + GN2_BASE_DIR=$GUIX_SITE + export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION) +else + echo INFO: GN2 is running from a source tree + GIT_HASH=$(git rev-parse HEAD) + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + export GN_VERSION=$GN2_ID:$(cat $GN2_BASE_DIR/etc/VERSION)-$GIT_BRANCH-${GIT_HASH:0:9} +fi echo GN_VERSION=$GN_VERSION # Handle settings parameter (can be .py or .json) @@ -41,13 +46,8 @@ else shift fi if [ ! -e $settings ]; then - GUIX_ETC=$GN2_BASE_DIR/lib/python2.7/site-packages/genenetwork2-2.0-py2.7.egg - if [ -d $GUIX_ETC ]; then - echo INFO: GN2 is actually running from GNU Guix - else - echo "ERROR: can not locate settings file - pass it in the command line" - exit 1 - fi + echo "ERROR: can not locate settings file - pass it in the command line" + exit 1 fi export WQFLASK_SETTINGS=$settings # Python export WQFLASK_OVERRIDES=$overrides # JSON @@ -55,30 +55,40 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides -# ---- Checks and balances -if [ -z $GUIX_ETC ]; then - if [ -z $GN2_PROFILE ] ; then - echo "WARNING: GN2_PROFILE has not been set - I hope you know what you are doing!" - else - export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages - export R_LIBS_SITE=$GN2_PROFILE/site-library - export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" - export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" - export XDG_DATA_DIRS="$GN2_PROFILE/share" - export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" - export LC_ALL=C - fi - if [ -z $PYTHONPATH ] ; then - echo "WARNING PYTHONPATH has not been set - use GN2_PROFILE!" - fi - if [ ! -d $R_LIBS_SITE ] ; then - echo "ERROR: R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" - echo "Paste into your shell the output of (for example)" - echo "guix package -p \$GN2_PROFILE --search-paths" - exit 1 +if [ -z $GN2_PROFILE ] ; then + echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" + export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) + if [ -d $GN2_PROFILE ]; then + echo "Best guess is $GN2_PROFILE" fi fi +if [ -z $GN2_PROFILE ]; then + read -p "PRESS [ENTER] TO CONTINUE..." +else + export PATH=$GN2_PROFILE/bin:$PATH + export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages + export R_LIBS_SITE=$GN2_PROFILE/site-library + export GUIX_JS_PATH=$GN2_PROFILE/share/genenetwork2/javascript + export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" + export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" + export XDG_DATA_DIRS="$GN2_PROFILE/share" + export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" + export LC_ALL=C # FIXME + export GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" + export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" + export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" + export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" +fi +if [ -z $PYTHONPATH ] ; then + echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!" + exit 1 +fi +if [ ! -d $R_LIBS_SITE ] ; then + echo "ERROR R_LIBS_SITE has not been set correctly (we only allow one path) - use GN2_PROFILE!" + echo "Paste into your shell the output of (for example)" + echo "guix package -p \$GN2_PROFILE --search-paths" + exit 1 +fi # We may change this one: export PYTHONPATH=$GN2_BASE_DIR/wqflask:$PYTHONPATH @@ -88,6 +98,8 @@ if [ -z $TMPDIR ]; then TMPDIR="/tmp" fi +# Show environment settings +set|grep guix set|grep $GN2_PROFILE set|grep TMPDIR @@ -97,7 +109,7 @@ if [ "$1" = '-c' ] ; then cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH echo RUNNING COMMAND $cmd - /usr/bin/env python $cmd + python $cmd exit 0 fi @@ -109,4 +121,4 @@ dbfilename gn2.rdb # Start the flask server running GN2 cd $GN2_BASE_DIR/wqflask echo "Starting with $settings" -/usr/bin/env python runserver.py +python runserver.py -- cgit v1.2.3 From 755b9aaa31c73cc5fe8c1a75d74846f1d8594419 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 31 May 2017 07:12:28 +0000 Subject: JS module handling --- VERSION | 2 +- bin/genenetwork2 | 2 +- doc/development.org | 20 ++++++++++++++++++++ etc/default_settings.py | 2 +- wqflask/utility/tools.py | 37 +++++++++++++++++++++++++++++-------- wqflask/wqflask/views.py | 6 +++--- 6 files changed, 55 insertions(+), 14 deletions(-) (limited to 'bin') diff --git a/VERSION b/VERSION index 144b089a..3e0b7cab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10-pre1 +2.10-pre4 diff --git a/bin/genenetwork2 b/bin/genenetwork2 index df688989..b3190f1e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -68,7 +68,7 @@ else export PATH=$GN2_PROFILE/bin:$PATH export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages export R_LIBS_SITE=$GN2_PROFILE/site-library - export GUIX_JS_PATH=$GN2_PROFILE/share/genenetwork2/javascript + export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" export XDG_DATA_DIRS="$GN2_PROFILE/share" diff --git a/doc/development.org b/doc/development.org index bbb6084f..5e6e318b 100644 --- a/doc/development.org +++ b/doc/development.org @@ -1,5 +1,7 @@ * Development +** Using GN2_PROFILE + After cloning the git source tree you can run the contained GN2 using an existing GN2_PROFILE, i.e., use a profile that was create to run a binary installation of GN2. This profile may be found by typing @@ -21,3 +23,21 @@ and use that instead. Note that the genenetwork2 script sets up the environment for running the webserver. This includes path to R modules and python modules. These are output on startup. To make sure there is no environment pollution you can + +** Javascript modules + +As of release 2.10-pre4 we Javascript modules are installed in three places: + +1. JS_GUIX_PATH: the Guix store - these are Guix pre-packaged modules +2. The git source tree (./wqflask/wqflask/static/packages/) +3. JS_GN_PATH: a local directory containing (temporary) development modules + +Packages currently in git (2) will move to JS_GUIX_PATH (1) over +time. This is to keep better track of origin updates. Putting packages +in git (2) is actively discouraged(!), unless there are GN2 specific +adaptations to the original Javascript modules. + +JS_GN_PATH (3) is for development purposes. By default is is set to +$HOME/genenetwork/javascript. Say you are working on an updated +version of a JS module not yet in (1) you can simply check out that +module in that path and it should show up. diff --git a/etc/default_settings.py b/etc/default_settings.py index 4f3a6f5c..1c5b10b4 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -61,7 +61,7 @@ HOME = os.environ['HOME'] # PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) # ---- Local path to JS libraries - for development modules (only) -# GN2_JS_PATH = os.environ['HOME']+"/genenetwork/javascript" (unused) +JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" # ---- GN2 Executables (overwrite for testing only) # PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index d46a84ba..c1c6fd36 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -13,6 +13,11 @@ logger = logging.getLogger(__name__ ) OVERRIDES = {} +def app_set(command_id, value): + """Set application wide value""" + app.config.setdefault(command_id,value) + value + def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with valid_path is a function checking whether the @@ -40,6 +45,7 @@ def get_setting(command_id,guess=None): def value(command): if command: # sys.stderr.write("Found "+command+"\n") + app_set(command_id,command) return command else: return None @@ -89,6 +95,18 @@ def valid_path(dir): return dir return None +def js_path(module=None): + """ + Find the JS module in the two paths + """ + try_gn = get_setting("JS_GN_PATH")+"/"+module + if valid_path(try_gn): + return try_gn + try_guix = get_setting("JS_GUIX_PATH")+"/"+module + if valid_path(try_guix): + return try_guix + raise "No JS path found for "+module+" (check JS_GN_PATH)" + def pylmm_command(guess=None): return valid_bin(get_setting("PYLMM_COMMAND",guess)) @@ -212,16 +230,19 @@ USE_REDIS = get_setting_bool('USE_REDIS') USE_GN_SERVER = get_setting_bool('USE_GN_SERVER') GENENETWORK_FILES = get_setting('GENENETWORK_FILES') -GUIX_JS_PATH = get_setting('GUIX_JS_PATH') -assert_dir(GUIX_JS_PATH) - -PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = gemma_command() -PLINK_COMMAND = plink_command() +JS_GUIX_PATH = get_setting('JS_GUIX_PATH') +assert_dir(JS_GUIX_PATH) +JS_GN_PATH = get_setting('JS_GN_PATH') +assert_dir(JS_GN_PATH) + +PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) TEMPDIR = tempdir() # defaults to UNIX TMPDIR -TWITTER_POST_FETCHER_JS_PATH = GUIX_JS_PATH + "/Twitter-Post-Fetcher" -assert_dir(TWITTER_POST_FETCHER_JS_PATH) +# ---- Handle specific JS modules +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher")) +assert_dir(JS_TWITTER_POST_FETCHER_PATH) from six import string_types diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3882e60a..07151425 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -53,7 +53,7 @@ from wqflask.ctl import ctl_analysis #from wqflask.trait_submission import submit_trait from utility import temp_data -from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,TWITTER_POST_FETCHER_JS_PATH +from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH from utility.helper_functions import get_species_groups from base import webqtlFormData @@ -147,8 +147,8 @@ def tmp_page(img_path): img_base64 = bytesarray ) @app.route("/twitter/") -def bd_files(filename): - return send_from_directory(TWITTER_POST_FETCHER_JS_PATH, filename) +def twitter(filename): + return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) #@app.route("/data_sharing") #def data_sharing_page(): -- cgit v1.2.3 From e467c3f19824415d9e9a3fdd4073d44439c83cbf Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 2 Jun 2017 10:21:05 +0000 Subject: settings: GENENETWORK_FILES --- bin/genenetwork2 | 2 +- etc/default_settings.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index b3190f1e..236d8f63 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -74,7 +74,7 @@ else export XDG_DATA_DIRS="$GN2_PROFILE/share" export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" export LC_ALL=C # FIXME - export GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" + export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" diff --git a/etc/default_settings.py b/etc/default_settings.py index 1c5b10b4..1b609414 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -53,11 +53,13 @@ USE_REDIS = True # REDIS caching (note that redis will be phased out) USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls HOME = os.environ['HOME'] +# ---- Default locations +GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files + # ---- Path overrides for Genenetwork - the defaults are normally # picked up from Guix or in the HOME directory # TMPDIR is normally picked up from the environment -# GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files # PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) # ---- Local path to JS libraries - for development modules (only) -- cgit v1.2.3