From 8677d1b7bd5b32791c440bf0de2fc4c61c98299f Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Sun, 21 May 2017 07:21:44 +0000 Subject: Merge from the diet branch. * refactored ./bin/genenetwork2 and /etc/default_settings - better detection of Guix - removed unused parameters, such as GEMMA_RESULT_PATH and TEMP_TRAITS - removing some default settings, such as PYLMM_COMMAND - rename some settings, such as SQL_URI and LOG_SQL_ALCHEMY - added GUIX_JS_PATH for JS modules * Added documentation on installing binary GN2 * Updated Twitter board to latest version in Guix - it had broken * Updated many tools in the latest Guix profile --- etc/VERSION | 1 + etc/default_settings.py | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 etc/VERSION (limited to 'etc') diff --git a/etc/VERSION b/etc/VERSION new file mode 100644 index 00000000..1785aa28 --- /dev/null +++ b/etc/VERSION @@ -0,0 +1 @@ +2.10rc3 diff --git a/etc/default_settings.py b/etc/default_settings.py index 6acea637..4f3a6f5c 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -9,6 +9,8 @@ # env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2 # env LOG_LEVEL=DEBUG ./bin/genenetwork2 ~/gn2_settings.py # +# Typically you need to set GN2_PROFILE too. +# # Note also that in the near future we will additionally fetch # settings from a JSON file # @@ -18,10 +20,9 @@ import os import sys -GN_VERSION = open("../VERSION","r").read() +GN_VERSION = open("../etc/VERSION","r").read() SQL_URI = "mysql://gn2:mysql_password@localhost/db_webqtl_s" -SQLALCHEMY_DATABASE_URI = 'mysql://gn2:mysql_password@localhost/db_webqtl_s' -SQLALCHEMY_POOL_RECYCLE = 3600 +SQL_ALCHEMY_POOL_RECYCLE = 3600 GN_SERVER_URL = "http://localhost:8880/" # ---- Flask configuration (see website) @@ -45,24 +46,24 @@ WEBSERVER_DEPLOY = None # Deployment specifics (nyi) LOG_LEVEL = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL) LOG_LEVEL_DEBUG = '0' # logger.debugf log level (0-5, 5 = show all) LOG_SQL = 'False' # Log SQL/backend and GN_SERVER calls -LOG_SQLALCHEMY = 'False' +LOG_SQL_ALCHEMY = 'False' LOG_BENCH = True # Log bench marks USE_REDIS = True # REDIS caching (note that redis will be phased out) USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls +HOME = os.environ['HOME'] -# Paths to JS libraries - -TWITTER_POST_FETCHER_JS_PATH = os.environ['HOME']+"/genenetwork/Twitter-Post-Fetcher" +# ---- Path overrides for Genenetwork - the defaults are normally +# picked up from Guix or in the HOME directory -# ---- Path overrides for Genenetwork # TMPDIR is normally picked up from the environment -HOME=os.environ['HOME'] -LOGFILE = HOME+"/genenetwork2.log" -GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files -LOCAL_PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files +# GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files +# PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) + +# ---- Local path to JS libraries - for development modules (only) +# GN2_JS_PATH = os.environ['HOME']+"/genenetwork/javascript" (unused) -# ---- GN2 Executables -PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) -PLINK_COMMAND = str.strip(os.popen("which plink2").read()) -GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) +# ---- GN2 Executables (overwrite for testing only) +# PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) +# PLINK_COMMAND = str.strip(os.popen("which plink2").read()) +# GEMMA_COMMAND = str.strip(os.popen("which gemma").read()) -- cgit v1.2.3 From 755b9aaa31c73cc5fe8c1a75d74846f1d8594419 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Wed, 31 May 2017 07:12:28 +0000 Subject: JS module handling --- VERSION | 2 +- bin/genenetwork2 | 2 +- doc/development.org | 20 ++++++++++++++++++++ etc/default_settings.py | 2 +- wqflask/utility/tools.py | 37 +++++++++++++++++++++++++++++-------- wqflask/wqflask/views.py | 6 +++--- 6 files changed, 55 insertions(+), 14 deletions(-) (limited to 'etc') diff --git a/VERSION b/VERSION index 144b089a..3e0b7cab 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.10-pre1 +2.10-pre4 diff --git a/bin/genenetwork2 b/bin/genenetwork2 index df688989..b3190f1e 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -68,7 +68,7 @@ else export PATH=$GN2_PROFILE/bin:$PATH export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages export R_LIBS_SITE=$GN2_PROFILE/site-library - export GUIX_JS_PATH=$GN2_PROFILE/share/genenetwork2/javascript + export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0" export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0" export XDG_DATA_DIRS="$GN2_PROFILE/share" diff --git a/doc/development.org b/doc/development.org index bbb6084f..5e6e318b 100644 --- a/doc/development.org +++ b/doc/development.org @@ -1,5 +1,7 @@ * Development +** Using GN2_PROFILE + After cloning the git source tree you can run the contained GN2 using an existing GN2_PROFILE, i.e., use a profile that was create to run a binary installation of GN2. This profile may be found by typing @@ -21,3 +23,21 @@ and use that instead. Note that the genenetwork2 script sets up the environment for running the webserver. This includes path to R modules and python modules. These are output on startup. To make sure there is no environment pollution you can + +** Javascript modules + +As of release 2.10-pre4 we Javascript modules are installed in three places: + +1. JS_GUIX_PATH: the Guix store - these are Guix pre-packaged modules +2. The git source tree (./wqflask/wqflask/static/packages/) +3. JS_GN_PATH: a local directory containing (temporary) development modules + +Packages currently in git (2) will move to JS_GUIX_PATH (1) over +time. This is to keep better track of origin updates. Putting packages +in git (2) is actively discouraged(!), unless there are GN2 specific +adaptations to the original Javascript modules. + +JS_GN_PATH (3) is for development purposes. By default is is set to +$HOME/genenetwork/javascript. Say you are working on an updated +version of a JS module not yet in (1) you can simply check out that +module in that path and it should show up. diff --git a/etc/default_settings.py b/etc/default_settings.py index 4f3a6f5c..1c5b10b4 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -61,7 +61,7 @@ HOME = os.environ['HOME'] # PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) # ---- Local path to JS libraries - for development modules (only) -# GN2_JS_PATH = os.environ['HOME']+"/genenetwork/javascript" (unused) +JS_GN_PATH = os.environ['HOME']+"/genenetwork/javascript" # ---- GN2 Executables (overwrite for testing only) # PYLMM_COMMAND = str.strip(os.popen("which pylmm_redis").read()) diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index d46a84ba..c1c6fd36 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -13,6 +13,11 @@ logger = logging.getLogger(__name__ ) OVERRIDES = {} +def app_set(command_id, value): + """Set application wide value""" + app.config.setdefault(command_id,value) + value + def get_setting(command_id,guess=None): """Resolve a setting from the environment or the global settings in app.config, with valid_path is a function checking whether the @@ -40,6 +45,7 @@ def get_setting(command_id,guess=None): def value(command): if command: # sys.stderr.write("Found "+command+"\n") + app_set(command_id,command) return command else: return None @@ -89,6 +95,18 @@ def valid_path(dir): return dir return None +def js_path(module=None): + """ + Find the JS module in the two paths + """ + try_gn = get_setting("JS_GN_PATH")+"/"+module + if valid_path(try_gn): + return try_gn + try_guix = get_setting("JS_GUIX_PATH")+"/"+module + if valid_path(try_guix): + return try_guix + raise "No JS path found for "+module+" (check JS_GN_PATH)" + def pylmm_command(guess=None): return valid_bin(get_setting("PYLMM_COMMAND",guess)) @@ -212,16 +230,19 @@ USE_REDIS = get_setting_bool('USE_REDIS') USE_GN_SERVER = get_setting_bool('USE_GN_SERVER') GENENETWORK_FILES = get_setting('GENENETWORK_FILES') -GUIX_JS_PATH = get_setting('GUIX_JS_PATH') -assert_dir(GUIX_JS_PATH) - -PYLMM_COMMAND = pylmm_command() -GEMMA_COMMAND = gemma_command() -PLINK_COMMAND = plink_command() +JS_GUIX_PATH = get_setting('JS_GUIX_PATH') +assert_dir(JS_GUIX_PATH) +JS_GN_PATH = get_setting('JS_GN_PATH') +assert_dir(JS_GN_PATH) + +PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) +GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) +PLINK_COMMAND = app_set("PLINK_COMMAND",plink_command()) TEMPDIR = tempdir() # defaults to UNIX TMPDIR -TWITTER_POST_FETCHER_JS_PATH = GUIX_JS_PATH + "/Twitter-Post-Fetcher" -assert_dir(TWITTER_POST_FETCHER_JS_PATH) +# ---- Handle specific JS modules +JS_TWITTER_POST_FETCHER_PATH = get_setting("JS_TWITTER_POST_FETCHER_PATH",js_path("Twitter-Post-Fetcher")) +assert_dir(JS_TWITTER_POST_FETCHER_PATH) from six import string_types diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py index 3882e60a..07151425 100644 --- a/wqflask/wqflask/views.py +++ b/wqflask/wqflask/views.py @@ -53,7 +53,7 @@ from wqflask.ctl import ctl_analysis #from wqflask.trait_submission import submit_trait from utility import temp_data -from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,TWITTER_POST_FETCHER_JS_PATH +from utility.tools import SQL_URI,TEMPDIR,USE_REDIS,USE_GN_SERVER,GN_SERVER_URL,GN_VERSION,JS_TWITTER_POST_FETCHER_PATH from utility.helper_functions import get_species_groups from base import webqtlFormData @@ -147,8 +147,8 @@ def tmp_page(img_path): img_base64 = bytesarray ) @app.route("/twitter/") -def bd_files(filename): - return send_from_directory(TWITTER_POST_FETCHER_JS_PATH, filename) +def twitter(filename): + return send_from_directory(JS_TWITTER_POST_FETCHER_PATH, filename) #@app.route("/data_sharing") #def data_sharing_page(): -- cgit v1.2.3 From e467c3f19824415d9e9a3fdd4073d44439c83cbf Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Fri, 2 Jun 2017 10:21:05 +0000 Subject: settings: GENENETWORK_FILES --- bin/genenetwork2 | 2 +- etc/default_settings.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'etc') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index b3190f1e..236d8f63 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -74,7 +74,7 @@ else export XDG_DATA_DIRS="$GN2_PROFILE/share" export GIO_EXTRA_MODULES="$GN2_PROFILE/lib/gio/modules" export LC_ALL=C # FIXME - export GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" + export GUIX_GENENETWORK_FILES="$GN2_PROFILE/share/genenetwork2" export PLINK_COMMAND="$GN2_PROFILE/bin/plink2" export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" diff --git a/etc/default_settings.py b/etc/default_settings.py index 1c5b10b4..1b609414 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -53,11 +53,13 @@ USE_REDIS = True # REDIS caching (note that redis will be phased out) USE_GN_SERVER = 'False' # Use GN_SERVER SQL calls HOME = os.environ['HOME'] +# ---- Default locations +GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files + # ---- Path overrides for Genenetwork - the defaults are normally # picked up from Guix or in the HOME directory # TMPDIR is normally picked up from the environment -# GENENETWORK_FILES = HOME+"/gn2_data" # base dir for all static data files # PRIVATE_FILES = HOME+"/gn2_private_data" # private static data files (unused) # ---- Local path to JS libraries - for development modules (only) -- cgit v1.2.3