From 941e42ef72bd6aa6761c35cf78e64df41977c5d3 Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Fri, 19 Jan 2018 10:36:19 +0300
Subject: Add elasticsearch module to the path
* Add some code to set up the path for the python-elasticsearch module.
---
bin/genenetwork2 | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
(limited to 'bin')
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 8886e4bc..3fe8baa0 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -95,6 +95,11 @@ export WQFLASK_OVERRIDES=$overrides # JSON
echo WQFLASK_SETTINGS=$settings
echo WQFLASK_OVERRIDES=$overrides
+if [ -z $ELASTICSEARCH_PROFILE ]; then
+ echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m";
+else
+ PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
+fi
if [ -z $GN2_PROFILE ] ; then
echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
@@ -108,7 +113,7 @@ if [ -z $GN2_PROFILE ]; then
read -p "PRESS [ENTER] TO CONTINUE..."
else
export PATH=$GN2_PROFILE/bin:$PATH
- export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages
+ export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH"
export R_LIBS_SITE=$GN2_PROFILE/site-library
export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0
export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
--
cgit v1.2.3
From 193a0c19647b37eeb9d18df02e1436d422b4f843 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 26 Mar 2018 09:28:26 +0000
Subject: Fix conflict
---
bin/genenetwork2 | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
(limited to 'bin')
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 3fe8baa0..74ed7f9b 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -129,7 +129,11 @@ else
if [ -z $GEMMA_WRAPPER_COMMAND ]; then
export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
fi
- if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi
+ while IFS=":" read -ra PPATH; do
+ for PPART in "${PPATH[@]}"; do
+ if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi
+ done
+ done <<< "$PYTHONPATH"
if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi
fi
--
cgit v1.2.3
From 3de1ecfa37b73b4cb011b634c8b4afc2362f858c Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Thu, 15 Feb 2018 10:30:52 +0000
Subject: Fixing authentication stuff so it uses parameters properly. Also no
PYTHONPATH needed as it is now in the Guix build.
---
bin/genenetwork2 | 12 +++++++-----
etc/default_settings.py | 18 ++++++++++++++++++
wqflask/run_gunicorn.py | 3 +++
wqflask/utility/elasticsearch_tools.py | 16 ++++++++++++----
wqflask/utility/tools.py | 29 +++++++++++------------------
wqflask/wqflask/user_manager.py | 7 +++++--
6 files changed, 56 insertions(+), 29 deletions(-)
(limited to 'bin')
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 74ed7f9b..31fefbd3 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -95,11 +95,13 @@ export WQFLASK_OVERRIDES=$overrides # JSON
echo WQFLASK_SETTINGS=$settings
echo WQFLASK_OVERRIDES=$overrides
-if [ -z $ELASTICSEARCH_PROFILE ]; then
- echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m";
-else
- PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
-fi
+# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead
+# if [ -z $ELASTICSEARCH_PROFILE ]; then
+# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE";
+# else
+# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
+# fi
+
if [ -z $GN2_PROFILE ] ; then
echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 699d21f1..a70d8aec 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -41,6 +41,24 @@ SECURITY_POST_LOGIN_VIEW = "/thank_you"
SERVER_PORT = 5003 # running on localhost
SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
+GITHUB_CLIENT_ID = "UNKNOWN"
+GITHUB_CLIENT_SECRET = "UNKNOWN"
+GITHUB_AUTH_URL = "UNKNOWN"
+GITHUB_API_URL = "UNKNOWN"
+
+ORCID_CLIENT_ID = "UNKNOWN"
+ORCID_CLIENT_SECRET = "UNKNOWN"
+ORCID_AUTH_URL = "UNKNOWN"
+ORCID_TOKEN_URL = "UNKNOWN"
+
+ELASTICSEARCH_HOST = "localhost"
+ELASTICSEARCH_PORT = '9200'
+
+SMTP_CONNECT = "UNKNOWN"
+SMTP_USERNAME = "UNKNOWN"
+SMTP_PASSWORD = "UNKNOWN"
+
+
# ---- Behavioural settings (defaults) note that logger and log levels can
# be overridden at the module level and with enviroment settings
WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD)
diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py
index 14a2d689..ebe3add5 100644
--- a/wqflask/run_gunicorn.py
+++ b/wqflask/run_gunicorn.py
@@ -11,6 +11,9 @@ print "Starting up Gunicorn process"
from wqflask import app
+app.config['SESSION_TYPE'] = 'filesystem'
+app.config['SECRET_KEY'] = 'super secret key'
+
@app.route("/gunicorn")
def hello():
return "
Hello There!
"
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index a964b025..2d3d5add 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -1,10 +1,18 @@
from elasticsearch import Elasticsearch, TransportError
import logging
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
+
def get_elasticsearch_connection():
+ logger.info("get_elasticsearch_connection")
es = None
try:
- from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
+ assert(ELASTICSEARCH_HOST)
+ assert(ELASTICSEARCH_PORT)
+ logger.info("ES HOST",ELASTICSEARCH_HOST)
es = Elasticsearch([{
"host": ELASTICSEARCH_HOST
@@ -31,12 +39,12 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
response = es.search(
index = index
, doc_type = doc_type
- , body = {
- "query": { "match": { column_name: column_value } }
+ , body = {
+ "query": { "match": { column_name: column_value } }
})
if len(response["hits"]["hits"]) > 0:
item_details = response["hits"]["hits"][0]["_source"]
- except TransportError as te:
+ except TransportError as te:
pass
return item_details
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 005f9b0f..8c9fed96 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -251,32 +251,25 @@ assert_dir(JS_GUIX_PATH)
JS_GN_PATH = get_setting('JS_GN_PATH')
# assert_dir(JS_GN_PATH)
-def get_setting_safe(setting):
- try:
- return get_setting(setting)
- except:
- print("Could not find the setting '", setting, "'. Continuing with value unset")
- return None
-
-GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID')
-GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET')
+GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID')
+GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET')
GITHUB_AUTH_URL = None
if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET:
GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET
-GITHUB_API_URL = get_setting_safe('GITHUB_API_URL')
-ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID')
-ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET')
+GITHUB_API_URL = get_setting('GITHUB_API_URL')
+ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID')
+ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET')
ORCID_AUTH_URL = None
if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET:
ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET
-ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL')
+ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
-ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST')
-ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT')
+ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
+ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
-SMTP_CONNECT = get_setting_safe('SMTP_CONNECT')
-SMTP_USERNAME = get_setting_safe('SMTP_USERNAME')
-SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD')
+SMTP_CONNECT = get_setting('SMTP_CONNECT')
+SMTP_USERNAME = get_setting('SMTP_USERNAME')
+SMTP_PASSWORD = get_setting('SMTP_PASSWORD')
PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command())
GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command())
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 6b667615..c8471cb1 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -55,9 +55,8 @@ logger = getLogger(__name__)
from base.data_set import create_datasets_list
import requests
-from utility.elasticsearch_tools import *
+from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user
-es = get_elasticsearch_connection()
THREE_DAYS = 60 * 60 * 24 * 3
#THREE_DAYS = 45
@@ -479,6 +478,7 @@ def password_reset_step2():
password = request.form['password']
set_password(password, user)
+ es = get_elasticsearch_connection()
es.update(
index = "users"
, doc_type = "local"
@@ -620,6 +620,7 @@ class LoginUser(object):
"""Login through the normal form"""
params = request.form if request.form else request.args
logger.debug("in login params are:", params)
+ es = get_elasticsearch_connection()
if not params:
from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL
external_login = None
@@ -628,6 +629,7 @@ class LoginUser(object):
"github": GITHUB_AUTH_URL,
"orcid": ORCID_AUTH_URL
}
+ assert(es is not None)
return render_template(
"new_security/login_user.html"
, external_login=external_login
@@ -822,6 +824,7 @@ def register():
params = request.form if request.form else request.args
params = params.to_dict(flat=True)
+ es = get_elasticsearch_connection()
params["es_connection"] = es
if params:
--
cgit v1.2.3
From 375935b23778148eba40ebce4a4f4def823aa3f0 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Thu, 15 Feb 2018 15:00:47 +0000
Subject: Requests added
---
bin/genenetwork2 | 10 ++--
bin/mechnical-rob | 111 ++++++++++++++++++++++++++++++++++++++++
bin/test-website | 114 ++----------------------------------------
test/requests/test-website.py | 20 ++++++++
4 files changed, 142 insertions(+), 113 deletions(-)
create mode 100755 bin/mechnical-rob
create mode 100755 test/requests/test-website.py
(limited to 'bin')
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 31fefbd3..ceafeedd 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -168,8 +168,9 @@ if [ "$1" = '-c' ] ; then
cd $GN2_BASE_DIR/wqflask
cmd=${2#wqflask/}
echo PYTHONPATH=$PYTHONPATH
- echo RUNNING COMMAND $cmd
- python $cmd
+ shift ; shift
+ echo RUNNING COMMAND $cmd $*
+ python $cmd $*
exit $?
fi
# Now handle command parameter -cli which runs in bash
@@ -178,8 +179,9 @@ if [ "$1" = "-cli" ] ; then
cd $GN2_BASE_DIR/wqflask
cmd=$2
echo PYTHONPATH=$PYTHONPATH
- echo RUNNING COMMAND $cmd
- $cmd
+ shift ; shift
+ echo RUNNING COMMAND $cmd $*
+ $cmd $*
exit $?
fi
if [ "$1" = '-gunicorn' ] ; then
diff --git a/bin/mechnical-rob b/bin/mechnical-rob
new file mode 100755
index 00000000..be223d94
--- /dev/null
+++ b/bin/mechnical-rob
@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+
+
+USAGE = <0
+ ARGV.shift
+ else
+ "http://localhost:5003"
+ end
+
+$stderr.print "Testing <",$host,">\n"
+
+require 'mechanize'
+require 'minitest/spec'
+require 'minitest/autorun'
+
+# These are the actual testing modules
+
+libpath = File.dirname(File.dirname(__FILE__))
+$: << File.join(libpath,'test/lib')
+
+require 'main_web_functionality'
+
+if options[:all] or options[:mapping]
+ require 'mapping'
+end
+
+if options[:all] or options[:link_checker]
+ require 'link_checker'
+end
+
+if options[:all] or options[:navigation]
+ require 'navigation'
+end
diff --git a/bin/test-website b/bin/test-website
index be223d94..5935f016 100755
--- a/bin/test-website
+++ b/bin/test-website
@@ -1,111 +1,7 @@
-#!/usr/bin/env ruby
+#! /bin/bash
-
-USAGE = <0
- ARGV.shift
- else
- "http://localhost:5003"
- end
-
-$stderr.print "Testing <",$host,">\n"
-
-require 'mechanize'
-require 'minitest/spec'
-require 'minitest/autorun'
-
-# These are the actual testing modules
-
-libpath = File.dirname(File.dirname(__FILE__))
-$: << File.join(libpath,'test/lib')
-
-require 'main_web_functionality'
-
-if options[:all] or options[:mapping]
- require 'mapping'
-end
-
-if options[:all] or options[:link_checker]
- require 'link_checker'
-end
-
-if options[:all] or options[:navigation]
- require 'navigation'
-end
+fi
diff --git a/test/requests/test-website.py b/test/requests/test-website.py
new file mode 100755
index 00000000..d02b71aa
--- /dev/null
+++ b/test/requests/test-website.py
@@ -0,0 +1,20 @@
+# Run with something like
+#
+# env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
+#
+# Mostly to pick up the Guix GN2_PROFILE and python modules
+
+import requests as req
+import sys
+
+print "Mechanical Rob firing up..."
+
+if len(sys.argv)<1:
+ raise "Problem with arguments"
+
+url = sys.argv[1]
+print url
+
+r = req.get(url)
+
+print r
--
cgit v1.2.3
From 3c8089e2c3957ca2d4a5906249b4818d367221dc Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 19 Mar 2018 13:06:13 +0000
Subject: - Disabled PYTHONPATH injection - fix Guix instead, see
https://github.com/pjotrp/genenetwork2/commit/63a5c8a42ad02e9126bb207465ff5eca98f6515d
- Renamed WQFLASK_SETTINGS to GN2_SETTINGS
---
bin/genenetwork2 | 33 +++++++++++++++------------------
wqflask/utility/tools.py | 15 +--------------
wqflask/wqflask/__init__.py | 4 ++--
3 files changed, 18 insertions(+), 34 deletions(-)
(limited to 'bin')
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index ceafeedd..3f06e7f9 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -68,32 +68,29 @@ if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then
echo "Can not use $1 switch without default settings file"
exit 1
fi
-# Handle settings parameter (can be .py or .json)
-if [ ! -z $1 ]; then
- settings=$(realpath "$1")
- if [ ! -e $settings ]; then
- settings=$GN2_BASE_DIR/etc/default_settings.py
- else
- shift
- fi
-fi
-ext="${settings##*.}"
-if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
- overrides=$settings
+settings=$1
+if [ -z $settings ]; then
+ settings=$GN2_BASE_DIR/etc/default_settings.py
else
- echo $settings
+ shift
fi
+settings=$(realpath $settings)
+
+# ext="${settings##*.}"
+# if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
+# overrides=$settings
+# else
+# echo $settings
+# fi
if [ ! -e $settings ]; then
echo "ERROR: can not locate settings file - pass it in the command line"
exit 1
fi
-export WQFLASK_SETTINGS=$settings # Python
-export WQFLASK_OVERRIDES=$overrides # JSON
-echo WQFLASK_SETTINGS=$settings
-echo WQFLASK_OVERRIDES=$overrides
+export GN2_SETTINGS=$settings # Python
+echo GN2_SETTINGS=$settings
# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead
# if [ -z $ELASTICSEARCH_PROFILE ]; then
@@ -115,7 +112,7 @@ if [ -z $GN2_PROFILE ]; then
read -p "PRESS [ENTER] TO CONTINUE..."
else
export PATH=$GN2_PROFILE/bin:$PATH
- export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH"
+ export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!!
export R_LIBS_SITE=$GN2_PROFILE/site-library
export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0
export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 8c9fed96..bae3a7f4 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -54,7 +54,7 @@ def get_setting(command_id,guess=None):
# print("Looking for "+command_id+"\n")
command = value(os.environ.get(command_id))
if command is None or command == "":
- command = OVERRIDES.get(command_id)
+ command = OVERRIDES.get(command_id) # currently not in use
if command is None:
# ---- Check whether setting exists in app
command = value(app.config.get(command_id))
@@ -285,18 +285,5 @@ assert_dir(JS_TWITTER_POST_FETCHER_PATH)
from six import string_types
-if os.environ.get('WQFLASK_OVERRIDES'):
- jsonfn = get_setting('WQFLASK_OVERRIDES')
- logger.info("WQFLASK_OVERRIDES: %s" % jsonfn)
- with open(jsonfn) as data_file:
- overrides = json.load(data_file)
- for k in overrides:
- cmd = overrides[k]
- if isinstance(cmd, string_types):
- OVERRIDES[k] = eval(cmd)
- else:
- OVERRIDES[k] = cmd
- logger.debug(OVERRIDES)
-
# assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData")
assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js")
diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py
index 2188ce17..bc8e9900 100644
--- a/wqflask/wqflask/__init__.py
+++ b/wqflask/wqflask/__init__.py
@@ -13,8 +13,8 @@ logging.basicConfig(level=logging.INFO)
app = Flask(__name__)
app.config.from_object('cfg.default_settings') # Get the defaults from cfg.default_settings
-app.config.from_envvar('WQFLASK_SETTINGS') # See http://flask.pocoo.org/docs/config/#configuring-from-files
-# Note we also use WQFLASK_OVERRIDES
+app.config.from_envvar('GN2_SETTINGS') # See http://flask.pocoo.org/docs/config/#configuring-from-files
+# Note no longer use the badly named WQFLASK_OVERRIDES (nyi)
app.jinja_env.globals.update(
undefined = jinja2.StrictUndefined,
--
cgit v1.2.3