From 63a5c8a42ad02e9126bb207465ff5eca98f6515d Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Fri, 19 Jan 2018 10:36:19 +0300 Subject: Add elasticsearch module to the path * Add some code to set up the path for the python-elasticsearch module. --- bin/genenetwork2 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index a7edb1c2..ccd9b1d9 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -61,6 +61,11 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides +if [ -z $ELASTICSEARCH_PROFILE ]; then + echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; +else + PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +fi if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) @@ -72,7 +77,7 @@ if [ -z $GN2_PROFILE ]; then read -p "PRESS [ENTER] TO CONTINUE..." else export PATH=$GN2_PROFILE/bin:$PATH - export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages + export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH" export R_LIBS_SITE=$GN2_PROFILE/site-library export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0 export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript -- cgit v1.2.3 From 6ec9dcc365b899f7f4aaab739103f4559c2c1620 Mon Sep 17 00:00:00 2001 From: Muriithi Frederick Muriuki Date: Mon, 5 Feb 2018 16:19:13 +0300 Subject: Check each part of the PYTHONPATH * PYTHONPATH may be composed of multiple locations in the filesystem, thus the need to test that each location exists and is a directory. --- bin/genenetwork2 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 34fbc72e..145ce395 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -126,7 +126,11 @@ else export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis" export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma" export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper" - if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + while IFS=":" read -ra PPATH; do + for PPart in "${PPATH[@]}"; do + if [ ! -d $PPart ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi + done + done <<< "$PYTHONPATH" if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi fi -- cgit v1.2.3 From 9d47ad572f46397bdd778d8f6e1c37d40e12aa2a Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 10:30:52 +0000 Subject: Fixing authentication stuff so it uses parameters properly. Also no PYTHONPATH needed as it is now in the Guix build. --- bin/genenetwork2 | 12 +++++++----- etc/default_settings.py | 18 ++++++++++++++++++ wqflask/run_gunicorn.py | 3 +++ wqflask/utility/elasticsearch_tools.py | 16 ++++++++++++---- wqflask/utility/tools.py | 29 +++++++++++------------------ wqflask/wqflask/user_manager.py | 7 +++++-- 6 files changed, 56 insertions(+), 29 deletions(-) (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 5438c1c0..42f79650 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -95,11 +95,13 @@ export WQFLASK_OVERRIDES=$overrides # JSON echo WQFLASK_SETTINGS=$settings echo WQFLASK_OVERRIDES=$overrides -if [ -z $ELASTICSEARCH_PROFILE ]; then - echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m"; -else - PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" -fi +# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead +# if [ -z $ELASTICSEARCH_PROFILE ]; then +# echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE"; +# else +# PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages" +# fi + if [ -z $GN2_PROFILE ] ; then echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!" export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2))) diff --git a/etc/default_settings.py b/etc/default_settings.py index 699d21f1..a70d8aec 100644 --- a/etc/default_settings.py +++ b/etc/default_settings.py @@ -41,6 +41,24 @@ SECURITY_POST_LOGIN_VIEW = "/thank_you" SERVER_PORT = 5003 # running on localhost SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc' +GITHUB_CLIENT_ID = "UNKNOWN" +GITHUB_CLIENT_SECRET = "UNKNOWN" +GITHUB_AUTH_URL = "UNKNOWN" +GITHUB_API_URL = "UNKNOWN" + +ORCID_CLIENT_ID = "UNKNOWN" +ORCID_CLIENT_SECRET = "UNKNOWN" +ORCID_AUTH_URL = "UNKNOWN" +ORCID_TOKEN_URL = "UNKNOWN" + +ELASTICSEARCH_HOST = "localhost" +ELASTICSEARCH_PORT = '9200' + +SMTP_CONNECT = "UNKNOWN" +SMTP_USERNAME = "UNKNOWN" +SMTP_PASSWORD = "UNKNOWN" + + # ---- Behavioural settings (defaults) note that logger and log levels can # be overridden at the module level and with enviroment settings WEBSERVER_MODE = 'DEV' # Python webserver mode (DEBUG|DEV|PROD) diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py index 14a2d689..ebe3add5 100644 --- a/wqflask/run_gunicorn.py +++ b/wqflask/run_gunicorn.py @@ -11,6 +11,9 @@ print "Starting up Gunicorn process" from wqflask import app +app.config['SESSION_TYPE'] = 'filesystem' +app.config['SECRET_KEY'] = 'super secret key' + @app.route("/gunicorn") def hello(): return "

Hello There!

" diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py index a964b025..2d3d5add 100644 --- a/wqflask/utility/elasticsearch_tools.py +++ b/wqflask/utility/elasticsearch_tools.py @@ -1,10 +1,18 @@ from elasticsearch import Elasticsearch, TransportError import logging +from utility.logger import getLogger +logger = getLogger(__name__) + +from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + def get_elasticsearch_connection(): + logger.info("get_elasticsearch_connection") es = None try: - from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT + assert(ELASTICSEARCH_HOST) + assert(ELASTICSEARCH_PORT) + logger.info("ES HOST",ELASTICSEARCH_HOST) es = Elasticsearch([{ "host": ELASTICSEARCH_HOST @@ -31,12 +39,12 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type): response = es.search( index = index , doc_type = doc_type - , body = { - "query": { "match": { column_name: column_value } } + , body = { + "query": { "match": { column_name: column_value } } }) if len(response["hits"]["hits"]) > 0: item_details = response["hits"]["hits"][0]["_source"] - except TransportError as te: + except TransportError as te: pass return item_details diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py index 005f9b0f..8c9fed96 100644 --- a/wqflask/utility/tools.py +++ b/wqflask/utility/tools.py @@ -251,32 +251,25 @@ assert_dir(JS_GUIX_PATH) JS_GN_PATH = get_setting('JS_GN_PATH') # assert_dir(JS_GN_PATH) -def get_setting_safe(setting): - try: - return get_setting(setting) - except: - print("Could not find the setting '", setting, "'. Continuing with value unset") - return None - -GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID') -GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET') +GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID') +GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET') GITHUB_AUTH_URL = None if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET: GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET -GITHUB_API_URL = get_setting_safe('GITHUB_API_URL') -ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID') -ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET') +GITHUB_API_URL = get_setting('GITHUB_API_URL') +ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID') +ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET') ORCID_AUTH_URL = None if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET: ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET -ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL') +ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL') -ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST') -ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT') +ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST') +ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT') -SMTP_CONNECT = get_setting_safe('SMTP_CONNECT') -SMTP_USERNAME = get_setting_safe('SMTP_USERNAME') -SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD') +SMTP_CONNECT = get_setting('SMTP_CONNECT') +SMTP_USERNAME = get_setting('SMTP_USERNAME') +SMTP_PASSWORD = get_setting('SMTP_PASSWORD') PYLMM_COMMAND = app_set("PYLMM_COMMAND",pylmm_command()) GEMMA_COMMAND = app_set("GEMMA_COMMAND",gemma_command()) diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py index 6b667615..c8471cb1 100644 --- a/wqflask/wqflask/user_manager.py +++ b/wqflask/wqflask/user_manager.py @@ -55,9 +55,8 @@ logger = getLogger(__name__) from base.data_set import create_datasets_list import requests -from utility.elasticsearch_tools import * +from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user -es = get_elasticsearch_connection() THREE_DAYS = 60 * 60 * 24 * 3 #THREE_DAYS = 45 @@ -479,6 +478,7 @@ def password_reset_step2(): password = request.form['password'] set_password(password, user) + es = get_elasticsearch_connection() es.update( index = "users" , doc_type = "local" @@ -620,6 +620,7 @@ class LoginUser(object): """Login through the normal form""" params = request.form if request.form else request.args logger.debug("in login params are:", params) + es = get_elasticsearch_connection() if not params: from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL external_login = None @@ -628,6 +629,7 @@ class LoginUser(object): "github": GITHUB_AUTH_URL, "orcid": ORCID_AUTH_URL } + assert(es is not None) return render_template( "new_security/login_user.html" , external_login=external_login @@ -822,6 +824,7 @@ def register(): params = request.form if request.form else request.args params = params.to_dict(flat=True) + es = get_elasticsearch_connection() params["es_connection"] = es if params: -- cgit v1.2.3 From 3a26e0fb7b88cd9c105a1f7e9ca9d5a8b2130d82 Mon Sep 17 00:00:00 2001 From: Pjotr Prins Date: Thu, 15 Feb 2018 15:00:47 +0000 Subject: Requests added --- bin/genenetwork2 | 10 ++-- bin/mechnical-rob | 111 ++++++++++++++++++++++++++++++++++++++++ bin/test-website | 114 ++---------------------------------------- test/requests/test-website.py | 20 ++++++++ 4 files changed, 142 insertions(+), 113 deletions(-) create mode 100755 bin/mechnical-rob create mode 100755 test/requests/test-website.py (limited to 'bin') diff --git a/bin/genenetwork2 b/bin/genenetwork2 index 42f79650..2a83a1cd 100755 --- a/bin/genenetwork2 +++ b/bin/genenetwork2 @@ -168,8 +168,9 @@ if [ "$1" = '-c' ] ; then cd $GN2_BASE_DIR/wqflask cmd=${2#wqflask/} echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - python $cmd + shift ; shift + echo RUNNING COMMAND $cmd $* + python $cmd $* exit $? fi # Now handle command parameter -cli which runs in bash @@ -178,8 +179,9 @@ if [ "$1" = "-cli" ] ; then cd $GN2_BASE_DIR/wqflask cmd=$2 echo PYTHONPATH=$PYTHONPATH - echo RUNNING COMMAND $cmd - $cmd + shift ; shift + echo RUNNING COMMAND $cmd $* + $cmd $* exit $? fi if [ "$1" = '-gunicorn' ] ; then diff --git a/bin/mechnical-rob b/bin/mechnical-rob new file mode 100755 index 00000000..be223d94 --- /dev/null +++ b/bin/mechnical-rob @@ -0,0 +1,111 @@ +#!/usr/bin/env ruby + + +USAGE = <0 + ARGV.shift + else + "http://localhost:5003" + end + +$stderr.print "Testing <",$host,">\n" + +require 'mechanize' +require 'minitest/spec' +require 'minitest/autorun' + +# These are the actual testing modules + +libpath = File.dirname(File.dirname(__FILE__)) +$: << File.join(libpath,'test/lib') + +require 'main_web_functionality' + +if options[:all] or options[:mapping] + require 'mapping' +end + +if options[:all] or options[:link_checker] + require 'link_checker' +end + +if options[:all] or options[:navigation] + require 'navigation' +end diff --git a/bin/test-website b/bin/test-website index be223d94..5935f016 100755 --- a/bin/test-website +++ b/bin/test-website @@ -1,111 +1,7 @@ -#!/usr/bin/env ruby +#! /bin/bash - -USAGE = <0 - ARGV.shift - else - "http://localhost:5003" - end - -$stderr.print "Testing <",$host,">\n" - -require 'mechanize' -require 'minitest/spec' -require 'minitest/autorun' - -# These are the actual testing modules - -libpath = File.dirname(File.dirname(__FILE__)) -$: << File.join(libpath,'test/lib') - -require 'main_web_functionality' - -if options[:all] or options[:mapping] - require 'mapping' -end - -if options[:all] or options[:link_checker] - require 'link_checker' -end - -if options[:all] or options[:navigation] - require 'navigation' -end +fi diff --git a/test/requests/test-website.py b/test/requests/test-website.py new file mode 100755 index 00000000..d02b71aa --- /dev/null +++ b/test/requests/test-website.py @@ -0,0 +1,20 @@ +# Run with something like +# +# env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003 +# +# Mostly to pick up the Guix GN2_PROFILE and python modules + +import requests as req +import sys + +print "Mechanical Rob firing up..." + +if len(sys.argv)<1: + raise "Problem with arguments" + +url = sys.argv[1] +print url + +r = req.get(url) + +print r -- cgit v1.2.3