From 941e42ef72bd6aa6761c35cf78e64df41977c5d3 Mon Sep 17 00:00:00 2001
From: Muriithi Frederick Muriuki
Date: Fri, 19 Jan 2018 10:36:19 +0300
Subject: Add elasticsearch module to the path

* Add some code to set up the path for the python-elasticsearch module.
---
 bin/genenetwork2 | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 8886e4bc..3fe8baa0 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -95,6 +95,11 @@ export WQFLASK_OVERRIDES=$overrides   # JSON
 echo WQFLASK_SETTINGS=$settings
 echo WQFLASK_OVERRIDES=$overrides
 
+if [ -z $ELASTICSEARCH_PROFILE ]; then
+    echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m";
+else
+    PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
+fi
 if [ -z $GN2_PROFILE ] ; then
     echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
     export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
@@ -108,7 +113,7 @@ if [ -z $GN2_PROFILE ]; then
     read -p "PRESS [ENTER] TO CONTINUE..."
 else
     export PATH=$GN2_PROFILE/bin:$PATH
-    export PYTHONPATH=$GN2_PROFILE/lib/python2.7/site-packages
+    export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH"
     export R_LIBS_SITE=$GN2_PROFILE/site-library
     export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0
     export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
-- 
cgit v1.2.3


From 193a0c19647b37eeb9d18df02e1436d422b4f843 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 26 Mar 2018 09:28:26 +0000
Subject: Fix conflict

---
 bin/genenetwork2 | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 3fe8baa0..74ed7f9b 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -129,7 +129,11 @@ else
     if [ -z $GEMMA_WRAPPER_COMMAND ]; then
         export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
     fi
-    if [ ! -d $PYTHONPATH ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi
+    while IFS=":" read -ra PPATH; do
+	for PPART in "${PPATH[@]}"; do
+	    if [ ! -d $PPART ] ; then echo "$PPART in PYTHONPATH not valid $PYTHONPATH" ; exit 1 ; fi
+	done
+    done <<< "$PYTHONPATH"
     if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
     if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi
 fi
-- 
cgit v1.2.3


From 3de1ecfa37b73b4cb011b634c8b4afc2362f858c Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Thu, 15 Feb 2018 10:30:52 +0000
Subject: Fixing authentication stuff so it uses parameters properly. Also no
 PYTHONPATH needed as it is now in the Guix build.

---
 bin/genenetwork2                       | 12 +++++++-----
 etc/default_settings.py                | 18 ++++++++++++++++++
 wqflask/run_gunicorn.py                |  3 +++
 wqflask/utility/elasticsearch_tools.py | 16 ++++++++++++----
 wqflask/utility/tools.py               | 29 +++++++++++------------------
 wqflask/wqflask/user_manager.py        |  7 +++++--
 6 files changed, 56 insertions(+), 29 deletions(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 74ed7f9b..31fefbd3 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -95,11 +95,13 @@ export WQFLASK_OVERRIDES=$overrides   # JSON
 echo WQFLASK_SETTINGS=$settings
 echo WQFLASK_OVERRIDES=$overrides
 
-if [ -z $ELASTICSEARCH_PROFILE ]; then
-    echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m";
-else
-    PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
-fi
+# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead
+# if [ -z $ELASTICSEARCH_PROFILE ]; then
+#     echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE";
+# else
+#     PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
+# fi
+
 if [ -z $GN2_PROFILE ] ; then
     echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
     export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 699d21f1..a70d8aec 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -41,6 +41,24 @@ SECURITY_POST_LOGIN_VIEW = "/thank_you"
 SERVER_PORT = 5003          # running on localhost
 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
 
+GITHUB_CLIENT_ID = "UNKNOWN"
+GITHUB_CLIENT_SECRET = "UNKNOWN"
+GITHUB_AUTH_URL = "UNKNOWN"
+GITHUB_API_URL = "UNKNOWN"
+
+ORCID_CLIENT_ID = "UNKNOWN"
+ORCID_CLIENT_SECRET = "UNKNOWN"
+ORCID_AUTH_URL = "UNKNOWN"
+ORCID_TOKEN_URL = "UNKNOWN"
+
+ELASTICSEARCH_HOST = "localhost"
+ELASTICSEARCH_PORT = '9200'
+
+SMTP_CONNECT = "UNKNOWN"
+SMTP_USERNAME = "UNKNOWN"
+SMTP_PASSWORD = "UNKNOWN"
+
+
 # ---- Behavioural settings (defaults) note that logger and log levels can
 #      be overridden at the module level and with enviroment settings
 WEBSERVER_MODE  = 'DEV'     # Python webserver mode (DEBUG|DEV|PROD)
diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py
index 14a2d689..ebe3add5 100644
--- a/wqflask/run_gunicorn.py
+++ b/wqflask/run_gunicorn.py
@@ -11,6 +11,9 @@ print "Starting up Gunicorn process"
 
 from wqflask import app
 
+app.config['SESSION_TYPE'] = 'filesystem'
+app.config['SECRET_KEY'] = 'super secret key'
+
 @app.route("/gunicorn")
 def hello():
     return "<h1 style='color:blue'>Hello There!</h1>"
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index a964b025..2d3d5add 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -1,10 +1,18 @@
 from elasticsearch import Elasticsearch, TransportError
 import logging
 
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
+
 def get_elasticsearch_connection():
+    logger.info("get_elasticsearch_connection")
     es = None
     try:
-        from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
+        assert(ELASTICSEARCH_HOST)
+        assert(ELASTICSEARCH_PORT)
+        logger.info("ES HOST",ELASTICSEARCH_HOST)
 
         es = Elasticsearch([{
             "host": ELASTICSEARCH_HOST
@@ -31,12 +39,12 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
         response = es.search(
             index = index
             , doc_type = doc_type
-            , body = { 
-                "query": { "match": { column_name: column_value } } 
+            , body = {
+                "query": { "match": { column_name: column_value } }
             })
         if len(response["hits"]["hits"]) > 0:
             item_details = response["hits"]["hits"][0]["_source"]
-    except TransportError as te: 
+    except TransportError as te:
         pass
     return item_details
 
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 005f9b0f..8c9fed96 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -251,32 +251,25 @@ assert_dir(JS_GUIX_PATH)
 JS_GN_PATH         = get_setting('JS_GN_PATH')
 # assert_dir(JS_GN_PATH)
 
-def get_setting_safe(setting):
-    try:
-        return get_setting(setting)
-    except:
-        print("Could not find the setting '", setting, "'. Continuing with value unset")
-        return None
-
-GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID')
-GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET')
+GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID')
+GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET')
 GITHUB_AUTH_URL = None
 if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET:
     GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET
-GITHUB_API_URL = get_setting_safe('GITHUB_API_URL')
-ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID')
-ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET')
+GITHUB_API_URL = get_setting('GITHUB_API_URL')
+ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID')
+ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET')
 ORCID_AUTH_URL = None
 if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET:
     ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET
-ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL')
+ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
 
-ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST')
-ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT')
+ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
+ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
 
-SMTP_CONNECT = get_setting_safe('SMTP_CONNECT')
-SMTP_USERNAME = get_setting_safe('SMTP_USERNAME')
-SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD')
+SMTP_CONNECT = get_setting('SMTP_CONNECT')
+SMTP_USERNAME = get_setting('SMTP_USERNAME')
+SMTP_PASSWORD = get_setting('SMTP_PASSWORD')
 
 PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
 GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 6b667615..c8471cb1 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -55,9 +55,8 @@ logger = getLogger(__name__)
 from base.data_set import create_datasets_list
 
 import requests
-from utility.elasticsearch_tools import *
+from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user
 
-es = get_elasticsearch_connection()
 THREE_DAYS = 60 * 60 * 24 * 3
 #THREE_DAYS = 45
 
@@ -479,6 +478,7 @@ def password_reset_step2():
     password = request.form['password']
     set_password(password, user)
 
+    es = get_elasticsearch_connection()
     es.update(
         index = "users"
         , doc_type = "local"
@@ -620,6 +620,7 @@ class LoginUser(object):
         """Login through the normal form"""
         params = request.form if request.form else request.args
         logger.debug("in login params are:", params)
+        es = get_elasticsearch_connection()
         if not params:
             from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL
             external_login = None
@@ -628,6 +629,7 @@ class LoginUser(object):
                     "github": GITHUB_AUTH_URL,
                     "orcid": ORCID_AUTH_URL
                 }
+            assert(es is not None)
             return render_template(
                 "new_security/login_user.html"
                 , external_login=external_login
@@ -822,6 +824,7 @@ def register():
 
     params = request.form if request.form else request.args
     params = params.to_dict(flat=True)
+    es = get_elasticsearch_connection()
     params["es_connection"] = es
 
     if params:
-- 
cgit v1.2.3


From 375935b23778148eba40ebce4a4f4def823aa3f0 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Thu, 15 Feb 2018 15:00:47 +0000
Subject: Requests added

---
 bin/genenetwork2              |  10 ++--
 bin/mechnical-rob             | 111 ++++++++++++++++++++++++++++++++++++++++
 bin/test-website              | 114 ++----------------------------------------
 test/requests/test-website.py |  20 ++++++++
 4 files changed, 142 insertions(+), 113 deletions(-)
 create mode 100755 bin/mechnical-rob
 create mode 100755 test/requests/test-website.py

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 31fefbd3..ceafeedd 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -168,8 +168,9 @@ if [ "$1" = '-c' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=${2#wqflask/}
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND $cmd
-    python $cmd
+    shift ; shift
+    echo RUNNING COMMAND $cmd $*
+    python $cmd $*
     exit $?
 fi
 # Now handle command parameter -cli which runs in bash
@@ -178,8 +179,9 @@ if [ "$1" = "-cli" ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=$2
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND $cmd
-    $cmd
+    shift ; shift
+    echo RUNNING COMMAND $cmd $*
+    $cmd $*
     exit $?
 fi
 if [ "$1" = '-gunicorn' ] ; then
diff --git a/bin/mechnical-rob b/bin/mechnical-rob
new file mode 100755
index 00000000..be223d94
--- /dev/null
+++ b/bin/mechnical-rob
@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+
+
+USAGE = <<EOT
+This is Mechanical-Rob - an automated web server tester for
+                         Genenetwork.org that uses the brilliant
+                         mechanize gem with minitest.
+
+To use this software you need to install mechanize. Run it from
+the root of the genenetwork2 source tree as, for example,
+
+  ./bin/test-website http://localhost:5003/ (default)
+
+If you are using the small deployment database you can use
+
+  ./bin/test-website --skip -n
+
+To run all tests
+
+  ./bin/test-website --all
+
+To run individual tests on localhost you can do
+
+  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
+
+For more information see http://genenetwork.org/
+
+EOT
+$stderr.print USAGE
+
+require 'optparse'
+
+options = { database: :small, link_checker: false}
+opts = OptionParser.new do |o|
+  o.banner = "Usage: #{File.basename($0)} [options] URL"
+
+  o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s|
+    options[:database] =
+      case s
+      when 'xx'
+        :unknown
+      else
+        :small
+      end
+  end
+
+  o.on('--all', 'Run all tests') do
+    options[:all] = true
+  end
+
+  o.on('-l','--link-checker', 'Check for dead links') do
+    options[:link_checker] = true
+  end
+
+  o.on('--navigation', 'Check for navigation') do
+    options[:navigation] = true
+  end
+
+  o.on('--mapping', 'Check for mapping') do
+    options[:mapping] = true
+  end
+
+  o.on('--skip-broken', 'Skip tests that are known to be broken') do
+    options[:skip_broken] = true
+  end
+
+  o.separator ""
+  o.on_tail('-h', '--help', 'display this help and exit') do
+    options[:show_help] = true
+  end
+end
+
+opts.parse!(ARGV)
+
+if options[:show_help]
+  print opts
+  exit 1
+end
+
+$options = options  # we are using a global here
+$host =
+  if ARGV.size>0
+    ARGV.shift
+  else
+    "http://localhost:5003"
+  end
+
+$stderr.print "Testing <",$host,">\n"
+
+require 'mechanize'
+require 'minitest/spec'
+require 'minitest/autorun'
+
+# These are the actual testing modules
+
+libpath = File.dirname(File.dirname(__FILE__))
+$: << File.join(libpath,'test/lib')
+
+require 'main_web_functionality'
+
+if options[:all] or options[:mapping]
+  require 'mapping'
+end
+
+if options[:all] or options[:link_checker]
+  require 'link_checker'
+end
+
+if options[:all] or options[:navigation]
+  require 'navigation'
+end
diff --git a/bin/test-website b/bin/test-website
index be223d94..5935f016 100755
--- a/bin/test-website
+++ b/bin/test-website
@@ -1,111 +1,7 @@
-#!/usr/bin/env ruby
+#! /bin/bash
 
-
-USAGE = <<EOT
-This is Mechanical-Rob - an automated web server tester for
-                         Genenetwork.org that uses the brilliant
-                         mechanize gem with minitest.
-
-To use this software you need to install mechanize. Run it from
-the root of the genenetwork2 source tree as, for example,
-
-  ./bin/test-website http://localhost:5003/ (default)
-
-If you are using the small deployment database you can use
-
-  ./bin/test-website --skip -n
-
-To run all tests
-
-  ./bin/test-website --all
-
-To run individual tests on localhost you can do
-
-  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
-
-For more information see http://genenetwork.org/
-
-EOT
-$stderr.print USAGE
-
-require 'optparse'
-
-options = { database: :small, link_checker: false}
-opts = OptionParser.new do |o|
-  o.banner = "Usage: #{File.basename($0)} [options] URL"
-
-  o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s|
-    options[:database] =
-      case s
-      when 'xx'
-        :unknown
-      else
-        :small
-      end
-  end
-
-  o.on('--all', 'Run all tests') do
-    options[:all] = true
-  end
-
-  o.on('-l','--link-checker', 'Check for dead links') do
-    options[:link_checker] = true
-  end
-
-  o.on('--navigation', 'Check for navigation') do
-    options[:navigation] = true
-  end
-
-  o.on('--mapping', 'Check for mapping') do
-    options[:mapping] = true
-  end
-
-  o.on('--skip-broken', 'Skip tests that are known to be broken') do
-    options[:skip_broken] = true
-  end
-
-  o.separator ""
-  o.on_tail('-h', '--help', 'display this help and exit') do
-    options[:show_help] = true
-  end
-end
-
-opts.parse!(ARGV)
-
-if options[:show_help]
-  print opts
+if [ -z $GN2_PROFILE ]; then
+  echo "Run request tests with something like"
+  echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
   exit 1
-end
-
-$options = options  # we are using a global here
-$host =
-  if ARGV.size>0
-    ARGV.shift
-  else
-    "http://localhost:5003"
-  end
-
-$stderr.print "Testing <",$host,">\n"
-
-require 'mechanize'
-require 'minitest/spec'
-require 'minitest/autorun'
-
-# These are the actual testing modules
-
-libpath = File.dirname(File.dirname(__FILE__))
-$: << File.join(libpath,'test/lib')
-
-require 'main_web_functionality'
-
-if options[:all] or options[:mapping]
-  require 'mapping'
-end
-
-if options[:all] or options[:link_checker]
-  require 'link_checker'
-end
-
-if options[:all] or options[:navigation]
-  require 'navigation'
-end
+fi
diff --git a/test/requests/test-website.py b/test/requests/test-website.py
new file mode 100755
index 00000000..d02b71aa
--- /dev/null
+++ b/test/requests/test-website.py
@@ -0,0 +1,20 @@
+# Run with something like
+#
+#   env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
+#
+# Mostly to pick up the Guix GN2_PROFILE and python modules
+
+import requests as req
+import sys
+
+print "Mechanical Rob firing up..."
+
+if len(sys.argv)<1:
+    raise "Problem with arguments"
+
+url = sys.argv[1]
+print url
+
+r = req.get(url)
+
+print r
-- 
cgit v1.2.3


From 3c8089e2c3957ca2d4a5906249b4818d367221dc Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 19 Mar 2018 13:06:13 +0000
Subject: - Disabled PYTHONPATH injection - fix Guix instead, see
 https://github.com/pjotrp/genenetwork2/commit/63a5c8a42ad02e9126bb207465ff5eca98f6515d
 - Renamed WQFLASK_SETTINGS to GN2_SETTINGS

---
 bin/genenetwork2            | 33 +++++++++++++++------------------
 wqflask/utility/tools.py    | 15 +--------------
 wqflask/wqflask/__init__.py |  4 ++--
 3 files changed, 18 insertions(+), 34 deletions(-)

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index ceafeedd..3f06e7f9 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -68,32 +68,29 @@ if [ "$1" = "-c" -o "$1" = "-gunicorn" ]; then
     echo "Can not use $1 switch without default settings file"
     exit 1
 fi
-# Handle settings parameter (can be .py or .json)
-if [ ! -z $1 ]; then
-    settings=$(realpath "$1")
-    if [ ! -e $settings ]; then
-        settings=$GN2_BASE_DIR/etc/default_settings.py
-    else
-        shift
-    fi
-fi
 
-ext="${settings##*.}"
-if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
-    overrides=$settings
+settings=$1
+if [ -z $settings ]; then
+    settings=$GN2_BASE_DIR/etc/default_settings.py
 else
-    echo $settings
+    shift
 fi
+settings=$(realpath $settings)
+
+# ext="${settings##*.}"
+# if [ "$ext" = "json" -o "$ext" = "JSON" ]; then
+#     overrides=$settings
+# else
+#     echo $settings
+# fi
 
 if [ ! -e $settings ]; then
     echo "ERROR: can not locate settings file - pass it in the command line"
     exit 1
 fi
-export WQFLASK_SETTINGS=$settings     # Python
-export WQFLASK_OVERRIDES=$overrides   # JSON
 
-echo WQFLASK_SETTINGS=$settings
-echo WQFLASK_OVERRIDES=$overrides
+export GN2_SETTINGS=$settings     # Python
+echo GN2_SETTINGS=$settings
 
 # This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead
 # if [ -z $ELASTICSEARCH_PROFILE ]; then
@@ -115,7 +112,7 @@ if [ -z $GN2_PROFILE ]; then
     read -p "PRESS [ENTER] TO CONTINUE..."
 else
     export PATH=$GN2_PROFILE/bin:$PATH
-    export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages${PYTHONPATH:+:}$PYTHONPATH"
+    export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!!
     export R_LIBS_SITE=$GN2_PROFILE/site-library
     export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0
     export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index 8c9fed96..bae3a7f4 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -54,7 +54,7 @@ def get_setting(command_id,guess=None):
     # print("Looking for "+command_id+"\n")
     command = value(os.environ.get(command_id))
     if command is None or command == "":
-        command = OVERRIDES.get(command_id)
+        command = OVERRIDES.get(command_id) # currently not in use
         if command is None:
             # ---- Check whether setting exists in app
             command = value(app.config.get(command_id))
@@ -285,18 +285,5 @@ assert_dir(JS_TWITTER_POST_FETCHER_PATH)
 
 from six import string_types
 
-if os.environ.get('WQFLASK_OVERRIDES'):
-    jsonfn = get_setting('WQFLASK_OVERRIDES')
-    logger.info("WQFLASK_OVERRIDES: %s" % jsonfn)
-    with open(jsonfn) as data_file:
-        overrides = json.load(data_file)
-        for k in overrides:
-            cmd = overrides[k]
-            if isinstance(cmd, string_types):
-                OVERRIDES[k] = eval(cmd)
-            else:
-                OVERRIDES[k] = cmd
-            logger.debug(OVERRIDES)
-
 # assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData")
 assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js")
diff --git a/wqflask/wqflask/__init__.py b/wqflask/wqflask/__init__.py
index 2188ce17..bc8e9900 100644
--- a/wqflask/wqflask/__init__.py
+++ b/wqflask/wqflask/__init__.py
@@ -13,8 +13,8 @@ logging.basicConfig(level=logging.INFO)
 app = Flask(__name__)
 
 app.config.from_object('cfg.default_settings')   # Get the defaults from cfg.default_settings
-app.config.from_envvar('WQFLASK_SETTINGS')       # See http://flask.pocoo.org/docs/config/#configuring-from-files
-# Note we also use WQFLASK_OVERRIDES
+app.config.from_envvar('GN2_SETTINGS')       # See http://flask.pocoo.org/docs/config/#configuring-from-files
+# Note no longer use the badly named WQFLASK_OVERRIDES (nyi)
 
 app.jinja_env.globals.update(
     undefined = jinja2.StrictUndefined,
-- 
cgit v1.2.3


From 9961d9af171ac61a72b69a845c8a8e438676b995 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Mon, 2 Apr 2018 08:51:04 +0000
Subject: Testing moved to Python

---
 bin/mechnical-rob | 111 ------------------------------------------------------
 1 file changed, 111 deletions(-)
 delete mode 100755 bin/mechnical-rob

(limited to 'bin')

diff --git a/bin/mechnical-rob b/bin/mechnical-rob
deleted file mode 100755
index be223d94..00000000
--- a/bin/mechnical-rob
+++ /dev/null
@@ -1,111 +0,0 @@
-#!/usr/bin/env ruby
-
-
-USAGE = <<EOT
-This is Mechanical-Rob - an automated web server tester for
-                         Genenetwork.org that uses the brilliant
-                         mechanize gem with minitest.
-
-To use this software you need to install mechanize. Run it from
-the root of the genenetwork2 source tree as, for example,
-
-  ./bin/test-website http://localhost:5003/ (default)
-
-If you are using the small deployment database you can use
-
-  ./bin/test-website --skip -n
-
-To run all tests
-
-  ./bin/test-website --all
-
-To run individual tests on localhost you can do
-
-  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
-
-For more information see http://genenetwork.org/
-
-EOT
-$stderr.print USAGE
-
-require 'optparse'
-
-options = { database: :small, link_checker: false}
-opts = OptionParser.new do |o|
-  o.banner = "Usage: #{File.basename($0)} [options] URL"
-
-  o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s|
-    options[:database] =
-      case s
-      when 'xx'
-        :unknown
-      else
-        :small
-      end
-  end
-
-  o.on('--all', 'Run all tests') do
-    options[:all] = true
-  end
-
-  o.on('-l','--link-checker', 'Check for dead links') do
-    options[:link_checker] = true
-  end
-
-  o.on('--navigation', 'Check for navigation') do
-    options[:navigation] = true
-  end
-
-  o.on('--mapping', 'Check for mapping') do
-    options[:mapping] = true
-  end
-
-  o.on('--skip-broken', 'Skip tests that are known to be broken') do
-    options[:skip_broken] = true
-  end
-
-  o.separator ""
-  o.on_tail('-h', '--help', 'display this help and exit') do
-    options[:show_help] = true
-  end
-end
-
-opts.parse!(ARGV)
-
-if options[:show_help]
-  print opts
-  exit 1
-end
-
-$options = options  # we are using a global here
-$host =
-  if ARGV.size>0
-    ARGV.shift
-  else
-    "http://localhost:5003"
-  end
-
-$stderr.print "Testing <",$host,">\n"
-
-require 'mechanize'
-require 'minitest/spec'
-require 'minitest/autorun'
-
-# These are the actual testing modules
-
-libpath = File.dirname(File.dirname(__FILE__))
-$: << File.join(libpath,'test/lib')
-
-require 'main_web_functionality'
-
-if options[:all] or options[:mapping]
-  require 'mapping'
-end
-
-if options[:all] or options[:link_checker]
-  require 'link_checker'
-end
-
-if options[:all] or options[:navigation]
-  require 'navigation'
-end
-- 
cgit v1.2.3


From 8145507d6d617554cf996e6cebf286d30ae64df0 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Wed, 4 Apr 2018 16:58:21 +0000
Subject: ES: doc

---
 bin/genenetwork2                       | 18 ++++++++++-----
 doc/elasticsearch.org                  | 41 ++++++++++++++++++++++++++++++++++
 wqflask/utility/elasticsearch_tools.py |  9 ++++----
 wqflask/wqflask/user_manager.py        | 23 +++++--------------
 4 files changed, 62 insertions(+), 29 deletions(-)
 create mode 100644 doc/elasticsearch.org

(limited to 'bin')

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 3f06e7f9..18e02388 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -21,10 +21,18 @@
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_settings.py
 #
-# To run a maintenance script with settings (instead of the webserver) add that with
-# a -c switch, e.g.
+# To run a maintenance python script with settings (instead of the
+# webserver) add that with a -c switch, e.g.
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 -c ./wqflask/maintenance/gen_select_dataset.py
+#
+# To run any script in the environment
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD"
+#
+# To get a python REPL(!)
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -cli python
 #
 # For development you may want to run
 #
@@ -114,7 +122,6 @@ else
     export PATH=$GN2_PROFILE/bin:$PATH
     export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!!
     export R_LIBS_SITE=$GN2_PROFILE/site-library
-    export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0
     export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
     export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0"
     export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0"
@@ -134,7 +141,6 @@ else
 	done
     done <<< "$PYTHONPATH"
     if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
-    if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi
 fi
 if [ -z $PYTHONPATH ] ; then
     echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!"
@@ -170,9 +176,9 @@ if [ "$1" = '-c' ] ; then
     python $cmd $*
     exit $?
 fi
+
 # Now handle command parameter -cli which runs in bash
 if [ "$1" = "-cli" ] ; then
-    echo "HERE"
     cd $GN2_BASE_DIR/wqflask
     cmd=$2
     echo PYTHONPATH=$PYTHONPATH
diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org
new file mode 100644
index 00000000..18adfc8b
--- /dev/null
+++ b/doc/elasticsearch.org
@@ -0,0 +1,41 @@
+* Elasticsearch
+
+To get the right environment, first you can get a python REPL with something like
+
+: env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python
+
+(make sure to use the correct GN2_PROFILE!)
+
+Next try
+
+#+BEGIN_SRC python
+
+from elasticsearch import Elasticsearch, TransportError
+
+es = Elasticsearch([{ "host": 'localhost', "port": '9200' }])
+
+# Dump all data
+
+es.search("*")
+
+# To fetch an E-mail record from the users index
+
+record = es.search(
+            index = 'users', doc_type = 'local', body = {
+                "query": { "match": { "email_address": "myname@email.com" } }
+            })
+
+# It is also possible to do wild card matching
+
+q = { "query": { "wildcard" : { "full_name" : "pjot*" } }}
+es.search(index = 'users', doc_type = 'local', body = q)
+
+# To get elements from that record:
+
+record['hits']['hits'][0][u'_source']['full_name']
+u'Pjotr'
+
+record['hits']['hits'][0][u'_source']['email_address']
+u"myname@email.com"
+
+#+END_SRC
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index 734379f7..1dba357d 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -12,6 +12,7 @@ def test_elasticsearch_connection():
         logger.warning("Elasticsearch is DOWN")
 
 def get_elasticsearch_connection():
+    """Return a connection to ES. Returns None on failure"""
     logger.info("get_elasticsearch_connection")
     es = None
     try:
@@ -20,14 +21,14 @@ def get_elasticsearch_connection():
         logger.info("ES HOST",ELASTICSEARCH_HOST)
 
         es = Elasticsearch([{
-            "host": ELASTICSEARCH_HOST
-            , "port": ELASTICSEARCH_PORT
+            "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
         }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
 
         es_logger = logging.getLogger("elasticsearch")
         es_logger.setLevel(logging.INFO)
         es_logger.addHandler(logging.NullHandler())
     except:
+        logger.error("Failed to get elasticsearch connection")
         es = None
 
     return es
@@ -42,9 +43,7 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
     item_details = None
     try:
         response = es.search(
-            index = index
-            , doc_type = doc_type
-            , body = {
+            index = index, doc_type = doc_type, body = {
                 "query": { "match": { column_name: column_value } }
             })
         if len(response["hits"]["hits"]) > 0:
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index ac3824a7..ead919fc 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -1,45 +1,30 @@
 from __future__ import print_function, division, absolute_import
 
-"""Used to Access things in template like this:
-(BUT NOW OUT OF DATE)
-
-    x: {{ g.identity.name }}
-    security: {{ security.__dict__ }}
-
-"""
-
 import os
 import hashlib
 import datetime
 import time
 import logging
-
 import uuid
 import hashlib
 import hmac
 import base64
-
 import urlparse
 
 import simplejson as json
 
 #from redis import StrictRedis
-import redis
+import redis # used for collections
 Redis = redis.StrictRedis()
 
-
 from flask import (Flask, g, render_template, url_for, request, make_response,
                    redirect, flash, abort)
 
 from wqflask import app
-
-
 from pprint import pformat as pf
 
-from wqflask import pbkdf2
-
+from wqflask import pbkdf2 # password hashing
 from wqflask.database import db_session
-
 from wqflask import model
 
 from utility import Bunch, Struct, after
@@ -62,8 +47,8 @@ THREE_DAYS = 60 * 60 * 24 * 3
 def timestamp():
     return datetime.datetime.utcnow().isoformat()
 
-
 class AnonUser(object):
+    """Anonymous user handling"""
     cookie_name = 'anon_user_v8'
 
     def __init__(self):
@@ -169,6 +154,8 @@ def create_signed_cookie():
     return the_uuid, uuid_signed
 
 class UserSession(object):
+    """Logged in user handling"""
+
     cookie_name = 'session_id_v2'
 
     def __init__(self):
-- 
cgit v1.2.3


From 518be96a310963a4127a0b569a73c91187c02b64 Mon Sep 17 00:00:00 2001
From: Pjotr Prins
Date: Thu, 12 Apr 2018 08:20:30 +0000
Subject: README and docs

---
 README.md                                 | 13 ++++++++-----
 bin/genenetwork2                          | 15 ++++++++-------
 wqflask/maintenance/gen_select_dataset.py |  3 ---
 3 files changed, 16 insertions(+), 15 deletions(-)

(limited to 'bin')

diff --git a/README.md b/README.md
index 8221ad65..3e7e64d0 100644
--- a/README.md
+++ b/README.md
@@ -25,7 +25,11 @@ Once installed GN2 can be run online through a browser interface
 ./bin/genenetwork2
 ```
 
-(default is http://localhost:5003/). We are building up automated
+(default is http://localhost:5003/). For more examples, including running scripts and a Python REPL
+see the startup script [./bin/genenetwork2](https://github.com/genenetwork/genenetwork2/blob/testing/bin/genenetwork2).
+
+
+We are building up automated
 testing using [mechanize](https://github.com/genenetwork/genenetwork2/tree/master/test/lib) which can be run with
 
 ```sh
@@ -70,16 +74,15 @@ For more information visit http://www.genenetwork.org/
 
 [![JOSS](http://joss.theoj.org/papers/10.21105/joss.00025/status.svg)](http://joss.theoj.org/papers/10.21105/joss.00025)
 
-GeneNetwork was published in the Journal of Open Source Software as 'GeneNetwork: framework for web-based genetics' by Zachary Sloan, Danny Arends, Karl W. Broman, Arthur Centeno, Nicholas Furlotte, Harm Nijveen, Lei Yan, Xiang Zhou, Robert W. WIlliams and Pjotr Prins 
+GeneNetwork was published in the Journal of Open Source Software as 'GeneNetwork: framework for web-based genetics' by Zachary Sloan, Danny Arends, Karl W. Broman, Arthur Centeno, Nicholas Furlotte, Harm Nijveen, Lei Yan, Xiang Zhou, Robert W. WIlliams and Pjotr Prins
 
 You may also cite the software using
 
-[![DOI](https://zenodo.org/badge/5591/genenetwork/genenetwork2.svg)](https://zenodo.org/badge/latestdoi/5591/genenetwork/genenetwork2). 
+[![DOI](https://zenodo.org/badge/5591/genenetwork/genenetwork2.svg)](https://zenodo.org/badge/latestdoi/5591/genenetwork/genenetwork2).
 
 ## Contact
 
 IRC on #genenetwork on irc.freenode.net.
 
 Code and primary web service managed by Dr. Robert W. Williams and the
-University of Tennessee Health Science Center, Memphis TN, USA. 
-
+University of Tennessee Health Science Center, Memphis TN, USA.
diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 18e02388..b3a8d8c2 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -15,28 +15,29 @@
 # installation path of genenetwork). Say your profile is in
 # ~/opt/gn-latest-guix
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2
 #
 # You can pass in your own settings file, e.g.
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_settings.py
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ~/my_settings.py
 #
 # To run a maintenance python script with settings (instead of the
-# webserver) add that with a -c switch, e.g.
+# webserver) run from the base-dir with settings file and add that
+# script with a -c switch, e.g.
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 -c ./wqflask/maintenance/gen_select_dataset.py
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ./maintenance/gen_select_dataset.py
 #
 # To run any script in the environment
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD"
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD"
 #
 # To get a python REPL(!)
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -cli python
+#   env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -cli python
 #
 # For development you may want to run
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
+#   env GN2_PROFILE=~/opt/gn-latest WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG ./bin/genenetwork2
 #
 # For staging and production we use gunicorn. Run with something like
 # (note you have to provide the server port). Provide a settings file!
diff --git a/wqflask/maintenance/gen_select_dataset.py b/wqflask/maintenance/gen_select_dataset.py
index f62d0cc1..79242661 100644
--- a/wqflask/maintenance/gen_select_dataset.py
+++ b/wqflask/maintenance/gen_select_dataset.py
@@ -63,9 +63,6 @@ from pprint import pformat as pf
 
 #conn = Engine.connect()
 
-print('WARNING: This conversion is now OBSOLETE as the menu gets built from the database in Javascript using GN_SERVER instead!')
-
-
 def parse_db_uri(db_uri):
     """Converts a database URI to the db name, host name, user name, and password"""
 
-- 
cgit v1.2.3