Merge branch 'pjotrp-gn-testing-es' into testing

17 files changed, 333 insertions, 330 deletions

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 145ce395..2a83a1cd 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -45,6 +45,7 @@
 
 SCRIPT=$(realpath "$0")
 echo SCRIPT=$SCRIPT
+echo GN2_PROFILE=$GN2_PROFILE
 GN2_BASE_DIR=$(dirname $(dirname "$SCRIPT"))
 GN2_ID=$(cat /etc/hostname):$(basename $GN2_BASE_DIR)
 
@@ -94,11 +95,13 @@ export WQFLASK_OVERRIDES=$overrides   # JSON
 echo WQFLASK_SETTINGS=$settings
 echo WQFLASK_OVERRIDES=$overrides
 
-if [ -z $ELASTICSEARCH_PROFILE ]; then
-    echo -e "\033[1;33mWARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE\033[0m";
-else
-    PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
-fi
+# This is a temporary hack to inject ES - should have added python2-elasticsearch package to guix instead
+# if [ -z $ELASTICSEARCH_PROFILE ]; then
+#     echo -e "WARNING: Elastic Search profile has not been set - use ELASTICSEARCH_PROFILE";
+# else
+#     PYTHONPATH="$PYTHONPATH${PYTHONPATH:+:}$ELASTICSEARCH_PROFILE/lib/python2.7/site-packages"
+# fi
+
 if [ -z $GN2_PROFILE ] ; then
     echo "WARNING: GN2_PROFILE has not been set - you need the environment, so I hope you know what you are doing!"
     export GN2_PROFILE=$(dirname $(dirname $(which genenetwork2)))
@@ -125,7 +128,9 @@ else
     export PLINK_COMMAND="$GN2_PROFILE/bin/plink2"
     export PYLMM_COMMAND="$GN2_PROFILE/bin/pylmm_redis"
     export GEMMA_COMMAND="$GN2_PROFILE/bin/gemma"
-    export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
+    if [ -z $GEMMA_WRAPPER_COMMAND ]; then
+        export GEMMA_WRAPPER_COMMAND="$GN2_PROFILE/bin/gemma-wrapper"
+    fi
     while IFS=":" read -ra PPATH; do
 	for PPart in "${PPATH[@]}"; do
 	    if [ ! -d $PPart ] ; then echo "PYTHONPATH not valid "$PYTHONPATH ; exit 1 ; fi
@@ -163,8 +168,9 @@ if [ "$1" = '-c' ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=${2#wqflask/}
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND $cmd
-    python $cmd
+    shift ; shift
+    echo RUNNING COMMAND $cmd $*
+    python $cmd $*
     exit $?
 fi
 # Now handle command parameter -cli which runs in bash
@@ -173,8 +179,9 @@ if [ "$1" = "-cli" ] ; then
     cd $GN2_BASE_DIR/wqflask
     cmd=$2
     echo PYTHONPATH=$PYTHONPATH
-    echo RUNNING COMMAND $cmd
-    $cmd
+    shift ; shift
+    echo RUNNING COMMAND $cmd $*
+    $cmd $*
     exit $?
 fi
 if [ "$1" = '-gunicorn' ] ; then
@@ -198,7 +205,8 @@ if [ "$1" = '-gunicorn-prod' ] ; then
     cd $GN2_BASE_DIR/wqflask
     echo PYTHONPATH=$PYTHONPATH
     if [ -z $SERVER_PORT ]; then echo "ERROR: Provide a SERVER_PORT" ; exit 1 ; fi
-    cmd="--bind 0.0.0.0:$SERVER_PORT --workers=32 --max-requests 1000 --timeout 1200 wsgi"
+    PID=$TMPDIR/gunicorn.$USER.pid
+    cmd="--bind 0.0.0.0:$SERVER_PORT --pid $PID -k eventlet --workers 20 --keep-alive 1200 --max-requests 1000 --timeout 1200 wsgi"
     echo RUNNING gunicorn $cmd
     gunicorn $cmd
     exit $?
diff --git a/bin/mechnical-rob b/bin/mechnical-rob
new file mode 100755
index 00000000..be223d94
--- /dev/null
+++ b/bin/mechnical-rob
@@ -0,0 +1,111 @@
+#!/usr/bin/env ruby
+
+
+USAGE = <<EOT
+This is Mechanical-Rob - an automated web server tester for
+                         Genenetwork.org that uses the brilliant
+                         mechanize gem with minitest.
+
+To use this software you need to install mechanize. Run it from
+the root of the genenetwork2 source tree as, for example,
+
+  ./bin/test-website http://localhost:5003/ (default)
+
+If you are using the small deployment database you can use
+
+  ./bin/test-website --skip -n
+
+To run all tests
+
+  ./bin/test-website --all
+
+To run individual tests on localhost you can do
+
+  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
+
+For more information see http://genenetwork.org/
+
+EOT
+$stderr.print USAGE
+
+require 'optparse'
+
+options = { database: :small, link_checker: false}
+opts = OptionParser.new do |o|
+  o.banner = "Usage: #{File.basename($0)} [options] URL"
+
+  o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s|
+    options[:database] =
+      case s
+      when 'xx'
+        :unknown
+      else
+        :small
+      end
+  end
+
+  o.on('--all', 'Run all tests') do
+    options[:all] = true
+  end
+
+  o.on('-l','--link-checker', 'Check for dead links') do
+    options[:link_checker] = true
+  end
+
+  o.on('--navigation', 'Check for navigation') do
+    options[:navigation] = true
+  end
+
+  o.on('--mapping', 'Check for mapping') do
+    options[:mapping] = true
+  end
+
+  o.on('--skip-broken', 'Skip tests that are known to be broken') do
+    options[:skip_broken] = true
+  end
+
+  o.separator ""
+  o.on_tail('-h', '--help', 'display this help and exit') do
+    options[:show_help] = true
+  end
+end
+
+opts.parse!(ARGV)
+
+if options[:show_help]
+  print opts
+  exit 1
+end
+
+$options = options  # we are using a global here
+$host =
+  if ARGV.size>0
+    ARGV.shift
+  else
+    "http://localhost:5003"
+  end
+
+$stderr.print "Testing <",$host,">\n"
+
+require 'mechanize'
+require 'minitest/spec'
+require 'minitest/autorun'
+
+# These are the actual testing modules
+
+libpath = File.dirname(File.dirname(__FILE__))
+$: << File.join(libpath,'test/lib')
+
+require 'main_web_functionality'
+
+if options[:all] or options[:mapping]
+  require 'mapping'
+end
+
+if options[:all] or options[:link_checker]
+  require 'link_checker'
+end
+
+if options[:all] or options[:navigation]
+  require 'navigation'
+end
diff --git a/bin/test-website b/bin/test-website
index be223d94..5935f016 100755
--- a/bin/test-website
+++ b/bin/test-website
@@ -1,111 +1,7 @@
-#!/usr/bin/env ruby
+#! /bin/bash
 
-
-USAGE = <<EOT
-This is Mechanical-Rob - an automated web server tester for
-                         Genenetwork.org that uses the brilliant
-                         mechanize gem with minitest.
-
-To use this software you need to install mechanize. Run it from
-the root of the genenetwork2 source tree as, for example,
-
-  ./bin/test-website http://localhost:5003/ (default)
-
-If you are using the small deployment database you can use
-
-  ./bin/test-website --skip -n
-
-To run all tests
-
-  ./bin/test-website --all
-
-To run individual tests on localhost you can do
-
-  ruby -Itest -Itest/lib test/lib/mapping.rb --name="/Mapping/"
-
-For more information see http://genenetwork.org/
-
-EOT
-$stderr.print USAGE
-
-require 'optparse'
-
-options = { database: :small, link_checker: false}
-opts = OptionParser.new do |o|
-  o.banner = "Usage: #{File.basename($0)} [options] URL"
-
-  o.on('-d','--database', String, 'Use database (default db_webqtl_s)') do |s|
-    options[:database] =
-      case s
-      when 'xx'
-        :unknown
-      else
-        :small
-      end
-  end
-
-  o.on('--all', 'Run all tests') do
-    options[:all] = true
-  end
-
-  o.on('-l','--link-checker', 'Check for dead links') do
-    options[:link_checker] = true
-  end
-
-  o.on('--navigation', 'Check for navigation') do
-    options[:navigation] = true
-  end
-
-  o.on('--mapping', 'Check for mapping') do
-    options[:mapping] = true
-  end
-
-  o.on('--skip-broken', 'Skip tests that are known to be broken') do
-    options[:skip_broken] = true
-  end
-
-  o.separator ""
-  o.on_tail('-h', '--help', 'display this help and exit') do
-    options[:show_help] = true
-  end
-end
-
-opts.parse!(ARGV)
-
-if options[:show_help]
-  print opts
+if [ -z $GN2_PROFILE ]; then
+  echo "Run request tests with something like"
+  echo env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
   exit 1
-end
-
-$options = options  # we are using a global here
-$host =
-  if ARGV.size>0
-    ARGV.shift
-  else
-    "http://localhost:5003"
-  end
-
-$stderr.print "Testing <",$host,">\n"
-
-require 'mechanize'
-require 'minitest/spec'
-require 'minitest/autorun'
-
-# These are the actual testing modules
-
-libpath = File.dirname(File.dirname(__FILE__))
-$: << File.join(libpath,'test/lib')
-
-require 'main_web_functionality'
-
-if options[:all] or options[:mapping]
-  require 'mapping'
-end
-
-if options[:all] or options[:link_checker]
-  require 'link_checker'
-end
-
-if options[:all] or options[:navigation]
-  require 'navigation'
-end
+fi
diff --git a/etc/default_settings.py b/etc/default_settings.py
index 699d21f1..a70d8aec 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -41,6 +41,24 @@ SECURITY_POST_LOGIN_VIEW = "/thank_you"
 SERVER_PORT = 5003          # running on localhost
 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
 
+GITHUB_CLIENT_ID = "UNKNOWN"
+GITHUB_CLIENT_SECRET = "UNKNOWN"
+GITHUB_AUTH_URL = "UNKNOWN"
+GITHUB_API_URL = "UNKNOWN"
+
+ORCID_CLIENT_ID = "UNKNOWN"
+ORCID_CLIENT_SECRET = "UNKNOWN"
+ORCID_AUTH_URL = "UNKNOWN"
+ORCID_TOKEN_URL = "UNKNOWN"
+
+ELASTICSEARCH_HOST = "localhost"
+ELASTICSEARCH_PORT = '9200'
+
+SMTP_CONNECT = "UNKNOWN"
+SMTP_USERNAME = "UNKNOWN"
+SMTP_PASSWORD = "UNKNOWN"
+
+
 # ---- Behavioural settings (defaults) note that logger and log levels can
 #      be overridden at the module level and with enviroment settings
 WEBSERVER_MODE  = 'DEV'     # Python webserver mode (DEBUG|DEV|PROD)
diff --git a/test/requests/test-website.py b/test/requests/test-website.py
new file mode 100755
index 00000000..d02b71aa
--- /dev/null
+++ b/test/requests/test-website.py
@@ -0,0 +1,20 @@
+# Run with something like
+#
+#   env GN2_PROFILE=/home/wrk/opt/gn-latest ./bin/genenetwork2 ./etc/default_settings.py -c ../test/requests/test-website.py http://localhost:5003
+#
+# Mostly to pick up the Guix GN2_PROFILE and python modules
+
+import requests as req
+import sys
+
+print "Mechanical Rob firing up..."
+
+if len(sys.argv)<1:
+    raise "Problem with arguments"
+
+url = sys.argv[1]
+print url
+
+r = req.get(url)
+
+print r
diff --git a/test/unittest/test_registration.py b/test/unittest/test_registration.py
new file mode 100644
index 00000000..98d0cdff
--- /dev/null
+++ b/test/unittest/test_registration.py
@@ -0,0 +1,27 @@
+# Run test with something like
+#
+# env GN2_PROFILE=~/opt/gn-latest GENENETWORK_FILES=$HOME/gn2_data ./bin/genenetwork2 ./etc/default_settings.py -c ../test/unittest/test_registration.py
+#
+
+import unittest
+import mock.es_double as es
+from wqflask.user_manager import RegisterUser
+
+class TestRegisterUser(unittest.TestCase):
+    def setUp(self):
+        self.es = es.ESDouble()
+
+    def testRegisterUserWithCorrectData(self):
+        data = {
+            "email_address": "user@example.com"
+            , "full_name": "A.N. Other"
+            , "organization": "Some Organisation"
+            , "password": "testing"
+            , "password_confirm": "testing"
+            , "es_connection": self.es
+        }
+        result = RegisterUser(data)
+        self.assertEqual(len(result.errors), 0, "Errors were not expected")
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/wqflask/base/webqtlConfig.py b/wqflask/base/webqtlConfig.py
index 1ef2bc26..1e66e957 100644
--- a/wqflask/base/webqtlConfig.py
+++ b/wqflask/base/webqtlConfig.py
@@ -82,6 +82,7 @@ assert_writable_dir(GENERATED_TEXT_DIR)
 # Flat file directories
 GENODIR              = flat_files('genotype')+'/'
 assert_dir(GENODIR)
+assert_dir(GENODIR+'bimbam') # for gemma
 
 # JSON genotypes are OBSOLETE
 JSON_GENODIR         = flat_files('genotype/json')+'/'
diff --git a/wqflask/tests/__init__.py b/wqflask/mock/__init__.py
index e69de29b..e69de29b 100644
--- a/wqflask/tests/__init__.py
+++ b/wqflask/mock/__init__.py
diff --git a/wqflask/mock/es_double.py b/wqflask/mock/es_double.py
new file mode 100644
index 00000000..6ef8a1b9
--- /dev/null
+++ b/wqflask/mock/es_double.py
@@ -0,0 +1,15 @@
+class ESDouble(object):
+    def __init__(self):
+        self.items = {}
+
+    def ping(self):
+        return true
+
+    def create(self, index, doc_type, body, id):
+        self.items["index"] = {doc_type: {"id": id, "_source": data}}
+
+    def search(self, index, doc_type, body):
+        return {
+            "hits": {
+                "hits": self.items[index][doc_type][body]
+            }}
diff --git a/wqflask/run_gunicorn.py b/wqflask/run_gunicorn.py
index 14a2d689..ebe3add5 100644
--- a/wqflask/run_gunicorn.py
+++ b/wqflask/run_gunicorn.py
@@ -11,6 +11,9 @@ print "Starting up Gunicorn process"
 
 from wqflask import app
 
+app.config['SESSION_TYPE'] = 'filesystem'
+app.config['SECRET_KEY'] = 'super secret key'
+
 @app.route("/gunicorn")
 def hello():
     return "<h1 style='color:blue'>Hello There!</h1>"
diff --git a/wqflask/tests/es_double.py b/wqflask/tests/es_double.py
deleted file mode 100644
index 00739016..00000000
--- a/wqflask/tests/es_double.py
+++ /dev/null
@@ -1,30 +0,0 @@
-class ESDouble(object):
-    def __init__(self):
-        self.items = {
-            "users": {
-                "local": []
-            }}
-
-    def ping(self):
-        return true
-
-    def create(self, index, doc_type, body, id):
-        item = {"id": id, "_source": body}
-        if not self.items.get("index", None):
-            self.items[index] = {doc_type: [item]}
-        else:
-            self.items[index][doc_type].append(item)
-
-    def search(self, index, doc_type, body):
-        d = body["query"]["match"]
-        column = [(key, d[key]) for key in d]
-
-        items = []
-        for thing in self.items[index][doc_type]:
-            if thing["_source"][column[0][0]] == column[0][1]:
-                items.append(thing)
-                break
-        return {
-            "hits": {
-                "hits": items
-            }}
diff --git a/wqflask/tests/test_registration.py b/wqflask/tests/test_registration.py
deleted file mode 100644
index 50a2a84c..00000000
--- a/wqflask/tests/test_registration.py
+++ /dev/null
@@ -1,113 +0,0 @@
-import unittest
-import es_double
-import wqflask.user_manager
-from wqflask.user_manager import RegisterUser
-
-class TestRegisterUser(unittest.TestCase):
-    def setUp(self):
-        # Mock elasticsearch
-        self.es = es_double.ESDouble()
-
-        # Patch method
-        wqflask.user_manager.basic_info = lambda : {"basic_info": "some info"}
-
-    def tearDown(self):
-        self.es = None
-
-    def testRegisterUserWithNoData(self):
-        data = {}
-        result = RegisterUser(data)
-        self.assertNotEqual(len(result.errors), 0, "Data was not provided. Error was expected")
-
-    def testRegisterUserWithNoEmail(self):
-        data = {
-            "email_address": ""
-            , "full_name": "A.N. Other"
-            , "organization": "Some Organisation"
-            , "password": "testing"
-            , "password_confirm": "testing"
-            , "es_connection": self.es
-        }
-
-        result = RegisterUser(data)
-        self.assertNotEqual(len(result.errors), 0, "Email not provided. Error was expected")
-
-    def testRegisterUserWithNoName(self):
-        data = {
-            "email_address": "user@example.com"
-            , "full_name": ""
-            , "organization": "Some Organisation"
-            , "password": "testing"
-            , "password_confirm": "testing"
-            , "es_connection": self.es
-        }
-
-        result = RegisterUser(data)
-        self.assertNotEqual(len(result.errors), 0, "Name not provided. Error was expected")
-
-    def testRegisterUserWithNoOrganisation(self):
-        data = {
-            "email_address": "user@example.com"
-            , "full_name": "A.N. Other"
-            , "organization": ""
-            , "password": "testing"
-            , "password_confirm": "testing"
-            , "es_connection": self.es
-        }
-        
-        result = RegisterUser(data)
-        self.assertEqual(len(result.errors), 0, "Organisation not provided. Error not expected")
-
-    def testRegisterUserWithShortOrganisation(self):
-        data = {
-            "email_address": "user@example.com"
-            , "full_name": "A.N. Other"
-            , "organization": "SO"
-            , "password": "testing"
-            , "password_confirm": "testing"
-            , "es_connection": self.es
-        }
-        
-        result = RegisterUser(data)
-        self.assertNotEqual(len(result.errors), 0, "Organisation name too short. Error expected")
-
-    def testRegisterUserWithNoPassword(self):
-        data = {
-            "email_address": "user@example.com"
-            , "full_name": "A.N. Other"
-            , "organization": "Some Organisation"
-            , "password": None
-            , "password_confirm": None
-            , "es_connection": self.es
-        }
-
-        result = RegisterUser(data)
-        self.assertNotEqual(len(result.errors), 0, "Password not provided. Error was expected")
-
-    def testRegisterUserWithNonMatchingPasswords(self):
-        data = {
-            "email_address": "user@example.com"
-            , "full_name": "A.N. Other"
-            , "organization": "Some Organisation"
-            , "password": "testing"
-            , "password_confirm": "stilltesting"
-            , "es_connection": self.es
-        }
-
-        result = RegisterUser(data)
-        self.assertNotEqual(len(result.errors), 0, "Password mismatch. Error was expected")
-
-    def testRegisterUserWithCorrectData(self):
-        data = {
-            "email_address": "user@example.com"
-            , "full_name": "A.N. Other"
-            , "organization": "Some Organisation"
-            , "password": "testing"
-            , "password_confirm": "testing"
-            , "es_connection": self.es
-        }
-        result = RegisterUser(data)
-        self.assertEqual(len(result.errors), 0, "All data items provided. Errors were not expected")
-
-if __name__ == "__main__":
-    unittest.main()
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index a964b025..2d3d5add 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -1,10 +1,18 @@
 from elasticsearch import Elasticsearch, TransportError
 import logging
 
+from utility.logger import getLogger
+logger = getLogger(__name__)
+
+from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
+
 def get_elasticsearch_connection():
+    logger.info("get_elasticsearch_connection")
     es = None
     try:
-        from utility.tools import ELASTICSEARCH_HOST, ELASTICSEARCH_PORT
+        assert(ELASTICSEARCH_HOST)
+        assert(ELASTICSEARCH_PORT)
+        logger.info("ES HOST",ELASTICSEARCH_HOST)
 
         es = Elasticsearch([{
             "host": ELASTICSEARCH_HOST
@@ -31,12 +39,12 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
         response = es.search(
             index = index
             , doc_type = doc_type
-            , body = { 
-                "query": { "match": { column_name: column_value } } 
+            , body = {
+                "query": { "match": { column_name: column_value } }
             })
         if len(response["hits"]["hits"]) > 0:
             item_details = response["hits"]["hits"][0]["_source"]
-    except TransportError as te: 
+    except TransportError as te:
         pass
     return item_details
 
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index ec673cf5..8c9fed96 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -16,7 +16,7 @@ OVERRIDES = {}
 def app_set(command_id, value):
     """Set application wide value"""
     app.config.setdefault(command_id,value)
-    value
+    return value
 
 def get_setting(command_id,guess=None):
     """Resolve a setting from the environment or the global settings in
@@ -51,7 +51,7 @@ def get_setting(command_id,guess=None):
             return None
 
     # ---- Check whether environment exists
-    logger.debug("Looking for "+command_id+"\n")
+    # print("Looking for "+command_id+"\n")
     command = value(os.environ.get(command_id))
     if command is None or command == "":
         command = OVERRIDES.get(command_id)
@@ -63,7 +63,7 @@ def get_setting(command_id,guess=None):
                 if command is None or command == "":
                     # print command
                     raise Exception(command_id+' setting unknown or faulty (update default_settings.py?).')
-    logger.debug("Set "+command_id+"="+str(command))
+    # print("Set "+command_id+"="+str(command))
     return command
 
 def get_setting_bool(id):
@@ -251,35 +251,29 @@ assert_dir(JS_GUIX_PATH)
 JS_GN_PATH         = get_setting('JS_GN_PATH')
 # assert_dir(JS_GN_PATH)
 
-def get_setting_safe(setting):
-    try:
-        return get_setting(setting)
-    except:
-        print("Could not find the setting '", setting, "'. Continuing with value unset")
-        return None
-
-GITHUB_CLIENT_ID = get_setting_safe('GITHUB_CLIENT_ID')
-GITHUB_CLIENT_SECRET = get_setting_safe('GITHUB_CLIENT_SECRET')
+GITHUB_CLIENT_ID = get_setting('GITHUB_CLIENT_ID')
+GITHUB_CLIENT_SECRET = get_setting('GITHUB_CLIENT_SECRET')
 GITHUB_AUTH_URL = None
 if GITHUB_CLIENT_ID and GITHUB_CLIENT_SECRET:
     GITHUB_AUTH_URL = "https://github.com/login/oauth/authorize?client_id="+GITHUB_CLIENT_ID+"&client_secret="+GITHUB_CLIENT_SECRET
-GITHUB_API_URL = get_setting_safe('GITHUB_API_URL')
-ORCID_CLIENT_ID = get_setting_safe('ORCID_CLIENT_ID')
-ORCID_CLIENT_SECRET = get_setting_safe('ORCID_CLIENT_SECRET')
+GITHUB_API_URL = get_setting('GITHUB_API_URL')
+ORCID_CLIENT_ID = get_setting('ORCID_CLIENT_ID')
+ORCID_CLIENT_SECRET = get_setting('ORCID_CLIENT_SECRET')
 ORCID_AUTH_URL = None
 if ORCID_CLIENT_ID and ORCID_CLIENT_SECRET:
     ORCID_AUTH_URL = "https://sandbox.orcid.org/oauth/authorize?response_type=code&scope=/authenticate&show_login=true&client_id="+ORCID_CLIENT_ID+"&client_secret="+ORCID_CLIENT_SECRET
-ORCID_TOKEN_URL = get_setting_safe('ORCID_TOKEN_URL')
+ORCID_TOKEN_URL = get_setting('ORCID_TOKEN_URL')
 
-ELASTICSEARCH_HOST = get_setting_safe('ELASTICSEARCH_HOST')
-ELASTICSEARCH_PORT = get_setting_safe('ELASTICSEARCH_PORT')
+ELASTICSEARCH_HOST = get_setting('ELASTICSEARCH_HOST')
+ELASTICSEARCH_PORT = get_setting('ELASTICSEARCH_PORT')
 
-SMTP_CONNECT = get_setting_safe('SMTP_CONNECT')
-SMTP_USERNAME = get_setting_safe('SMTP_USERNAME')
-SMTP_PASSWORD = get_setting_safe('SMTP_PASSWORD')
+SMTP_CONNECT = get_setting('SMTP_CONNECT')
+SMTP_USERNAME = get_setting('SMTP_USERNAME')
+SMTP_PASSWORD = get_setting('SMTP_PASSWORD')
 
 PYLMM_COMMAND      = app_set("PYLMM_COMMAND",pylmm_command())
 GEMMA_COMMAND      = app_set("GEMMA_COMMAND",gemma_command())
+assert(GEMMA_COMMAND is not None)
 PLINK_COMMAND      = app_set("PLINK_COMMAND",plink_command())
 GEMMA_WRAPPER_COMMAND = gemma_wrapper_command()
 TEMPDIR            = tempdir() # defaults to UNIX TMPDIR
@@ -293,7 +287,7 @@ from six import string_types
 
 if os.environ.get('WQFLASK_OVERRIDES'):
     jsonfn = get_setting('WQFLASK_OVERRIDES')
-    logger.error("WQFLASK_OVERRIDES: %s" % jsonfn)
+    logger.info("WQFLASK_OVERRIDES: %s" % jsonfn)
     with open(jsonfn) as data_file:
         overrides = json.load(data_file)
         for k in overrides:
@@ -305,8 +299,4 @@ if os.environ.get('WQFLASK_OVERRIDES'):
             logger.debug(OVERRIDES)
 
 # assert_file(PHEWAS_FILES+"/auwerx/PheWAS_pval_EMMA_norm.RData")
-# assert_dir(get_setting("JS_BIODALLIANCE"))
-# assert_file(get_setting("JS_BIODALLIANCE")+"/build/dalliance-all.js")
-# assert_file(get_setting("JS_BIODALLIANCE")+"/build/worker-all.js")
-# assert_dir(get_setting("JS_TWITTER_POST_FETCHER"))
 assert_file(JS_TWITTER_POST_FETCHER_PATH+"/js/twitterFetcher_min.js")
diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 24432ad0..3d1c0d17 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -75,6 +75,46 @@ def print_mem(stage=""):
     mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
     #print("{}: {}".format(stage, mem/1024))
 
+def is_float(value):
+    try:
+        float(value)
+        return True
+    except:
+        return False
+
+def is_int(value):
+    try:
+        int(value)
+        return True
+    except:
+        return False
+
+def is_str(value):
+    if value is None:
+        return False
+    try:
+        str(value)
+        return True
+    except:
+        return False
+
+def get_float(vars,name,default=None):
+    if name in vars:
+        if is_float(vars[name]):
+            return float(vars[name])
+    return None
+
+def get_int(vars,name,default=None):
+    if name in vars:
+        if is_int(vars[name]):
+            return float(vars[name])
+    return default
+
+def get_string(vars,name,default=None):
+    if name in vars:
+        if not vars[name] is None:
+            return str(vars[name])
+    return default
 
 class AuthException(Exception):
     pass
@@ -96,7 +136,19 @@ class CorrelationResults(object):
         # get trait list from db (database name)
         # calculate correlation with Base vector and targets
 
-        print("TESTING...")
+        # Check parameters
+        assert('corr_type' in start_vars)
+        assert(is_str(start_vars['corr_type']))
+        assert('dataset' in start_vars)
+        # assert('group' in start_vars) permitted to be empty?
+        assert('corr_sample_method' in start_vars)
+        assert('corr_samples_group' in start_vars)
+        assert('corr_dataset' in start_vars)
+        assert('min_expr' in start_vars)
+        assert('corr_return_results' in start_vars)
+        if 'loc_chr' in start_vars:
+            assert('min_loc_mb' in start_vars)
+            assert('max_loc_mb' in start_vars)
 
         with Bench("Doing correlations"):
             if start_vars['dataset'] == "Temp":
@@ -115,27 +167,17 @@ class CorrelationResults(object):
             self.sample_data = {}
             self.corr_type = start_vars['corr_type']
             self.corr_method = start_vars['corr_sample_method']
-            if 'min_expr' in start_vars:
-                if start_vars['min_expr'] != "":
-                    self.min_expr = float(start_vars['min_expr'])
-                else:
-                    self.min_expr = None
-            self.p_range_lower = float(start_vars['p_range_lower'])
-            self.p_range_upper = float(start_vars['p_range_upper'])
+            self.min_expr = get_float(start_vars,'min_expr')
+            self.p_range_lower = get_float(start_vars,'p_range_lower',-1.0)
+            self.p_range_upper = get_float(start_vars,'p_range_upper',1.0)
 
             if ('loc_chr' in start_vars and
                 'min_loc_mb' in start_vars and
                 'max_loc_mb' in start_vars):
 
-                self.location_chr = start_vars['loc_chr']
-                if start_vars['min_loc_mb'].isdigit():
-                    self.min_location_mb = start_vars['min_loc_mb']
-                else:
-                    self.min_location_mb = None
-                if start_vars['max_loc_mb'].isdigit():
-                    self.max_location_mb = start_vars['max_loc_mb']
-                else:
-                    self.max_location_mb = None
+                self.location_chr = get_string(start_vars,'loc_chr')
+                self.min_location_mb = get_int(start_vars,'min_loc_mb')
+                self.max_location_mb = get_int(start_vars,'max_loc_mb')
 
             self.get_formatted_corr_type()
             self.return_number = int(start_vars['corr_return_results'])
@@ -183,7 +225,7 @@ class CorrelationResults(object):
                 else:
                     for trait, values in self.target_dataset.trait_data.iteritems():
                         self.get_sample_r_and_p_values(trait, values)
-                        
+
             elif self.corr_type == "lit":
                 self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId")
                 lit_corr_data = self.do_lit_correlation_for_all_traits()
@@ -564,7 +606,7 @@ class CorrelationResults(object):
                 self.this_trait_vals.append(sample_value)
                 target_vals.append(target_sample_value)
 
-        self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals)	
+        self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals)
 
         #ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/
         if self.corr_method == 'pearson':
@@ -574,8 +616,8 @@ class CorrelationResults(object):
 
         if num_overlap > 5:
             self.correlation_data[trait] = [sample_r, sample_p, num_overlap]
-		
-		
+
+
         """
         correlations = []
 
@@ -673,8 +715,8 @@ class CorrelationResults(object):
                         method=self.method)
 
         return trait_list
-        """		
-		
+        """
+
 
     def do_tissue_corr_for_all_traits_2(self):
         """Comments Possibly Out of Date!!!!!
@@ -1089,7 +1131,7 @@ class CorrelationResults(object):
             totalTraits = len(traits) #XZ, 09/18/2008: total trait number
 
         return traits
-			
+
     def calculate_corr_for_all_tissues(self, tissue_dataset_id=None):
 
         symbol_corr_dict = {}
@@ -1129,7 +1171,7 @@ class CorrelationResults(object):
                     values_2.append(target_value)
             correlation = calCorrelation(values_1, values_2)
             self.correlation_data[trait] = correlation
-			
+
     def getFileName(self, target_db_name):  ### dcrowell  August 2008
         """Returns the name of the reference database file with which correlations are calculated.
         Takes argument cursor which is a cursor object of any instance of a subclass of templatePage
@@ -1144,7 +1186,7 @@ class CorrelationResults(object):
         return FileName
 
     def do_parallel_correlation(self, db_filename, num_overlap):
-	
+
         #XZ, 01/14/2009: This method is for parallel computing only.
         #XZ: It is supposed to be called when "Genetic Correlation, Pearson's r" (method 1)
         #XZ: or "Genetic Correlation, Spearman's rho" (method 2) is selected
@@ -1313,7 +1355,7 @@ class CorrelationResults(object):
                         z_value = z_value*math.sqrt(nOverlap-3)
                         sample_p = 2.0*(1.0 - reaper.normp(abs(z_value)))
 
-                correlation_data[traitdataName] = [sample_r, sample_p, nOverlap]	
+                correlation_data[traitdataName] = [sample_r, sample_p, nOverlap]
 
                 # traitinfo = [traitdataName, sample_r, nOverlap]
                 # allcorrelations.append(traitinfo)
@@ -1321,7 +1363,7 @@ class CorrelationResults(object):
             return correlation_data
             # return allcorrelations
 
-	
+
         datasetFile = open(webqtlConfig.GENERATED_TEXT_DIR+db_filename,'r')
 
         print("Invoking parallel computing")
@@ -1378,5 +1420,3 @@ class CorrelationResults(object):
         # for one_result in results:
             # for one_traitinfo in one_result:
                 # allcorrelations.append( one_traitinfo )
-
-
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index a24e43d4..68920130 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -3,7 +3,7 @@ import os, math, string, random, json
 from base import webqtlConfig
 from base.trait import GeneralTrait
 from base.data_set import create_dataset
-from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR
+from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin, assert_file
 
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
@@ -11,6 +11,7 @@ logger = utility.logger.getLogger(__name__ )
 def run_gemma(this_dataset, samples, vals, covariates, method, use_loco):
     """Generates p-values for each marker using GEMMA"""
 
+    assert_bin(GEMMA_COMMAND);
     if this_dataset.group.genofile != None:
         genofile_name = this_dataset.group.genofile[:-5]
     else:
@@ -27,7 +28,7 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco):
         if i < (len(this_chromosomes) - 1):
             chr_list_string += this_chromosomes[i+1].name + ","
         else:
-            chr_list_string += this_chromosomes[i+1].name  
+            chr_list_string += this_chromosomes[i+1].name
 
     if covariates != "":
         gen_covariates_file(this_dataset, covariates)
@@ -209,8 +210,13 @@ def parse_gemma_output(genofile_name):
 def parse_loco_output(this_dataset, gwa_output_filename):
 
     output_filelist = []
-    with open("{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json") as data_file:
-       data = json.load(data_file)
+    jsonfn = "{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json"
+    assert_file(jsonfn)
+    try:
+        with open(jsonfn) as data_file:
+            data = json.load(data_file)
+    except:
+        logger.error("Can not parse "+jsonfn)
 
     files = data['files']
     for file in files:
@@ -247,4 +253,4 @@ def parse_loco_output(this_dataset, gwa_output_filename):
                     included_markers.append(line.split("\t")[1])
                     p_values.append(float(line.split("\t")[10]))
 
-    return marker_obs
\ No newline at end of file
+    return marker_obs
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 6b667615..c8471cb1 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -55,9 +55,8 @@ logger = getLogger(__name__)
 from base.data_set import create_datasets_list
 
 import requests
-from utility.elasticsearch_tools import *
+from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user
 
-es = get_elasticsearch_connection()
 THREE_DAYS = 60 * 60 * 24 * 3
 #THREE_DAYS = 45
 
@@ -479,6 +478,7 @@ def password_reset_step2():
     password = request.form['password']
     set_password(password, user)
 
+    es = get_elasticsearch_connection()
     es.update(
         index = "users"
         , doc_type = "local"
@@ -620,6 +620,7 @@ class LoginUser(object):
         """Login through the normal form"""
         params = request.form if request.form else request.args
         logger.debug("in login params are:", params)
+        es = get_elasticsearch_connection()
         if not params:
             from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL
             external_login = None
@@ -628,6 +629,7 @@ class LoginUser(object):
                     "github": GITHUB_AUTH_URL,
                     "orcid": ORCID_AUTH_URL
                 }
+            assert(es is not None)
             return render_template(
                 "new_security/login_user.html"
                 , external_login=external_login
@@ -822,6 +824,7 @@ def register():
 
     params = request.form if request.form else request.args
     params = params.to_dict(flat=True)
+    es = get_elasticsearch_connection()
     params["es_connection"] = es
 
     if params: