Merge pull request #303 from pjotrp/testing

Minor edits and document on ES

4 files changed, 67 insertions, 31 deletions

diff --git a/bin/genenetwork2 b/bin/genenetwork2
index 3f06e7f9..18e02388 100755
--- a/bin/genenetwork2
+++ b/bin/genenetwork2
@@ -21,10 +21,18 @@
 #
 #   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_settings.py
 #
-# To run a maintenance script with settings (instead of the webserver) add that with
-# a -c switch, e.g.
+# To run a maintenance python script with settings (instead of the
+# webserver) add that with a -c switch, e.g.
 #
-#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ~/my_overrides.json -c ./wqflask/maintenance/gen_select_dataset.py
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 -c ./wqflask/maintenance/gen_select_dataset.py
+#
+# To run any script in the environment
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -cli echo "HELLO WORLD"
+#
+# To get a python REPL(!)
+#
+#   env GN2_PROFILE=~/opt/gn-latest-guix ./bin/genenetwork2 ./etc/default_settings.py -cli python
 #
 # For development you may want to run
 #
@@ -114,7 +122,6 @@ else
     export PATH=$GN2_PROFILE/bin:$PATH
     export PYTHONPATH="$GN2_PROFILE/lib/python2.7/site-packages" # never inject another PYTHONPATH!!
     export R_LIBS_SITE=$GN2_PROFILE/site-library
-    export GEM_PATH=$GN2_PROFILE/lib/ruby/gems/2.4.0
     export JS_GUIX_PATH=$GN2_PROFILE/share/genenetwork2/javascript
     export GUIX_GTK3_PATH="$GN2_PROFILE/lib/gtk-3.0"
     export GI_TYPELIB_PATH="$GN2_PROFILE/lib/girepository-1.0"
@@ -134,7 +141,6 @@ else
 	done
     done <<< "$PYTHONPATH"
     if [ ! -d $R_LIBS_SITE ] ; then echo "R_LIBS_SITE not valid "$R_LIBS_SITE ; exit 1 ; fi
-    if [ ! -d $GEM_PATH ] ; then echo "GEM_PATH not valid "$GEM_PATH ; exit 1 ; fi
 fi
 if [ -z $PYTHONPATH ] ; then
     echo "ERROR PYTHONPATH has not been set - use GN2_PROFILE!"
@@ -170,9 +176,9 @@ if [ "$1" = '-c' ] ; then
     python $cmd $*
     exit $?
 fi
+
 # Now handle command parameter -cli which runs in bash
 if [ "$1" = "-cli" ] ; then
-    echo "HERE"
     cd $GN2_BASE_DIR/wqflask
     cmd=$2
     echo PYTHONPATH=$PYTHONPATH
diff --git a/doc/elasticsearch.org b/doc/elasticsearch.org
new file mode 100644
index 00000000..18adfc8b
--- /dev/null
+++ b/doc/elasticsearch.org
@@ -0,0 +1,41 @@
+* Elasticsearch
+
+To get the right environment, first you can get a python REPL with something like
+
+: env GN2_PROFILE=~/opt/gn-latest ./bin/genenetwork2 ../etc/default_settings.py -cli python
+
+(make sure to use the correct GN2_PROFILE!)
+
+Next try
+
+#+BEGIN_SRC python
+
+from elasticsearch import Elasticsearch, TransportError
+
+es = Elasticsearch([{ "host": 'localhost', "port": '9200' }])
+
+# Dump all data
+
+es.search("*")
+
+# To fetch an E-mail record from the users index
+
+record = es.search(
+            index = 'users', doc_type = 'local', body = {
+                "query": { "match": { "email_address": "myname@email.com" } }
+            })
+
+# It is also possible to do wild card matching
+
+q = { "query": { "wildcard" : { "full_name" : "pjot*" } }}
+es.search(index = 'users', doc_type = 'local', body = q)
+
+# To get elements from that record:
+
+record['hits']['hits'][0][u'_source']['full_name']
+u'Pjotr'
+
+record['hits']['hits'][0][u'_source']['email_address']
+u"myname@email.com"
+
+#+END_SRC
diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index 734379f7..1dba357d 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -12,6 +12,7 @@ def test_elasticsearch_connection():
         logger.warning("Elasticsearch is DOWN")
 
 def get_elasticsearch_connection():
+    """Return a connection to ES. Returns None on failure"""
     logger.info("get_elasticsearch_connection")
     es = None
     try:
@@ -20,14 +21,14 @@ def get_elasticsearch_connection():
         logger.info("ES HOST",ELASTICSEARCH_HOST)
 
         es = Elasticsearch([{
-            "host": ELASTICSEARCH_HOST
-            , "port": ELASTICSEARCH_PORT
+            "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
         }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
 
         es_logger = logging.getLogger("elasticsearch")
         es_logger.setLevel(logging.INFO)
         es_logger.addHandler(logging.NullHandler())
     except:
+        logger.error("Failed to get elasticsearch connection")
         es = None
 
     return es
@@ -42,9 +43,7 @@ def get_item_by_unique_column(es, column_name, column_value, index, doc_type):
     item_details = None
     try:
         response = es.search(
-            index = index
-            , doc_type = doc_type
-            , body = {
+            index = index, doc_type = doc_type, body = {
                 "query": { "match": { column_name: column_value } }
             })
         if len(response["hits"]["hits"]) > 0:
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index ac3824a7..5f6c818e 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -1,45 +1,30 @@
 from __future__ import print_function, division, absolute_import
 
-"""Used to Access things in template like this:
-(BUT NOW OUT OF DATE)
-
-    x: {{ g.identity.name }}
-    security: {{ security.__dict__ }}
-
-"""
-
 import os
 import hashlib
 import datetime
 import time
 import logging
-
 import uuid
 import hashlib
 import hmac
 import base64
-
 import urlparse
 
 import simplejson as json
 
 #from redis import StrictRedis
-import redis
+import redis # used for collections
 Redis = redis.StrictRedis()
 
-
 from flask import (Flask, g, render_template, url_for, request, make_response,
                    redirect, flash, abort)
 
 from wqflask import app
-
-
 from pprint import pformat as pf
 
-from wqflask import pbkdf2
-
+from wqflask import pbkdf2 # password hashing
 from wqflask.database import db_session
-
 from wqflask import model
 
 from utility import Bunch, Struct, after
@@ -62,8 +47,8 @@ THREE_DAYS = 60 * 60 * 24 * 3
 def timestamp():
     return datetime.datetime.utcnow().isoformat()
 
-
 class AnonUser(object):
+    """Anonymous user handling"""
     cookie_name = 'anon_user_v8'
 
     def __init__(self):
@@ -169,6 +154,8 @@ def create_signed_cookie():
     return the_uuid, uuid_signed
 
 class UserSession(object):
+    """Logged in user handling"""
+
     cookie_name = 'session_id_v2'
 
     def __init__(self):
@@ -437,6 +424,7 @@ def verify_email():
 
 @app.route("/n/password_reset", methods=['GET'])
 def password_reset():
+    """Entry point after user clicks link in E-mail"""
     logger.debug("in password_reset request.url is:", request.url)
     # We do this mainly just to assert that it's in proper form for displaying next page
     # Really not necessary but doesn't hurt
@@ -467,6 +455,7 @@ def password_reset():
 
 @app.route("/n/password_reset_step2", methods=('POST',))
 def password_reset_step2():
+    """Handle confirmation E-mail for password reset"""
     logger.debug("in password_reset request.url is:", request.url)
 
     errors = []
@@ -663,8 +652,6 @@ class LoginUser(object):
             VerificationEmail(user)
             return render_template("new_security/verification_still_needed.html",
                                    subject=VerificationEmail.subject)
-
-
         if valid:
             if params.get('remember'):
                 logger.debug("I will remember you")
@@ -742,12 +729,15 @@ def logout():
 
 @app.route("/n/forgot_password")
 def forgot_password():
+    """Entry point for forgotten password"""
     return render_template("new_security/forgot_password.html")
 
 @app.route("/n/forgot_password_submit", methods=('POST',))
 def forgot_password_submit():
+    """When a forgotten password form is submitted we get here"""
     params = request.form
     email_address = params['email_address']
+    logger.debug("Wants to send password E-mail to ",email_address)
     es = get_elasticsearch_connection()
     user_details = get_user_by_unique_column(es, "email_address", email_address)
     if user_details: