Introduce behaviour parameters to facilitate development

4 files changed, 65 insertions, 39 deletions

diff --git a/etc/default_settings.py b/etc/default_settings.py
index 07ba2e48..bc464861 100644
--- a/etc/default_settings.py
+++ b/etc/default_settings.py
@@ -1,3 +1,13 @@
+# Default settings file defines a single Flask process for the Python
+# webserver running in developer mode with limited console
+# output. Copy this file and run it from ./bin/genenetwork2 configfile
+#
+# Note that these settings are fetched in ./wqflask/utilities/tools.py
+# which has support for overriding them through environment variables,
+# e.g.
+#
+#   env LOG_SQL=True USE_REDIS=False ./bin/genenetwork2 
+
 import os
 import sys
 
@@ -23,10 +33,13 @@ SQLALCHEMY_POOL_RECYCLE = 3600
 SERVER_PORT = 5003
 SECRET_HMAC_CODE = '\x08\xdf\xfa\x93N\x80\xd9\\H@\\\x9f`\x98d^\xb4a;\xc6OM\x946a\xbc\xfc\x80:*\xebc'
 
-# Other settings (defaults)
-LOGGING        = WARNING        # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL)
-DEBUG_MODE     = 1              # Debug level (0-5)
-USE_REDIS      = True           # REDIS caching (redis will be phased out)
+# Behavioural settings (defaults) note that logger and log levels can
+# be overridden at the module level
+WEBSERVER_MODE  = 'DEV'     # Python webserver mode (DEBUG|DEV|PROD)
+LOGGING         = 'WARNING' # Logger mode (DEBUG|INFO|WARNING|ERROR|CRITICAL)
+DEBUG_LOG_LEVEL = 1         # Debug log level (0-5)
+LOG_SQL         = False     # Log SQL/backend calls
+USE_REDIS       = True      # REDIS caching (note that redis will be phased out)
 
 # Path overrides for Genenetwork
 GENENETWORK_FILES = HOME+"/gn2_data"
diff --git a/wqflask/utility/tools.py b/wqflask/utility/tools.py
index dd8c4a1e..955f3bdd 100644
--- a/wqflask/utility/tools.py
+++ b/wqflask/utility/tools.py
@@ -7,12 +7,12 @@ from wqflask import app
 
 def get_setting(command_id,guess=None):
     """Resolve a setting from the environment or the global settings in
-    app.config, with get_valid_path is a function checking whether the
+    app.config, with valid_path is a function checking whether the
     path points to an expected directory and returns the full path to
     the binary command
 
       guess = os.environ.get('HOME')+'/pylmm'
-      get_setting('PYLMM_PATH',guess)
+      valid_path(get_setting('PYLMM_PATH',guess))
 
     first tries the environment variable in +id+, next gets the Flask
     app setting for the same +id+ and finally does an educated
@@ -31,13 +31,13 @@ def get_setting(command_id,guess=None):
     """
     def value(command):
         if command:
-            sys.stderr.write("Found path "+command+"\n")
+            # sys.stderr.write("Found "+command+"\n")
             return command
         else:
             return None
     
     # ---- Check whether environment exists
-    sys.stderr.write("Looking for "+command_id+"\n")
+    # sys.stderr.write("Looking for "+command_id+"\n")
     command = value(os.environ.get(command_id))
     if not command:
         # ---- Check whether setting exists in app
@@ -45,7 +45,8 @@ def get_setting(command_id,guess=None):
         if not command:
             command = value(guess)
             if not command:
-                raise Exception(command_id+' path unknown or faulty (update settings.py?). '+command_id+' should point to the path')
+                raise Exception(command_id+' setting unknown or faulty (update settings.py?).')
+    sys.stderr.write("Set "+command_id+"="+str(command)+"\n")
     return command
 
 def valid_bin(bin):
@@ -127,11 +128,17 @@ def locate_ignore_error(name, subdir=None):
 
 def tempdir():
     return valid_path(get_setting("TEMPDIR","/tmp"))
-
     
 # Cached values
-PYLMM_COMMAND = pylmm_command()
-GEMMA_COMMAND = gemma_command()
-PLINK_COMMAND = plink_command()
-FLAT_FILES    = flat_files()
-TEMPDIR       = tempdir()
+WEBSERVER_MODE     = get_setting('WEBSERVER_MODE')
+LOGGING            = get_setting('LOGGING')
+DEBUG_LOG_LEVEL    = get_setting('DEBUG_LOG_LEVEL')
+LOG_SQL            = get_setting('LOG_SQL') in [True,'TRUE','True','true']
+USE_REDIS          = get_setting('USE_REDIS') in [True,'TRUE','True','true']
+
+PYLMM_COMMAND      = pylmm_command()
+GEMMA_COMMAND      = gemma_command()
+PLINK_COMMAND      = plink_command()
+FLAT_FILES         = flat_files()
+TEMPDIR            = tempdir()
+
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 8a0485b3..39f6d62c 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -31,7 +31,7 @@ from base.data_set import create_dataset
 from base.trait import GeneralTrait
 from wqflask import parser
 from wqflask import do_search
-from utility import webqtlUtil
+from utility import webqtlUtil,tools
 from dbFunction import webqtlDatabaseFunction
 
 from utility import formatting
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index df1f77bc..ba250d4f 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -52,7 +52,7 @@ from wqflask.wgcna import wgcna_analysis
 from wqflask.ctl import ctl_analysis
 
 from utility import temp_data
-from utility.tools import TEMPDIR
+from utility.tools import TEMPDIR,USE_REDIS
 
 from base import webqtlFormData
 from base.webqtlConfig import GENERATED_IMAGE_DIR
@@ -138,11 +138,16 @@ def search_page():
     else:
         key = "search_results:v1:" + json.dumps(request.args, sort_keys=True)
         print("key is:", pf(key))
-        with Bench("Loading cache"):
-            result = Redis.get(key)
+        if USE_REDIS:
+            with Bench("Trying Redis cache"):
+                result = Redis.get(key)
+        else:
+            print("Skipping Redis cache (USE_REDIS=False)")
+            result = None
 
         if result:
-            print("Cache hit!!!")
+            print("Cache hit on search results!!!")
+            print("USE_REDIS=",USE_REDIS)
             with Bench("Loading results"):
                 result = pickle.loads(result)
         else:
@@ -152,8 +157,9 @@ def search_page():
             result = the_search.__dict__
 
             print("result: ", pf(result))
-            Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
-            Redis.expire(key, 60*60)
+            if USE_REDIS:
+                Redis.set(key, pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
+                Redis.expire(key, 60*60)
 
         if result['search_term_exists']:
             return render_template("search_result_page.html", **result)
@@ -168,7 +174,7 @@ def gsearchact():
         return render_template("gsearch_gene.html", **result)
     elif type == "phenotype":
         return render_template("gsearch_pheno.html", **result)
-        
+
 @app.route("/gsearch_updating", methods=('POST',))
 def gsearch_updating():
     print("REQUEST ARGS:", request.values)
@@ -179,7 +185,7 @@ def gsearch_updating():
         # return render_template("gsearch_gene_updating.html", **result)
     # elif type == "phenotype":
         # return render_template("gsearch_pheno.html", **result)
-	
+
 @app.route("/docedit")
 def docedit():
     doc = docs.Docs(request.args['entry'])
@@ -294,13 +300,13 @@ def export_trait_csv():
     return Response(csv_data,
                     mimetype='text/csv',
                     headers={"Content-Disposition":"attachment;filename=sample_data.csv"})
-                    
+
 @app.route('/export_perm_data', methods=('POST',))
 def export_perm_data():
     """CSV file consisting of the permutation data for the mapping results"""
     num_perm = float(request.form['num_perm'])
     perm_data = json.loads(request.form['perm_results'])
-    
+
     buff = StringIO.StringIO()
     writer = csv.writer(buff)
     writer.writerow(["Suggestive LRS (p=0.63) = " + str(perm_data[int(num_perm*0.37-1)])])
@@ -338,10 +344,10 @@ def show_trait_page():
 @app.route("/heatmap", methods=('POST',))
 def heatmap_page():
     print("In heatmap, request.form is:", pf(request.form))
-    
+
     start_vars = request.form
     temp_uuid = uuid.uuid4()
-    
+
     traits = [trait.strip() for trait in start_vars['trait_list'].split(',')]
     if traits[0] != "":
         version = "v5"
@@ -349,33 +355,33 @@ def heatmap_page():
         print("key is:", pf(key))
         with Bench("Loading cache"):
             result = Redis.get(key)
-        
+
         if result:
             print("Cache hit!!!")
             with Bench("Loading results"):
                 result = pickle.loads(result)
-    
+
         else:
             print("Cache miss!!!")
-    
+
             template_vars = heatmap.Heatmap(request.form, temp_uuid)
             template_vars.js_data = json.dumps(template_vars.js_data,
                                                default=json_default_handler,
                                                indent="   ")
-        
+
             result = template_vars.__dict__
 
             for item in template_vars.__dict__.keys():
                 print("  ---**--- {}: {}".format(type(template_vars.__dict__[item]), item))
-    
+
             pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
             print("pickled result length:", len(pickled_result))
             Redis.set(key, pickled_result)
             Redis.expire(key, 60*60)
-    
+
         with Bench("Rendering template"):
             rendered_template = render_template("heatmap.html", **result)
-     
+
     else:
         rendered_template = render_template("empty_collection.html", **{'tool':'Heatmap'})
 
@@ -468,18 +474,18 @@ def marker_regression_page():
                 imgB64 = imgdata.encode("base64")
                 bytesarray = array.array('B', imgB64)
                 result['pair_scan_array'] = bytesarray
-                rendered_template = render_template("pair_scan_results.html", **result)        
+                rendered_template = render_template("pair_scan_results.html", **result)
         else:
             #for item in template_vars.__dict__.keys():
             #    print("  ---**--- {}: {}".format(type(template_vars.__dict__[item]), item))
-            
+
             gn1_template_vars = marker_regression_gn1.MarkerRegression(result).__dict__
 
             pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
             print("pickled result length:", len(pickled_result))
             Redis.set(key, pickled_result)
             Redis.expire(key, 1*60)
-            
+
             with Bench("Rendering template"):
                 rendered_template = render_template("marker_regression_gn1.html", **gn1_template_vars)
 
@@ -542,7 +548,7 @@ def corr_matrix_page():
         template_vars.js_data = json.dumps(template_vars.js_data,
                                            default=json_default_handler,
                                            indent="   ")
-    
+
         return render_template("correlation_matrix.html", **template_vars.__dict__)
     else:
         return render_template("empty_collection.html", **{'tool':'Correlation Matrix'})