Merge branch 'pjotrp-gn-testing-es' into testing

3 files changed, 86 insertions, 37 deletions

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 24432ad0..3d1c0d17 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -75,6 +75,46 @@ def print_mem(stage=""):
     mem = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
     #print("{}: {}".format(stage, mem/1024))
 
+def is_float(value):
+    try:
+        float(value)
+        return True
+    except:
+        return False
+
+def is_int(value):
+    try:
+        int(value)
+        return True
+    except:
+        return False
+
+def is_str(value):
+    if value is None:
+        return False
+    try:
+        str(value)
+        return True
+    except:
+        return False
+
+def get_float(vars,name,default=None):
+    if name in vars:
+        if is_float(vars[name]):
+            return float(vars[name])
+    return None
+
+def get_int(vars,name,default=None):
+    if name in vars:
+        if is_int(vars[name]):
+            return float(vars[name])
+    return default
+
+def get_string(vars,name,default=None):
+    if name in vars:
+        if not vars[name] is None:
+            return str(vars[name])
+    return default
 
 class AuthException(Exception):
     pass
@@ -96,7 +136,19 @@ class CorrelationResults(object):
         # get trait list from db (database name)
         # calculate correlation with Base vector and targets
 
-        print("TESTING...")
+        # Check parameters
+        assert('corr_type' in start_vars)
+        assert(is_str(start_vars['corr_type']))
+        assert('dataset' in start_vars)
+        # assert('group' in start_vars) permitted to be empty?
+        assert('corr_sample_method' in start_vars)
+        assert('corr_samples_group' in start_vars)
+        assert('corr_dataset' in start_vars)
+        assert('min_expr' in start_vars)
+        assert('corr_return_results' in start_vars)
+        if 'loc_chr' in start_vars:
+            assert('min_loc_mb' in start_vars)
+            assert('max_loc_mb' in start_vars)
 
         with Bench("Doing correlations"):
             if start_vars['dataset'] == "Temp":
@@ -115,27 +167,17 @@ class CorrelationResults(object):
             self.sample_data = {}
             self.corr_type = start_vars['corr_type']
             self.corr_method = start_vars['corr_sample_method']
-            if 'min_expr' in start_vars:
-                if start_vars['min_expr'] != "":
-                    self.min_expr = float(start_vars['min_expr'])
-                else:
-                    self.min_expr = None
-            self.p_range_lower = float(start_vars['p_range_lower'])
-            self.p_range_upper = float(start_vars['p_range_upper'])
+            self.min_expr = get_float(start_vars,'min_expr')
+            self.p_range_lower = get_float(start_vars,'p_range_lower',-1.0)
+            self.p_range_upper = get_float(start_vars,'p_range_upper',1.0)
 
             if ('loc_chr' in start_vars and
                 'min_loc_mb' in start_vars and
                 'max_loc_mb' in start_vars):
 
-                self.location_chr = start_vars['loc_chr']
-                if start_vars['min_loc_mb'].isdigit():
-                    self.min_location_mb = start_vars['min_loc_mb']
-                else:
-                    self.min_location_mb = None
-                if start_vars['max_loc_mb'].isdigit():
-                    self.max_location_mb = start_vars['max_loc_mb']
-                else:
-                    self.max_location_mb = None
+                self.location_chr = get_string(start_vars,'loc_chr')
+                self.min_location_mb = get_int(start_vars,'min_loc_mb')
+                self.max_location_mb = get_int(start_vars,'max_loc_mb')
 
             self.get_formatted_corr_type()
             self.return_number = int(start_vars['corr_return_results'])
@@ -183,7 +225,7 @@ class CorrelationResults(object):
                 else:
                     for trait, values in self.target_dataset.trait_data.iteritems():
                         self.get_sample_r_and_p_values(trait, values)
-                        
+
             elif self.corr_type == "lit":
                 self.trait_geneid_dict = self.dataset.retrieve_genes("GeneId")
                 lit_corr_data = self.do_lit_correlation_for_all_traits()
@@ -564,7 +606,7 @@ class CorrelationResults(object):
                 self.this_trait_vals.append(sample_value)
                 target_vals.append(target_sample_value)
 
-        self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals)	
+        self.this_trait_vals, target_vals, num_overlap = corr_result_helpers.normalize_values(self.this_trait_vals, target_vals)
 
         #ZS: 2015 could add biweight correlation, see http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3465711/
         if self.corr_method == 'pearson':
@@ -574,8 +616,8 @@ class CorrelationResults(object):
 
         if num_overlap > 5:
             self.correlation_data[trait] = [sample_r, sample_p, num_overlap]
-		
-		
+
+
         """
         correlations = []
 
@@ -673,8 +715,8 @@ class CorrelationResults(object):
                         method=self.method)
 
         return trait_list
-        """		
-		
+        """
+
 
     def do_tissue_corr_for_all_traits_2(self):
         """Comments Possibly Out of Date!!!!!
@@ -1089,7 +1131,7 @@ class CorrelationResults(object):
             totalTraits = len(traits) #XZ, 09/18/2008: total trait number
 
         return traits
-			
+
     def calculate_corr_for_all_tissues(self, tissue_dataset_id=None):
 
         symbol_corr_dict = {}
@@ -1129,7 +1171,7 @@ class CorrelationResults(object):
                     values_2.append(target_value)
             correlation = calCorrelation(values_1, values_2)
             self.correlation_data[trait] = correlation
-			
+
     def getFileName(self, target_db_name):  ### dcrowell  August 2008
         """Returns the name of the reference database file with which correlations are calculated.
         Takes argument cursor which is a cursor object of any instance of a subclass of templatePage
@@ -1144,7 +1186,7 @@ class CorrelationResults(object):
         return FileName
 
     def do_parallel_correlation(self, db_filename, num_overlap):
-	
+
         #XZ, 01/14/2009: This method is for parallel computing only.
         #XZ: It is supposed to be called when "Genetic Correlation, Pearson's r" (method 1)
         #XZ: or "Genetic Correlation, Spearman's rho" (method 2) is selected
@@ -1313,7 +1355,7 @@ class CorrelationResults(object):
                         z_value = z_value*math.sqrt(nOverlap-3)
                         sample_p = 2.0*(1.0 - reaper.normp(abs(z_value)))
 
-                correlation_data[traitdataName] = [sample_r, sample_p, nOverlap]	
+                correlation_data[traitdataName] = [sample_r, sample_p, nOverlap]
 
                 # traitinfo = [traitdataName, sample_r, nOverlap]
                 # allcorrelations.append(traitinfo)
@@ -1321,7 +1363,7 @@ class CorrelationResults(object):
             return correlation_data
             # return allcorrelations
 
-	
+
         datasetFile = open(webqtlConfig.GENERATED_TEXT_DIR+db_filename,'r')
 
         print("Invoking parallel computing")
@@ -1378,5 +1420,3 @@ class CorrelationResults(object):
         # for one_result in results:
             # for one_traitinfo in one_result:
                 # allcorrelations.append( one_traitinfo )
-
-
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index a24e43d4..68920130 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -3,7 +3,7 @@ import os, math, string, random, json
 from base import webqtlConfig
 from base.trait import GeneralTrait
 from base.data_set import create_dataset
-from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR
+from utility.tools import flat_files, GEMMA_COMMAND, GEMMA_WRAPPER_COMMAND, TEMPDIR, assert_bin, assert_file
 
 import utility.logger
 logger = utility.logger.getLogger(__name__ )
@@ -11,6 +11,7 @@ logger = utility.logger.getLogger(__name__ )
 def run_gemma(this_dataset, samples, vals, covariates, method, use_loco):
     """Generates p-values for each marker using GEMMA"""
 
+    assert_bin(GEMMA_COMMAND);
     if this_dataset.group.genofile != None:
         genofile_name = this_dataset.group.genofile[:-5]
     else:
@@ -27,7 +28,7 @@ def run_gemma(this_dataset, samples, vals, covariates, method, use_loco):
         if i < (len(this_chromosomes) - 1):
             chr_list_string += this_chromosomes[i+1].name + ","
         else:
-            chr_list_string += this_chromosomes[i+1].name  
+            chr_list_string += this_chromosomes[i+1].name
 
     if covariates != "":
         gen_covariates_file(this_dataset, covariates)
@@ -209,8 +210,13 @@ def parse_gemma_output(genofile_name):
 def parse_loco_output(this_dataset, gwa_output_filename):
 
     output_filelist = []
-    with open("{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json") as data_file:
-       data = json.load(data_file)
+    jsonfn = "{}/gn2/".format(TEMPDIR) + gwa_output_filename + ".json"
+    assert_file(jsonfn)
+    try:
+        with open(jsonfn) as data_file:
+            data = json.load(data_file)
+    except:
+        logger.error("Can not parse "+jsonfn)
 
     files = data['files']
     for file in files:
@@ -247,4 +253,4 @@ def parse_loco_output(this_dataset, gwa_output_filename):
                     included_markers.append(line.split("\t")[1])
                     p_values.append(float(line.split("\t")[10]))
 
-    return marker_obs
\ No newline at end of file
+    return marker_obs
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 6b667615..c8471cb1 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -55,9 +55,8 @@ logger = getLogger(__name__)
 from base.data_set import create_datasets_list
 
 import requests
-from utility.elasticsearch_tools import *
+from utility.elasticsearch_tools import get_elasticsearch_connection, get_user_by_unique_column, save_user
 
-es = get_elasticsearch_connection()
 THREE_DAYS = 60 * 60 * 24 * 3
 #THREE_DAYS = 45
 
@@ -479,6 +478,7 @@ def password_reset_step2():
     password = request.form['password']
     set_password(password, user)
 
+    es = get_elasticsearch_connection()
     es.update(
         index = "users"
         , doc_type = "local"
@@ -620,6 +620,7 @@ class LoginUser(object):
         """Login through the normal form"""
         params = request.form if request.form else request.args
         logger.debug("in login params are:", params)
+        es = get_elasticsearch_connection()
         if not params:
             from utility.tools import GITHUB_AUTH_URL, ORCID_AUTH_URL
             external_login = None
@@ -628,6 +629,7 @@ class LoginUser(object):
                     "github": GITHUB_AUTH_URL,
                     "orcid": ORCID_AUTH_URL
                 }
+            assert(es is not None)
             return render_template(
                 "new_security/login_user.html"
                 , external_login=external_login
@@ -822,6 +824,7 @@ def register():
 
     params = request.form if request.form else request.args
     params = params.to_dict(flat=True)
+    es = get_elasticsearch_connection()
     params["es_connection"] = es
 
     if params: