Added option to transform/normalize trait sample values (log2 and qnorm) on trait page

Got figures mostly working with transformed values, except for changing the y-axis range of one

Fixed issue with the home page "Make Default" button position (as well as the drop-downs in general)

Added timeout to Elasticsearch connection to fix related error

10 files changed, 147 insertions, 39 deletions

diff --git a/wqflask/utility/elasticsearch_tools.py b/wqflask/utility/elasticsearch_tools.py
index 293a9ae6..15cdd0bc 100644
--- a/wqflask/utility/elasticsearch_tools.py
+++ b/wqflask/utility/elasticsearch_tools.py
@@ -63,7 +63,7 @@ def get_elasticsearch_connection(for_user=True):
 
         es = Elasticsearch([{
             "host": ELASTICSEARCH_HOST, "port": ELASTICSEARCH_PORT
-        }]) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
+        }], timeout=30, retry_on_timeout=True) if (ELASTICSEARCH_HOST and ELASTICSEARCH_PORT) else None
 
         if for_user:
             setup_users_index(es)
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index dc13afc3..5ebab611 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -144,13 +144,16 @@ def parse_gemma_output(genofile_name):
 
     with open("{}{}_output.assoc.txt".format(webqtlConfig.GENERATED_IMAGE_DIR, genofile_name)) as output_file:
         for line in output_file:
-            if line.startswith("chr"):
+            if line.startswith("chr\t"):
                 continue
             else:
                 marker = {}
                 marker['name'] = line.split("\t")[1]
                 if line.split("\t")[0] != "X" and line.split("\t")[0] != "X/Y":
-                    marker['chr'] = int(line.split("\t")[0])
+                    if "chr" in line.split("\t")[0]:
+                        marker['chr'] = int(line.split("\t")[0][3:])
+                    else:
+                        marker['chr'] = int(line.split("\t")[0])
                 else:
                     marker['chr'] = line.split("\t")[0]
                 marker['Mb'] = float(line.split("\t")[2]) / 1000000
@@ -186,13 +189,16 @@ def parse_loco_output(this_dataset, gwa_output_filename):
     for this_file in output_filelist:
         with open(this_file) as output_file:
             for line in output_file:
-                if line.startswith("chr"):
+                if line.startswith("chr\t"):
                     continue
                 else:
                     marker = {}
                     marker['name'] = line.split("\t")[1]
                     if line.split("\t")[0] != "X" and line.split("\t")[0] != "X/Y":
-                        marker['chr'] = int(line.split("\t")[0])
+                        if "chr" in line.split("\t")[0]:
+                            marker['chr'] = int(line.split("\t")[0][3:])
+                        else:
+                            marker['chr'] = int(line.split("\t")[0])
                         if marker['chr'] > previous_chr:
                             previous_chr = marker['chr']
                         elif marker['chr'] < previous_chr:
diff --git a/wqflask/wqflask/show_trait/SampleList.py b/wqflask/wqflask/show_trait/SampleList.py
index 78bb3b42..8dbba530 100644
--- a/wqflask/wqflask/show_trait/SampleList.py
+++ b/wqflask/wqflask/show_trait/SampleList.py
@@ -37,7 +37,7 @@ class SampleList(object):
 
         self.get_attributes()
 
-        self.sample_qnorm = get_transform_vals(self.dataset, this_trait)
+        #self.sample_qnorm = get_transform_vals(self.dataset, this_trait)
 
         if self.this_trait and self.dataset and self.dataset.type == 'ProbeSet':
             self.get_extra_attribute_values()
diff --git a/wqflask/wqflask/show_trait/show_trait.py b/wqflask/wqflask/show_trait/show_trait.py
index d6d83c02..7b952af4 100644
--- a/wqflask/wqflask/show_trait/show_trait.py
+++ b/wqflask/wqflask/show_trait/show_trait.py
@@ -12,6 +12,8 @@ from collections import OrderedDict
 import redis
 Redis = redis.StrictRedis()
 
+import scipy.stats as ss
+
 from flask import Flask, g
 
 from htmlgen import HTMLgen2 as HT
@@ -137,6 +139,8 @@ class ShowTrait(object):
 
         self.make_sample_lists()
 
+        self.qnorm_vals = quantile_normalize_vals(self.sample_groups)
+
         # Todo: Add back in the ones we actually need from below, as we discover we need them
         hddn = OrderedDict()
 
@@ -281,6 +285,44 @@ class ShowTrait(object):
             self.sample_groups = (primary_samples,)
         self.dataset.group.allsamples = all_samples_ordered
 
+def quantile_normalize_vals(sample_groups):
+    def normf(trait_vals):
+        ranked_vals = ss.rankdata(trait_vals)
+        p_list = []
+        for i, val in enumerate(trait_vals):
+            p_list.append(((i+1) - 0.5)/len(trait_vals))
+
+        z = ss.norm.ppf(p_list)
+        normed_vals = []
+        for rank in ranked_vals:
+            normed_vals.append("%0.3f" % z[int(rank)-1])
+
+        return normed_vals
+
+    qnorm_by_group = []
+    for sample_type in sample_groups:
+        trait_vals = []
+        for sample in sample_type.sample_list:
+            try:
+                trait_vals.append(float(sample.value))
+            except:
+                continue
+
+        qnorm_vals = normf(trait_vals)
+
+        qnorm_vals_with_x = []
+        counter = 0
+        for sample in sample_type.sample_list:
+            if sample.display_value == "x":
+                qnorm_vals_with_x.append("x")
+            else:
+                qnorm_vals_with_x.append(qnorm_vals[counter])
+                counter += 1
+
+        qnorm_by_group.append(qnorm_vals_with_x)
+
+    return qnorm_by_group
+
 def get_nearest_marker(this_trait, this_db):
     this_chr = this_trait.locus_chr
     logger.debug("this_chr:", this_chr)
diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js
index 17afc814..5e2ecc33 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait.js
@@ -549,6 +549,7 @@
     };
     $('#block_outliers').click(block_outliers);
     reset_samples_table = function() {
+      $('input[name="transform"]').val("");
       return $('.trait_value_input').each((function(_this) {
         return function(_index, element) {
           console.log("value is:", $(element).val());
@@ -559,6 +560,52 @@
       })(this));
     };
     $('#reset').click(reset_samples_table);
+
+    log_normalize_data = function() {
+      return $('.trait_value_input').each((function(_this) {
+        return function(_index, element) {
+          current_value = $(element).data("value");
+          if(isNaN(current_value)) {
+            return current_value
+          } else {
+            $(element).val(Math.log2(current_value).toFixed(3));
+            return Math.log2(current_value).toFixed(3)
+          }
+        };
+      })(this));
+    };
+
+    qnorm_data = function() {
+      return $('.trait_value_input').each((function(_this) {
+        return function(_index, element) {
+          current_value = $(element).data("value");
+          if(isNaN(current_value)) {
+            return current_value
+          } else {
+            $(element).val($(element).data("qnorm"));
+            return $(element).data("qnorm");
+          }
+        };
+      })(this));
+    };
+
+    normalize_data = function() {
+      if ($('#norm_method option:selected').val() == 'log2'){
+        if ($('input[name="transform"]').val() != "log2") {
+          log_normalize_data()
+          $('input[name="transform"]').val("log2")
+        }
+      }
+      else if ($('#norm_method option:selected').val() == 'qnorm'){
+        if ($('input[name="transform"]').val() != "qnorm") {
+          qnorm_data()
+          $('input[name="transform"]').val("qnorm")
+        }
+      }
+    }
+
+    $('#normalize').click(normalize_data);
+
     switch_qnorm_data = function() {
       return $('.trait_value_input').each((function(_this) {
         return function(_index, element) {
@@ -734,7 +781,7 @@
         box_data = [trace1, trace2, trace3]
     } else {
         var box_layout = {
-            width: 500,
+            width: 300,
             height: 500,
             margin: {
                 l: 50,
@@ -834,7 +881,7 @@
 
     var layout = {
         yaxis: {
-            range: [range_bottom, range_top]
+            range: [range_bottom, range_top],
         },
         width: 1200,
         height: 500,
@@ -884,6 +931,7 @@
     $('#block_outliers').click(edit_data_change);
     $('#reset').click(edit_data_change);
     $('#qnorm').click(edit_data_change);
+    $('#normalize').click(edit_data_change);
     return console.log("end");
   });
 
diff --git a/wqflask/wqflask/templates/index_page_orig.html b/wqflask/wqflask/templates/index_page_orig.html
index 2a5556ea..dba3e266 100755
--- a/wqflask/wqflask/templates/index_page_orig.html
+++ b/wqflask/wqflask/templates/index_page_orig.html
@@ -34,11 +34,11 @@
 
                                 <div class="form-group">
                                     <label for="species" class="col-xs-1 control-label" style="width: 65px !important;">Species:</label>
-                                    <div class="col-xs-10 controls input-append" style="padding-right: 0px;">
-                                        <div class="col-xs-8">
-                                          <select name="species" id="species" class="form-control span3" style="width: 280px !important;"></select>
+                                    <div class="col-xs-10 controls input-append" style="display: flex; padding-left: 20px;">
+                                        <div class="col-8">
+                                          <select name="species" id="species" class="form-control" style="width: 280px !important;"></select>
                                         </div>
-                                        <div class="col-xs-4">
+                                        <div class="col-4" style="margin-left: 10px;">
                                           <button type="button" id="make_default" class="btn btn-primary form-control">Make Default</button>
                                         </div>
                                     </div>
@@ -46,9 +46,9 @@
 
                                 <div class="form-group">
                                     <label for="group" class="col-xs-1 control-label" style="width: 65px !important;">Group:</label>
-                                    <div class="col-xs-10 controls input-append">
-                                        <div class="col-xs-8">
-                                          <select name="group" id="group" class="form-control span3" style="width: 280px !important;"></select>
+                                    <div class="col-xs-10 controls input-append" style="padding-left: 20px;">
+                                        <div class="col-8">
+                                          <select name="group" id="group" class="form-control" style="width: 280px !important;"></select>
                                           <i class="icon-question-sign"></i>
                                         </div>
                                     </div>
@@ -56,21 +56,21 @@
 
                                 <div class="form-group">
                                     <label for="tissue" class="col-xs-1 control-label" style="width: 65px !important;">Type:</label>
-                                    <div class="col-xs-10 controls">
-                                        <div class="col-xs-8">
-                                          <select name="type" id="type" class="form-control span3" style="width: 280px !important;"></select>
+                                    <div class="col-xs-10 controls" style="padding-left: 20px;">
+                                        <div class="col-8">
+                                          <select name="type" id="type" class="form-control" style="width: 280px !important;"></select>
                                         </div>
                                     </div>
                                 </div>
 
                                 <div class="form-group">
                                     <label for="dataset" class="col-xs-1 control-label" style="width: 65px !important;">Dataset:</label>
-                                    <div class="col-xs-10 controls input-append">
-                                        <div class="col-xs-10">
-                                          <select name="dataset" id="dataset" class="form-control span5" style="width: 340px !important;"></select>
+                                    <div class="col-xs-10 controls input-append" style="display: flex; padding-left: 20px;">
+                                        <div class="col-9">
+                                          <select name="dataset" id="dataset" class="form-control" style="width: 340px !important;"></select>
                                           <i class="icon-question-sign"></i>
                                         </div>
-                                        <div class="col-xs-2">
+                                        <div class="col-3" style="margin-left: 10px;">
                                           <button type="button" id="dataset_info" class="btn btn-primary form-control" style="width: 75px !important;">Info</button>
                                         </div>
                                     </div>
@@ -85,8 +85,8 @@
 
                                 <div class="form-group">
                                     <label for="or_search" class="col-xs-1 control-label" style="padding-left: 0px; padding-right: 0px; width: 65px !important;">Get Any:</label>
-                                    <div class="col-xs-10 controls">
-                                        <div class="col-xs-8">
+                                    <div class="col-xs-10 controls" style="padding-left: 20px;">
+                                        <div class="col-8">
                                           <textarea onkeydown="pressed(event)" name="search_terms_or" rows="1" class="form-control search-query" style="max-width: 550px; width: 450px !important;" id="or_search"></textarea>
                                         </div>
                                     </div>
@@ -95,8 +95,8 @@
                                 <!--  GET ANY HELP   -->
                                 <div class="form-group">
                                     <label for="btsearch" class="col-xs-1 control-label" style="width: 65px !important;"></label>
-                                    <div class="col-xs-10 controls">
-                                        <div class="col-xs-12 controls">
+                                    <div class="col-xs-10 controls" style="padding-left: 20px;">
+                                        <div class="col-12 controls">
                                             Enter terms, genes, ID numbers in the <b>Search</b> field.<br>
                                             Use <b>*</b> or <b>?</b> wildcards (Cyp*a?, synap*).<br>
                                             Use <b>quotes</b> for terms such as <i>"tyrosine kinase"</i>.
@@ -106,8 +106,8 @@
 
                                 <div class="form-group">
                                     <label for="and_search" class="col-xs-1 control-label" style="padding-left: 0px; padding-right: 0px; width: 65px !important;">Combined:</label>
-                                    <div class="col-xs-10 controls">
-                                        <div class="col-xs-8">
+                                    <div class="col-xs-10 controls" style="padding-left: 20px;">
+                                        <div class="col-8">
                                           <textarea onkeydown="pressed(event)" name="search_terms_and" rows="1" class="form-control search-query" style="max-width: 550px; width: 450px !important;" id="and_search"></textarea>
                                         </div>
                                     </div>
@@ -115,8 +115,8 @@
 
                                 <div class="form-group">
                                     <label for="btsearch" class="col-xs-1 control-label" style="width: 65px !important;"></label>
-                                    <div class="col-xs-10 controls">
-                                        <div class="col-xs-2 controls" style="width: 100px !important;">
+                                    <div class="col-xs-10 controls" style="padding-left: 20px;">
+                                        <div class="col-2 controls" style="width: 100px !important;">
                                             <input id="btsearch" type="submit" class="btn btn-primary form-control" value="Search">
                                         </div>
                                     </div>
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index 4aad4242..f67fff90 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -35,6 +35,7 @@
         <input type="hidden" name="temp_uuid" id="temp_uuid" value="{{ temp_uuid }}">
         <input type="hidden" name="genofile" value="">
         <input type="hidden" name="covariates" value="">
+        <input type="hidden" name="transform" value="">
 
         <div class="container">
             <div class="panel-group" id="accordion">
diff --git a/wqflask/wqflask/templates/show_trait_edit_data.html b/wqflask/wqflask/templates/show_trait_edit_data.html
index 1402db47..482d1d88 100644
--- a/wqflask/wqflask/templates/show_trait_edit_data.html
+++ b/wqflask/wqflask/templates/show_trait_edit_data.html
@@ -9,7 +9,7 @@
                 needed.
             </p>
 
-            <div id="blockMenuSpan" class="input-append">
+            <div id="blockMenuSpan" class="input-append" style="margin-bottom: 10px;">
                 <label for="remove_samples_field">Block samples by index:</label>
                 <input type="text" id="remove_samples_field">
                 <select id="block_group" size="1">
@@ -25,11 +25,8 @@
             <div id="remove_samples_invalid" class="alert alert-error" style="display:none;">
                   Please check that your input is formatted correctly, e.g. <strong>3, 5-10, 12</strong>
             </div>
-
-            <br>
-
             {% if sample_groups[0].attributes %}
-            <div class="input-append">
+            <div class="input-append" style="margin-top:10px; margin-bottom:10px;">
                 <label for="exclude_menu">Block samples by group:</label>
                 <select id="exclude_menu" size=1>
                   {% for attribute in sample_groups[0].attributes %}
@@ -42,7 +39,6 @@
                 <input type="button" id="exclude_group" class="btn" value="Block">
             </div>
             {% endif %}
-            <br>
             <div>
               <input type="button" id="hide_no_value" class="btn btn-default" value="Hide No Value">
               <input type="button" id="block_outliers" class="btn btn-default" value="Block Outliers">
@@ -55,9 +51,18 @@
                 </select>
               </span>
               <br>
+              <div style="margin-top:10px;">
+              <input type="button" id="normalize" class="btn btn-default" value="Normalize">
+              <select id="norm_method" class="select optional span2">
+                <option value="log2">Log2</option>
+                <option value="qnorm">Quantile</option>
+              </select>
+              </div>
+              <!--
               {% if sample_groups[0].sample_qnorm is not none %}
               <input type="button" id="qnorm" class="btn btn-default" value="Quantile Normalize">
               {% endif %}
+              -->
             </div>
         </fieldset>
         <br>
@@ -79,6 +84,7 @@
 
     <!--<div id="edit_sample_lists">-->
     {% for sample_type in sample_groups %}
+        {% set outer_loop = loop %}
         <div class="sample_group" style="width:{{ trait_table_width }}%;">
             <h3>{{ sample_type.header }}</h3>
             <hr>
@@ -115,7 +121,7 @@
         
                         {# Todo: Add IDs #}
                         <td class="column_name-Value" align="right">
-                          <input type="text" data-value="{{ sample.display_value }}" data-transform="{{ sample_type.sample_qnorm[sample.name] }}" name="{{ 'value:' + sample.name }}"
+                          <input type="text" data-value="{{ sample.display_value }}" data-qnorm="{{ qnorm_vals[outer_loop.index - 1][loop.index - 1] }}" name="{{ 'value:' + sample.name }}"
                                  style="text-align:right;"
                                  class="trait_value_input edit_sample_value"
                                  value="{{ sample.display_value }}"
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index 9c19028c..a54b595b 100644
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -71,8 +71,10 @@
                         <div class="mapping_method_fields form-group">
                             <label style="text-align: right;" class="col-xs-3 control-label">Covariates</label>
                             <div style="margin-left:20px;" class="col-xs-7">
-                              {% if g.user_session.logged_in and (g.user_session.num_collections > 0) %}
+                              {% if g.user_session.logged_in %}
+                              {% if g.user_session.num_collections < 1 %}
                               No collections available. Please add traits to a collection to use them as covariates.
+                              {% endif %}
                               {% elif g.cookie_session.display_num_collections() == "" %}
                               No collections available. Please add traits to a collection to use them as covariates.
                               {% else %}
diff --git a/wqflask/wqflask/user_manager.py b/wqflask/wqflask/user_manager.py
index 5d388d66..830c7864 100644
--- a/wqflask/wqflask/user_manager.py
+++ b/wqflask/wqflask/user_manager.py
@@ -67,7 +67,10 @@ class AnonUser(object):
         #    but wouldn't set cookies for staging and my branch. The new code (using @app.after_request) seems to work.
         @app.after_request
         def set_cookie(response):
-            response.set_cookie(self.cookie_name, self.cookie)
+            if self.cookie:
+                pass
+            else:
+                response.set_cookie(self.cookie_name, self.cookie)
             return response
 
         #@after.after_this_request