Merge branch 'testing' of github.com:genenetwork/genenetwork2 into feature/pass_sample_vals_as_json

7 files changed, 139 insertions, 115 deletions

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index be983c87..a36b947c 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -71,7 +71,6 @@ class CorrelationResults(object):
         assert('corr_sample_method' in start_vars)
         assert('corr_samples_group' in start_vars)
         assert('corr_dataset' in start_vars)
-        #assert('min_expr' in start_vars)
         assert('corr_return_results' in start_vars)
         if 'loc_chr' in start_vars:
             assert('min_loc_mb' in start_vars)
@@ -195,15 +194,15 @@ class CorrelationResults(object):
                 if (float(self.correlation_data[trait][0]) >= self.p_range_lower and
                     float(self.correlation_data[trait][0]) <= self.p_range_upper):
 
-                    if self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Geno":
-
+                    if (self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Publish") and bool(trait_object.mean):
                         if (self.min_expr != None) and (float(trait_object.mean) < self.min_expr):
                             continue
-                        elif range_chr_as_int != None and (chr_as_int != range_chr_as_int):
+                    if self.target_dataset.type == "ProbeSet" or self.target_dataset.type == "Geno":
+                        if range_chr_as_int != None and (chr_as_int != range_chr_as_int):
                             continue
-                        elif (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)):
+                        if (self.min_location_mb != None) and (float(trait_object.mb) < float(self.min_location_mb)):
                             continue
-                        elif (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)):
+                        if (self.max_location_mb != None) and (float(trait_object.mb) > float(self.max_location_mb)):
                             continue
 
                     (trait_object.sample_r,
@@ -517,6 +516,7 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap
         elif target_dataset.type == "Publish":
             results_dict['abbreviation_display'] = "N/A"
             results_dict['description'] = "N/A"
+            results_dict['mean'] = "N/A"
             results_dict['authors_display'] = "N/A"
             results_dict['additive'] = "N/A"
             if for_api:
@@ -530,6 +530,8 @@ def generate_corr_json(corr_results, this_trait, dataset, target_dataset, for_ap
                 results_dict['abbreviation_display'] = trait.abbreviation
             if bool(trait.description_display):
                 results_dict['description'] = trait.description_display
+            if bool(trait.mean):
+                results_dict['mean'] = f"{float(trait.mean):.3f}"
             if bool(trait.authors):
                 authors_list = trait.authors.split(',')
                 if len(authors_list) > 6:
@@ -603,6 +605,7 @@ def get_header_fields(data_type, corr_method):
                             'Record',
                             'Abbreviation',
                             'Description',
+                            'Mean',
                             'Authors',
                             'Year',
                             'Sample rho',
@@ -616,6 +619,7 @@ def get_header_fields(data_type, corr_method):
                             'Record',
                             'Abbreviation',
                             'Description',
+                            'Mean',
                             'Authors',
                             'Year',
                             'Sample r',
diff --git a/wqflask/wqflask/marker_regression/gemma_mapping.py b/wqflask/wqflask/marker_regression/gemma_mapping.py
index 630a3afa..ab3a7278 100644
--- a/wqflask/wqflask/marker_regression/gemma_mapping.py
+++ b/wqflask/wqflask/marker_regression/gemma_mapping.py
@@ -54,7 +54,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, maf
                                                                                           TEMPDIR,
                                                                                           trait_filename)
           if covariates != "":
-              gemma_command += ' -c %s/%s_covariates.txt -a %s/%s_snps.txt -lmm 2 -maf %s > %s/gn2/%s.json' % (flat_files('mapping'),
+              gemma_command += ' -c %s/%s_covariates.txt -a %s/%s_snps.txt -lmm 9 -maf %s > %s/gn2/%s.json' % (flat_files('mapping'),
                                                                                                                 this_dataset.group.name,
                                                                                                                 flat_files('genotype/bimbam'),
                                                                                                                 genofile_name,
@@ -62,7 +62,7 @@ def run_gemma(this_trait, this_dataset, samples, vals, covariates, use_loco, maf
                                                                                                                 TEMPDIR,
                                                                                                                 gwa_output_filename)
           else:
-              gemma_command += ' -a %s/%s_snps.txt -lmm 2 -maf %s > %s/gn2/%s.json' % (flat_files('genotype/bimbam'),
+              gemma_command += ' -a %s/%s_snps.txt -lmm 9 -maf %s > %s/gn2/%s.json' % (flat_files('genotype/bimbam'),
                                                                                                                genofile_name,
                                                                                                                maf,
                                                                                                                TEMPDIR,
@@ -184,11 +184,8 @@ def parse_loco_output(this_dataset, gwa_output_filename, loco="True"):
                     else:
                         marker['chr'] = line.split("\t")[0]
                     marker['Mb'] = float(line.split("\t")[2]) / 1000000
-                    if loco == "True":
-                        marker['p_value'] = float(line.split("\t")[9])
-                    else:
-                        marker['p_value'] = float(line.split("\t")[10])
-                        marker['additive'] = float(line.split("\t")[7])
+                    marker['p_value'] = float(line.split("\t")[10])
+                    marker['additive'] = float(line.split("\t")[7])
                     if math.isnan(marker['p_value']) or (marker['p_value'] <= 0):
                         marker['lod_score'] = 0
                         #marker['lrs_value'] = 0
diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js
index b71a9dd8..2b8a4d30 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait.js
@@ -100,15 +100,7 @@ d3.select("#select_compare_trait").on("click", (function(_this) {
     return open_trait_selection();
   };
 })(this));
-d3.select("#select_covariates").on("click", (function(_this) {
-  return function() {
-    return open_covariate_selection();
-  };
-})(this));
-$("#remove_covariates").click(function () {
-    $("input[name=covariates]").val("")
-    $(".selected-covariates").val("")
-});
+
 $(".select_covariates").click(function () {
   open_covariate_selection();
 });
@@ -475,7 +467,7 @@ edit_data_change = function() {
         if (is_number(sample_val) && sample_val !== "") {
           sample_val = parseFloat(sample_val);
           sample_sets[table].add_value(sample_val);
-          if (typeof var_nodes !== 'undefined'){
+          if (typeof var_nodes === 'undefined'){
             sample_var = null;
           } else {
             sample_var = var_nodes[_j].childNodes[0].value
@@ -498,6 +490,7 @@ edit_data_change = function() {
         }
       }
     }
+
   }
 
   update_stat_values(sample_sets);
@@ -543,6 +536,24 @@ on_corr_method_change = function() {
 };
 $('select[name=corr_type]').change(on_corr_method_change);
 
+on_dataset_change = function() {
+  let dataset_type = $('select[name=corr_dataset] option:selected').data('type');
+
+  if (dataset_type == "mrna_assay"){
+    $('#min_expr_filter').show();
+    $('#location_filter').show();
+  }
+  else if (dataset_type == "pheno"){
+    $('#min_expr_filter').show();
+    $('#location_filter').hide();
+  }
+  else {
+    $('#min_expr_filter').hide();
+    $('#location_filter').show();
+  }
+}
+$('select[name=corr_dataset]').change(on_dataset_change);
+
 submit_special = function(url) {
   $("input[name=sample_vals]").val(JSON.stringify(fetch_sample_values()))
   $("#trait_data_form").attr("action", url);
@@ -650,6 +661,8 @@ block_by_attribute_value = function() {
       this_val_node.value = "x";
     }
   }
+
+  edit_data_change();
 };
 $('#exclude_by_attr').click(block_by_attribute_value);
 
diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html
index 8e2a23fd..6188c0e7 100644
--- a/wqflask/wqflask/templates/correlation_page.html
+++ b/wqflask/wqflask/templates/correlation_page.html
@@ -95,14 +95,15 @@
           {% elif target_dataset.type == 'Publish' %}
           <button class="toggle-vis" data-column="3">Abbreviation</button>
           <button class="toggle-vis" data-column="4">Description</button>
-          <button class="toggle-vis" data-column="5">Authors</button>
-          <button class="toggle-vis" data-column="6">Year</button>
-          <button class="toggle-vis" data-column="7">Sample {% if corr_method == 'pearson' %}r{% else %}rho{% endif %}</button>
-          <button class="toggle-vis" data-column="8">N</button>
-          <button class="toggle-vis" data-column="9">Sample p({% if corr_method == 'pearson' %}r{% else %}rho{% endif %})</button>
-          <button class="toggle-vis" data-column="10">Peak LOD</button>
-          <button class="toggle-vis" data-column="11">Peak Location</button>
-          <button class="toggle-vis" data-column="12">Effect Size</button>
+          <button class="toggle-vis" data-column="5">Mean</button>
+          <button class="toggle-vis" data-column="6">Authors</button>
+          <button class="toggle-vis" data-column="7">Year</button>
+          <button class="toggle-vis" data-column="8">Sample {% if corr_method == 'pearson' %}r{% else %}rho{% endif %}</button>
+          <button class="toggle-vis" data-column="9">N</button>
+          <button class="toggle-vis" data-column="10">Sample p({% if corr_method == 'pearson' %}r{% else %}rho{% endif %})</button>
+          <button class="toggle-vis" data-column="11">Peak LOD</button>
+          <button class="toggle-vis" data-column="12">Peak Location</button>
+          <button class="toggle-vis" data-column="13">Effect Size</button>
           {% else %}
           <button class="toggle-vis" data-column="3">Location</button>
           <button class="toggle-vis" data-column="4">Sample {% if corr_method == 'pearson' %}r{% else %}rho{% endif %}</button>
@@ -399,6 +400,13 @@
                       }
                     },
                     {
+                      'title': "Mean",
+                      'type': "natural-minus-na",
+                      'width': "40px",
+                      'data': "mean",
+                      'orderSequence': [ "desc", "asc"]
+                    },
+                    {
                       'title': "Authors",
                       'type': "natural",
                       'width': "400px",
@@ -514,6 +522,8 @@
                 } ],
                 {% if target_dataset.type == 'Geno' %}
                 "order": [[6, "asc" ]],
+                {% elif target_dataset.type == 'Publish' %}
+                "order": [[10, "asc" ]],
                 {% else %}
                 "order": [[9, "asc" ]],
                 {% endif %}
diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
index ba72ff27..eaa0c308 100644
--- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html
+++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
@@ -2,86 +2,85 @@
     <div class="col-xs-3 correlation-options">
       <div class="form-horizontal section-form-div">
 
-          <div class="form-group">
-              <label for="corr_type" class="col-xs-2 control-label">Method</label>
-              <div class="col-xs-3 controls">
-                  <select name="corr_type" class="form-control">
-                      <option value="sample">Sample r</option>
-                      <option value="lit">Literature r</option>
-                      <option value="tissue">Tissue r</option>
-                  </select>
-              </div>
-          </div>
+        <div class="form-group">
+            <label for="corr_type" class="col-xs-2 control-label">Method</label>
+            <div class="col-xs-3 controls">
+                <select name="corr_type" class="form-control">
+                    <option value="sample">Sample r</option>
+                    <option value="lit">Literature r</option>
+                    <option value="tissue">Tissue r</option>
+                </select>
+            </div>
+        </div>
 
-          <div class="form-group">
-              <label for="corr_dataset" class="col-xs-2 control-label">Database</label>
-              <div class="col-xs-10 controls">
-                  <select name="corr_dataset" class="form-control">
-                      {% for tissue in corr_tools.dataset_menu %}
-                          {% if tissue.tissue %}
-                              <optgroup label="{{ tissue.tissue }} ------">
-                          {% endif %}
-                          {% for dataset in tissue.datasets %}
-                              <option value="{{ dataset[1] }}"
-                              {% if corr_tools.dataset_menu_selected == dataset[1] %}
-                                  selected
-                              {% endif %}>
-                                {{ dataset[0] }}
-                              </option>
-                          {% endfor %}
-                          {% if tissue.tissue %}
-                              </optgroup>
-                          {% endif %}
-                      {% endfor %}
-                  </select>
-              </div>
-          </div>
+        <div class="form-group">
+            <label for="corr_dataset" class="col-xs-2 control-label">Database</label>
+            <div class="col-xs-10 controls">
+                <select name="corr_dataset" class="form-control">
+                    {% for tissue in corr_tools.dataset_menu %}
+                        {% if tissue.tissue %}
+                        <optgroup label="{{ tissue.tissue }} ------">
+                        {% endif %}
+                        {% for dataset in tissue.datasets %}
+                        <option data-type="{% if tissue.tissue %}mrna_assay{% elif dataset[1][-4:] == 'Geno' %}geno{% else %}pheno{% endif %}" value="{{ dataset[1] }}"
+                        {% if corr_tools.dataset_menu_selected == dataset[1] %}
+                            selected
+                        {% endif %}>
+                        {{ dataset[0] }}
+                        </option>
+                        {% endfor %}
+                        {% if tissue.tissue %}
+                        </optgroup>
+                        {% endif %}
+                    {% endfor %}
+                </select>
+            </div>
+        </div>
 
-          <div class="form-group">
-              <label for="corr_return_results" class="col-xs-2 control-label">Limit to</label>
-              <div class="col-xs-4 controls">
-                  <select name="corr_return_results" class="form-control">
-                      {% for return_result in corr_tools.return_results_menu %}
-                          <option value="{{ return_result }}"
-                          {% if corr_tools.return_results_menu_selected == return_result %}
-                              selected
-                          {% endif %}>
-                          Top {{ return_result }}
-                          </option>
-                      {% endfor %}
-                  </select>
-              </div>
-          </div>
+        <div class="form-group">
+            <label for="corr_return_results" class="col-xs-2 control-label">Limit to</label>
+            <div class="col-xs-4 controls">
+                <select name="corr_return_results" class="form-control">
+                    {% for return_result in corr_tools.return_results_menu %}
+                        <option value="{{ return_result }}"
+                        {% if corr_tools.return_results_menu_selected == return_result %}
+                            selected
+                        {% endif %}>
+                        Top {{ return_result }}
+                        </option>
+                    {% endfor %}
+                </select>
+            </div>
+        </div>
 
-          <div class="form-group">
-              <label for="corr_samples_group" class="col-xs-2 control-label">Samples</label>
-              <div class="col-xs-4 controls">
-                  <select name="corr_samples_group" class="form-control">
-                      {% for group, pretty_group in sample_group_types.items() %}
-                          <option value="{{ group }}">{{ pretty_group }}</option>
-                      {% endfor %}
-                  </select>
-              </div>
-          </div>
+        <div class="form-group">
+            <label for="corr_samples_group" class="col-xs-2 control-label">Samples</label>
+            <div class="col-xs-4 controls">
+                <select name="corr_samples_group" class="form-control">
+                    {% for group, pretty_group in sample_group_types.items() %}
+                        <option value="{{ group }}">{{ pretty_group }}</option>
+                    {% endfor %}
+                </select>
+            </div>
+        </div>
 
-          <div id="corr_sample_method" class="form-group">
-              <label for="corr_sample_method" class="col-xs-2 control-label">Type</label>
-              <div class="col-xs-4 controls">
-                  <select name="corr_sample_method" class="form-control">
-                      <option value="pearson">Pearson</option>
-                      <option value="spearman">Spearman Rank</option>
-                      <option value="bicor">Biweight Midcorrelation</option>
-                  </select>
-              </div>
-          </div>
-          {% if dataset.type != "Publish" %}
-          <div class="form-group">
-              <label class="col-xs-2 control-label">Min Expr</label>
-              <div class="col-xs-4 controls">
-                  <input name="min_expr" value="" type="text" class="form-control min-expr-field">
-              </div>
-          </div>
-          <div class="form-group">
+        <div id="corr_sample_method" class="form-group">
+            <label for="corr_sample_method" class="col-xs-2 control-label">Type</label>
+            <div class="col-xs-4 controls">
+                <select name="corr_sample_method" class="form-control">
+                    <option value="pearson">Pearson</option>
+                    <option value="spearman">Spearman Rank</option>
+                    <option value="bicor">Biweight Midcorrelation</option>
+                </select>
+            </div>
+        </div>
+        <div id="min_expr_filter" class="form-group" style="display: {% if dataset.type != 'Geno' %}block{% else %}none{% endif %};">
+            <label class="col-xs-2 control-label">Min Expr</label>
+            <div class="col-xs-4 controls">
+                <input name="min_expr" value="" type="text" class="form-control min-expr-field">
+            </div>
+        </div>
+        <div id="location_filter" class="form-group" style="display: {% if dataset.type != 'Publish' %}block{% else %}none{% endif %};">
               <label class="col-xs-2 control-label">Location</label>
               <div class="col-xs-6 controls">
                   <span>
@@ -91,7 +90,6 @@
                   <br>
               </div>
           </div>
-          {% endif %}
           <div class="form-group">
               <label class="col-xs-2 control-label">Range</label>
               <div class="col-xs-5 controls">
diff --git a/wqflask/wqflask/templates/show_trait_mapping_tools.html b/wqflask/wqflask/templates/show_trait_mapping_tools.html
index eca436c6..c42fe4aa 100755
--- a/wqflask/wqflask/templates/show_trait_mapping_tools.html
+++ b/wqflask/wqflask/templates/show_trait_mapping_tools.html
@@ -74,8 +74,8 @@
                               No collections available. Please add traits to a collection to use them as covariates.
                               {% else %}
                               <div class="select-covar-div">
-                                <button type="button" id="select_covariates" class="btn btn-default select-covar-button">Select</button>
-                                <button type="button" id="remove_covariates" class="btn btn-default select-covar-button">Remove</button>
+                                <button type="button" class="btn btn-default select-covar-button select_covariates">Select</button>
+                                <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button>
                               </div>
                               <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea>
                               {% endif %}
@@ -317,8 +317,8 @@
                             No collections available. Please add traits to a collection to use them as covariates.
                             {% else %}
                             <div class="select-covar-div">
-                              <button type="button" id="select_covariates" class="btn btn-default select-covar-button">Select</button>
-                              <button type="button" id="remove_covariates" class="btn btn-default select-covar-button">Remove</button>
+                              <button type="button" class="btn btn-default select-covar-button select_covariates">Select</button>
+                              <button type="button" class="btn btn-default select-covar-button remove_covariates">Remove</button>
                             </div>
                             <textarea rows="3" cols="50" readonly placeholder="No covariates selected" class="selected-covariates"></textarea>
                             {% endif %}
diff --git a/wqflask/wqflask/user_login.py b/wqflask/wqflask/user_login.py
index f25ebc32..bc608e84 100644
--- a/wqflask/wqflask/user_login.py
+++ b/wqflask/wqflask/user_login.py
@@ -25,7 +25,7 @@ from utility.logger import getLogger
 logger = getLogger(__name__)
 
 from smtplib import SMTP
-from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY
+from utility.tools import SMTP_CONNECT, SMTP_USERNAME, SMTP_PASSWORD, LOG_SQL_ALCHEMY, GN2_BRANCH_URL
 
 THREE_DAYS = 60 * 60 * 24 * 3
 
@@ -239,7 +239,7 @@ def github_oauth2():
     }
 
     result = requests.post("https://github.com/login/oauth/access_token", json=data)
-    result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in [token.encode("utf-8") for token in result.text.split("&")]]}
+    result_dict = {arr[0]:arr[1] for arr in [tok.split("=") for tok in result.text.split("&")]}
 
     github_user = get_github_user_details(result_dict["access_token"])
 
@@ -277,9 +277,11 @@ def orcid_oauth2():
         data = {
             "client_id": ORCID_CLIENT_ID, 
             "client_secret": ORCID_CLIENT_SECRET, 
-            "grant_type": "authorization_code", 
+            "grant_type": "authorization_code",
+            "redirect_uri": GN2_BRANCH_URL + "n/login/orcid_oauth2",
             "code": code
         }
+
         result = requests.post(ORCID_TOKEN_URL, data=data)
         result_dict = json.loads(result.text.encode("utf-8"))