Merge pull request #535 from zsloan/feature/pass_sample_vals_as_json

Feature/pass sample vals as json

8 files changed, 59 insertions, 85 deletions

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 298a3559..a36b947c 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -86,8 +86,6 @@ class CorrelationResults(object):
             else:
                 helper_functions.get_species_dataset_trait(self, start_vars)
 
-            #self.dataset.group.read_genotype_file()
-
             corr_samples_group = start_vars['corr_samples_group']
 
             self.sample_data = {}
@@ -453,13 +451,13 @@ class CorrelationResults(object):
         if not excluded_samples:
             excluded_samples = ()
 
+        sample_val_dict = json.loads(start_vars['sample_vals'])
         for sample in sample_names:
             if sample not in excluded_samples:
-                # print("Looking for",sample,"in",start_vars)
-                value = start_vars.get('value:' + sample)
-                if value:
-                    if not value.strip().lower() == 'x':
-                        self.sample_data[str(sample)] = float(value)
+                value = sample_val_dict[sample]
+                if not value.strip().lower() == 'x':
+                    self.sample_data[str(sample)] = float(value)
+
 
 def do_bicor(this_trait_vals, target_trait_vals):
     r_library = ro.r["library"]             # Map the library function
diff --git a/wqflask/wqflask/marker_regression/display_mapping_results.py b/wqflask/wqflask/marker_regression/display_mapping_results.py
index 08c2d750..9f1b050d 100644
--- a/wqflask/wqflask/marker_regression/display_mapping_results.py
+++ b/wqflask/wqflask/marker_regression/display_mapping_results.py
@@ -270,8 +270,9 @@ class DisplayMappingResults(object):
 
         # Needing for form submission when doing single chr
         # mapping or remapping after changing options
-        self.samples = start_vars['samples']
-        self.vals = start_vars['vals']
+        self.sample_vals = start_vars['sample_vals']
+        self.sample_vals_dict = json.loads(self.sample_vals)
+
         self.transform = start_vars['transform']
         self.mapping_method = start_vars['mapping_method']
         self.mapping_results_path = start_vars['mapping_results_path']
@@ -492,11 +493,10 @@ class DisplayMappingResults(object):
 ## count the amount of individuals to be plotted, and increase self.graphHeight
         if self.haplotypeAnalystChecked and self.selectedChr > -1:
             thisTrait = self.this_trait
-            _strains, _vals, _vars, _aliases = thisTrait.export_informative()
             smd=[]
-            for ii, _val in enumerate(self.vals):
-                if _val != "x":
-                    temp = GeneralObject(name=self.samples[ii], value=float(_val))
+            for sample in self.sample_vals_dict.keys():
+                if self.sample_vals_dict[sample] != "x":
+                    temp = GeneralObject(name=sample, value=float(self.sample_vals_dict[sample]))
                     smd.append(temp)
                 else:
                     continue
@@ -1464,12 +1464,11 @@ class DisplayMappingResults(object):
         yPaddingTop = yTopOffset
 
         thisTrait = self.this_trait
-        _strains, _vals, _vars, _aliases = thisTrait.export_informative()
 
         smd=[]
-        for ii, _val in enumerate(self.vals):
-            if _val != "x":
-                temp = GeneralObject(name=self.samples[ii], value=float(_val))
+        for sample in self.sample_vals_dict.keys():
+            if self.sample_vals_dict[sample] != "x":
+                temp = GeneralObject(name=sample, value=float(self.sample_vals_dict[sample]))
                 smd.append(temp)
             else:
                 continue
diff --git a/wqflask/wqflask/marker_regression/run_mapping.py b/wqflask/wqflask/marker_regression/run_mapping.py
index 891fcc66..a1f87b61 100644
--- a/wqflask/wqflask/marker_regression/run_mapping.py
+++ b/wqflask/wqflask/marker_regression/run_mapping.py
@@ -71,56 +71,22 @@ class RunMapping(object):
         all_samples_ordered = self.dataset.group.all_samples_ordered()
 
         self.vals = []
-        if 'samples' in start_vars:
-            self.samples = start_vars['samples'].split(",")
-            if (len(genofile_samplelist) != 0):
-                for sample in genofile_samplelist:
-                    if sample in self.samples:
-                        value = start_vars.get('value:' + sample)
-                        if value:
-                            self.vals.append(value)
-                    else:
-                        self.vals.append("x")
-            else:
-                for sample in self.samples:
-                    value = start_vars.get('value:' + sample)
-                    if value:
-                        self.vals.append(value)
+        self.samples = []
+        self.sample_vals = start_vars['sample_vals']
+        sample_val_dict = json.loads(self.sample_vals)
+        samples = sample_val_dict.keys()
+        if (len(genofile_samplelist) != 0):
+            for sample in genofile_samplelist:
+                self.samples.append(sample)
+                if sample in samples:
+                    self.vals.append(sample_val_dict[sample])
+                else:
+                    self.vals.append("x")
         else:
-            self.samples = []
-            if (len(genofile_samplelist) != 0):
-                for sample in genofile_samplelist:
-                    if sample in self.dataset.group.samplelist:
-                        in_trait_data = False
-                        for item in self.this_trait.data:
-                            if self.this_trait.data[item].name == sample:
-                                value = start_vars['value:' + self.this_trait.data[item].name]
-                                self.samples.append(self.this_trait.data[item].name)
-                                self.vals.append(value)
-                                in_trait_data = True
-                                break
-                        if not in_trait_data:
-                            value = start_vars.get('value:' + sample)
-                            if value:
-                                self.samples.append(sample)
-                                self.vals.append(value)
-                    else:
-                        self.vals.append("x")
-            else:
-                for sample in self.dataset.group.samplelist: # sample is actually the name of an individual
-                    in_trait_data = False
-                    for item in self.this_trait.data:
-                        if self.this_trait.data[item].name == sample:
-                            value = start_vars['value:' + self.this_trait.data[item].name]
-                            self.samples.append(self.this_trait.data[item].name)
-                            self.vals.append(value)
-                            in_trait_data = True
-                            break
-                    if not in_trait_data:
-                        value = start_vars.get('value:' + sample)
-                        if value:
-                            self.samples.append(sample)
-                            self.vals.append(value)
+            for sample in self.dataset.group.samplelist:
+                if sample in samples:
+                    self.vals.append(sample_val_dict[sample])
+                    self.samples.append(sample)
 
         if 'n_samples' in start_vars:
             self.n_samples = start_vars['n_samples']
diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js
index 765f2f2d..b8ffa8e8 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait.js
@@ -409,6 +409,25 @@ process_id = function() {
   }
   return processed;
 };
+
+fetch_sample_values = function() {
+  // This is meant to fetch all sample values using DataTables API, since they can't all be submitted with the form when using Scroller (and this should also be faster)
+  sample_val_dict = {};
+
+  table = 'samples_primary';
+  if ($('#' + table).length){
+    table_api = $('#' + table).DataTable();
+    val_nodes = table_api.column(3).nodes().to$();
+    for (_j = 0; _j < val_nodes.length; _j++){
+      sample_name = val_nodes[_j].childNodes[0].name.split(":")[1]
+      sample_val = val_nodes[_j].childNodes[0].value
+      sample_val_dict[sample_name] = sample_val
+    }
+  }
+
+  return sample_val_dict;
+}
+
 edit_data_change = function() {
   var already_seen, checkbox, name, real_dict, real_value, real_variance, row, rows, sample_sets, table, tables, _i, _j, _len, _len1;
   already_seen = {};
@@ -526,7 +545,7 @@ on_dataset_change = function() {
 $('select[name=corr_dataset]').change(on_dataset_change);
 
 submit_special = function(url) {
-  get_table_contents_for_form_submit("trait_data_form");
+  $("input[name=sample_vals]").val(JSON.stringify(fetch_sample_values()))
   $("#trait_data_form").attr("action", url);
   $("#trait_data_form").submit();
 };
@@ -550,7 +569,7 @@ get_table_contents_for_form_submit = function(form_id) {
  });
 }
 
-var corr_input_list = ['corr_type', 'primary_samples', 'trait_id', 'dataset', 'group', 'tool_used', 'form_url', 'corr_sample_method', 'corr_samples_group', 'corr_dataset', 'min_expr',
+var corr_input_list = ['sample_vals', 'corr_type', 'primary_samples', 'trait_id', 'dataset', 'group', 'tool_used', 'form_url', 'corr_sample_method', 'corr_samples_group', 'corr_dataset', 'min_expr',
                         'corr_return_results', 'loc_chr', 'min_loc_mb', 'max_loc_mb', 'p_range_lower', 'p_range_upper']
 
 $(".corr_compute").on("click", (function(_this) {
diff --git a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
index 34582f21..3ae52975 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait_mapping_tools.js
@@ -141,7 +141,7 @@ $('input[name=display_all]').change((function(_this) {
 })(this));
 
 //ZS: This is a list of inputs to be passed to the loading page, since not all inputs on the trait page are relevant to mapping
-var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale',
+var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals',
                           'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results',
                           'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'control_marker', 'control_marker_db', 'do_control', 'genofile', 
                           'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 
diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html
index 985d542c..c2256e47 100644
--- a/wqflask/wqflask/templates/mapping_results.html
+++ b/wqflask/wqflask/templates/mapping_results.html
@@ -31,10 +31,7 @@
         {% endif %}
         <input type="hidden" name="results_path" value="{{ mapping_results_path }}">
         <input type="hidden" name="method" value="{{ mapping_method }}">
-        <input type="hidden" name="samples" value="{{ samples|join(",") }}">
-        {% for sample in samples %}
-        <input type="hidden" name="value:{{ sample }}" value="{{ vals[loop.index - 1] }}">
-        {% endfor %}
+        <input type="hidden" name="sample_vals" value="{{ sample_vals }}">
         <input type="hidden" name="n_samples" value="{{ n_samples }}">
         <input type="hidden" name="maf" value="{{ maf }}">
         <input type="hidden" name="use_loco" value="{{ use_loco }}">
@@ -464,7 +461,7 @@
 
         });
 
-        var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale',
+        var mapping_input_list = ['temp_uuid', 'trait_id', 'dataset', 'tool_used', 'form_url', 'method', 'transform', 'trimmed_markers', 'selected_chr', 'chromosomes', 'mapping_scale', 'sample_vals',
                             'score_type', 'suggestive', 'significant', 'num_perm', 'permCheck', 'perm_output', 'perm_strata', 'categorical_vars', 'num_bootstrap', 'bootCheck', 'bootstrap_results',
                             'LRSCheck', 'covariates', 'maf', 'use_loco', 'manhattan_plot', 'color_scheme', 'manhattan_single_color', 'control_marker', 'control_marker_db', 'do_control', 'genofile',
                             'pair_scan', 'startMb', 'endMb', 'graphWidth', 'lrsMax', 'additiveCheck', 'showSNP', 'showGenes', 'viewLegend', 'haplotypeAnalystCheck', 
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index c881eb76..77fc9342 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -35,6 +35,7 @@
         <input type="hidden" name="genofile" value="">
         <input type="hidden" name="covariates" value="">
         <input type="hidden" name="transform" value="">
+        <input type="hidden" name="sample_vals" value="">
 
         <div class="container showtrait-main-div">
             <div class="panel-group" id="accordion">
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 632e0d2d..2b3e2a37 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -650,12 +650,13 @@ def loading_page():
         wanted = initial_start_vars['wanted_inputs'].split(",")
         start_vars = {}
         for key, value in list(initial_start_vars.items()):
-            if key in wanted or key.startswith(('value:')):
+            if key in wanted:
                 start_vars[key] = value
 
         if 'n_samples' in start_vars:
             n_samples = int(start_vars['n_samples'])
         else:
+            sample_vals_dict = json.loads(start_vars['sample_vals'])
             if 'group' in start_vars:
                 dataset = create_dataset(start_vars['dataset'], group_name = start_vars['group'])
             else:
@@ -671,8 +672,7 @@ def loading_page():
                         samples = genofile_samples
 
             for sample in samples:
-                value = start_vars.get('value:' + sample)
-                if value != "x":
+                if sample_vals_dict[sample] != "x":
                     n_samples += 1
 
         start_vars['n_samples'] = n_samples
@@ -689,7 +689,6 @@ def loading_page():
 @app.route("/run_mapping", methods=('POST',))
 def mapping_results_page():
     initial_start_vars = request.form
-    #logger.debug("Mapping called with initial_start_vars:", initial_start_vars.items())
     logger.info(request.url)
     temp_uuid = initial_start_vars['temp_uuid']
     wanted = (
@@ -699,6 +698,7 @@ def mapping_results_page():
         'species',
         'samples',
         'vals',
+        'sample_vals',
         'first_run',
         'output_files',
         'geno_db_exists',
@@ -752,13 +752,11 @@ def mapping_results_page():
     )
     start_vars = {}
     for key, value in list(initial_start_vars.items()):
-        if key in wanted or key.startswith(('value:')):
+        if key in wanted:
             start_vars[key] = value
-    #logger.debug("Mapping called with start_vars:", start_vars)
 
     version = "v3"
     key = "mapping_results:{}:".format(version) + json.dumps(start_vars, sort_keys=True)
-    #logger.info("key is:", pf(key))
     with Bench("Loading cache"):
         result = None # Just for testing
         #result = Redis.get(key)
@@ -804,10 +802,6 @@ def mapping_results_page():
                     rendered_template = render_template("pair_scan_results.html", **result)
             else:
                 gn1_template_vars = display_mapping_results.DisplayMappingResults(result).__dict__
-                #pickled_result = pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
-                #logger.info("pickled result length:", len(pickled_result))
-                #Redis.set(key, pickled_result)
-                #Redis.expire(key, 1*60)
 
                 with Bench("Rendering template"):
                     #if (gn1_template_vars['mapping_method'] == "gemma") or (gn1_template_vars['mapping_method'] == "plink"):