add server side processing due to large tables in DataTables; work on server-side processing API pending for sorting, paging etc

3 files changed, 59 insertions, 16 deletions

diff --git a/wqflask/wqflask/snp_browser/snp_browser.py b/wqflask/wqflask/snp_browser/snp_browser.py
index a2fb7195..9ae8892b 100644
--- a/wqflask/wqflask/snp_browser/snp_browser.py
+++ b/wqflask/wqflask/snp_browser/snp_browser.py
@@ -4,7 +4,7 @@ import string
 from PIL import (Image)
 
 from utility.logger import getLogger
-logger = getLogger(__name__ )
+logger = getLogger(__name__)
 
 from base import species
 from base import webqtlConfig
@@ -14,16 +14,16 @@ class SnpBrowser(object):
     def __init__(self, start_vars):
         self.strain_lists = get_browser_sample_lists()
         self.initialize_parameters(start_vars)
-        self.limit_number = 10000
 
         if self.first_run == "false":
             self.filtered_results = self.get_browser_results()
+            self.table_rows = self.get_table_rows()
+            self.rows_count = len(self.table_rows)
 
-            if len(self.filtered_results) <= self.limit_number:
-                self.table_rows = self.get_table_rows()
-            else:
-                self.empty_columns = None
-                self.table_rows = []
+            del self.filtered_results
+
+            if 'sEcho' not in start_vars:
+                self.table_rows = self.table_rows[:500]
 
             if self.limit_strains == "true":
                 self.header_fields, self.empty_field_count = get_header_list(variant_type = self.variant_type, strains = self.chosen_strains, empty_columns = self.empty_columns)
@@ -380,7 +380,7 @@ class SnpBrowser(object):
 
         the_rows = []
         for i, result in enumerate(self.filtered_results):
-            this_row = []
+            this_row = {}
             if self.variant_type == "SNP":
                 snp_name, rs, chr, mb, alleles, gene, transcript, exon, domain, function, function_details, snp_source, conservation_score, snp_id = result[:14]
                 allele_value_list = result[14:]
@@ -520,8 +520,6 @@ class SnpBrowser(object):
                     "source_name": str(source_name)
                 }
                 #this_row = [indel_name, indel_chr, indel_mb_s, indel_mb_e, indel_strand, indel_type, indel_size, indel_sequence, source_name]
-            else:
-                this_row = {}
 
             the_rows.append(this_row)
 
@@ -643,6 +641,27 @@ class SnpBrowser(object):
         #for i in range(n_click):
         #    href = url_for('snp_browser', first_run="false", chosen_strains_mouse=self.chosen_strains_mouse, chosen_strains_rat=self.chosen_strains_rat, variant=self.variant_type, species=self.species_name, gene_name=self.gene_name, chr=self.chr, start_mb=self.start_mb, end_mb=self.end_mb, limit_strains=self.limit_strains, domain=self.domain, function=self.function, criteria=self.criteria, score=self.score, diff_alleles=self.diff_alleles)
 
+class SnpPage(object):
+
+    def __init__(self, start_vars):
+        self.snp_browser = SnpBrowser(start_vars)
+        # self.table_rows = self.filter_rows()
+        self.rows_count = self.snp_browser.rows_count
+        self.sEcho = start_vars['sEcho']
+
+    def filter_rows(self):
+        pass
+
+    def get_page(self):
+        output = {}
+        output['sEcho'] = str(self.sEcho)
+        output['iTotalRecords'] = str(self.rows_count)
+        output['iTotalDisplayRecords'] = str(100)
+        # logger.info(len(self.table_rows), type(self.table_rows), self.table_rows[0])
+        logger.info(self.snp_browser.rows_count, len(self.snp_browser.table_rows))
+        output['data'] = self.snp_browser.table_rows[:100]
+        return output
+
 def get_browser_sample_lists(species_id=1):
     strain_lists = {}
     mouse_strain_list = []
diff --git a/wqflask/wqflask/templates/snp_browser.html b/wqflask/wqflask/templates/snp_browser.html
index 83606804..f3e928af 100644
--- a/wqflask/wqflask/templates/snp_browser.html
+++ b/wqflask/wqflask/templates/snp_browser.html
@@ -188,10 +188,7 @@
     </div>
 
     <div style="margin-top: 20px;">
-    {% if filtered_results is defined %}
-    {% if filtered_results|length > limit_number %}
-    There are more than 10000 results. Consider limiting your search to a smaller range.
-    {% else %}
+    {% if table_rows is defined %}
     <table class="dataTable cell-border nowrap" id="results_table" style="float: left;">
       <thead>
         <tr>
@@ -214,7 +211,6 @@
       <td colspan="100%" align="center"><br><b><font size="15">Loading...</font></b><br></td>
       </tbody>
     </table>
-    {% endif %}
     {% endif %}     
     </div>
   </div>
@@ -229,11 +225,23 @@
 
   <script type='text/javascript'>
       var json_rows = {{ table_rows|safe }};
+      var json_rows_count = {{ rows_count|safe }};
       var empty_columns = {{ empty_columns|safe }};
   </script>
 
   <script language="javascript">
 
+    var getParams = function(url) {
+      var parser = document.createElement('a');
+      parser.href = url;
+      var params = parser.search.substring(1);
+      console.log(params);
+      if(params.length > 0) {
+        return ('?'+params);
+      }
+      return params;
+    };
+
     var substringMatcher = function(strs) {
       return function findMatches(q, cb) {
         var matches, substringRegex;
@@ -266,7 +274,7 @@
       source: substringMatcher(rat_genes)
     });
 
-    {% if filtered_results is defined %}
+    {% if table_rows is defined %}
     $("#results_table").DataTable( {
       'data': json_rows,
       {% if variant_type == "SNP" %}
@@ -423,6 +431,10 @@
       'order': [[1, "asc" ]],
       'sDom': "rtip",
       'iDisplayLength': 500,
+      'bServerSide': true,
+      'bProcessing': true,
+      'sAjaxSource': '/snp_browser_table'+getParams(window.location.href),
+      'deferLoading': json_rows_count,
       'processing': true,
       'language': {
         'loadingRecords': '&nbsp;',
@@ -525,6 +537,8 @@
       $("input[name=chosen_strains]").val(strain_list.join(","));
     });
 
+    
+
     $("input[name=export_csv]").click(function() {
       var csv = [];
       var rows = document.querySelectorAll("table tr");
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 08673f79..a2c66d53 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -897,6 +897,7 @@ def corr_scatter_plot_page():
 def snp_browser_page():
     logger.info(request.url)
     template_vars = snp_browser.SnpBrowser(request.args)
+    logger.info(template_vars.__dict__.keys())
 
     return render_template("snp_browser.html", **template_vars.__dict__)
 
@@ -906,6 +907,15 @@ def db_info_page():
 
     return render_template("info_page.html", **template_vars.__dict__)
 
+@app.route("/snp_browser_table", methods=('GET',))
+def snp_browser_table():
+    logger.info(request.url)
+    logger.info(request.args)
+    
+    current_page = snp_browser.SnpPage(request.args).get_page()
+
+    return flask.jsonify(current_page)
+
 @app.route("/tutorial/WebQTLTour", methods=('GET',))
 def tutorial_page():
     #ZS: Currently just links to GN1