Merge branch 'flask' of git://github.com/zsloan/genenetwork into flask

Conflicts:
	wqflask/base/data_set.py

11 files changed, 625 insertions, 179 deletions

diff --git a/wqflask/wqflask/correlation/correlation_plot.py b/wqflask/wqflask/correlation/correlation_plot.py
index 4b043fc3..c2b64d70 100644
--- a/wqflask/wqflask/correlation/correlation_plot.py
+++ b/wqflask/wqflask/correlation/correlation_plot.py
@@ -24,6 +24,10 @@ class CorrelationPlot(object):
         self.samples_1 = self.get_samples(self.dataset1, sample_names_1, self.trait1)
         self.samples_2 = self.get_samples(self.dataset2, sample_names_2, self.trait2)
 
+        coords = {}
+        for sample in self.samples_1:
+            coords[sample.name] = (sample.val)
+
 
     def get_sample_names(self, dataset):
         if dataset.group.parlist:
@@ -45,4 +49,5 @@ class CorrelationPlot(object):
                             sample_group_type='primary',
                             header="%s Only" % (dataset.group.name))
 
-        return samples
\ No newline at end of file
+        return samples
+        
\ No newline at end of file
diff --git a/wqflask/wqflask/do_search.py b/wqflask/wqflask/do_search.py
index 1b1b56fb..7b3e0869 100644
--- a/wqflask/wqflask/do_search.py
+++ b/wqflask/wqflask/do_search.py
@@ -36,7 +36,7 @@ class DoSearch(object):
     def execute(self, query):
         """Executes query and returns results"""
         query = self.normalize_spaces(query)
-        print("in do_search query is:", pf(query))
+        #print("in do_search query is:", pf(query))
         results = g.db.execute(query, no_parameters=True).fetchall()
         #results = self.cursor.fetchall()
         return results
@@ -90,7 +90,7 @@ class QuickMrnaAssaySearch(DoSearch):
                     AGAINST ('%s' IN BOOLEAN MODE))
                             """ % (escape(self.search_term[0]))
 
-        print("final query is:", pf(query))
+        #print("final query is:", pf(query))
 
         return self.execute(query)
 
@@ -134,7 +134,7 @@ class MrnaAssaySearch(DoSearch):
                                     where_clause,
                                     escape(self.dataset.id)))        
 
-        print("query is:", pf(query))
+        #print("query is:", pf(query))
 
         return query
 
@@ -155,7 +155,7 @@ class MrnaAssaySearch(DoSearch):
                             """ % (escape(self.search_term[0]),
                             escape(str(self.dataset.id)))
 
-        print("final query is:", pf(query))
+        #print("final query is:", pf(query))
 
         return self.execute(query)
 
diff --git a/wqflask/wqflask/marker_regression/marker_regression.py b/wqflask/wqflask/marker_regression/marker_regression.py
index 6ae1318e..334ce631 100755
--- a/wqflask/wqflask/marker_regression/marker_regression.py
+++ b/wqflask/wqflask/marker_regression/marker_regression.py
@@ -78,8 +78,9 @@ class MarkerRegression(object):
             
             genotype_matrix = np.array(trimmed_genotype_data).T
             
-            print("pheno_vector is: ", pf(pheno_vector))
-            print("genotype_matrix is: ", pf(genotype_matrix))
+            print("pheno_vector: ", pf(pheno_vector))
+            print("genotype_matrix: ", pf(genotype_matrix))
+            print("genotype_matrix.shape: ", pf(genotype_matrix.shape))
             
             t_stats, p_values = lmm.run(
                 pheno_vector,
diff --git a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py
index a3ba8fdb..5bfc3a01 100644
--- a/wqflask/wqflask/my_pylmm/pyLMM/lmm.py
+++ b/wqflask/wqflask/my_pylmm/pyLMM/lmm.py
@@ -28,6 +28,8 @@ from scipy import stats
 import pdb
 
 import gzip
+import zlib
+import datetime
 import cPickle as pickle
 import simplejson as json
 
@@ -38,6 +40,9 @@ from utility import temp_data
 
 from wqflask.my_pylmm.pyLMM import chunks
 
+import redis
+Redis = redis.Redis()
+
 #np.seterr('raise')
 
 def run_human(pheno_vector,
@@ -51,6 +56,20 @@ def run_human(pheno_vector,
     keep = True - v
     keep = keep.reshape((len(keep),))
 
+    identifier = str(uuid.uuid4())
+    
+    print("pheno_vector: ", pf(pheno_vector))
+    print("kinship_matrix: ", pf(kinship_matrix))
+    print("kinship_matrix.shape: ", pf(kinship_matrix.shape))
+
+    lmm_vars = pickle.dumps(dict(
+        pheno_vector = pheno_vector,
+        covariate_matrix = covariate_matrix,
+        kinship_matrix = kinship_matrix
+    ))
+    Redis.hset(identifier, "lmm_vars", lmm_vars)
+    Redis.expire(identifier, 60*60)
+
     if v.sum():
         pheno_vector = pheno_vector[keep]
         #print("pheno_vector shape is now: ", pf(pheno_vector.shape))
@@ -60,16 +79,18 @@ def run_human(pheno_vector,
         kinship_matrix = kinship_matrix[keep,:][:,keep]
 
     n = kinship_matrix.shape[0]
+    #print("n is:", n)
     lmm_ob = LMM(pheno_vector,
                 kinship_matrix,
                 covariate_matrix)
     lmm_ob.fit()
 
+
     # Buffers for pvalues and t-stats
     p_values = []
     t_stats = []
 
-    print("input_file: ", plink_input_file)
+    #print("input_file: ", plink_input_file)
 
     with Bench("Opening and loading pickle file"):
         with gzip.open(plink_input_file, "rb") as input_file:
@@ -87,45 +108,65 @@ def run_human(pheno_vector,
         with Bench("Create list of inputs"):
             inputs = list(plink_input)
             
+        print("len(genotypes): ", len(inputs))
+
         with Bench("Divide into chunks"):
             results = chunks.divide_into_chunks(inputs, 64)
 
         result_store = []
-        identifier = uuid.uuid4()
+
+        key = "plink_inputs"
+        
+        # Todo: Delete below line when done testing
+        Redis.delete(key)
+        
+        timestamp = datetime.datetime.utcnow().isoformat()
+
+        #print("Starting adding loop")
         for part, result in enumerate(results):
-            # todo: Don't use TempData here. Instead revert old one and store this stuff as a list
-            data_store = temp_data.TempData(identifier, "plink", part)
+            #data = pickle.dumps(result, pickle.HIGHEST_PROTOCOL)
+            holder = pickle.dumps(dict(
+                identifier = identifier,
+                part = part,
+                timestamp = timestamp,
+                result = result
+            ), pickle.HIGHEST_PROTOCOL)
             
-            data_store.store("data", pickle.dumps(result, pickle.HIGHEST_PROTOCOL))
-            result_store.append(data_store)
-
+            #print("Adding:", part)
+            Redis.rpush(key, zlib.compress(holder))
+        #print("End adding loop")
+        #print("***** Added to {} queue *****".format(key))
         for snp, this_id in plink_input:
-            with Bench("part before association"):
-                if count > 2000:
-                    break
-                count += 1
-
-                percent_complete = (float(count) / total_snps) * 100
-                #print("percent_complete: ", percent_complete)
-                loading_progress.store("percent_complete", percent_complete)
-        
-            with Bench("actual association"):
-                ps, ts = human_association(snp,
-                                           n,
-                                           keep,
-                                           lmm_ob,
-                                           pheno_vector,
-                                           covariate_matrix,
-                                           kinship_matrix,
-                                           refit)
-
-            with Bench("after association"):
-                p_values.append(ps)
-                t_stats.append(ts)
+            #with Bench("part before association"):
+            if count > 2000:
+                break
+            count += 1
+
+            percent_complete = (float(count) / total_snps) * 100
+            #print("percent_complete: ", percent_complete)
+            loading_progress.store("percent_complete", percent_complete)
+
+            #with Bench("actual association"):
+            ps, ts = human_association(snp,
+                                       n,
+                                       keep,
+                                       lmm_ob,
+                                       pheno_vector,
+                                       covariate_matrix,
+                                       kinship_matrix,
+                                       refit)
+
+            #with Bench("after association"):
+            p_values.append(ps)
+            t_stats.append(ts)
         
     return p_values, t_stats
 
 
+#class HumanAssociation(object):
+#    def __init__(self):
+#        
+
 def human_association(snp,
                       n,
                       keep,
@@ -202,6 +243,8 @@ def run(pheno_vector,
     with Bench("LMM_ob fitting"):
         lmm_ob.fit()
 
+    print("genotype_matrix: ", genotype_matrix.shape)
+
     with Bench("Doing GWAS"):
         t_stats, p_values = GWAS(pheno_vector,
                                 genotype_matrix,
@@ -304,7 +347,7 @@ def GWAS(pheno_vector,
     covariate_matrix - n x q covariate matrix
     restricted_max_likelihood - use restricted maximum likelihood
     refit - refit the variance component for each SNP
-      
+    
     """
     if kinship_eigen_vals == None:
         kinship_eigen_vals = []
diff --git a/wqflask/wqflask/my_pylmm/pyLMM/process_plink.py b/wqflask/wqflask/my_pylmm/pyLMM/process_plink.py
new file mode 100644
index 00000000..e47c18e1
--- /dev/null
+++ b/wqflask/wqflask/my_pylmm/pyLMM/process_plink.py
@@ -0,0 +1,127 @@
+from __future__ import absolute_import, print_function, division
+
+import sys
+sys.path.append("../../..")
+
+print("sys.path: ", sys.path)
+
+import numpy as np
+
+import zlib
+import cPickle as pickle
+import redis
+Redis = redis.Redis()
+
+import lmm
+
+class ProcessLmmChunk(object):
+    
+    def __init__(self):
+        self.get_snp_data()
+        self.get_lmm_vars()
+        
+        keep = self.trim_matrices()
+        
+        self.do_association(keep)
+        
+        print("p_values is: ", self.p_values)
+        
+    def get_snp_data(self):
+        plink_pickled = zlib.decompress(Redis.lpop("plink_inputs"))
+        plink_data = pickle.loads(plink_pickled)
+        
+        self.snps = np.array(plink_data['result'])
+        self.identifier = plink_data['identifier']
+        
+    def get_lmm_vars(self):
+        lmm_vars_pickled = Redis.hget(self.identifier, "lmm_vars")
+        lmm_vars = pickle.loads(lmm_vars_pickled)
+        
+        self.pheno_vector = np.array(lmm_vars['pheno_vector'])
+        self.covariate_matrix = np.array(lmm_vars['covariate_matrix'])
+        self.kinship_matrix = np.array(lmm_vars['kinship_matrix'])
+        
+    def trim_matrices(self):
+        v = np.isnan(self.pheno_vector)
+        keep = True - v
+        keep = keep.reshape((len(keep),))
+        
+        if v.sum():
+            self.pheno_vector = self.pheno_vector[keep]
+            self.covariate_matrix = self.covariate_matrix[keep,:]
+            self.kinship_matrix = self.kinship_matrix[keep,:][:,keep]
+
+        return keep
+    
+    def do_association(self, keep):
+        n = self.kinship_matrix.shape[0]
+        lmm_ob = lmm.LMM(self.pheno_vector,
+                    self.kinship_matrix,
+                    self.covariate_matrix)
+        lmm_ob.fit()
+    
+        self.p_values = []
+        
+        for snp in self.snps:
+            snp = snp[0]
+            p_value, t_stat = lmm.human_association(snp,
+                                        n,
+                                        keep,
+                                        lmm_ob,
+                                        self.pheno_vector,
+                                        self.covariate_matrix,
+                                        self.kinship_matrix,
+                                        False)
+        
+            self.p_values.append(p_value)
+            
+
+#plink_pickled = zlib.decompress(Redis.lpop("plink_inputs"))
+#
+#plink_data = pickle.loads(plink_pickled)
+#result = np.array(plink_data['result'])
+#print("snp size is: ", result.shape)
+#identifier = plink_data['identifier']
+#
+#lmm_vars_pickled = Redis.hget(identifier, "lmm_vars")
+#lmm_vars = pickle.loads(lmm_vars_pickled)
+#
+#pheno_vector = np.array(lmm_vars['pheno_vector'])
+#covariate_matrix = np.array(lmm_vars['covariate_matrix'])
+#kinship_matrix = np.array(lmm_vars['kinship_matrix'])
+#
+#v = np.isnan(pheno_vector)
+#keep = True - v
+#keep = keep.reshape((len(keep),))
+#print("keep is: ", keep)
+#
+#if v.sum():
+#    pheno_vector = pheno_vector[keep]
+#    covariate_matrix = covariate_matrix[keep,:]
+#    kinship_matrix = kinship_matrix[keep,:][:,keep]
+#
+#n = kinship_matrix.shape[0]
+#print("n is: ", n)
+#lmm_ob = lmm.LMM(pheno_vector,
+#            kinship_matrix,
+#            covariate_matrix)
+#lmm_ob.fit()
+#
+#p_values = []
+#
+#for snp in result:
+#    snp = snp[0]
+#    p_value, t_stat = lmm.human_association(snp,
+#                                n,
+#                                keep,
+#                                lmm_ob,
+#                                pheno_vector,
+#                                covariate_matrix,
+#                                kinship_matrix,
+#                                False)
+#
+#    p_values.append(p_value)
+    
+
+
+
diff --git a/wqflask/wqflask/search_results.py b/wqflask/wqflask/search_results.py
index 4238aa7f..bb0313b3 100644
--- a/wqflask/wqflask/search_results.py
+++ b/wqflask/wqflask/search_results.py
@@ -26,8 +26,7 @@ from MySQLdb import escape_string as escape
 from htmlgen import HTMLgen2 as HT
 
 from base import webqtlConfig
-from utility.THCell import THCell
-from utility.TDCell import TDCell
+from utility.benchmark import Bench
 from base.data_set import create_dataset
 from base.trait import GeneralTrait
 from wqflask import parser
@@ -74,6 +73,7 @@ class SearchResultPage(object):
             self.quick = True
             self.search_terms = kw['q']
             print("self.search_terms is: ", self.search_terms)
+            self.trait_type = kw['trait_type']
             self.quick_search()
         else:
             self.results = []
@@ -124,24 +124,53 @@ class SearchResultPage(object):
                     FROM QuickSearch
                     WHERE MATCH (terms)
                           AGAINST ('{}' IN BOOLEAN MODE) """.format(search_terms)
-        dbresults = g.db.execute(query, no_parameters=True).fetchall()
+        
+        with Bench("Doing QuickSearch Query: "):
+            dbresults = g.db.execute(query, no_parameters=True).fetchall()
         #print("results: ", pf(results))
         
         self.results = collections.defaultdict(list)
         
         type_dict = {'PublishXRef': 'phenotype',
-                   'ProbesetXRef': 'mrna_assay',
+                   'ProbeSetXRef': 'mrna_assay',
                    'GenoXRef': 'genotype'}
-
+        
+        self.species_groups = {}
+        
         for dbresult in dbresults:
             this_result = {}
             this_result['table_name'] = dbresult.table_name
-            this_result['key'] = dbresult.the_key
-            this_result['result_fields'] = json.loads(dbresult.result_fields)
-            
-            self.results[type_dict[dbresult.table_name]].append(this_result)
+            if self.trait_type == type_dict[dbresult.table_name] or self.trait_type == 'all':
+                this_result['key'] = dbresult.the_key
+                this_result['result_fields'] = json.loads(dbresult.result_fields)
+                this_species = this_result['result_fields']['species']
+                this_group = this_result['result_fields']['group_name']
+                if this_species not in self.species_groups:
+                    self.species_groups[this_species] = {}
+                if type_dict[dbresult.table_name] not in self.species_groups[this_species]:
+                    self.species_groups[this_species][type_dict[dbresult.table_name]] = []
+                if this_group not in self.species_groups[this_species][type_dict[dbresult.table_name]]:
+                    self.species_groups[this_species][type_dict[dbresult.table_name]].append(this_group)
+                #if type_dict[dbresult.table_name] not in self.species_groups:
+                #    self.species_groups[type_dict[dbresult.table_name]] = {}
+                #if this_species not in self.species_groups[type_dict[dbresult.table_name]]:
+                #    self.species_groups[type_dict[dbresult.table_name]][this_species] = []
+                #if this_group not in self.species_groups[type_dict[dbresult.table_name]][this_species]:
+                #    self.species_groups[type_dict[dbresult.table_name]][this_species].append(this_group)
+                self.results[type_dict[dbresult.table_name]].append(this_result)
             
-        print("results: ", pf(self.results['phenotype']))
+        import redis
+        Redis = redis.Redis()
+        
+        
+        
+    #def get_group_species_tree(self):
+    #    self.species_groups = collections.default_dict(list)
+    #    for key in self.results:
+    #        for item in self.results[key]:
+    #            self.species_groups[item['result_fields']['species']].append(
+    #                                        item['result_fields']['group_name'])
+
 
     #def quick_search(self):
     #    self.search_terms = parser.parse(self.search_terms)
@@ -209,6 +238,6 @@ class SearchResultPage(object):
                                     self.dataset,
                                     )
             self.results.extend(the_search.run())
-            print("in the search results are:", self.results)
+            #print("in the search results are:", self.results)
 
         self.header_fields = the_search.header_fields
diff --git a/wqflask/wqflask/templates/all_results.html b/wqflask/wqflask/templates/all_results.html
new file mode 100644
index 00000000..a42e42d8
--- /dev/null
+++ b/wqflask/wqflask/templates/all_results.html
@@ -0,0 +1,134 @@
+            <ul class="nav nav-tabs">

+                {% for species in species_groups %}

+                <li> <a href="#tab{{ loop.index }}" data-toggle="tab">{{ species }}</a></li>

+                {% endfor %}

+            </ul>

+            <div class="tab-content">

+                {% for species in species_groups %}

+                <div class="tab-pane active" id="tab{{ loop.index }}">

+                    <div class="tabbable tabs-left">

+                        <ul class="nav nav-tabs">

+                            {% if species_groups[species]['phenotype'] %}

+                            <li class="active"> <a href="#tab{{ loop.index }}_1" data-toggle="tab">Phenotype</a></li>

+                            {% endif %}

+                            {% if species_groups[species]['mrna_assay'] %}

+                            <li> <a href="#tab{{ loop.index }}_2" data-toggle="tab">mRNA Assay</a></li>

+                            {% endif %}

+                            {% if species_groups[species]['genotype'] %}

+                            <li> <a href="#tab{{ loop.index }}_3" data-toggle="tab">Genotype</a></li>

+                            {% endif %}

+                        </ul>

+                        <div class="tab-content">

+                            {% if species_groups[species]['phenotype'] %}

+                            <div class="tab-pane active" id="tab{{ loop.index }}_1">

+                                <table id="pheno_results" class="table table-hover table-striped table-bordered">

+                                    <thead>

+                                        <tr>

+                                        <th>Id</th>

+                                        <th>Species</th>

+                                        <th>Group</th>

+                                        <th>Description</th>

+                                        <th>LRS</th>

+                                        <th>Year</th>

+                                        <th>Authors</th>

+                                        </tr>

+                                    </thead>

+                                    <tbody>

+                                        {% for result in results.phenotype %}

+                                        {% if result.result_fields['species'] == species %}

+                                        <tr>

+                                            <td>{{ result.result_fields['phenotype_id'] }}</td>

+                                            <td>{{ result.result_fields['species'] }}</td>

+                                            <td>{{ result.result_fields['group_name'] }}</td>

+                                            <td>{{ result.result_fields['description'] }}</td>

+                                            <td>{{ result.result_fields['lrs'] }}</td>

+                                            <td>

+                                                <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ result.result_fields['pubmed_id'] }}&dopt=Abstract">

+                                                    {{ result.result_fields['year'] }}

+                                                </a>

+                                            </td>

+                                            <td>{{ result.result_fields['authors'] }}</td>

+                                        </tr>

+                                        {% endif %}

+                                        {% endfor %}

+                                    </tbody>

+                                </table>

+                            </div>

+                            {% endif %}

+                            {% if species_groups[species]['mrna_assay'] %}

+                            <div class="tab-pane" id="tab{{ loop.index }}_2">

+                                <table id="mrna_assay_results" class="table table-hover table-striped table-bordered">

+                                    <thead>

+                                        <tr>

+                                        <th>Record ID</th>

+                                        <th>Species</th>

+                                        <th>Group</th>

+                                        <th>Data Set</th>

+                                        <th>Symbol</th>

+                                        <th>Description</th>

+                                        <th>Location</th>

+                                        <th>Mean Expr</th>

+                                        <th>Max LRS</th>

+                                        </tr>

+                                    </thead>

+                                    <tbody>

+                                        {% for result in results.mrna_assay %}

+                                        {% if result.result_fields['species'] == species %}

+                                        <tr>

+                                            <td>

+                                                <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['name'] }}&dataset={{ result.result_fields['dataset'] }}"

+                                                   {{ result.result_fields['name'] }}

+                                                </a>

+                                            </td>

+                                            <td>{{ result.result_fields['species'] }}</td>

+                                            <td>{{ result.result_fields['group_name'] }}</td>

+                                            <td>{{ result.result_fields['dataset_name'] }}</td>

+                                            <td>{{ result.result_fields['symbol'] }}</td>

+                                            <td>{{ result.result_fields['description'] }}</td>

+                                            <td>{{ result.result_fields['chr'] }} : {{ result['mb'] }}</td>

+                                            <td>{{ result.result_fields['mean'] }}</td>

+                                            <td>{{ result.result_fields['lrs'] }}</td>

+                                        </tr>

+                                        {% endif %}

+                                        {% endfor %}

+                                    </tbody>

+                                </table>       

+                            </div>

+                            {% endif %}

+                            {% if species_groups[species]['genotype'] %}

+                            <div class="tab-pane" id="tab{{ loop.index }}_3">

+                                <table id="geno_results" class="table table-hover table-striped table-bordered">

+                                    <thead>

+                                        <tr>

+                                        <th>Marker</th>

+                                        <th>Species</th>

+                                        <th>Group</th>

+                                        <th>Data Set</th>

+                                        <th>Location</th>

+                                        </tr>

+                                    </thead>

+                                    <tbody>

+                                        {% for result in results.genotype %}

+                                        {% if result.result_fields['species'] == species %}

+                                        <tr>

+                                            <td>

+                                                <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['marker_name'] }}&dataset={{ result.result_fields['dataset'] }}">

+                                                    {{ result.result_fields['marker_name'] }}

+                                                </a>

+                                            </td>

+                                            <td>{{ result.result_fields['species'] }}</td>

+                                            <td>{{ result.result_fields['group_name'] }}</td>

+                                            <td>{{ result.result_fields['dataset_name'] }}</td>

+                                            <td>{{ result.result_fields['chr'] }} : {{ result.result_fields['mb'] }}</td>

+                                        </tr>

+                                        {% endif %}

+                                        {% endfor %}

+                                    </tbody>

+                                </table>   

+                            </div>

+                            {% endif %}

+                        </div>

+                    </div>

+                </div>

+                {% endfor %}

+            </div>
\ No newline at end of file
diff --git a/wqflask/wqflask/templates/index_page.html b/wqflask/wqflask/templates/index_page.html
index 0cc1c353..a9f2cb24 100644
--- a/wqflask/wqflask/templates/index_page.html
+++ b/wqflask/wqflask/templates/index_page.html
@@ -9,7 +9,6 @@
         <div class="container">
             <h1>GeneNetwork</h1>
             <p class="lead">Open source bioinformatics for systems genetics</p>
-            <p>- Lei Yan</p>
         </div>
     </header>
 
@@ -33,7 +32,14 @@
                     </div>
                     <form method="get" action="/search"  name="SEARCHFORM">
                         <fieldset>
-                            <label for="quick">Search:</label>
+                            <div class="input-prepend">
+                                <select id="trait_type" name="trait_type" class="span2">
+                                    <option value="all">All</option>
+                                    <option value="mrna_assay">Molecular Assays</option>
+                                    <option value="phenotype">Phenotypes</option>
+                                    <option value="genotype">Genotypes</option>
+                                </select>
+                            </div>
                             <div class="input-append">
                                 <input class="input-xlarge"
                                     id="quick" name="q"
@@ -43,11 +49,9 @@
                                     class="btn btn-primary" value="Search">
                             </div>
                         </fieldset>
-                       
                     </form>
                 </section>
                 <section id="search">
-                    
                     <div class="page-header">
                         <h1>Select and search</h1>
                     </div>
diff --git a/wqflask/wqflask/templates/quick_search.html b/wqflask/wqflask/templates/quick_search.html
index d50b4937..b0e38708 100644
--- a/wqflask/wqflask/templates/quick_search.html
+++ b/wqflask/wqflask/templates/quick_search.html
@@ -21,30 +21,158 @@
         <ul>
             {% if search_terms %}
             <li>
-                {% for word in search_terms %}
-                    <strong>{{word}}</strong> {% if not loop.last %} or {% endif %}
-                {% endfor %}
+                <strong>{{search_terms}}</strong> 
             </li>
             {% endif %}
         </ul>
-        
-        <p>To study a record, click on its ID below.<br />
-        Check records below and click Add button to add to selection.</p>
+
 
         <div class="tabbable"> <!-- Only required for left/right tabs -->
+            {% if trait_type == 'all' %}
+                {% include 'all_results.html' %}
+            {% else %}
             <ul class="nav nav-tabs">
-                <li class="active"> <a href="#tab1" data-toggle="tab">Phenotype</a></li>
-                <li>                <a href="#tab2" data-toggle="tab">mRNA Assay</a></li>
-                <li>                <a href="#tab3" data-toggle="tab">Genotype</a></li>
+                {% for species in species_groups %}
+                <li> <a href="#tab{{ loop.index }}" data-toggle="tab">{{ species }}</a></li>
+                {% endfor %}
             </ul>
             <div class="tab-content">
-                <div class="tab-pane active" id="tab1">
-                    <table class="table table-hover">
+                {% for species in species_groups %}
+                <div class="tab-pane active" id="tab{{ loop.index }}">
+                    <div>
+                        {#
+                        <ul class="nav nav-tabs">
+                            {% if species_groups[species]['phenotype'] %}
+                            <li class="active"> <a href="#tab{{ loop.index }}_1" data-toggle="tab">Phenotype</a></li>
+                            {% endif %}
+                            {% if species_groups[species]['mrna_assay'] %}
+                            <li> <a href="#tab{{ loop.index }}_2" data-toggle="tab">mRNA Assay</a></li>
+                            {% endif %}
+                            {% if species_groups[species]['genotype'] %}
+                            <li> <a href="#tab{{ loop.index }}_3" data-toggle="tab">Genotype</a></li>
+                            {% endif %}
+                        </ul>
+                        #}
+                        <div>
+                            {% if trait_type == 'phenotype' and species_groups[species]['phenotype'] %}
+                                <table id="pheno_results" class="table table-hover table-striped table-bordered">
+                                    <thead>
+                                        <tr>
+                                        <th>Id</th>
+                                        <th>Species</th>
+                                        <th>Group</th>
+                                        <th>Description</th>
+                                        <th>LRS</th>
+                                        <th>Year</th>
+                                        <th>Authors</th>
+                                        </tr>
+                                    </thead>
+                                    <tbody>
+                                        {% for result in results.phenotype %}
+                                        {% if result.result_fields['species'] == species %}
+                                        <tr>
+                                            <td>{{ result.result_fields['phenotype_id'] }}</td>
+                                            <td>{{ result.result_fields['species'] }}</td>
+                                            <td>{{ result.result_fields['group_name'] }}</td>
+                                            <td>{{ result.result_fields['description'] }}</td>
+                                            <td>{{ result.result_fields['lrs'] }}</td>
+                                            <td>
+                                                <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ result.result_fields['pubmed_id'] }}&dopt=Abstract">
+                                                    {{ result.result_fields['year'] }}
+                                                </a>
+                                            </td>
+                                            <td>{{ result.result_fields['authors'] }}</td>
+                                        </tr>
+                                        {% endif %}
+                                        {% endfor %}
+                                    </tbody>
+                                </table>
+                            {% endif %}
+                            {% if trait_type == 'mrna_assay' and species_groups[species]['mrna_assay'] %}
+                                <table id="mrna_assay_results" class="table table-hover table-striped table-bordered">
+                                    <thead>
+                                        <tr>
+                                        <th>Record ID</th>
+                                        <th>Species</th>
+                                        <th>Group</th>
+                                        <th>Data Set</th>
+                                        <th>Symbol</th>
+                                        <th>Description</th>
+                                        <th>Location</th>
+                                        <th>Mean Expr</th>
+                                        <th>Max LRS</th>
+                                        </tr>
+                                    </thead>
+                                    <tbody>
+                                        {% for result in results.mrna_assay %}
+                                        {% if result.result_fields['species'] == species %}
+                                        <tr>
+                                            <td>
+                                                <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['name'] }}&dataset={{ result.result_fields['dataset'] }}"
+                                                   {{ result.result_fields['name'] }}
+                                                </a>
+                                            </td>
+                                            <td>{{ result.result_fields['species'] }}</td>
+                                            <td>{{ result.result_fields['group_name'] }}</td>
+                                            <td>{{ result.result_fields['dataset_name'] }}</td>
+                                            <td>{{ result.result_fields['symbol'] }}</td>
+                                            <td>{{ result.result_fields['description'] }}</td>
+                                            <td>{{ result.result_fields['chr'] }} : {{ result['mb'] }}</td>
+                                            <td>{{ result.result_fields['mean'] }}</td>
+                                            <td>{{ result.result_fields['lrs'] }}</td>
+                                        </tr>
+                                        {% endif %}
+                                        {% endfor %}
+                                    </tbody>
+                                </table>
+                            {% endif %}
+                            {% if trait_type == 'genotype' and species_groups[species]['genotype'] %}
+                                <table id="geno_results" class="table table-hover table-striped table-bordered">
+                                    <thead>
+                                        <tr>
+                                        <th>Marker</th>
+                                        <th>Species</th>
+                                        <th>Group</th>
+                                        <th>Data Set</th>
+                                        <th>Location</th>
+                                        </tr>
+                                    </thead>
+                                    <tbody>
+                                        {% for result in results.genotype %}
+                                        {% if result.result_fields['species'] == species %}
+                                        <tr>
+                                            <td>
+                                                <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['marker_name'] }}&dataset={{ result.result_fields['dataset'] }}">
+                                                    {{ result.result_fields['marker_name'] }}
+                                                </a>
+                                            </td>
+                                            <td>{{ result.result_fields['species'] }}</td>
+                                            <td>{{ result.result_fields['group_name'] }}</td>
+                                            <td>{{ result.result_fields['dataset_name'] }}</td>
+                                            <td>{{ result.result_fields['chr'] }} : {{ result.result_fields['mb'] }}</td>
+                                        </tr>
+                                        {% endif %}
+                                        {% endfor %}
+                                    </tbody>
+                                </table>
+                            {% endif %}
+                        </div>
+                    </div>
+                </div>
+                {% endfor %}
+            </div>
+            {% endif %}
+        </div>
+    </div>
+
+<!-- End of body -->
+
+{% endblock %}
+
+                {#
+                    <table id="pheno_results" class="table table-hover table-striped table-bordered">
                         <thead>
                             <tr>
-                       <!--     {% for key, _value in results.phenotype[0].result_fields.items() %}
-                                <th>{{key}}</th>
-                            {% endfor %}-->
                             <th>Id</th>
                             <th>Species</th>
                             <th>Group</th>
@@ -57,26 +185,24 @@
                         <tbody>
                             {% for result in results.phenotype %}
                             <tr>
-                                {% for result in result.result_fields.items() %}
-                                <td>{{ result['phenotype_id'] }}</td>
-                                <td>{{ result['species'] }}</td>
-                                <td>{{ result['group_name'] }}</td>
-                                <td>{{ result['description'] }}</td>
-                                <td>{{ result['lrs'] }}</td>
+                                <td>{{ result.result_fields['phenotype_id'] }}</td>
+                                <td>{{ result.result_fields['species'] }}</td>
+                                <td>{{ result.result_fields['group_name'] }}</td>
+                                <td>{{ result.result_fields['description'] }}</td>
+                                <td>{{ result.result_fields['lrs'] }}</td>
                                 <td>
-                                    <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ result['pubmed_id'] }}&dopt=Abstract">
-                                        {{ result['year'] }}
+                                    <a href="http://www.ncbi.nlm.nih.gov/entrez/query.fcgi?cmd=Retrieve&db=PubMed&list_uids={{ result.result_fields['pubmed_id'] }}&dopt=Abstract">
+                                        {{ result.result_fields['year'] }}
                                     </a>
                                 </td>
-                                <td>{{ result['authors'] }}</td>
-                                {% endfor %}
+                                <td>{{ result.result_fields['authors'] }}</td>
                             </tr>
                             {% endfor %}
                         </tbody>
                     </table>
                 </div>
                 <div class="tab-pane" id="tab2">
-                    <table class="table table-hover">
+                    <table id="mrna_assay_results" class="table table-hover table-striped table-bordered">
                         <thead>
                             <tr>
                             <th>Record ID</th>
@@ -93,28 +219,26 @@
                         <tbody>
                             {% for result in results.mrna_assay %}
                             <tr>
-                                {% for result in result.result_fields.items() %}
                                 <td>
-                                    <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result['name'] }}&dataset={{ result['dataset'] }}"
-                                       {{ result['name'] }}
+                                    <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['name'] }}&dataset={{ result.result_fields['dataset'] }}"
+                                       {{ result.result_fields['name'] }}
                                     </a>
                                 </td>
-                                <td>{{ result['species'] }}</td>
-                                <td>{{ result['group_name'] }}</td>
-                                <td>{{ result['dataset_name'] }}</td>
-                                <td>{{ result['symbol'] }}</td>
-                                <td>{{ result['description'] }}</td>
-                                <td>{{ result['chr'] }} : {{ result['mb'] }}</td>
-                                <td>{{ result['mean'] }}</td>
-                                <td>{{ result['lrs'] }}</td>
-                                {% endfor %}
+                                <td>{{ result.result_fields['species'] }}</td>
+                                <td>{{ result.result_fields['group_name'] }}</td>
+                                <td>{{ result.result_fields['dataset_name'] }}</td>
+                                <td>{{ result.result_fields['symbol'] }}</td>
+                                <td>{{ result.result_fields['description'] }}</td>
+                                <td>{{ result.result_fields['chr'] }} : {{ result['mb'] }}</td>
+                                <td>{{ result.result_fields['mean'] }}</td>
+                                <td>{{ result.result_fields['lrs'] }}</td>
                             </tr>
                             {% endfor %}
                         </tbody>
                     </table>                    
                 </div>
                 <div class="tab-pane" id="tab3">
-                    <table class="table table-hover">
+                    <table id="geno_results" class="table table-hover table-striped table-bordered">
                         <thead>
                             <tr>
                             <th>Marker</th>
@@ -125,91 +249,58 @@
                             </tr>
                         </thead>
                         <tbody>
-                            {% for result in results.mrna_assay %}
+                            {% for result in results.genotype %}
                             <tr>
-                                {% for result in result.result_fields.items() %}
                                 <td>
-                                    <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result['marker_name'] }}&dataset={{ result['dataset'] }}">
-                                        {{ result['marker_name'] }}
+                                    <a href="http://gn2python.genenetwork.org/show_trait?trait_id={{ result.result_fields['marker_name'] }}&dataset={{ result.result_fields['dataset'] }}">
+                                        {{ result.result_fields['marker_name'] }}
                                     </a>
                                 </td>
-                                <td>{{ result['species'] }}</td>
-                                <td>{{ result['group_name'] }}</td>
-                                <td>{{ result['dataset_name'] }}</td>
-                                <td>{{ result['chr'] }} : {{ result['mb'] }}</td>
-                                {% endfor %}
+                                <td>{{ result.result_fields['species'] }}</td>
+                                <td>{{ result.result_fields['group_name'] }}</td>
+                                <td>{{ result.result_fields['dataset_name'] }}</td>
+                                <td>{{ result.result_fields['chr'] }} : {{ result.result_fields['mb'] }}</td>
                             </tr>
                             {% endfor %}
                         </tbody>
                     </table>                     
                 </div>
-            </div>
-        </div>
-        
-        
-
-        <!--<div class="bs-docs-example">
-            <table class="table table-hover">
-                <thead>
-                    <tr>
-                    {% for header in header_fields %}
-                        <th>{{header}}</th>
-                    {% endfor %}
-                    </tr>
-                </thead>
-
-                <tbody>
-                    {% for this_trait in trait_list %}
-                    <TR id="{{ this_trait }}">
-                        <TD>
-                            <INPUT TYPE="checkbox" NAME="searchResult" class="checkbox"
-                                   VALUE="{{ this_trait }}">
-                        </TD>
-                        <TD>
-                            <a href="{{ url_for('show_trait_page',
-                                    trait_id = this_trait.name,
-                                    dataset = dataset.name
-                                    )}}">
-                                {{ this_trait.name }}
-                            </a>
-                        </TD>
-                        {% if dataset.type == 'ProbeSet' %}
-                            <TD>{{ this_trait.symbol }}</TD>
-                            <TD>{{ this_trait.description_display }}</TD>
-                            <TD>{{ this_trait.location_repr }}</TD>
-                            <TD>{{ this_trait.mean }}</TD>
-                            <TD>{{ this_trait.LRS_score_repr }}</TD>
-                            <TD>{{ this_trait.LRS_location_repr }}</TD>
-                        {% elif dataset.type == 'Publish' %}
-                            <TD>{{ this_trait.description_display }}</TD>
-                            <TD>{{ this_trait.authors }}</TD>
-                            <TD>
-                                <a href="{{ this_trait.pubmed_link }}">
-                                    {{ this_trait.pubmed_text }}
-                                </a>
-                            </TD>
-                            <TD>{{ this_trait.LRS_score_repr }}</TD>
-                            <TD>{{ this_trait.LRS_location_repr }}</TD>
-                        {% elif dataset.type == 'Geno' %}
-                            <TD>{{ this_trait.location_repr }}</TD>
-                        {% endif %}
-                    </TR>
-                {% endfor %}
-                </tbody>
+                #}
 
-            </table>
 
-            <br />
-
-            <button class="btn"><i class="icon-ok"></i> Select</button>
-            <button class="btn"><i class="icon-remove"></i> Deselect</button>
-            <button class="btn"><i class="icon-resize-vertical"></i> Invert</button>
-            <button class="btn"><i class="icon-plus-sign"></i> Add</button>
-            <button class="btn btn-primary pull-right"><i class="icon-download icon-white"></i> Download Table</button>
-        </div>-->
-
-    </div>
-
-<!-- End of body -->
+{% block js %}  
+    <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/new/packages/DataTables/js/jquery.dataTables.min.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/packages/DT_bootstrap/DT_bootstrap.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/packages/TableTools/media/js/TableTools.min.js"></script>
+    <script language="javascript" type="text/javascript" src="/static/packages/underscore/underscore-min.js"></script>
 
+    
+    <script type="text/javascript" charset="utf-8">
+        $(document).ready( function () {
+            console.time("Creating table");
+            $('#pheno_results, #mrna_assay_results, #geno_results').dataTable( {
+                //"sDom": "<<'span3'l><'span3'T><'span4'f>'row-fluid'r>t<'row-fluid'<'span6'i><'span6'p>>",
+                "sDom": "lTftipr",
+                "oTableTools": {
+                    "aButtons": [
+                        "copy",
+                        "print",
+                        {
+                            "sExtends":    "collection",
+                            "sButtonText": 'Save <span class="caret" />',
+                            "aButtons":    [ "csv", "xls", "pdf" ]
+                        }
+                    ],
+                    "sSwfPath": "/static/packages/TableTools/media/swf/copy_csv_xls_pdf.swf"
+                },
+                "iDisplayLength": 50,
+                "bLengthChange": true,
+                "bDeferRender": true,
+                "bSortClasses": false
+            } );
+            console.timeEnd("Creating table");
+        });
+    </script>
 {% endblock %}
+
diff --git a/wqflask/wqflask/templates/show_trait_details.html b/wqflask/wqflask/templates/show_trait_details.html
index c3abfc9f..b57c3c21 100644
--- a/wqflask/wqflask/templates/show_trait_details.html
+++ b/wqflask/wqflask/templates/show_trait_details.html
@@ -19,11 +19,11 @@
             BLAT Specifity
         </a>
     </dt>
-    <dd>{{ "%.1f" % (this_trait.probe_set_specificity) }}</dd>
+    <dd>{{ "%s" % (this_trait.probe_set_specificity) }}</dd>
     {% endif %}
     {% if this_trait.probe_set_blat_score %}
     <dt>BLAT Score</dt>
-    <dd>{{ "%i" % (this_trait.probe_set_blat_score) }}</dd>
+    <dd>{{ "%s" % (this_trait.probe_set_blat_score) }}</dd>
     {% endif %}
 </dl>
 
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index 102863f0..d50a9aa2 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -78,17 +78,28 @@ def search_page():
         else:
             return render_template("data_sharing.html", **template_vars.__dict__)
     else:
-        print("calling search_results.SearchResultPage")
-        the_search = search_results.SearchResultPage(request.args)
-        print("template_vars is:", pf(the_search.__dict__))
-        #print("trait_list is:", pf(the_search.__dict__['trait_list'][0].__dict__))
-        #for trait in the_search.trait_list:
-        #    print(" -", trait.description_display)
-
-        if the_search.quick:
-            return render_template("quick_search.html", **the_search.__dict__)
+        key = "search_results:v2:" + json.dumps(request.args, sort_keys=True)
+        print("key is:", pf(key))
+        with Bench("Loading cache"):
+            result = Redis.get(key)
+            
+        if result:
+            print("Cache hit!!!")
+            with Bench("Loading results"):
+                result = pickle.loads(result)
         else:
-            return render_template("search_result_page.html", **the_search.__dict__)
+            print("calling search_results.SearchResultPage")
+            the_search = search_results.SearchResultPage(request.args)
+            result = the_search.__dict__
+            
+            print("result: ", pf(result))
+            Redis.set(key, pickle.dumps(result))
+            Redis.expire(key, 60*60)
+
+        if result['quick']:
+            return render_template("quick_search.html", **result)
+        else:
+            return render_template("search_result_page.html", **result)
 
 
 @app.route("/whats_new")
@@ -173,9 +184,10 @@ def marker_regression_page():
     for key, value in initial_start_vars.iteritems():
         if key in wanted or key.startswith(('value:')):
             start_vars[key] = value
-    
+
     version = "v14"
     key = "marker_regression:{}:".format(version) + json.dumps(start_vars, sort_keys=True)
+    print("key is:", pf(key))
     with Bench("Loading cache"):
         result = Redis.get(key)
     
@@ -258,4 +270,4 @@ def json_default_handler(obj):
     
 #@app.after_request
 #def after_request(response):
-#    gc.collect()
\ No newline at end of file
+#    gc.collect()