Merge branch 'testing' of github.com:genenetwork/genenetwork2 into testing

17 files changed, 448 insertions, 49 deletions

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index a36abc0a..c6b56a81 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -11,7 +11,7 @@ on:
 jobs:
   unittest:
     runs-on: ubuntu-latest
-    container: bonfacekilz/python3-genenetwork2:ad741c1
+    container: bonfacekilz/python3-genenetwork2:00ba1f8
 
     steps:
     # First start with mariadb set then checkout. The checkout gives
@@ -31,7 +31,7 @@ jobs:
       run: |
         /gn2-profile/bin/screen -dmLS redisconn /gn2-profile/bin/redis-server
 
-    # Redis is required by some of the tests 6379
+    # Initialise the tables
     - name: Bootstrap tables
       run: |
         mysql -u root -e "SHOW DATABASES;"
@@ -39,28 +39,27 @@ jobs:
         mysql -u root -e "CREATE USER 'gn2'@'localhost' IDENTIFIED BY 'mysql_password';"
         mysql -u root -e "GRANT ALL PRIVILEGES ON *.* TO 'gn2'@'localhost';FLUSH PRIVILEGES;"
 
-    - name: Run the unit tests
+    - name: Start Genenetwork as a  Background Task
       run: |
-        env GN2_PROFILE=/gn2-profile \
+        /gn2-profile/bin/screen -dm bash -c "env GN2_PROFILE=/gn2-profile \
         TMPDIR=/tmp SERVER_PORT=5004 \
         WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \
         GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \
-        etc/default_settings.py -c -m unittest discover -v
+        etc/default_settings.py"
 
-    - name: Start Genenetwork as a  Background Task
+    - name: Run the unit tests
       run: |
         env GN2_PROFILE=/gn2-profile \
         TMPDIR=/tmp SERVER_PORT=5004 \
         WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \
         GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \
-        etc/default_settings.py&
+        etc/default_settings.py -c -m unittest discover -v
 
     - name: Test for Broken Links
       run: |
-
         env GN2_PROFILE=/gn2-profile \
         TMPDIR=/tmp\
         WEBSERVER_MODE=DEBUG LOG_LEVEL=DEBUG \
         GENENETWORK_FILES=/genotype_files/ bin/genenetwork2 \
-        etc/default_settings.py -c  /__w/genenetwork2/genenetwork2/test/requests/links_scraper/genelinks.py 
-
+        etc/default_settings.py -c \
+        $PWD/test/requests/links_scraper/genelinks.py
diff --git a/doc/docker-container.org b/doc/docker-container.org
index ec91824a..c894c4b5 100644
--- a/doc/docker-container.org
+++ b/doc/docker-container.org
@@ -59,11 +59,11 @@ RUN tar -xzf /tmp/gn2.tar.gz -C / && rm -f /tmp/gn2.tar.gz && \
 
 Build the image(Note the fullstop at the end):
 
-: sudo docker build -t python2-genenetwork2:latest -f Dockerfile .
+: sudo docker build -t python3-genenetwork2:latest -f Dockerfile .
 
 To load the image interactively you've just created:
 
-: docker run -ti "python2-genenetwork2:latest" bash
+: docker run -ti "python3-genenetwork2:latest" bash
 
 Assuming you have a docker instance running, you could always run
 commands in it e.g:
@@ -78,7 +78,7 @@ CI environment using Github Actions.
 To push to dockerhub, first get the image name by running =docker
 images=. Push to dockerhub using a command similar to:
 
-: docker push bonfacekilz/python2-genenetwork2:latest
+: docker push bonfacekilz/python3-genenetwork2:latest
 
 Right now, we have 2 images on DockerHub:
 
diff --git a/test/requests/links_scraper/genelinks.py b/test/requests/links_scraper/genelinks.py
index 12300f4a..52c13489 100644
--- a/test/requests/links_scraper/genelinks.py
+++ b/test/requests/links_scraper/genelinks.py
@@ -2,7 +2,6 @@ import re
 import requests
 import urllib3
 import os
-import logging
 
 from urllib.request import urlopen as uReq
 from bs4 import BeautifulSoup as soup
@@ -26,7 +25,6 @@ def search_templates():
                 parsed_page = soup(
                     open(file_path, encoding="utf8"), "html.parser")
                 html_parsed_pages.append(parsed_page)
-
     return html_parsed_pages
 
 
@@ -34,7 +32,7 @@ def is_valid_link(url_link):
     try:
         result = urlparse(url_link)
         return all([result.scheme, result.netloc, result.path])
-    except Exception as e:
+    except Exception:
         return False
 
 
@@ -42,13 +40,10 @@ def test_link(link):
     print(f'Checking -->{link}')
     results = None
     try:
-
         results = requests.get(link, verify=False, timeout=10)
         status_code = results.status_code
-
-    except Exception as e:
+    except Exception:
         status_code = 408
-
     return int(status_code) > 403
 
 
@@ -56,14 +51,11 @@ def fetch_css_links(parsed_page):
     print("fetching css links")
     for link in parsed_page.findAll("link"):
         full_path = None
-
         link_url = link.attrs.get("href")
         if is_valid_link(link_url):
             full_path = link_url
-
         elif re.match(r"^/css", link_url) or re.match(r"^/js", link_url):
             full_path = urljoin('http://localhost:5004/', link_url)
-
         if full_path is not None:
             if test_link(full_path):
                 BROKEN_LINKS.add(full_path)
@@ -71,16 +63,13 @@ def fetch_css_links(parsed_page):
 
 def fetch_html_links(parsed_page):
     print("fetching a tags ")
-
     for link in parsed_page.findAll("a"):
         full_path = None
         link_url = link.attrs.get("href")
         if re.match(r"^/", link_url):
             full_path = urljoin('http://localhost:5004/', link_url)
-
         elif is_valid_link(link_url):
             full_path = link_url
-
         if full_path is not None:
             if test_link(full_path):
                 BROKEN_LINKS.add(full_path)
@@ -92,8 +81,11 @@ def fetch_script_tags(parsed_page):
         js_link = link.attrs.get("src")
         if js_link is not None:
             if is_valid_link(js_link):
-                raise SystemExit("Failed,the library should be packaged in guix.\
-                                Please contact,http://genenetwork.org/ for more details")
+                raise SystemExit("Failed,the library should be "
+                                 "packaged in guix. "
+                                 "Please contact, "
+                                 "http://genenetwork.org/ "
+                                 "for more details")
 
             elif re.match(r"^/css", js_link) or re.match(r"^/js", js_link):
                 full_path = urljoin('http://localhost:5004/', js_link)
@@ -102,11 +94,9 @@ def fetch_script_tags(parsed_page):
 
 
 def fetch_page_links(page_url):
-
     urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
     html_page = uReq(page_url)
     parsed_page = soup(html_page, "html.parser")
-
     fetch_script_tags(parsed_page=parsed_page)
     fetch_css_links(parsed_page=parsed_page)
     fetch_html_links(parsed_page=parsed_page)
@@ -114,13 +104,10 @@ def fetch_page_links(page_url):
 
 def webpages_to_check():
     pages = [f"http://localhost:{PORT}/"]
-
     return pages
 
 
 if __name__ == '__main__':
-    # results = search_templates()
-
     for page in webpages_to_check():
         fetch_page_links(page)
         if len(BROKEN_LINKS) > 0:
@@ -130,4 +117,5 @@ if __name__ == '__main__':
 
     if len(BROKEN_LINKS) > 0:
         raise SystemExit(
-            "The links Above are broken.Please contact genenetwork.org<<<<<<<<")
+            "The links Above are broken. "
+            "Please contact genenetwork.org<<<<<<<<")
diff --git a/wqflask/base/data_set.py b/wqflask/base/data_set.py
index e20f2f98..75ddf278 100644
--- a/wqflask/base/data_set.py
+++ b/wqflask/base/data_set.py
@@ -124,6 +124,7 @@ class DatasetType:
                                 self.datasets[short_dataset_name] = new_type
             except Exception:  # Do nothing
                 pass
+
             self.redis_instance.set("dataset_structure",
                                     json.dumps(self.datasets))
         self.data = data
@@ -165,6 +166,7 @@ class DatasetType:
         if t in ['pheno', 'other_pheno']:
             group_name = name.replace("Publish", "")
 
+
         results = g.db.execute(sql_query_mapping[t] % group_name).fetchone()
         if results:
             self.datasets[name] = dataset_name_mapping[t]
@@ -646,6 +648,8 @@ class DataSet:
                 "Dataset {} is not yet available in GeneNetwork.".format(self.name))
             pass
 
+
+
     def get_trait_data(self, sample_list=None):
         if sample_list:
             self.samplelist = sample_list
diff --git a/wqflask/base/mrna_assay_tissue_data.py b/wqflask/base/mrna_assay_tissue_data.py
index 882ae911..8f8e2b0a 100644
--- a/wqflask/base/mrna_assay_tissue_data.py
+++ b/wqflask/base/mrna_assay_tissue_data.py
@@ -6,6 +6,7 @@ from utility import db_tools
 from utility import Bunch
 
 from utility.db_tools import escape
+from gn3.db_utils import database_connector
 
 
 from utility.logger import getLogger
@@ -45,16 +46,18 @@ class MrnaAssayTissueData:
                 and t.Mean = x.maxmean;
                     '''.format(in_clause)
 
-        results = g.db.execute(query).fetchall()
 
-        lower_symbols = []
+        # lower_symbols = []
+        lower_symbols = {}
         for gene_symbol in gene_symbols:
+            # lower_symbols[gene_symbol.lower()] = True
             if gene_symbol != None:
-                lower_symbols.append(gene_symbol.lower())
-
+                lower_symbols[gene_symbol.lower()] = True
+        results = list(g.db.execute(query).fetchall())
         for result in results:
             symbol = result[0]
-            if symbol.lower() in lower_symbols:
+            if symbol  is not None and lower_symbols.get(symbol.lower()):
+
                 symbol = symbol.lower()
 
                 self.data[symbol].gene_id = result.GeneId
@@ -83,6 +86,7 @@ class MrnaAssayTissueData:
                        WHERE TissueProbeSetData.Id IN {} and
                              TissueProbeSetXRef.DataId = TissueProbeSetData.Id""".format(db_tools.create_in_clause(id_list))
 
+
             results = g.db.execute(query).fetchall()
             for result in results:
                 if result.Symbol.lower() not in symbol_values_dict:
diff --git a/wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py b/wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py
new file mode 100644
index 00000000..e1bd6d86
--- /dev/null
+++ b/wqflask/tests/unit/wqflask/correlation/test_correlation_gn3.py
@@ -0,0 +1,14 @@
+"""this module contains tests for code used in integrating to gn3 api"""
+from unittest import TestCase
+from base.data_set import create_dataset
+
+class TestCorrelation(TestCase):
+
+    def test_create_dataset(self):
+        """test for creating datasets"""
+
+        pass
+    def test_fetch_dataset_info(self):
+        """test for fetching dataset info data"""
+
+        pass
diff --git a/wqflask/wqflask/correlation/correlation_functions.py b/wqflask/wqflask/correlation/correlation_functions.py
index 0f24241a..2d25fbd8 100644
--- a/wqflask/wqflask/correlation/correlation_functions.py
+++ b/wqflask/wqflask/correlation/correlation_functions.py
@@ -82,6 +82,6 @@ def cal_zero_order_corr_for_tiss(primaryValue=[], targetValue=[], method='pearso
 
 def get_trait_symbol_and_tissue_values(symbol_list=None):
     tissue_data = MrnaAssayTissueData(gene_symbols=symbol_list)
-
-    if len(tissue_data.gene_symbols):
-        return tissue_data.get_symbol_values_pairs()
+    if len(tissue_data.gene_symbols) >0:
+        results = tissue_data.get_symbol_values_pairs()
+        return results 
diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
new file mode 100644
index 00000000..98d52591
--- /dev/null
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -0,0 +1,205 @@
+"""module that calls the gn3 api's to do the correlation """
+import json
+
+from wqflask.correlation import correlation_functions
+
+from base import data_set
+
+from base.trait import create_trait
+from base.trait import retrieve_sample_data
+
+from gn3.computations.correlations import compute_all_sample_correlation
+from gn3.computations.correlations import map_shared_keys_to_values
+from gn3.computations.correlations import compute_all_lit_correlation
+from gn3.computations.correlations import compute_tissue_correlation
+from gn3.db_utils import database_connector
+
+
+def create_target_this_trait(start_vars):
+    """this function creates the required trait and target dataset for correlation"""
+
+    this_dataset = data_set.create_dataset(dataset_name=start_vars['dataset'])
+    target_dataset = data_set.create_dataset(
+        dataset_name=start_vars['corr_dataset'])
+    this_trait = create_trait(dataset=this_dataset,
+                              name=start_vars['trait_id'])
+    sample_data = ()
+    return (this_dataset, this_trait, target_dataset, sample_data)
+
+
+def process_samples(start_vars, sample_names, excluded_samples=None):
+    """process samples"""
+    sample_data = {}
+    if not excluded_samples:
+        excluded_samples = ()
+        sample_vals_dict = json.loads(start_vars["sample_vals"])
+        for sample in sample_names:
+            if sample not in excluded_samples:
+                val = sample_vals_dict[sample]
+                if not val.strip().lower() == "x":
+                    sample_data[str(sample)] = float(val)
+    return sample_data
+
+
+def sample_for_trait_lists(corr_results, target_dataset,
+                           this_trait, this_dataset, start_vars):
+    """interface function for correlation on top results"""
+
+    sample_data = process_samples(
+        start_vars, this_dataset.group.samplelist)
+    target_dataset.get_trait_data(list(sample_data.keys()))
+    # should filter target traits from here
+    _corr_results = corr_results
+
+    this_trait = retrieve_sample_data(this_trait, this_dataset)
+
+    this_trait_data = {
+        "trait_sample_data": sample_data,
+        "trait_id": start_vars["trait_id"]
+    }
+    results = map_shared_keys_to_values(
+        target_dataset.samplelist, target_dataset.trait_data)
+    correlation_results = compute_all_sample_correlation(corr_method="pearson",
+                                                         this_trait=this_trait_data,
+                                                         target_dataset=results)
+
+    return correlation_results
+
+
+def tissue_for_trait_lists(corr_results, this_dataset, this_trait):
+    """interface function for doing tissue corr_results on trait_list"""
+    trait_lists = dict([(list(corr_result)[0], True)
+                        for corr_result in corr_results])
+    # trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results}
+    traits_symbol_dict = this_dataset.retrieve_genes("Symbol")
+    traits_symbol_dict = dict({trait_name: symbol for (
+        trait_name, symbol) in traits_symbol_dict.items() if trait_lists.get(trait_name)})
+    primary_tissue_data, target_tissue_data = get_tissue_correlation_input(
+        this_trait, traits_symbol_dict)
+    corr_results = compute_tissue_correlation(
+        primary_tissue_dict=primary_tissue_data,
+        target_tissues_data=target_tissue_data,
+        corr_method="pearson")
+    return corr_results
+
+
+def lit_for_trait_list(corr_results, this_dataset, this_trait):
+    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
+        this_trait, this_dataset)
+
+    # trait_lists = {list(corr_results)[0]: 1 for corr_result in corr_results}
+    trait_lists = dict([(list(corr_result)[0], True)
+                        for corr_result in corr_results])
+
+    geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if
+                   trait_lists.get(trait_name)}
+
+    conn, _cursor_object = database_connector()
+
+    with conn:
+
+        correlation_results = compute_all_lit_correlation(
+            conn=conn, trait_lists=list(geneid_dict.items()),
+            species=species, gene_id=this_trait_geneid)
+
+    return correlation_results
+
+
+def compute_correlation(start_vars, method="pearson"):
+    """compute correlation for to call gn3  api"""
+    # pylint: disable-msg=too-many-locals
+
+    corr_type = start_vars['corr_type']
+
+    (this_dataset, this_trait, target_dataset,
+     sample_data) = create_target_this_trait(start_vars)
+
+    method = start_vars['corr_sample_method']
+    corr_return_results = int(start_vars.get("corr_return_results", 100))
+    corr_input_data = {}
+
+    if corr_type == "sample":
+
+        sample_data = process_samples(
+            start_vars, this_dataset.group.samplelist)
+        target_dataset.get_trait_data(list(sample_data.keys()))
+        this_trait = retrieve_sample_data(this_trait, this_dataset)
+        this_trait_data = {
+            "trait_sample_data": sample_data,
+            "trait_id": start_vars["trait_id"]
+        }
+        results = map_shared_keys_to_values(
+            target_dataset.samplelist, target_dataset.trait_data)
+        correlation_results = compute_all_sample_correlation(corr_method=method,
+                                                             this_trait=this_trait_data,
+                                                             target_dataset=results)
+
+        # do tissue correaltion
+
+        # code to be use later
+
+        # tissue_result = tissue_for_trait_lists(
+        #     correlation_results, this_dataset, this_trait)
+        # # lit spoils the party so slow
+        # lit_result = lit_for_trait_list(
+        #     correlation_results, this_dataset, this_trait)
+
+
+    elif corr_type == "tissue":
+        trait_symbol_dict = this_dataset.retrieve_genes("Symbol")
+        primary_tissue_data, target_tissue_data = get_tissue_correlation_input(
+            this_trait, trait_symbol_dict)
+
+        corr_input_data = {
+            "primary_tissue": primary_tissue_data,
+            "target_tissues_dict": target_tissue_data
+        }
+        correlation_results = compute_tissue_correlation(
+            primary_tissue_dict=corr_input_data["primary_tissue"],
+            target_tissues_data=corr_input_data[
+                "target_tissues_dict"],
+            corr_method=method
+
+        )
+
+    elif corr_type == "lit":
+        (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
+            this_trait, this_dataset)
+
+        conn, _cursor_object = database_connector()
+        with conn:
+            correlation_results = compute_all_lit_correlation(
+                conn=conn, trait_lists=list(geneid_dict.items()),
+                species=species, gene_id=this_trait_geneid)
+
+    return correlation_results[0:corr_return_results]
+
+
+def do_lit_correlation(this_trait, this_dataset):
+    """function for fetching lit inputs"""
+    geneid_dict = this_dataset.retrieve_genes("GeneId")
+    species = this_dataset.group.species.lower()
+    trait_geneid = this_trait.geneid
+    return (trait_geneid, geneid_dict, species)
+
+
+def get_tissue_correlation_input(this_trait, trait_symbol_dict):
+    """Gets tissue expression values for the primary trait and target tissues values"""
+    primary_trait_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(
+        symbol_list=[this_trait.symbol])
+    if this_trait.symbol.lower() in primary_trait_tissue_vals_dict:
+        primary_trait_tissue_values = primary_trait_tissue_vals_dict[this_trait.symbol.lower(
+        )]
+        corr_result_tissue_vals_dict = correlation_functions.get_trait_symbol_and_tissue_values(
+            symbol_list=list(trait_symbol_dict.values()))
+        primary_tissue_data = {
+            "this_id": this_trait.name,
+            "tissue_values": primary_trait_tissue_values
+
+        }
+        target_tissue_data = {
+            "trait_symbol_dict": trait_symbol_dict,
+            "symbol_tissue_vals_dict": corr_result_tissue_vals_dict
+        }
+        return (primary_tissue_data, target_tissue_data)
+    return None
diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 9b0b6118..7fc9f955 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -96,6 +96,7 @@ class CorrelationResults:
             self.p_range_lower = get_float(start_vars, 'p_range_lower', -1.0)
             self.p_range_upper = get_float(start_vars, 'p_range_upper', 1.0)
 
+
             if ('loc_chr' in start_vars
                 and 'min_loc_mb' in start_vars
                     and 'max_loc_mb' in start_vars):
@@ -200,6 +201,7 @@ class CorrelationResults:
                         if chr_info.name == trait_object.chr:
                             chr_as_int = order_id
 
+
                 if (float(self.correlation_data[trait][0]) >= self.p_range_lower
                         and float(self.correlation_data[trait][0]) <= self.p_range_upper):
 
diff --git a/wqflask/wqflask/static/new/javascript/show_trait.js b/wqflask/wqflask/static/new/javascript/show_trait.js
index 6e9d68c4..569046d3 100644
--- a/wqflask/wqflask/static/new/javascript/show_trait.js
+++ b/wqflask/wqflask/static/new/javascript/show_trait.js
@@ -585,6 +585,16 @@ get_table_contents_for_form_submit = function(form_id) {
 var corr_input_list = ['sample_vals', 'corr_type', 'primary_samples', 'trait_id', 'dataset', 'group', 'tool_used', 'form_url', 'corr_sample_method', 'corr_samples_group', 'corr_dataset', 'min_expr',
                         'corr_return_results', 'location_type', 'loc_chr', 'min_loc_mb', 'max_loc_mb', 'p_range_lower', 'p_range_upper']
 
+$(".test_corr_compute").on("click", (function(_this) {
+  return function() {
+    $('input[name=tool_used]').val("Correlation");
+    $('input[name=form_url]').val("/test_corr_compute");
+    $('input[name=wanted_inputs]').val(corr_input_list.join(","));
+    url = "/loading";
+    return submit_special(url);
+  };
+})(this));
+
 $(".corr_compute").on("click", (function(_this) {
   return function() {
     $('input[name=tool_used]').val("Correlation");
diff --git a/wqflask/wqflask/templates/correlation_page.html b/wqflask/wqflask/templates/correlation_page.html
index a9a3e1a0..4cad2749 100644
--- a/wqflask/wqflask/templates/correlation_page.html
+++ b/wqflask/wqflask/templates/correlation_page.html
@@ -4,7 +4,7 @@
     <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" />
     <link rel="stylesheet" type="text/css" href="{{ url_for('js', filename='DataTablesExtensions/buttonsBootstrap/css/buttons.bootstrap.css') }}" />
     <link rel="stylesheet" type="text/css" href="{{ url_for('js', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css') }}">
-    <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css">
+    <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='fontawesome/css/all.min.css') }}"/>
     <link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" />
     <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" />
 {% endblock %}
@@ -141,7 +141,7 @@
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/dataTables.buttons.min.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/buttons.html5.min.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/natural.js') }}"></script>
-    <script language="javascript" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/js/all.min.js"></script>
+    <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='fontawesome/js/all.min.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script>
     
 
diff --git a/wqflask/wqflask/templates/mapping_results.html b/wqflask/wqflask/templates/mapping_results.html
index 73d7501b..d6fc6e37 100644
--- a/wqflask/wqflask/templates/mapping_results.html
+++ b/wqflask/wqflask/templates/mapping_results.html
@@ -357,7 +357,7 @@
     {% endif %}
 
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script>
-    <script language="javascript" type="text/javascript" src="https://cdn.datatables.net/buttons/1.0.0/js/dataTables.buttons.min.js"></script>
+   <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/dataTables.buttons.min.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/scientific.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/natural.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='purescript-genome-browser/js/purescript-genetics-browser.js') }}"></script>
diff --git a/wqflask/wqflask/templates/search_result_page.html b/wqflask/wqflask/templates/search_result_page.html
index e7a7bc51..827bad98 100644
--- a/wqflask/wqflask/templates/search_result_page.html
+++ b/wqflask/wqflask/templates/search_result_page.html
@@ -4,7 +4,7 @@
     <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" />
     <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='fontawesome/css/font-awesome.min.css') }}" />
     <link rel="stylesheet" type="text/css" href="{{ url_for('js', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css') }}">
-    <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css">
+    <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='fontawesome/css/all.min.css') }}"/>
     <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" />
     <link rel="stylesheet" type="text/css" href="static/new/css/trait_list.css" />
 {% endblock %}
@@ -150,7 +150,7 @@
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/natural.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/dataTables.buttons.min.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/buttons.colVis.min.js') }}"></script>
-    <script language="javascript" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/js/all.min.js"></script>
+    <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='fontawesome/js/all.min.js') }}"></script>
 
     <script language="javascript" type="text/javascript" src="/static/new/javascript/search_results.js"></script>
 
diff --git a/wqflask/wqflask/templates/show_trait.html b/wqflask/wqflask/templates/show_trait.html
index 09ecb7b6..fc14822c 100644
--- a/wqflask/wqflask/templates/show_trait.html
+++ b/wqflask/wqflask/templates/show_trait.html
@@ -148,7 +148,7 @@
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/scientific.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/natural.js') }}"></script>
-    <script language="javascript" type="text/javascript" src="https://cdn.datatables.net/scroller/2.0.3/js/dataTables.scroller.min.js"></script>
+    <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script>
     <script language="javascript" type="text/javascript" src="{{ url_for('js', filename='nouislider/nouislider.js') }}"></script>
     <script type="text/javascript" src="/static/new/javascript/initialize_show_trait_tables.js"></script>
     <script type="text/javascript" src="/static/new/javascript/show_trait_mapping_tools.js"></script>
diff --git a/wqflask/wqflask/templates/show_trait_calculate_correlations.html b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
index 50803978..ef784c84 100644
--- a/wqflask/wqflask/templates/show_trait_calculate_correlations.html
+++ b/wqflask/wqflask/templates/show_trait_calculate_correlations.html
@@ -117,6 +117,9 @@
               <div class="col-xs-3 controls">
                   <input type="button" class="btn corr_compute submit_special btn-success" data-url="/corr_compute" title="Compute Correlation" value="Compute">
               </div>
+                  <div class="col-xs-3 controls">
+                  <input type="button" class="btn test_corr_compute submit_special btn-success" data-url="/test_corr_compute" title="Compute Correlation" value="Test Compute">
+              </div>
           </div>
       </div>
     </div>
diff --git a/wqflask/wqflask/templates/test_correlation_page.html b/wqflask/wqflask/templates/test_correlation_page.html
new file mode 100644
index 00000000..037e9735
--- /dev/null
+++ b/wqflask/wqflask/templates/test_correlation_page.html
@@ -0,0 +1,159 @@
+{% extends "base.html" %}
+{% block title %}Correlation Results{% endblock %}
+{% block css %}
+    <link rel="stylesheet" type="text/css" href="{{ url_for('css', filename='DataTables/css/jquery.dataTables.css') }}" />
+    <link rel="stylesheet" type="text/css" href="{{ url_for('js', filename='DataTablesExtensions/buttonsBootstrap/css/buttons.bootstrap.css') }}" />
+    <link rel="stylesheet" type="text/css" href="{{ url_for('js', filename='DataTablesExtensions/buttonStyles/css/buttons.dataTables.min.css') }}">
+    <link rel="stylesheet" type="text/css" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/css/all.min.css">
+    <link rel="stylesheet" type="text/css" href="/static/new/css/trait_list.css" />
+    <link rel="stylesheet" type="text/css" href="/static/new/css/show_trait.css" />
+
+    <style type="text/css">
+    	.td-styles{
+    		height: 40px;
+    		text-align: center;
+    	}
+    	.trait_col {
+    		font-weight:bolder;
+    		text-align: center;
+    		color:#036ffc;
+    		/*font-size: 1.1em;*/
+    	}
+    	table th {
+    		font-weight: bolder;
+    		text-transform: uppercase;
+    	}
+    	.correlation-title {
+    		padding:25px 10px;
+    	}
+    	.correlation-title h3 span {
+          font-weight: bolder;
+    	}
+    	.header-toggle-vis {
+    		padding:10px 5px;
+    	}
+    	.header-toggle-vis button {
+    		border-radius: 5px;
+
+    	}
+    </style>
+{% endblock %}
+
+{% block content %}
+
+<div class="correlation-title">
+	<h3>Correlation Results for <span>Dataset_name</span> against <span><a href="">trait_name</a></span> for the top <span>all</span> Results</h3>
+</div>
+<div class="header-toggle-vis">
+	      <h4 style="font-weight: bolder;padding: 5px 3px;">Toggle Columns</h4>
+	      <button class="toggle-vis" data-column="1">Index</button>
+          <button class="toggle-vis" data-column="2">Trait Name</button>
+          <button class="toggle-vis" data-column="3">Sample r</button>
+          <button class="toggle-vis" data-column="4">Sample P(r)</button>
+          <button class="toggle-vis" data-column="5">Num overlap</button>
+</div>
+	<table id="example" class="display" width="100%">
+		        <thead>
+            <tr >
+            	<th></th>
+                <th>index</th>
+                <th>trait_name</th>
+                <th>Sample r</th>
+                <th>Sample r(p)</th>
+                <th>N</th>
+                <th>Tissue r</th>
+                <th>Tissue r(p)</th>
+                <th>Lit r</th>
+            </tr>
+        </thead>
+	</table>
+
+{% endblock %}
+
+{% block js %}
+<script type="text/javascript" src="{{ url_for('js', filename='js_alt/md5.min.js') }}"></script>
+<script type="text/javascript" src="/static/new/javascript/search_results.js"></script>
+
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='js_alt/underscore.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='jszip/jszip.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='js_alt/underscore.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTables/js/jquery.dataTables.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/dataTables.buttons.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/buttons/js/buttons.html5.min.js') }}"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/plugins/sorting/natural.js') }}"></script>
+<script language="javascript" type="text/javascript" src="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.1/js/all.min.js"></script>
+<script language="javascript" type="text/javascript" src="{{ url_for('js', filename='DataTablesExtensions/scroller/js/dataTables.scroller.min.js') }}"></script>
+<script type="text/javascript">
+	console.log("running this script")
+	let correlationResults = {{correlation_results|safe}}
+	// document.querySelector(".content").innerHTML =correlationResults
+	// parse the data
+	let  counter = 0;
+	let corr_type = "tissue";
+	correlationResults =correlationResults.map((trait_object)=>{
+		let trait_name = Object.keys(trait_object)[0]
+		
+		let new_dict = {
+			"index":counter,
+			"trait_name":trait_name,
+			...trait_object[trait_name]
+		}
+		counter++;
+		return new_dict;
+	})
+
+console.log(correlationResults) 
+	
+</script>
+
+<script type="text/javascript">
+	$(document).ready(function() {
+    let  table = $('#example').DataTable( {
+        "data": correlationResults,
+      "columns": [
+            {"data":corr_type=="sample"?null:"fd","width":"25px"},
+            { "data": "index","width":"120px","title":"Index" },
+            { "data": "trait_name","title":"TraitName"},
+            { "data": "corr_coeffient","defaultContent": "--"},
+            { "data": "p_value","defaultContent":"--"},
+            { "data": "num_overlap","defaultContent":"--"},
+            {"data":"tissue_corr","defaultContent":"--","title":"Tissue r"},
+            {"data":"tissue_p_val","defaultContent":"--","title":"Tissue r(p)"},
+            {"data":"lit_corr","defaultContent":"--","title":"Lit rho"}
+        ],
+        "columnDefs": [ 
+           {
+           	targets:0,
+           	data:null,
+           	defaultContent: '',
+           	orderable: false,
+           	className: 'select-checkbox',
+           	"render":(data,type,row)=>{
+           		return `<input type="checkbox" class="checkbox trait_checkbox" value="other">`
+           	}
+
+           },
+           {className:"trait_col",targets:2},
+           {className: "td-styles", targets: "_all"},
+           {
+           	"targets":2,
+           	"render":(data,type,row)=>{
+           		let urlLink = "/show_trait?trait_id=1453207_at&dataset=HC_M2_0606_P"
+           		let traitLink = `<a href=${urlLink}>${data}</a>`
+           		return traitLink
+           	},
+           }
+
+     ]
+    } );
+
+    $(":button.toggle-vis").on("click",function(e){
+    	e.preventDefault()
+    	let column = table.column($(this).attr("data-column"));
+    	column.visible(!column.visible())
+    	console.log($(this).attr("data-column"))
+    })
+} );
+</script>
+
+{% endblock %}
\ No newline at end of file
diff --git a/wqflask/wqflask/views.py b/wqflask/wqflask/views.py
index d2ede70f..3c875163 100644
--- a/wqflask/wqflask/views.py
+++ b/wqflask/wqflask/views.py
@@ -47,6 +47,7 @@ from wqflask.marker_regression import run_mapping
 from wqflask.marker_regression import display_mapping_results
 from wqflask.network_graph import network_graph
 from wqflask.correlation import show_corr_results
+from wqflask.correlation.correlation_gn3_api import compute_correlation
 from wqflask.correlation_matrix import show_corr_matrix
 from wqflask.correlation import corr_scatter_plot
 from wqflask.wgcna import wgcna_analysis
@@ -717,7 +718,7 @@ def mapping_results_container_page():
 
 @app.route("/loading", methods=('POST',))
 def loading_page():
-    logger.info(request.url)
+    # logger.info(request.url)
     initial_start_vars = request.form
     start_vars_container = {}
     n_samples = 0  # ZS: So it can be displayed on loading page
@@ -963,7 +964,17 @@ def corr_compute_page():
     template_vars = show_corr_results.CorrelationResults(request.form)
     return render_template("correlation_page.html", **template_vars.__dict__)
 
+    # to test/disable the new  correlation api uncomment these lines
+
+    # correlation_results = compute_correlation(request.form)
+    # return render_template("test_correlation_page.html", correlation_results=correlation_results)
+
 
+@app.route("/test_corr_compute", methods=["POST"])
+def test_corr_compute_page():
+    correlation_results = compute_correlation(request.form)
+    return render_template("test_correlation_page.html", correlation_results=correlation_results)
+    
 @app.route("/corr_matrix", methods=('POST',))
 def corr_matrix_page():
     logger.info("In corr_matrix, request.form is:", pf(request.form))