From eb9716633ed71108809029a9cb859714de66639f Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Tue, 9 Aug 2022 12:44:46 +0300
Subject: Integrate rust correlations. Deduplicate similar code.

- Use rust correlations rather than the slower python one
- Remove code duplicated in the
  `wqflask/wqflask/correlation/rust_correlation.py` module within the
  `compute_correlation_rust` function.
---
 wqflask/wqflask/correlation/correlation_gn3_api.py | 52 +++++-----------------
 1 file changed, 11 insertions(+), 41 deletions(-)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 05210d9b..f13dd27c 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -198,51 +198,21 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
 
     corr_type = start_vars['corr_type']
 
-    (this_dataset, this_trait, target_dataset,
-     sample_data) = create_target_this_trait(start_vars)
-
-    target_dataset_type = target_dataset.type
-    this_dataset_type = this_dataset.type
-
     method = start_vars['corr_sample_method']
     corr_return_results = int(start_vars.get("corr_return_results", 100))
     corr_input_data = {}
 
-    if corr_type == "sample":
-        (this_trait_data, target_dataset_data) = fetch_sample_data(
-            start_vars, this_trait, this_dataset, target_dataset)
-        ## This import has to be inside the function to prevent circular imports
-        from wqflask.correlation.rust_correlation import compute_correlation_rust
-        rust_correlation_results = compute_correlation_rust(
-            start_vars, corr_type, method, corr_return_results)
-        correlation_results = rust_correlation_results["correlation_results"]
-
-    elif corr_type == "tissue":
-        trait_symbol_dict = this_dataset.retrieve_genes("Symbol")
-        tissue_input = get_tissue_correlation_input(
-            this_trait, trait_symbol_dict)
-
-        if tissue_input is not None:
-            (primary_tissue_data, target_tissue_data) = tissue_input
-
-            corr_input_data = {
-                "primary_tissue": primary_tissue_data,
-                "target_tissues_dict": target_tissue_data
-            }
-            correlation_results = compute_tissue_correlation(
-                primary_tissue_dict=corr_input_data["primary_tissue"],
-                target_tissues_data=corr_input_data[
-                    "target_tissues_dict"],
-                corr_method=method
-
-            )
-        else:
-            return {"correlation_results": [],
-                    "this_trait": this_trait.name,
-                    "target_dataset": start_vars['corr_dataset'],
-                    "return_results": corr_return_results}
-
-    elif corr_type == "lit":
+    from wqflask.correlation.rust_correlation import compute_correlation_rust
+    rust_correlation_results =
+    return compute_correlation_rust(
+        start_vars, corr_type, method, corr_return_results)
+    correlation_results = rust_correlation_results["correlation_results"]
+
+    if corr_type == "lit":# elif corr_type == "lit":
+        (this_dataset, this_trait, target_dataset,
+         sample_data) = create_target_this_trait(start_vars)
+        target_dataset_type = target_dataset.type
+        this_dataset_type = this_dataset.type
         (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
             this_trait, this_dataset)
 
-- 
cgit v1.2.3


From 58f9dcb692741b15d3acbeb86a2dfce680234583 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Tue, 9 Aug 2022 13:18:51 +0300
Subject: Fix minor bug.

---
 wqflask/wqflask/correlation/correlation_gn3_api.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index f13dd27c..e1e2613d 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -203,8 +203,7 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
     corr_input_data = {}
 
     from wqflask.correlation.rust_correlation import compute_correlation_rust
-    rust_correlation_results =
-    return compute_correlation_rust(
+    rust_correlation_results = compute_correlation_rust(
         start_vars, corr_type, method, corr_return_results)
     correlation_results = rust_correlation_results["correlation_results"]
 
-- 
cgit v1.2.3


From 46c192365d17cce891f49c8b2d5ba2c222a40588 Mon Sep 17 00:00:00 2001
From: Alexander_Kabui
Date: Wed, 10 Aug 2022 07:13:13 +0300
Subject: add code to compute top n tissue results

---
 wqflask/wqflask/correlation/rust_correlation.py | 59 ++++++++++++++++++++++---
 1 file changed, 54 insertions(+), 5 deletions(-)

diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 4106d3f0..4bd2dd9d 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -2,9 +2,37 @@
 import json
 from wqflask.correlation.correlation_functions import get_trait_symbol_and_tissue_values
 from wqflask.correlation.correlation_gn3_api import create_target_this_trait
+from wqflask.correlation.correlation_gn3_api import lit_for_trait_list
+from wqflask.correlation.correlation_gn3_api import do_lit_correlation
+from gn3.computations.correlations import compute_all_lit_correlation
 from gn3.computations.rust_correlation import run_correlation
 from gn3.computations.rust_correlation import get_sample_corr_data
 from gn3.computations.rust_correlation import parse_tissue_corr_data
+from gn3.db_utils import database_connector
+
+
+
+
+
+def compute_top_n_tissue(this_dataset, this_trait, traits, method):
+
+    trait_symbol_dict = dict({trait_name: symbol for (
+        trait_name, symbol) in this_dataset.retrieve_genes("Symbol").items() if traits.get(trait_name)})
+
+    corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
+        symbol_list=list(trait_symbol_dict.values()))
+
+    data = parse_tissue_corr_data(symbol_name=this_trait.symbol,
+                                  symbol_dict=get_trait_symbol_and_tissue_values(
+                                      symbol_list=[this_trait.symbol]),
+                                  dataset_symbols=trait_symbol_dict,
+                                  dataset_vals=corr_result_tissue_vals_dict)
+
+    if data:
+        return run_correlation(
+            data[1], data[0], method, ",","tissue")
+
+    return {}
 
 
 def compute_correlation_rust(start_vars: dict, corr_type: str,
@@ -28,9 +56,29 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
             lts = [key] + [str(x) for x in val]
             r = ",".join(lts)
             target_data.append(r)
+        # breakpoint()
+
+        results_k = run_correlation(target_data, ",".join(
+            [str(x) for x in list(sample_data.values())]), method, ",")
+
+        tissue_top = compute_top_n_tissue(
+            this_dataset, this_trait, results_k, method)
+
+
+        lit_top = compute_top_n_lit(results_k,this_dataset,this_trait)
+
+
+        results = []
+
+        for (key,val) in results_k.items():
+            if key in tissue_top:
+                results_k[key].update(tissue_top[key])
+
+            if key in lit_top:
+                results_k[key].update(lit_top[key])
+
+            results.append({key:results_k[key]})
 
-        results = run_correlation(
-            target_data, list(sample_data.values()), method, ",")
 
 
     if corr_type == "tissue":
@@ -41,15 +89,16 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
 
         data = parse_tissue_corr_data(symbol_name=this_trait.symbol,
                                       symbol_dict=get_trait_symbol_and_tissue_values(
-                                          symbol_list=[this_trait.symbol]),
+                                          symbol_list=[this_trait.symbol]
+                                      ),
                                       dataset_symbols=trait_symbol_dict,
                                       dataset_vals=corr_result_tissue_vals_dict)
 
         if data:
             results = run_correlation(
-                data[1], data[0], method, ",")
+                data[1], data[0], method, ",","tissue")
 
-    return {"correlation_results": results[0:n_top],
+    return {"correlation_results": results,
             "this_trait": this_trait.name,
             "target_dataset": start_vars['corr_dataset'],
             "return_results": n_top
-- 
cgit v1.2.3


From b7b0996b3e4ef52ccfeec70dd731e54ce1dab8dc Mon Sep 17 00:00:00 2001
From: Alexander_Kabui
Date: Wed, 10 Aug 2022 07:15:40 +0300
Subject: compute top n tissue results

---
 wqflask/wqflask/correlation/rust_correlation.py | 39 ++++++++++++-------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 4bd2dd9d..161215c5 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -13,6 +13,24 @@ from gn3.db_utils import database_connector
 
 
 
+def compute_top_n_lit(corr_results, this_dataset, this_trait):
+    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
+        this_trait, this_dataset)
+
+    geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if
+                   corr_results.get(trait_name)}
+
+    conn = database_connector()
+
+    with conn:
+
+        correlation_results = compute_all_lit_correlation(
+            conn=conn, trait_lists=list(geneid_dict.items()),
+            species=species, gene_id=this_trait_geneid)
+
+    return correlation_results
+
+
 
 def compute_top_n_tissue(this_dataset, this_trait, traits, method):
 
@@ -56,29 +74,10 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
             lts = [key] + [str(x) for x in val]
             r = ",".join(lts)
             target_data.append(r)
-        # breakpoint()
 
-        results_k = run_correlation(target_data, ",".join(
+        results = run_correlation(target_data, ",".join(
             [str(x) for x in list(sample_data.values())]), method, ",")
 
-        tissue_top = compute_top_n_tissue(
-            this_dataset, this_trait, results_k, method)
-
-
-        lit_top = compute_top_n_lit(results_k,this_dataset,this_trait)
-
-
-        results = []
-
-        for (key,val) in results_k.items():
-            if key in tissue_top:
-                results_k[key].update(tissue_top[key])
-
-            if key in lit_top:
-                results_k[key].update(lit_top[key])
-
-            results.append({key:results_k[key]})
-
 
 
     if corr_type == "tissue":
-- 
cgit v1.2.3


From 3c117eebe6b0973db7f3fe6cf15f1109b883e629 Mon Sep 17 00:00:00 2001
From: Alexander_Kabui
Date: Wed, 10 Aug 2022 08:43:11 +0300
Subject: code to merge results;minor fixes

---
 wqflask/wqflask/correlation/rust_correlation.py | 51 +++++++++++++++++++++----
 1 file changed, 44 insertions(+), 7 deletions(-)

diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 161215c5..8431f179 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -11,8 +11,6 @@ from gn3.computations.rust_correlation import parse_tissue_corr_data
 from gn3.db_utils import database_connector
 
 
-
-
 def compute_top_n_lit(corr_results, this_dataset, this_trait):
     (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
         this_trait, this_dataset)
@@ -31,9 +29,10 @@ def compute_top_n_lit(corr_results, this_dataset, this_trait):
     return correlation_results
 
 
-
 def compute_top_n_tissue(this_dataset, this_trait, traits, method):
 
+    # refactor lots of rpt
+
     trait_symbol_dict = dict({trait_name: symbol for (
         trait_name, symbol) in this_dataset.retrieve_genes("Symbol").items() if traits.get(trait_name)})
 
@@ -48,11 +47,32 @@ def compute_top_n_tissue(this_dataset, this_trait, traits, method):
 
     if data:
         return run_correlation(
-            data[1], data[0], method, ",","tissue")
+            data[1], data[0], method, ",", "tissue")
 
     return {}
 
 
+def merge_results(dict_a, dict_b, dict_c):
+    """code to merge diff corr  into individual dicts
+    a"""
+
+    correlation_results = []
+
+    for (key, val) in dict_a.items():
+
+        if key in dict_b:
+
+            dict_a[key].update(dict_b[key])
+
+        if key in dict_c:
+
+            dict_a[key].update(dict_c[key])
+
+        correlation_results.append({key: dict_a[key]})
+
+    return correlation_results
+
+
 def compute_correlation_rust(start_vars: dict, corr_type: str,
                              method: str = "pearson", n_top: int = 500):
     """function to compute correlation"""
@@ -75,10 +95,27 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
             r = ",".join(lts)
             target_data.append(r)
 
-        results = run_correlation(target_data, ",".join(
-            [str(x) for x in list(sample_data.values())]), method, ",")
 
+        results = run_correlation(target_data,
+                                  list(sample_data.values()),
+                                  method,
+                                  ",",
+                                  corr_type,
+                                  n_top)
+
+        # example compute of compute both correlation
+
+
+
+        top_tissue_results = compute_top_n_tissue(this_dataset,this_trait,results,method)
+
+
+        top_lit_results = compute_top_n_lit(results,this_dataset,this_trait)
+
+
+        # merging the results
 
+        results = merge_results(results,top_tissue_results,top_lit_results)
 
     if corr_type == "tissue":
 
@@ -95,7 +132,7 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
 
         if data:
             results = run_correlation(
-                data[1], data[0], method, ",","tissue")
+                data[1], data[0], method, ",", "tissue")
 
     return {"correlation_results": results,
             "this_trait": this_trait.name,
-- 
cgit v1.2.3


From aebd4ec7ba5763953fefa9299e1cdcba19f7a673 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Wed, 10 Aug 2022 12:02:54 +0300
Subject: Fix formatting for legibility

---
 test/requests/correlation_tests.py                 | 44 ++++++++++++++++++++++
 test/requests/test-website.py                      |  2 +
 wqflask/wqflask/correlation/correlation_gn3_api.py | 23 +++++------
 wqflask/wqflask/correlation/rust_correlation.py    | 39 +++++++++----------
 4 files changed, 73 insertions(+), 35 deletions(-)
 create mode 100644 test/requests/correlation_tests.py

diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py
new file mode 100644
index 00000000..b0213ff5
--- /dev/null
+++ b/test/requests/correlation_tests.py
@@ -0,0 +1,44 @@
+import requests
+from lxml.html import parse
+from link_checker import check_page
+
+def do_request(host, data):
+    return request.get(
+        f"{host}/corr_compute",
+        params={
+            **data,
+            "corr_dataset": "HC_M2_0606_P",
+            "corr_return_results": "100",
+            "corr_samples_group": "samples_primary",})
+
+def check_sample_correlations(baseurl):
+    data = {
+        "corr_type": "sample",
+        "corr_sample_method": "pearson",
+        "location_type": "gene"
+    }
+    result = do_request(host, data)
+    assert result.status == 200
+    assert (result.text.find("Values of record 1435464_at") >= 0), ""
+
+def check_tissue_correlations(baseurl):
+    data = {
+        "corr_type": "tissue"
+    }
+    result = do_request(host, data)
+    assert False, "Not implemented yet."
+
+def check_lit_correlations(baseurl):
+    data = {
+        "corr_type": "lit"
+    }
+    result = do_request(host, data)
+    assert False, "Not implemented yet."
+
+def check_correlations(args_obj, parser):
+    print("")
+    print("Checking the correlations...")
+    host = args_obj.host
+    check_sample_correlations(host)
+    check_tissue_correlations(host)
+    check_lit_correlations(host)
diff --git a/test/requests/test-website.py b/test/requests/test-website.py
index 71055fca..8062bff1 100755
--- a/test/requests/test-website.py
+++ b/test/requests/test-website.py
@@ -9,6 +9,7 @@ from link_checker import check_links
 from link_checker import check_packaged_js_files
 from mapping_tests import check_mapping
 from navigation_tests import check_navigation
+from correlation_tests import check_correlations
 from main_web_functionality import check_main_web_functionality
 import link_checker
 import sys
@@ -28,6 +29,7 @@ def run_all(args_obj, parser):
     check_links(args_obj, parser)
     check_packaged_js_files(args_obj, parser)
     check_mapping(args_obj, parser)
+    check_correlations(args_obj, parser)
     # TODO: Add other functions as they are created.
 
 
diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index e1e2613d..6df4eafe 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -224,19 +224,16 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
     correlation_results = correlation_results[0:corr_return_results]
 
     if (compute_all):
-        correlation_results = compute_corr_for_top_results(start_vars,
-                                                           correlation_results,
-                                                           this_trait,
-                                                           this_dataset,
-                                                           target_dataset,
-                                                           corr_type)
-
-    correlation_data = {"correlation_results": correlation_results,
-                        "this_trait": this_trait.name,
-                        "target_dataset": start_vars['corr_dataset'],
-                        "return_results": corr_return_results}
-
-    return correlation_data
+        correlation_results = compute_corr_for_top_results(
+            start_vars, correlation_results, this_trait, this_dataset,
+            target_dataset, corr_type)
+
+    return {
+        "correlation_results": correlation_results,
+        "this_trait": this_trait.name,
+        "target_dataset": start_vars['corr_dataset'],
+        "return_results": corr_return_results
+    }
 
 
 def compute_corr_for_top_results(start_vars,
diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 8431f179..8a5021cc 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -33,8 +33,11 @@ def compute_top_n_tissue(this_dataset, this_trait, traits, method):
 
     # refactor lots of rpt
 
-    trait_symbol_dict = dict({trait_name: symbol for (
-        trait_name, symbol) in this_dataset.retrieve_genes("Symbol").items() if traits.get(trait_name)})
+    trait_symbol_dict = dict({
+        trait_name: symbol
+        for (trait_name, symbol)
+        in this_dataset.retrieve_genes("Symbol").items()
+        if traits.get(trait_name)})
 
     corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
         symbol_list=list(trait_symbol_dict.values()))
@@ -73,8 +76,9 @@ def merge_results(dict_a, dict_b, dict_c):
     return correlation_results
 
 
-def compute_correlation_rust(start_vars: dict, corr_type: str,
-                             method: str = "pearson", n_top: int = 500):
+def compute_correlation_rust(
+        start_vars: dict, corr_type: str, method: str = "pearson",
+        n_top: int = 500):
     """function to compute correlation"""
 
     (this_dataset, this_trait, target_dataset,
@@ -96,25 +100,15 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
             target_data.append(r)
 
 
-        results = run_correlation(target_data,
-                                  list(sample_data.values()),
-                                  method,
-                                  ",",
-                                  corr_type,
-                                  n_top)
+        results = run_correlation(
+            target_data, list(sample_data.values()), method, ",", corr_type,
+            n_top)
 
         # example compute of compute both correlation
-
-
-
         top_tissue_results = compute_top_n_tissue(this_dataset,this_trait,results,method)
-
-
         top_lit_results = compute_top_n_lit(results,this_dataset,this_trait)
 
-
         # merging the results
-
         results = merge_results(results,top_tissue_results,top_lit_results)
 
     if corr_type == "tissue":
@@ -134,8 +128,9 @@ def compute_correlation_rust(start_vars: dict, corr_type: str,
             results = run_correlation(
                 data[1], data[0], method, ",", "tissue")
 
-    return {"correlation_results": results,
-            "this_trait": this_trait.name,
-            "target_dataset": start_vars['corr_dataset'],
-            "return_results": n_top
-            }
+    return {
+        "correlation_results": results,
+        "this_trait": this_trait.name,
+        "target_dataset": start_vars['corr_dataset'],
+        "return_results": n_top
+    }
-- 
cgit v1.2.3


From ac9941b3cff7605500dc6ec6fda7a4f5db664a0a Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Wed, 10 Aug 2022 13:47:56 +0300
Subject: Check that the correlations work as expected.

---
 test/requests/correlation_tests.py | 61 +++++++++++++++++++++++++++-----------
 1 file changed, 44 insertions(+), 17 deletions(-)

diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py
index b0213ff5..070698fd 100644
--- a/test/requests/correlation_tests.py
+++ b/test/requests/correlation_tests.py
@@ -2,43 +2,70 @@ import requests
 from lxml.html import parse
 from link_checker import check_page
 
-def do_request(host, data):
-    return request.get(
-        f"{host}/corr_compute",
-        params={
-            **data,
+def sample_vals():
+    return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}'
+
+def do_request(url, data):
+    return requests.post(
+        url,
+        data={
+            "dataset": "HC_M2_0606_P",
+            "trait_id": "1435464_at",
             "corr_dataset": "HC_M2_0606_P",
             "corr_return_results": "100",
-            "corr_samples_group": "samples_primary",})
+            "corr_samples_group": "samples_primary",
+            "sample_vals": sample_vals(),
+            **data,
+        })
 
 def check_sample_correlations(baseurl):
     data = {
         "corr_type": "sample",
         "corr_sample_method": "pearson",
-        "location_type": "gene"
+        "location_type": "gene",
+        "corr_return_results": "200"
     }
-    result = do_request(host, data)
-    assert result.status == 200
-    assert (result.text.find("Values of record 1435464_at") >= 0), ""
+    top_n_message = "The top 200 correlations ranked by the Genetic Correlation"
+    result = do_request(f"{baseurl}/corr_compute", data)
+    assert result.status_code == 200
+    assert (result.text.find("Values of record 1435464_at") >= 0), result.text
+    assert (result.text.find(top_n_message) >= 0), result.text
 
 def check_tissue_correlations(baseurl):
+    print("\tChecking tissue correlations...", end="")
     data = {
-        "corr_type": "tissue"
+        "corr_type": "tissue",
+        "corr_sample_method": "pearson",
+        "location_type": "gene",
     }
-    result = do_request(host, data)
-    assert False, "Not implemented yet."
+    top_n_message = "The top 100 correlations ranked by the Genetic Correlation"
+    result = do_request(f"{baseurl}/corr_compute", data)
+    assert result.status_code == 200
+    assert (result.text.find("Values of record 1435464_at") >= 0), result.text
+    assert (result.text.find(top_n_message) >= 0), result.text
 
 def check_lit_correlations(baseurl):
+    print("\tChecking lit correlations...", end="")
     data = {
-        "corr_type": "lit"
+        "corr_type": "lit",
+        "corr_return_results": "200"
     }
-    result = do_request(host, data)
-    assert False, "Not implemented yet."
+    top_n_message = "The top 500 correlations ranked by the Genetic Correlation"
+    result = do_request(f"{baseurl}/corr_compute", data)
+    assert result.status_code == 200
+    assert (result.text.find("Values of record 1435464_at") >= 0), result.text
+    assert (result.text.find(top_n_message) >= 0), result.text
 
 def check_correlations(args_obj, parser):
     print("")
     print("Checking the correlations...")
+    corr_type_fns = {
+        "sample": check_sample_correlations,}
     host = args_obj.host
-    check_sample_correlations(host)
+    for corr_type, corr_type_fn in corr_type_fns.items():
+        print(f"\tChecking {corr_type} correlations...", end="")
+        corr_type_fn(host)
+        print(" ok")
     check_tissue_correlations(host)
     check_lit_correlations(host)
+    print("OK")
-- 
cgit v1.2.3


From 90195f350758902cecbd184bfa500eac2a39a263 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Thu, 11 Aug 2022 06:57:11 +0300
Subject: Update format to prevent tissue correlation from failing

Update the data format of returned values so that it conforms with
expectatitions.
---
 wqflask/wqflask/correlation/rust_correlation.py | 53 +++++++++++--------------
 1 file changed, 24 insertions(+), 29 deletions(-)

diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 8a5021cc..4a22af72 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -1,5 +1,6 @@
 """module contains integration code for rust-gn3"""
 import json
+from functools import reduce
 from wqflask.correlation.correlation_functions import get_trait_symbol_and_tissue_values
 from wqflask.correlation.correlation_gn3_api import create_target_this_trait
 from wqflask.correlation.correlation_gn3_api import lit_for_trait_list
@@ -11,22 +12,21 @@ from gn3.computations.rust_correlation import parse_tissue_corr_data
 from gn3.db_utils import database_connector
 
 
-def compute_top_n_lit(corr_results, this_dataset, this_trait):
+def compute_top_n_lit(corr_results, this_dataset, this_trait) -> dict:
     (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
         this_trait, this_dataset)
 
     geneid_dict = {trait_name: geneid for (trait_name, geneid) in geneid_dict.items() if
                    corr_results.get(trait_name)}
+    with database_connector() as conn:
+        return reduce(
+            lambda acc, corr: {**acc, **corr},
+            compute_all_lit_correlation(
+                conn=conn, trait_lists=list(geneid_dict.items()),
+                species=species, gene_id=this_trait_geneid),
+            {})
 
-    conn = database_connector()
-
-    with conn:
-
-        correlation_results = compute_all_lit_correlation(
-            conn=conn, trait_lists=list(geneid_dict.items()),
-            species=species, gene_id=this_trait_geneid)
-
-    return correlation_results
+    return {}
 
 
 def compute_top_n_tissue(this_dataset, this_trait, traits, method):
@@ -55,25 +55,19 @@ def compute_top_n_tissue(this_dataset, this_trait, traits, method):
     return {}
 
 
-def merge_results(dict_a, dict_b, dict_c):
+def merge_results(dict_a: dict, dict_b: dict, dict_c: dict) -> list[dict]:
     """code to merge diff corr  into individual dicts
     a"""
 
-    correlation_results = []
-
-    for (key, val) in dict_a.items():
-
-        if key in dict_b:
-
-            dict_a[key].update(dict_b[key])
-
-        if key in dict_c:
-
-            dict_a[key].update(dict_c[key])
-
-        correlation_results.append({key: dict_a[key]})
-
-    return correlation_results
+    def __merge__(trait_name, trait_corrs):
+        return {
+            trait_name: {
+                **trait_corrs,
+                **dict_b.get(trait_name, {}),
+                **dict_c.get(trait_name, {})
+            }
+        }
+    return [__merge__(tname, tcorrs) for tname, tcorrs in dict_a.items()]
 
 
 def compute_correlation_rust(
@@ -109,7 +103,7 @@ def compute_correlation_rust(
         top_lit_results = compute_top_n_lit(results,this_dataset,this_trait)
 
         # merging the results
-        results = merge_results(results,top_tissue_results,top_lit_results)
+        results = merge_results(results, top_tissue_results, top_lit_results)
 
     if corr_type == "tissue":
 
@@ -125,8 +119,9 @@ def compute_correlation_rust(
                                       dataset_vals=corr_result_tissue_vals_dict)
 
         if data:
-            results = run_correlation(
-                data[1], data[0], method, ",", "tissue")
+            results = merge_results(
+                run_correlation(data[1], data[0], method, ",", "tissue"),
+                {}, {})
 
     return {
         "correlation_results": results,
-- 
cgit v1.2.3


From 8658545c99a576229f9263414f5557545500f4a1 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Thu, 11 Aug 2022 08:12:54 +0300
Subject: Check for keys before using them.

---
 wqflask/wqflask/correlation/show_corr_results.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/wqflask/wqflask/correlation/show_corr_results.py b/wqflask/wqflask/correlation/show_corr_results.py
index 1c391386..988910c3 100644
--- a/wqflask/wqflask/correlation/show_corr_results.py
+++ b/wqflask/wqflask/correlation/show_corr_results.py
@@ -121,9 +121,12 @@ def correlation_json_for_table(correlation_data, this_trait, this_dataset, targe
         results_dict['dataset'] = target_dataset['name']
         results_dict['hmac'] = hmac.data_hmac(
             '{}:{}'.format(target_trait['name'], target_dataset['name']))
-        results_dict['sample_r'] = f"{float(trait['corr_coefficient']):.3f}"
-        results_dict['num_overlap'] = trait['num_overlap']
-        results_dict['sample_p'] = f"{float(trait['p_value']):.3e}"
+        if "corr_coefficient" in trait:
+            results_dict['sample_r'] = f"{float(trait['corr_coefficient']):.3f}"
+        if "num_overlap" in trait:
+            results_dict['num_overlap'] = trait['num_overlap']
+        if "p_value" in trait:
+            results_dict['sample_p'] = f"{float(trait['p_value']):.3e}"
         if target_dataset['type'] == "ProbeSet":
             results_dict['symbol'] = target_trait['symbol']
             results_dict['description'] = "N/A"
-- 
cgit v1.2.3


From 309785e95696567a35b42690b032808eaa59c86d Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Thu, 11 Aug 2022 11:01:05 +0300
Subject: Fix correlation tests: Fix the test message.

---
 test/requests/correlation_tests.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py
index 070698fd..0b8ba8cd 100644
--- a/test/requests/correlation_tests.py
+++ b/test/requests/correlation_tests.py
@@ -32,25 +32,23 @@ def check_sample_correlations(baseurl):
     assert (result.text.find(top_n_message) >= 0), result.text
 
 def check_tissue_correlations(baseurl):
-    print("\tChecking tissue correlations...", end="")
     data = {
         "corr_type": "tissue",
         "corr_sample_method": "pearson",
         "location_type": "gene",
     }
-    top_n_message = "The top 100 correlations ranked by the Genetic Correlation"
+    top_n_message = "The top 100 correlations ranked by the Tissue Correlation"
     result = do_request(f"{baseurl}/corr_compute", data)
     assert result.status_code == 200
     assert (result.text.find("Values of record 1435464_at") >= 0), result.text
     assert (result.text.find(top_n_message) >= 0), result.text
 
 def check_lit_correlations(baseurl):
-    print("\tChecking lit correlations...", end="")
     data = {
         "corr_type": "lit",
         "corr_return_results": "200"
     }
-    top_n_message = "The top 500 correlations ranked by the Genetic Correlation"
+    top_n_message = "The top 200 correlations ranked by the Lit Correlation"
     result = do_request(f"{baseurl}/corr_compute", data)
     assert result.status_code == 200
     assert (result.text.find("Values of record 1435464_at") >= 0), result.text
-- 
cgit v1.2.3


From 3c1cb6a94b64dae28c62f481e1f4499f8f5b89e7 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Thu, 11 Aug 2022 12:30:18 +0300
Subject: Refactor: separate the three correlation types

Refactor the code such that each correlation type (sample, tissue,
literature) is computed in its own function. This makes the code
clearer, and helps reduce repetition.
---
 wqflask/wqflask/correlation/correlation_gn3_api.py |  39 +------
 wqflask/wqflask/correlation/rust_correlation.py    | 121 +++++++++++++--------
 2 files changed, 78 insertions(+), 82 deletions(-)

diff --git a/wqflask/wqflask/correlation/correlation_gn3_api.py b/wqflask/wqflask/correlation/correlation_gn3_api.py
index 6df4eafe..1a375501 100644
--- a/wqflask/wqflask/correlation/correlation_gn3_api.py
+++ b/wqflask/wqflask/correlation/correlation_gn3_api.py
@@ -194,46 +194,13 @@ def compute_correlation(start_vars, method="pearson", compute_all=False):
     method -- Correlation method to be used (pearson, spearman, or bicor)
     compute_all -- Include sample, tissue, and literature correlations (when applicable)
     """
-    # pylint: disable-msg=too-many-locals
+    from wqflask.correlation.rust_correlation import compute_correlation_rust
 
     corr_type = start_vars['corr_type']
-
     method = start_vars['corr_sample_method']
     corr_return_results = int(start_vars.get("corr_return_results", 100))
-    corr_input_data = {}
-
-    from wqflask.correlation.rust_correlation import compute_correlation_rust
-    rust_correlation_results = compute_correlation_rust(
-        start_vars, corr_type, method, corr_return_results)
-    correlation_results = rust_correlation_results["correlation_results"]
-
-    if corr_type == "lit":# elif corr_type == "lit":
-        (this_dataset, this_trait, target_dataset,
-         sample_data) = create_target_this_trait(start_vars)
-        target_dataset_type = target_dataset.type
-        this_dataset_type = this_dataset.type
-        (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
-            this_trait, this_dataset)
-
-        conn = database_connector()
-        with conn:
-            correlation_results = compute_all_lit_correlation(
-                conn=conn, trait_lists=list(geneid_dict.items()),
-                species=species, gene_id=this_trait_geneid)
-
-    correlation_results = correlation_results[0:corr_return_results]
-
-    if (compute_all):
-        correlation_results = compute_corr_for_top_results(
-            start_vars, correlation_results, this_trait, this_dataset,
-            target_dataset, corr_type)
-
-    return {
-        "correlation_results": correlation_results,
-        "this_trait": this_trait.name,
-        "target_dataset": start_vars['corr_dataset'],
-        "return_results": corr_return_results
-    }
+    return compute_correlation_rust(
+        start_vars, corr_type, method, corr_return_results, compute_all)
 
 
 def compute_corr_for_top_results(start_vars,
diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index 4a22af72..b4435887 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -69,62 +69,91 @@ def merge_results(dict_a: dict, dict_b: dict, dict_c: dict) -> list[dict]:
         }
     return [__merge__(tname, tcorrs) for tname, tcorrs in dict_a.items()]
 
+def __compute_sample_corr__(
+        start_vars: dict, corr_type: str, method: str, n_top: int,
+        target_trait_info: tuple):
+    """Compute the sample correlations"""
+    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
+    all_samples = json.loads(start_vars["sample_vals"])
+    sample_data = get_sample_corr_data(
+        sample_type=start_vars["corr_samples_group"], all_samples=all_samples,
+        dataset_samples=this_dataset.group.all_samples_ordered())
+    target_dataset.get_trait_data(list(sample_data.keys()))
+
+    target_data = []
+    for (key, val) in target_dataset.trait_data.items():
+        lts = [key] + [str(x) for x in val]
+        r = ",".join(lts)
+        target_data.append(r)
+
+
+    return run_correlation(
+        target_data, list(sample_data.values()), method, ",", corr_type,
+        n_top)
+
+def __compute_tissue_corr__(
+        start_vars: dict, corr_type: str, method: str, n_top: int,
+        target_trait_info: tuple):
+    """Compute the tissue correlations"""
+    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
+    trait_symbol_dict = this_dataset.retrieve_genes("Symbol")
+    corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
+        symbol_list=list(trait_symbol_dict.values()))
 
-def compute_correlation_rust(
-        start_vars: dict, corr_type: str, method: str = "pearson",
-        n_top: int = 500):
-    """function to compute correlation"""
-
-    (this_dataset, this_trait, target_dataset,
-     sample_data) = create_target_this_trait(start_vars)
-
-    if corr_type == "sample":
-
-        all_samples = json.loads(start_vars["sample_vals"])
-        sample_data = get_sample_corr_data(sample_type=start_vars["corr_samples_group"],
-                                           all_samples=all_samples,
-                                           dataset_samples=this_dataset.group.all_samples_ordered())
+    data = parse_tissue_corr_data(
+        symbol_name=this_trait.symbol,
+        symbol_dict=get_trait_symbol_and_tissue_values(
+            symbol_list=[this_trait.symbol]),
+        dataset_symbols=trait_symbol_dict,
+        dataset_vals=corr_result_tissue_vals_dict)
 
-        target_dataset.get_trait_data(list(sample_data.keys()))
+    if data:
+        return run_correlation(data[1], data[0], method, ",", "tissue")
+    return {}
 
-        target_data = []
-        for (key, val) in target_dataset.trait_data.items():
-            lts = [key] + [str(x) for x in val]
-            r = ",".join(lts)
-            target_data.append(r)
+def __compute_lit_corr__(
+        start_vars: dict, corr_type: str, method: str, n_top: int,
+        target_trait_info: tuple):
+    """Compute the literature correlations"""
+    (this_dataset, this_trait, target_dataset, sample_data) = target_trait_info
+    target_dataset_type = target_dataset.type
+    this_dataset_type = this_dataset.type
+    (this_trait_geneid, geneid_dict, species) = do_lit_correlation(
+        this_trait, this_dataset)
 
+    with database_connector() as conn:
+        return compute_all_lit_correlation(
+            conn=conn, trait_lists=list(geneid_dict.items()),
+            species=species, gene_id=this_trait_geneid)
+    return {}
 
-        results = run_correlation(
-            target_data, list(sample_data.values()), method, ",", corr_type,
-            n_top)
+def compute_correlation_rust(
+        start_vars: dict, corr_type: str, method: str = "pearson",
+        n_top: int = 500, compute_all: bool = False):
+    """function to compute correlation"""
+    target_trait_info = create_target_this_trait(start_vars)
+    (this_dataset, this_trait, target_dataset, sample_data) = (
+        target_trait_info)
+
+    corr_type_fns = {
+        "sample": __compute_sample_corr__,
+        "tissue": __compute_tissue_corr__,
+        "lit": __compute_lit_corr__
+    }
+    results = corr_type_fns[corr_type](
+        start_vars, corr_type, method, n_top, target_trait_info)
 
+    top_tissue_results = {}
+    top_lit_results = {}
+    if compute_all:
         # example compute of compute both correlation
-        top_tissue_results = compute_top_n_tissue(this_dataset,this_trait,results,method)
+        top_tissue_results = compute_top_n_tissue(
+            this_dataset,this_trait,results,method)
         top_lit_results = compute_top_n_lit(results,this_dataset,this_trait)
 
-        # merging the results
-        results = merge_results(results, top_tissue_results, top_lit_results)
-
-    if corr_type == "tissue":
-
-        trait_symbol_dict = this_dataset.retrieve_genes("Symbol")
-        corr_result_tissue_vals_dict = get_trait_symbol_and_tissue_values(
-            symbol_list=list(trait_symbol_dict.values()))
-
-        data = parse_tissue_corr_data(symbol_name=this_trait.symbol,
-                                      symbol_dict=get_trait_symbol_and_tissue_values(
-                                          symbol_list=[this_trait.symbol]
-                                      ),
-                                      dataset_symbols=trait_symbol_dict,
-                                      dataset_vals=corr_result_tissue_vals_dict)
-
-        if data:
-            results = merge_results(
-                run_correlation(data[1], data[0], method, ",", "tissue"),
-                {}, {})
-
     return {
-        "correlation_results": results,
+        "correlation_results": merge_results(
+            results, top_tissue_results, top_lit_results),
         "this_trait": this_trait.name,
         "target_dataset": start_vars['corr_dataset'],
         "return_results": n_top
-- 
cgit v1.2.3


From fbc7ea257028f07751d7ebaa9f8656569215e124 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Fri, 12 Aug 2022 12:52:38 +0300
Subject: Fix some issues with the tests. Add notes in code for improvements.

---
 test/requests/correlation_tests.py              | 9 +++++----
 wqflask/wqflask/correlation/rust_correlation.py | 2 ++
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/requests/correlation_tests.py b/test/requests/correlation_tests.py
index 0b8ba8cd..2d314833 100644
--- a/test/requests/correlation_tests.py
+++ b/test/requests/correlation_tests.py
@@ -12,6 +12,7 @@ def do_request(url, data):
             "dataset": "HC_M2_0606_P",
             "trait_id": "1435464_at",
             "corr_dataset": "HC_M2_0606_P",
+            "corr_sample_method": "pearson",
             "corr_return_results": "100",
             "corr_samples_group": "samples_primary",
             "sample_vals": sample_vals(),
@@ -34,7 +35,6 @@ def check_sample_correlations(baseurl):
 def check_tissue_correlations(baseurl):
     data = {
         "corr_type": "tissue",
-        "corr_sample_method": "pearson",
         "location_type": "gene",
     }
     top_n_message = "The top 100 correlations ranked by the Tissue Correlation"
@@ -58,12 +58,13 @@ def check_correlations(args_obj, parser):
     print("")
     print("Checking the correlations...")
     corr_type_fns = {
-        "sample": check_sample_correlations,}
+        "sample": check_sample_correlations,
+        "tissue": check_tissue_correlations,
+        "tissue": check_lit_correlations
+    }
     host = args_obj.host
     for corr_type, corr_type_fn in corr_type_fns.items():
         print(f"\tChecking {corr_type} correlations...", end="")
         corr_type_fn(host)
         print(" ok")
-    check_tissue_correlations(host)
-    check_lit_correlations(host)
     print("OK")
diff --git a/wqflask/wqflask/correlation/rust_correlation.py b/wqflask/wqflask/correlation/rust_correlation.py
index b4435887..3628f549 100644
--- a/wqflask/wqflask/correlation/rust_correlation.py
+++ b/wqflask/wqflask/correlation/rust_correlation.py
@@ -135,6 +135,7 @@ def compute_correlation_rust(
     (this_dataset, this_trait, target_dataset, sample_data) = (
         target_trait_info)
 
+    ## Replace this with `match ...` once we hit Python 3.10
     corr_type_fns = {
         "sample": __compute_sample_corr__,
         "tissue": __compute_tissue_corr__,
@@ -142,6 +143,7 @@ def compute_correlation_rust(
     }
     results = corr_type_fns[corr_type](
         start_vars, corr_type, method, n_top, target_trait_info)
+    ## END: Replace this with `match ...` once we hit Python 3.10
 
     top_tissue_results = {}
     top_lit_results = {}
-- 
cgit v1.2.3


From 13b4f8d96a5bcbbd18d2f8847660c896605c8148 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Fri, 12 Aug 2022 13:01:50 +0300
Subject: Add a profiling script

Add a script to enable profiling the code.
---
 README.md                        | 19 ++++++++++
 wqflask/scripts/__init__.py      |  0
 wqflask/scripts/profile_corrs.py | 78 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 97 insertions(+)
 create mode 100644 wqflask/scripts/__init__.py
 create mode 100644 wqflask/scripts/profile_corrs.py

diff --git a/README.md b/README.md
index c5172f8b..38f54bf9 100644
--- a/README.md
+++ b/README.md
@@ -173,6 +173,25 @@ check the logs. If there is ERROR 1054 (42S22): Unknown column
 'InbredSet.Family' in 'field list' it may be you are trying the small
 database.
 
+### Run Scripts
+
+As part of the profiling effort, some scripts are added to run specific parts of the system under a profiler without running the entire web-server - as such, to run the script, you could do something like:
+
+```
+env HOME=/home/frederick \
+    GN2_PROFILE=~/opt/gn2-latest \
+    GN3_DEV_REPO_PATH=~/genenetwork/genenetwork3 \
+    SQL_URI="mysql://username:password@host-ip:host-port/db_webqtl" \
+    SERVER_PORT=5001 \
+    bin/genenetwork2 ../gn2_settings.py \
+    -cli python3 -m scripts.profile_corrs \
+	../performance_$(date +"%Y%m%dT%H:%M:%S").profile
+```
+
+and you can find the performance metrics at the file specified, in this case, a file starting with `performance_` with the date and time of the run, and ending with `.profile`.
+
+Please replace the environment variables in the sample command above with the appropriate values for your environment.
+
 ## Documentation
 
 User documentation can be found
diff --git a/wqflask/scripts/__init__.py b/wqflask/scripts/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/wqflask/scripts/profile_corrs.py b/wqflask/scripts/profile_corrs.py
new file mode 100644
index 00000000..7c43b8c7
--- /dev/null
+++ b/wqflask/scripts/profile_corrs.py
@@ -0,0 +1,78 @@
+import io
+import sys
+import pstats
+import cProfile
+
+import sqlalchemy
+from flask import g, request
+
+from utility.startup_config import app_config
+
+from wqflask import app
+from wqflask.user_session import UserSession
+from wqflask.views import connect_db, shutdown_session
+from wqflask.correlation.correlation_gn3_api import compute_correlation
+
+def sample_vals():
+    return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}'
+
+def simulated_form(corr_type: str = "sample"):
+    assert corr_type in ("sample", "tissue", "lit")
+    return {
+            "dataset": "HC_M2_0606_P",
+            "trait_id": "1435464_at",
+            "corr_dataset": "HC_M2_0606_P",
+            "corr_sample_method": "pearson",
+            "corr_return_results": "100",
+            "corr_samples_group": "samples_primary",
+            "sample_vals": sample_vals(),
+            "corr_type": corr_type,
+            "corr_sample_method": "pearson",
+            "location_type": "gene",
+            "corr_return_results": "20000"
+        }
+
+def profile_corrs():
+    "Profile tho correlations"
+    profiler = cProfile.Profile()
+    # compute_correlation(simulated_form(), compute_all=True)
+    profiler.enable()
+    correlation_results = compute_correlation(request.form, compute_all=True)
+    # correlation_results = set_template_vars(request.form, correlation_results)
+    profiler.disable()
+
+    return profiler
+
+def dump_stats(profiler):
+    iostr = io.StringIO()
+    sort_by = pstats.SortKey.CUMULATIVE
+    ps = pstats.Stats(profiler, stream=iostr).sort_stats(sort_by)
+    ps.print_stats()
+
+    cli_args = sys.argv
+    if len(cli_args) > 1:
+        with open(cli_args[1], "w+", encoding="utf-8") as output_file:
+            print(iostr.getvalue(), file=output_file)
+
+        return 0
+
+    print(iostr.getvalue())
+    return 0
+
+
+if __name__ == "__main__":
+    def main():
+        "Entry point for profiler script"
+        return dump_stats(profile_corrs())
+
+    app_config()
+    with app.app_context():
+        with app.test_request_context("/corr_compute", data=simulated_form()):
+            g.db = sqlalchemy.create_engine( ## Setup global db connection
+                app.config.get('SQL_URI'), encoding="latin1")
+            g.user_session = UserSession()
+            # g.user_session.user_id = 0
+            main()
+            ## dispose global db connection
+            g.db.dispose()
+            g.db.close()
-- 
cgit v1.2.3