Merge pull request #9 from genenetwork/Feature/speed-up-correlation

Feature/speed up correlation

3 files changed, 123 insertions, 37 deletions

diff --git a/gn3/api/correlation.py b/gn3/api/correlation.py
index 2339088..e7e89cf 100644
--- a/gn3/api/correlation.py
+++ b/gn3/api/correlation.py
@@ -23,7 +23,6 @@ def compute_sample_integration(corr_method="pearson"):
     this_trait_data = correlation_input.get("trait_data")
 
     results = map_shared_keys_to_values(target_samplelist, target_data_values)
-
     correlation_results = compute_all_sample_correlation(corr_method=corr_method,
                                                          this_trait=this_trait_data,
                                                          target_dataset=results)
@@ -33,9 +32,10 @@ def compute_sample_integration(corr_method="pearson"):
 
 @correlation.route("/sample_r/<string:corr_method>", methods=["POST"])
 def compute_sample_r(corr_method="pearson"):
-    """correlation endpoint for computing sample r correlations\
+    """Correlation endpoint for computing sample r correlations\
     api expects the trait data with has the trait and also the\
-    target_dataset  data"""
+    target_dataset  data
+    """
     correlation_input = request.get_json()
 
     # xtodo move code below to compute_all_sampl correlation
@@ -53,9 +53,10 @@ def compute_sample_r(corr_method="pearson"):
 
 @correlation.route("/lit_corr/<string:species>/<int:gene_id>", methods=["POST"])
 def compute_lit_corr(species=None, gene_id=None):
-    """api endpoint for doing lit correlation.results for lit correlation\
+    """Api endpoint for doing lit correlation.results for lit correlation\
     are fetched from the database this is the only case where the db\
-    might be needed for actual computing of the correlation results"""
+    might be needed for actual computing of the correlation results
+    """
 
     conn, _cursor_object = database_connector()
     target_traits_gene_ids = request.get_json()
@@ -72,7 +73,7 @@ def compute_lit_corr(species=None, gene_id=None):
 
 @correlation.route("/tissue_corr/<string:corr_method>", methods=["POST"])
 def compute_tissue_corr(corr_method="pearson"):
-    """api endpoint fr doing tissue correlation"""
+    """Api endpoint fr doing tissue correlation"""
     tissue_input_data = request.get_json()
     primary_tissue_dict = tissue_input_data["primary_tissue"]
     target_tissues_dict = tissue_input_data["target_tissues_dict"]
diff --git a/gn3/computations/correlations.py b/gn3/computations/correlations.py
index 26b7294..0d15d9b 100644
--- a/gn3/computations/correlations.py
+++ b/gn3/computations/correlations.py
@@ -1,4 +1,6 @@
 """module contains code for correlations"""
+import multiprocessing
+
 from typing import List
 from typing import Tuple
 from typing import Optional
@@ -7,11 +9,6 @@ from typing import Callable
 import scipy.stats
 
 
-def compute_sum(rhs: int, lhs: int) -> int:
-    """Initial tests to compute sum of two numbers"""
-    return rhs + lhs
-
-
 def map_shared_keys_to_values(target_sample_keys: List, target_sample_vals: dict)-> List:
     """Function to construct target dataset data items given commoned shared\
     keys and trait samplelist values for example given keys  >>>>>>>>>>\
@@ -73,14 +70,12 @@ pearson,spearman and biweight mid correlation return value is rho and p_value
     return (corr_coeffient, p_val)
 
 
-def compute_sample_r_correlation(
-        corr_method: str, trait_vals,
-        target_samples_vals) -> Optional[Tuple[float, float, int]]:
+def compute_sample_r_correlation(trait_name, corr_method, trait_vals,
+                                 target_samples_vals) -> Optional[Tuple[str, float, float, int]]:
     """Given a primary trait values and target trait values calculate the
     correlation coeff and p value
 
     """
-
     (sanitized_traits_vals, sanitized_target_vals,
      num_overlap) = normalize_values(trait_vals, target_samples_vals)
 
@@ -94,7 +89,7 @@ def compute_sample_r_correlation(
         # xtodo check if corr_coefficient is None
         # should use numpy.isNan scipy.isNan is deprecated
         if corr_coeffient is not None:
-            return (corr_coeffient, p_value, num_overlap)
+            return (trait_name, corr_coeffient, p_value, num_overlap)
 
     return None
 
@@ -104,15 +99,15 @@ def do_bicor(x_val, y_val) -> Tuple[float, float]:
 package :not packaged in guix
 
     """
-    return (x_val, y_val)
+    _corr_input = (x_val, y_val)
+    return (0.0, 0.0)
 
 
 def filter_shared_sample_keys(this_samplelist,
                               target_samplelist) -> Tuple[List, List]:
-    """Given primary and target samplelist for two base and target trait select
-filter the values using the shared keys
-
-    """
+    """Given primary and target samplelist\
+    for two base and target trait select\
+    filter the values using the shared keys"""
     this_vals = []
     target_vals = []
     for key, value in target_samplelist.items():
@@ -125,26 +120,70 @@ filter the values using the shared keys
 def compute_all_sample_correlation(this_trait,
                                    target_dataset,
                                    corr_method="pearson") -> List:
-    """Given a trait data samplelist and target__datasets compute all sample
-correlation"""
+    """Given a trait data samplelist and\
+    target__datasets compute all sample correlation
+    """
+    # xtodo fix trait_name currently returning single one
+    # pylint: disable-msg=too-many-locals
+
+    this_trait_samples = this_trait["trait_sample_data"]
+    corr_results = []
+    processed_values = []
+    for target_trait in target_dataset:
+        trait_name = target_trait.get("trait_id")
+        target_trait_data = target_trait["trait_sample_data"]
+        # this_vals, target_vals = filter_shared_sample_keys(
+        #     this_trait_samples, target_trait_data)
+
+        processed_values.append((trait_name, corr_method, *filter_shared_sample_keys(
+            this_trait_samples, target_trait_data)))
+    with multiprocessing.Pool(4) as pool:
+        results = pool.starmap(compute_sample_r_correlation, processed_values)
+
+        for sample_correlation in results:
+            if sample_correlation is not None:
+                (trait_name, corr_coeffient, p_value,
+                 num_overlap) = sample_correlation
+
+                corr_result = {
+                    "corr_coeffient": corr_coeffient,
+                    "p_value": p_value,
+                    "num_overlap": num_overlap
+                }
+
+                corr_results.append({trait_name: corr_result})
+
+    return sorted(
+        corr_results,
+        key=lambda trait_name: -abs(list(trait_name.values())[0]["corr_coeffient"]))
+
+
+def benchmark_compute_all_sample(this_trait,
+                                 target_dataset,
+                                 corr_method="pearson") ->List:
+    """Temp function to benchmark with compute_all_sample_r\
+    alternative to compute_all_sample_r where we use \
+    multiprocessing
+    """
 
     this_trait_samples = this_trait["trait_sample_data"]
 
     corr_results = []
 
     for target_trait in target_dataset:
-        trait_id = target_trait.get("trait_id")
+        trait_name = target_trait.get("trait_id")
         target_trait_data = target_trait["trait_sample_data"]
         this_vals, target_vals = filter_shared_sample_keys(
             this_trait_samples, target_trait_data)
 
         sample_correlation = compute_sample_r_correlation(
+            trait_name=trait_name,
             corr_method=corr_method,
             trait_vals=this_vals,
             target_samples_vals=target_vals)
 
         if sample_correlation is not None:
-            (corr_coeffient, p_value, num_overlap) = sample_correlation
+            (trait_name, corr_coeffient, p_value, num_overlap) = sample_correlation
 
         else:
             continue
@@ -155,7 +194,7 @@ correlation"""
             "num_overlap": num_overlap
         }
 
-        corr_results.append({trait_id: corr_result})
+        corr_results.append({trait_name: corr_result})
 
     return corr_results
 
@@ -187,6 +226,7 @@ def tissue_correlation_for_trait_list(
         primary_tissue_vals: List,
         target_tissues_values: List,
         corr_method: str,
+        trait_id: str,
         compute_corr_p_value: Callable = compute_corr_coeff_p_value) -> dict:
     """Given a primary tissue values for a trait and the target tissues values
     compute the correlation_cooeff and p value the input required are arrays
@@ -202,13 +242,12 @@ def tissue_correlation_for_trait_list(
                                      target_values=target_tissues_values,
                                      corr_method=corr_method)
 
-    lit_corr_result = {
+    tiss_corr_result = {trait_id: {
         "tissue_corr": tissue_corr_coeffient,
-        "p_value": p_value,
-        "tissue_number": len(primary_tissue_vals)
-    }
+        "tissue_number": len(primary_tissue_vals),
+        "p_value": p_value}}
 
-    return lit_corr_result
+    return tiss_corr_result
 
 
 def fetch_lit_correlation_data(
@@ -323,15 +362,17 @@ def compute_all_lit_correlation(conn, trait_lists: List,
                                 species: str, gene_id):
     """Function that acts as an abstraction for
     lit_correlation_for_trait_list"""
-    # xtodo to be refactored
 
     lit_results = lit_correlation_for_trait_list(
         conn=conn,
         target_trait_lists=trait_lists,
         species=species,
         trait_gene_id=gene_id)
+    sorted_lit_results = sorted(
+        lit_results,
+        key=lambda trait_name: -abs(list(trait_name.values())[0]["lit_corr"]))
 
-    return {"lit_results": lit_results}
+    return sorted_lit_results
 
 
 def compute_all_tissue_correlation(primary_tissue_dict: dict,
@@ -343,7 +384,7 @@ def compute_all_tissue_correlation(primary_tissue_dict: dict,
 
     """
 
-    tissues_results = {}
+    tissues_results = []
 
     primary_tissue_vals = primary_tissue_dict["tissue_values"]
     traits_symbol_dict = target_tissues_data["trait_symbol_dict"]
@@ -360,11 +401,17 @@ def compute_all_tissue_correlation(primary_tissue_dict: dict,
         tissue_result = tissue_correlation_for_trait_list(
             primary_tissue_vals=primary_tissue_vals,
             target_tissues_values=target_tissue_vals,
+            trait_id=trait_id,
             corr_method=corr_method)
 
-        tissues_results[trait_id] = tissue_result
+        tissue_result_dict = {trait_id: tissue_result}
+        tissues_results.append(tissue_result_dict)
 
-    return tissues_results
+    sorted_tissues_results = sorted(
+        tissues_results,
+        key=lambda trait_name: -abs(list(trait_name.values())[0]["tissue_corr"]))
+
+    return sorted_tissues_results
 
 
 def process_trait_symbol_dict(trait_symbol_dict, symbol_tissue_vals_dict) -> List:
@@ -384,3 +431,38 @@ def process_trait_symbol_dict(trait_symbol_dict, symbol_tissue_vals_dict) -> Lis
                 traits_tissue_vals.append(target_tissue_dict)
 
     return traits_tissue_vals
+
+
+def compute_tissue_correlation(primary_tissue_dict: dict,
+                               target_tissues_data: dict,
+                               corr_method: str):
+    """Experimental function that uses multiprocessing\
+    for computing tissue correlation
+    """
+
+    tissues_results = []
+
+    primary_tissue_vals = primary_tissue_dict["tissue_values"]
+    traits_symbol_dict = target_tissues_data["trait_symbol_dict"]
+    symbol_tissue_vals_dict = target_tissues_data["symbol_tissue_vals_dict"]
+
+    target_tissues_list = process_trait_symbol_dict(
+        traits_symbol_dict, symbol_tissue_vals_dict)
+    processed_values = []
+
+    for target_tissue_obj in target_tissues_list:
+        trait_id = target_tissue_obj.get("trait_id")
+
+        target_tissue_vals = target_tissue_obj.get("tissue_values")
+        processed_values.append(
+            (primary_tissue_vals, target_tissue_vals, corr_method, trait_id))
+
+    with multiprocessing.Pool(4) as pool:
+        results = pool.starmap(
+            tissue_correlation_for_trait_list, processed_values)
+        for result in results:
+            tissues_results.append(result)
+
+    return sorted(
+        tissues_results,
+        key=lambda trait_name: -abs(list(trait_name.values())[0]["tissue_corr"]))
diff --git a/gn3/settings.py b/gn3/settings.py
index e77a977..7b3ffb7 100644
--- a/gn3/settings.py
+++ b/gn3/settings.py
@@ -12,6 +12,9 @@ REDIS_JOB_QUEUE = "GN3::job-queue"
 TMPDIR = os.environ.get("TMPDIR", tempfile.gettempdir())
 
 # SQL confs
-SQL_URI = os.environ.get("SQL_URI", "mysql://kabui:1234@localhost/db_webqtl")
+SQL_URI = os.environ.get("SQL_URI", "mysql://webqtlout:webqtlout@localhost/db_webqtl")
 SECRET_KEY = "password"
 SQLALCHEMY_TRACK_MODIFICATIONS = False
+# gn2 results only used in fetching dataset info
+
+GN2_BASE_URL = "http://www.genenetwork.org/"