Run partial correlations with external script

Use new external script to run the partial correlations for both cases,
i.e.
- against an entire dataset, or
- against selected traits

2 files changed, 46 insertions, 31 deletions

diff --git a/gn3/api/correlation.py b/gn3/api/correlation.py
index 3aadcb9..1667302 100644
--- a/gn3/api/correlation.py
+++ b/gn3/api/correlation.py
@@ -16,8 +16,6 @@ from gn3.computations.correlations import map_shared_keys_to_values
 from gn3.computations.correlations import compute_tissue_correlation
 from gn3.computations.correlations import compute_all_lit_correlation
 from gn3.computations.correlations import compute_all_sample_correlation
-from gn3.computations.partial_correlations import (
-    partial_correlations_with_target_traits)
 
 correlation = Blueprint("correlation", __name__)
 
@@ -124,16 +122,25 @@ def partial_correlation():
             "messages": request_errors,
             "error_type": "Client Error"})
 
-    if with_target_db:
-        with redis.Redis() as conn:
-            queueing_results = run_async_cmd(
+    with redis.Redis() as conn:
+        if with_target_db:
+            command = compose_pcorrs_command(
+                trait_fullname(args["primary_trait"]),
+                tuple(
+                    trait_fullname(trait) for trait in args["control_traits"]),
+                args["method"], target_database=args["target_db"],
+                criteria = int(args.get("criteria", 500)))
+        else:
+            command = compose_pcorrs_command(
+                trait_fullname(args["primary_trait"]),
+                tuple(
+                    trait_fullname(trait) for trait in args["control_traits"]),
+                args["method"], target_traits=tuple(
+                    trait_fullname(trait) for trait in args["target_traits"]))
+
+        queueing_results = run_async_cmd(
                 conn=conn,
-                cmd=compose_pcorrs_command(
-                    trait_fullname(args["primary_trait"]),
-                    tuple(
-                        trait_fullname(trait) for trait in args["control_traits"]),
-                    args["method"], args["target_db"],
-                    int(args.get("criteria", 500))),
+                cmd=command,
                 job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
                 env = {"PYTHONPATH": ":".join(sys.path), "SQL_URI": SQL_URI})
         return build_response({
@@ -141,15 +148,3 @@ def partial_correlation():
             "results": queueing_results,
             "queued": True
         })
-
-    with database_connector() as conn:
-        results = partial_correlations_with_target_traits(
-            conn,
-            trait_fullname(args["primary_trait"]),
-            tuple(
-                trait_fullname(trait) for trait in args["control_traits"]),
-            args["method"],
-            tuple(
-                trait_fullname(trait) for trait in args["target_traits"]))
-
-    return build_response({"status": "success", "results": results})
diff --git a/gn3/commands.py b/gn3/commands.py
index 9684eeb..b1b3fc7 100644
--- a/gn3/commands.py
+++ b/gn3/commands.py
@@ -1,6 +1,5 @@
 """Procedures used to work with the various bio-informatics cli
 commands"""
-import os
 import sys
 import json
 import subprocess
@@ -51,15 +50,36 @@ def compose_rqtl_cmd(rqtl_wrapper_cmd: str,
 
     return cmd
 
+def compose_pcorrs_command_for_selected_traits(
+        prefix_cmd: Tuple[str, ...], target_traits: Tuple[str, ...]) -> Tuple[
+            str, ...]:
+    """Build command for partial correlations against selected traits."""
+    return prefix_cmd + ("against-traits", ",".join(target_traits))
+
+def compose_pcorrs_command_for_database(
+        prefix_cmd: Tuple[str, ...], target_database: str,
+        criteria: int = 500) -> Tuple[str, ...]:
+    """Build command for partial correlations against an entire dataset."""
+    return prefix_cmd + (
+        "against-db", f"{target_database}", f"--criteria={criteria}")
+
 def compose_pcorrs_command(
         primary_trait: str, control_traits: Tuple[str, ...], method: str,
-        target_database: str, criteria: int = 500):
+        **kwargs):
     """Compose the command to run partias correlations"""
-    rundir = os.path.abspath(".")
-    return (
-        f"{sys.executable}", f"{rundir}/scripts/partial_correlations.py",
-        primary_trait, ",".join(control_traits), f'"{method}"',
-        f"{target_database}", f"--criteria={criteria}")
+    print(f"KWARGS: {kwargs}")
+    prefix_cmd = (
+        f"{sys.executable}", "-m", "scripts.partial_correlations",
+        primary_trait, ",".join(control_traits), f'"{method}"')
+    if (
+            kwargs.get("target_database") is not None
+            and kwargs.get("target_traits") is None):
+        return compose_pcorrs_command_for_database(prefix_cmd, **kwargs)
+    if (
+            kwargs.get("target_database") is None
+            and kwargs.get("target_traits") is not None):
+        return compose_pcorrs_command_for_selected_traits(prefix_cmd, **kwargs)
+    raise Exception("Invalid state: I don't know what command to generate!")
 
 def queue_cmd(conn: Redis,
               job_queue: str,
@@ -111,5 +131,5 @@ def run_async_cmd(
     """A utility function to call `gn3.commands.queue_cmd` function and run the
     worker in the `one-shot` mode."""
     cmd_id = queue_cmd(conn, job_queue, cmd, email, env)
-    subprocess.Popen(["python3", "sheepdog/worker.py"]) # pylint: disable=[consider-using-with]
+    subprocess.Popen([f"{sys.executable}", "-m", "sheepdog.worker"]) # pylint: disable=[consider-using-with]
     return cmd_id