Replace "generate_gemma_computation_cmd" with "compose_gemma_cmd"

2 files changed, 113 insertions, 98 deletions

diff --git a/gn3/api/gemma.py b/gn3/api/gemma.py
index 7fcf2a2..e5a38e4 100644
--- a/gn3/api/gemma.py
+++ b/gn3/api/gemma.py
@@ -113,21 +113,24 @@ traitfile, and snpsfile are extracted from a metadata.json file.
         genofile, phenofile, snpsfile = [os.path.join(working_dir,
                                                       _dict.get(x))
                                          for x in ["geno", "pheno", "snps"]]
+        if not do_paths_exist([genofile, phenofile, snpsfile]):
+            raise FileNotFoundError
         gemma_kwargs = {"g": genofile, "p": phenofile, "a": snpsfile}
         _hash = get_hash_of_files([genofile, phenofile, snpsfile])
         k_output_filename = f"{_hash}-k-output.json"
-        k_computation_cmd = generate_gemma_computation_cmd(
-            gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
-            gemma_wrapper_kwargs=None,
-            gemma_kwargs=gemma_kwargs,
-            output_file=(f"{current_app.config.get('TMPDIR')}/"
-                         f"{token}/{k_output_filename}"))
         return jsonify(
             unique_id=queue_cmd(
                 conn=redis.Redis(),
                 email=(request.get_json() or {}).get('email'),
                 job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
-                cmd=f"{k_computation_cmd}"),
+                cmd=compose_gemma_cmd(
+                    gemma_wrapper_cmd=current_app.config.get("GEMMA_"
+                                                             "WRAPPER_CMD"),
+                    gemma_wrapper_kwargs=None,
+                    gemma_kwargs=gemma_kwargs,
+                    gemma_args=["-gk", ">",
+                                (f"{current_app.config.get('TMPDIR')}/"
+                                 f"{token}/{k_output_filename}")])),
             status="queued",
             output_file=k_output_filename)
     # pylint: disable=W0703
@@ -154,18 +157,19 @@ values.
         gemma_kwargs = {"g": genofile, "p": phenofile, "a": snpsfile}
         _hash = get_hash_of_files([genofile, phenofile, snpsfile])
         k_output_filename = f"{_hash}-k-output.json"
-        k_computation_cmd = generate_gemma_computation_cmd(
-            gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
-            gemma_wrapper_kwargs={"loco": f"--input {chromosomes}"},
-            gemma_kwargs=gemma_kwargs,
-            output_file=(f"{current_app.config.get('TMPDIR')}/"
-                         f"{token}/{k_output_filename}"))
         return jsonify(
             unique_id=queue_cmd(
                 conn=redis.Redis(),
                 email=(request.get_json() or {}).get('email'),
                 job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
-                cmd=f"{k_computation_cmd}"),
+                cmd=compose_gemma_cmd(
+                    gemma_wrapper_cmd=current_app.config.get("GEMMA_"
+                                                             "WRAPPER_CMD"),
+                    gemma_wrapper_kwargs={"loco": f"--input {chromosomes}"},
+                    gemma_kwargs=gemma_kwargs,
+                    gemma_args=["-gk", ">",
+                                (f"{current_app.config.get('TMPDIR')}/"
+                                 f"{token}/{k_output_filename}")])),
             status="queued",
             output_file=k_output_filename)
     # pylint: disable=W0703
@@ -198,14 +202,16 @@ def compute_gwa(k_filename, token):
                 conn=redis.Redis(),
                 email=(request.get_json() or {}).get('email'),
                 job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
-                cmd=generate_gemma_computation_cmd(
-                    gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
+                cmd=compose_gemma_cmd(
+                    gemma_wrapper_cmd=current_app.config.get("GEMMA_"
+                                                             "WRAPPER_CMD"),
                     gemma_wrapper_kwargs={
                         "input": os.path.join(working_dir, k_filename)
                     },
                     gemma_kwargs=gemma_kwargs,
-                    output_file=(f"{current_app.config.get('TMPDIR')}/"
-                                 f"{token}/{_output_filename}"))),
+                    gemma_args=["gk", ">",
+                                (f"{current_app.config.get('TMPDIR')}/"
+                                 f"{token}/{_output_filename}")])),
             status="queued",
             output_file=_output_filename)
     # pylint: disable=W0703
@@ -239,14 +245,16 @@ def compute_gwa_with_covar(k_filename, token):
                 conn=redis.Redis(),
                 email=(request.get_json() or {}).get('email'),
                 job_queue=current_app.config.get("REDIS_JOB_QUEUE"),
-                cmd=generate_gemma_computation_cmd(
-                    gemma_cmd=current_app.config.get("GEMMA_WRAPPER_CMD"),
+                cmd=compose_gemma_cmd(
+                    gemma_wrapper_cmd=current_app.config.get("GEMMA_"
+                                                             "WRAPPER_CMD"),
                     gemma_wrapper_kwargs={
                         "input": os.path.join(working_dir, k_filename)
                     },
                     gemma_kwargs=gemma_kwargs,
-                    output_file=(f"{current_app.config.get('TMPDIR')}/"
-                                 f"{token}/{_output_filename}"))),
+                    gemma_args=["-gk", ">",
+                                (f"{current_app.config.get('TMPDIR')}/"
+                                 f"{token}/{_output_filename}")])),
             status="queued",
             output_file=_output_filename)
     # pylint: disable=W0703
diff --git a/tests/integration/test_gemma.py b/tests/integration/test_gemma.py
index 45c9148..61421f2 100644
--- a/tests/integration/test_gemma.py
+++ b/tests/integration/test_gemma.py
@@ -1,3 +1,4 @@
+# pylint: disable=R0913
 """Integration tests for gemma API endpoints"""
 import os
 import unittest
@@ -151,12 +152,16 @@ class GemmaAPITest(unittest.TestCase):
                          {"status": "test"})
 
     @mock.patch("gn3.api.gemma.queue_cmd")
-    @mock.patch("gn3.api.gemma.generate_gemma_computation_cmd")
     @mock.patch("gn3.api.gemma.get_hash_of_files")
     @mock.patch("gn3.api.gemma.jsonfile_to_dict")
-    def test_k_compute(self, mock_json, mock_hash, mock_cmd,
+    @mock.patch("gn3.api.gemma.do_paths_exist")
+    @mock.patch("gn3.api.gemma.redis.Redis")
+    def test_k_compute(self, mock_redis, mock_path_exist,
+                       mock_json, mock_hash,
                        mock_queue_cmd):
         """Test /gemma/k-compute/<token>"""
+        mock_path_exist.return_value, _redis_conn = True, mock.MagicMock()
+        mock_redis.return_value = _redis_conn
         mock_queue_cmd.return_value = "my-unique-id"
         mock_json.return_value = {
             "geno": "genofile.txt",
@@ -164,13 +169,16 @@ class GemmaAPITest(unittest.TestCase):
             "snps": "snpfile.txt",
         }
         mock_hash.return_value = "hash"
-        mock_cmd.return_value = ("gemma-wrapper --json -- "
-                                 "-debug -g "
-                                 "genotype_name.txt "
-                                 "-p traitfilename.txt "
-                                 "-a genotype_snps.txt "
-                                 "-gk > k_output_filename.json")
         response = self.app.post("/api/gemma/k-compute/test-data")
+        mock_queue_cmd.assert_called_once_with(
+            conn=_redis_conn,
+            email=None,
+            job_queue='GN3::job-queue',
+            cmd=("gemma-wrapper --json -- "
+                 "-g /tmp/test-data/genofile.txt "
+                 "-p /tmp/test-data/phenofile.txt "
+                 "-a /tmp/test-data/snpfile.txt "
+                 "-gk > /tmp/test-data/hash-k-output.json"))
         self.assertEqual(response.get_json(), {
             "output_file": "hash-k-output.json",
             "status": "queued",
@@ -178,12 +186,16 @@ class GemmaAPITest(unittest.TestCase):
         })
 
     @mock.patch("gn3.api.gemma.queue_cmd")
-    @mock.patch("gn3.api.gemma.generate_gemma_computation_cmd")
     @mock.patch("gn3.api.gemma.get_hash_of_files")
     @mock.patch("gn3.api.gemma.jsonfile_to_dict")
-    def test_k_compute_loco(self, mock_json, mock_hash, mock_cmd,
+    @mock.patch("gn3.api.gemma.do_paths_exist")
+    @mock.patch("gn3.api.gemma.redis.Redis")
+    def test_k_compute_loco(self, mock_redis, mock_path_exist,
+                            mock_json, mock_hash,
                             mock_queue_cmd):
         """Test /gemma/k-compute/loco/<chromosomes>/<token>"""
+        mock_path_exist.return_value, _redis_conn = True, mock.MagicMock()
+        mock_redis.return_value = _redis_conn
         mock_queue_cmd.return_value = "my-unique-id"
         mock_json.return_value = {
             "geno": "genofile.txt",
@@ -191,14 +203,18 @@ class GemmaAPITest(unittest.TestCase):
             "snps": "snpfile.txt",
         }
         mock_hash.return_value = "hash"
-        mock_cmd.return_value = ("gemma-wrapper --json -- "
-                                 "-debug -g "
-                                 "genotype_name.txt "
-                                 "-p traitfilename.txt "
-                                 "-a genotype_snps.txt "
-                                 "-gk > k_output_filename.json")
         response = self.app.post(("/api/gemma/k-compute/loco/"
                                   "1%2C2%2C3%2C4%2C5%2C6/test-data"))
+        mock_queue_cmd.assert_called_once_with(
+            conn=_redis_conn,
+            email=None,
+            job_queue='GN3::job-queue',
+            cmd=("gemma-wrapper --json --loco "
+                 "--input 1,2,3,4,5,6 -- "
+                 "-g /tmp/test-data/genofile.txt "
+                 "-p /tmp/test-data/phenofile.txt "
+                 "-a /tmp/test-data/snpfile.txt "
+                 "-gk > /tmp/test-data/hash-k-output.json"))
         self.assertEqual(response.get_json(), {
             "output_file": "hash-k-output.json",
             "status": "queued",
@@ -206,12 +222,16 @@ class GemmaAPITest(unittest.TestCase):
         })
 
     @mock.patch("gn3.api.gemma.queue_cmd")
-    @mock.patch("gn3.api.gemma.generate_gemma_computation_cmd")
     @mock.patch("gn3.api.gemma.get_hash_of_files")
     @mock.patch("gn3.api.gemma.jsonfile_to_dict")
-    def test_gwa_compute(self, mock_json, mock_hash, mock_cmd,
+    @mock.patch("gn3.api.gemma.do_paths_exist")
+    @mock.patch("gn3.api.gemma.redis.Redis")
+    def test_gwa_compute(self, mock_redis, mock_path_exist,
+                         mock_json, mock_hash,
                          mock_queue_cmd):
         """Test /gemma/gwa-compute/<k-inputfile>/<token>"""
+        mock_path_exist.return_value, _redis_conn = True, mock.MagicMock()
+        mock_redis.return_value = _redis_conn
         mock_queue_cmd.return_value = "my-unique-id"
         mock_json.return_value = {
             "geno": "genofile.txt",
@@ -219,27 +239,21 @@ class GemmaAPITest(unittest.TestCase):
             "snps": "snpfile.txt",
         }
         mock_hash.return_value = "hash"
-        mock_cmd.return_value = ("gemma-wrapper --json -- "
-                                 "-debug -g "
-                                 "genotype_name.txt "
-                                 "-p traitfilename.txt "
-                                 "-a genotype_snps.txt "
-                                 "-gk > k_output_filename.json")
         response = self.app.post(("/api/gemma/gwa-compute/hash-k-output.json/"
                                   "my-token"))
         mock_hash.assert_called_once_with(['/tmp/my-token/genofile.txt',
                                            '/tmp/my-token/phenofile.txt',
                                            '/tmp/my-token/snpfile.txt'])
-        mock_cmd.assert_called_once_with(
-            gemma_cmd='gemma-wrapper',
-            gemma_wrapper_kwargs={
-                'input': '/tmp/my-token/hash-k-output.json'
-            },
-            gemma_kwargs={'g': '/tmp/my-token/genofile.txt',
-                          'p': '/tmp/my-token/phenofile.txt',
-                          'a': '/tmp/my-token/snpfile.txt',
-                          'lmm': 9},
-            output_file='/tmp/my-token/hash-gwa-output.json')
+        mock_queue_cmd.assert_called_once_with(
+            conn=_redis_conn,
+            email=None,
+            job_queue='GN3::job-queue',
+            cmd=("gemma-wrapper --json "
+                 "--input /tmp/my-token/hash-k-output.json"
+                 " -- -g /tmp/my-token/genofile.txt "
+                 "-p /tmp/my-token/phenofile.txt "
+                 "-a /tmp/my-token/snpfile.txt "
+                 "-lmm 9 gk > /tmp/my-token/hash-gwa-output.json"))
         self.assertEqual(response.get_json(), {
             "unique_id": "my-unique-id",
             "status": "queued",
@@ -247,13 +261,19 @@ class GemmaAPITest(unittest.TestCase):
         })
 
     @mock.patch("gn3.api.gemma.queue_cmd")
-    @mock.patch("gn3.api.gemma.generate_gemma_computation_cmd")
     @mock.patch("gn3.api.gemma.get_hash_of_files")
     @mock.patch("gn3.api.gemma.jsonfile_to_dict")
-    def test_gwa_compute_with_covars(self, mock_json, mock_hash, mock_cmd,
+    @mock.patch("gn3.api.gemma.do_paths_exist")
+    @mock.patch("gn3.api.gemma.redis.Redis")
+    def test_gwa_compute_with_covars(self, mock_redis,
+                                     mock_path_exist,
+                                     mock_json, mock_hash,
                                      mock_queue_cmd):
         """Test /gemma/gwa-compute/covars/<k-inputfile>/<token>"""
+        mock_path_exist.return_value, _redis_conn = True, mock.MagicMock()
+        mock_redis.return_value = _redis_conn
         mock_queue_cmd.return_value = "my-unique-id"
+        mock_path_exist.return_value = True
         mock_json.return_value = {
             "geno": "genofile.txt",
             "pheno": "phenofile.txt",
@@ -261,12 +281,6 @@ class GemmaAPITest(unittest.TestCase):
             "covar": "covarfile.txt",
         }
         mock_hash.return_value = "hash"
-        mock_cmd.return_value = ("gemma-wrapper --json -- "
-                                 "-debug -g "
-                                 "genotype_name.txt "
-                                 "-p traitfilename.txt "
-                                 "-a genotype_snps.txt "
-                                 "-gk > k_output_filename.json")
         response = self.app.post(("/api/gemma/gwa-compute/"
                                   "covars/hash-k-output.json/"
                                   "my-token"))
@@ -274,37 +288,37 @@ class GemmaAPITest(unittest.TestCase):
                                            '/tmp/my-token/phenofile.txt',
                                            '/tmp/my-token/snpfile.txt',
                                            '/tmp/my-token/covarfile.txt'])
-        mock_cmd.assert_called_once_with(
-            gemma_cmd='gemma-wrapper',
-            gemma_wrapper_kwargs={
-                'input': '/tmp/my-token/hash-k-output.json'
-            },
-            gemma_kwargs={'g': '/tmp/my-token/genofile.txt',
-                          'p': '/tmp/my-token/phenofile.txt',
-                          'a': '/tmp/my-token/snpfile.txt',
-                          'c': '/tmp/my-token/covarfile.txt',
-                          'lmm': 9},
-            output_file='/tmp/my-token/hash-gwa-output.json')
+        mock_queue_cmd.assert_called_once_with(
+            conn=_redis_conn,
+            email=None,
+            job_queue="GN3::job-queue",
+            cmd=("gemma-wrapper --json --input "
+                 "/tmp/my-token/hash-k-output.json -- "
+                 "-g /tmp/my-token/genofile.txt "
+                 "-p /tmp/my-token/phenofile.txt "
+                 "-a /tmp/my-token/snpfile.txt "
+                 "-c /tmp/my-token/covarfile.txt -lmm 9 "
+                 "-gk > /tmp/my-token/hash-gwa-output.json"))
         self.assertEqual(response.get_json(), {
             "unique_id": "my-unique-id",
             "status": "queued",
             "output_file": "hash-gwa-output.json"
         })
 
-    # pylint: disable=R0913
     @mock.patch("gn3.api.gemma.queue_cmd")
-    @mock.patch("gn3.api.gemma.compose_gemma_cmd")
     @mock.patch("gn3.api.gemma.get_hash_of_files")
     @mock.patch("gn3.api.gemma.jsonfile_to_dict")
     @mock.patch("gn3.api.gemma.do_paths_exist")
-    def test_gwa_compute_with_loco_only(self, mock_path_exist,
+    @mock.patch("gn3.api.gemma.redis.Redis")
+    def test_gwa_compute_with_loco_only(self, mock_redis,
+                                        mock_path_exist,
                                         mock_json, mock_hash,
-                                        mock_cmd,
                                         mock_queue_cmd):
         """Test /gemma/gwa-compute/<k-inputfile>/loco/maf/<maf>/<token>
 
         """
-        mock_path_exist.return_value = True
+        mock_path_exist.return_value, _redis_conn = True, mock.MagicMock()
+        mock_redis.return_value = _redis_conn
         mock_queue_cmd.return_value = "my-unique-id"
         mock_json.return_value = {
             "geno": "genofile.txt",
@@ -313,30 +327,23 @@ class GemmaAPITest(unittest.TestCase):
             "covar": "covarfile.txt",
         }
         mock_hash.return_value = "hash"
-        mock_cmd.return_value = ("gemma-wrapper --json -- "
-                                 "-debug -g "
-                                 "genotype_name.txt "
-                                 "-p traitfilename.txt "
-                                 "-a genotype_snps.txt "
-                                 "-gk > k_output_filename.json")
         response = self.app.post(("/api/gemma/gwa-compute/"
                                   "hash-k-output.json/loco/"
                                   "maf/21/my-token"))
         mock_hash.assert_called_once_with(['/tmp/my-token/genofile.txt',
                                            '/tmp/my-token/phenofile.txt',
                                            '/tmp/my-token/snpfile.txt'])
-        mock_cmd.assert_called_once_with(
-            gemma_wrapper_cmd='gemma-wrapper',
-            gemma_wrapper_kwargs={
-                'loco': '--input /tmp/my-token/hash-k-output.json'
-            },
-            gemma_kwargs={'g': '/tmp/my-token/genofile.txt',
-                          'p': '/tmp/my-token/phenofile.txt',
-                          'a': '/tmp/my-token/snpfile.txt',
-                          'lmm': 9,
-                          'maf': 21},
-            gemma_args=["-gk", ">",
-                        '/tmp/my-token/hash-gwa-output.json'])
+        mock_queue_cmd.assert_called_once_with(
+            conn=_redis_conn,
+            email=None,
+            job_queue='GN3::job-queue',
+            cmd=("gemma-wrapper --json --loco --input "
+                 "/tmp/my-token/hash-k-output.json -- "
+                 "-g /tmp/my-token/genofile.txt "
+                 "-p /tmp/my-token/phenofile.txt "
+                 "-a /tmp/my-token/snpfile.txt "
+                 "-lmm 9 -maf 21.0 "
+                 "-gk > /tmp/my-token/hash-gwa-output.json"))
         self.assertEqual(response.get_json(), {
             "unique_id": "my-unique-id",
             "status": "queued",