Init setup and script for async job to load phenotypes into databaseHEADmain

2 files changed, 92 insertions, 3 deletions

diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py
new file mode 100644
index 0000000..fa710a6
--- /dev/null
+++ b/scripts/load_phenotypes_to_db.py
@@ -0,0 +1,61 @@
+import uuid
+import logging
+import argparse
+from pathlib import Path
+
+from gn_libs import jobs, mysqldb, sqlite3
+
+logging.basicConfig(
+    format="%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def load_data(conn, job):
+    """Load the data attached in the given job."""
+    pass
+
+
+if __name__ == "__main__":
+    def parse_args():
+        """Setup command-line arguments."""
+        parser = argparse.ArgumentParser(
+            prog="load_phenotypes_to_db",
+            description="Process the phenotypes' data and load it into the database.")
+        parser.add_argument("db_uri", type=str, help="MariaDB/MySQL connection URL")
+        parser.add_argument(
+            "jobs_db_path", type=Path, help="Path to jobs' SQLite database.")
+        parser.add_argument("job_id", type=uuid.UUID, help="ID of the running job")
+        parser.add_argument(
+            "--log-level",
+            type=str,
+            help="Determines what is logged out.",
+            choices=("debug", "info", "warning", "error", "critical"),
+            default="info")
+        return parser.parse_args()
+
+    def setup_logging(log_level: str):
+        """Setup logging for the script."""
+        logger.setLevel(log_level)
+        logging.getLogger("uploader.phenotypes.models").setLevel(log_level)
+
+
+    def main():
+        """Entry-point for this script."""
+        args = parse_args()
+        setup_logging(args.log_level.upper())
+
+        with (mysqldb.database_connection(args.db_uri) as conn,
+              sqlite3.connection(args.jobs_db_path) as jobs_conn):
+            try:
+                return load_data(conn, jobs.job(jobs_conn, args.job_id))
+            except jobs.jobs.JobNotFound as _jne:
+                logger.error("Could not find job with ID: %s", args.job_id)
+            except Exception as _exc:
+                logger.error("Loading failed with general exception!",
+                             exc_info=True,
+                             stack_info=True)
+
+        return 1
+
+
+    main()
diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py
index 00ecadb..a54e1a3 100644
--- a/uploader/phenotypes/views.py
+++ b/uploader/phenotypes/views.py
@@ -17,7 +17,11 @@ from pymonad.either import Left
 from requests.models import Response
 from MySQLdb.cursors import DictCursor
 from werkzeug.utils import secure_filename
+
+from gn_libs import sqlite3
+from gn_libs import jobs as gnlibs_jobs
 from gn_libs.mysqldb import database_connection
+
 from flask import (flash,
                    request,
                    url_for,
@@ -622,7 +626,33 @@ def load_data_to_database(
         **kwargs
 ):# pylint: disable=[unused-argument]
     """Load the data from the given QC job into the database."""
-    raise NotImplemented("This feature is yet to be implemented!")
+    jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]
+    with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+          sqlite3.connection(jobs_db) as conn):
+        qc_job = jobs.job(rconn, jobs.jobsnamespace(), request.form["data-qc-job-id"])
+        load_job_id = uuid.uuid4()
+        command = [
+            sys.executable,
+            "-u",
+            "-m",
+            "scripts.load_phenotypes_to_db",
+            jobs_db,
+            str(load_job_id),
+            "--log-level",
+            logging.getLevelName(
+                app.logger.getEffectiveLevel()
+            ).lower()
+        ]
+        load_job = gnlibs_jobs.launch_job(
+            gnlibs_jobs.initialise_job(conn,
+                                       load_job_id,
+                                       command,
+                                       "load-new-phenotypes-data",
+                                       extra_meta={}),
+            jobs_db,
+            f"{app.config['UPLOAD_FOLDER']}/job_errors",
+            worker_manager="gn_libs.jobs.launcher")
+        return str(load_job)
 
 
 def update_phenotype_metadata(conn, metadata: dict):
@@ -996,8 +1026,6 @@ def edit_upload_phenotype_data(# pylint: disable=[unused-argument]
     edit_file = save_file(request.files["file-upload-bulk-edit-upload"],
                           Path(app.config["UPLOAD_FOLDER"]))
 
-    from gn_libs import jobs as gnlibs_jobs
-    from gn_libs import sqlite3
     jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]
     with sqlite3.connection(jobs_db) as conn:
         job_id = uuid.uuid4()