diff options
| -rw-r--r-- | scripts/load_phenotypes_to_db.py | 61 | ||||
| -rw-r--r-- | uploader/phenotypes/views.py | 34 |
2 files changed, 92 insertions, 3 deletions
diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py new file mode 100644 index 0000000..fa710a6 --- /dev/null +++ b/scripts/load_phenotypes_to_db.py @@ -0,0 +1,61 @@ +import uuid +import logging +import argparse +from pathlib import Path + +from gn_libs import jobs, mysqldb, sqlite3 + +logging.basicConfig( + format="%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + +def load_data(conn, job): + """Load the data attached in the given job.""" + pass + + +if __name__ == "__main__": + def parse_args(): + """Setup command-line arguments.""" + parser = argparse.ArgumentParser( + prog="load_phenotypes_to_db", + description="Process the phenotypes' data and load it into the database.") + parser.add_argument("db_uri", type=str, help="MariaDB/MySQL connection URL") + parser.add_argument( + "jobs_db_path", type=Path, help="Path to jobs' SQLite database.") + parser.add_argument("job_id", type=uuid.UUID, help="ID of the running job") + parser.add_argument( + "--log-level", + type=str, + help="Determines what is logged out.", + choices=("debug", "info", "warning", "error", "critical"), + default="info") + return parser.parse_args() + + def setup_logging(log_level: str): + """Setup logging for the script.""" + logger.setLevel(log_level) + logging.getLogger("uploader.phenotypes.models").setLevel(log_level) + + + def main(): + """Entry-point for this script.""" + args = parse_args() + setup_logging(args.log_level.upper()) + + with (mysqldb.database_connection(args.db_uri) as conn, + sqlite3.connection(args.jobs_db_path) as jobs_conn): + try: + return load_data(conn, jobs.job(jobs_conn, args.job_id)) + except jobs.jobs.JobNotFound as _jne: + logger.error("Could not find job with ID: %s", args.job_id) + except Exception as _exc: + logger.error("Loading failed with general exception!", + exc_info=True, + stack_info=True) + + return 1 + + + main() diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index 00ecadb..a54e1a3 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -17,7 +17,11 @@ from pymonad.either import Left from requests.models import Response from MySQLdb.cursors import DictCursor from werkzeug.utils import secure_filename + +from gn_libs import sqlite3 +from gn_libs import jobs as gnlibs_jobs from gn_libs.mysqldb import database_connection + from flask import (flash, request, url_for, @@ -622,7 +626,33 @@ def load_data_to_database( **kwargs ):# pylint: disable=[unused-argument] """Load the data from the given QC job into the database.""" - raise NotImplemented("This feature is yet to be implemented!") + jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + sqlite3.connection(jobs_db) as conn): + qc_job = jobs.job(rconn, jobs.jobsnamespace(), request.form["data-qc-job-id"]) + load_job_id = uuid.uuid4() + command = [ + sys.executable, + "-u", + "-m", + "scripts.load_phenotypes_to_db", + jobs_db, + str(load_job_id), + "--log-level", + logging.getLevelName( + app.logger.getEffectiveLevel() + ).lower() + ] + load_job = gnlibs_jobs.launch_job( + gnlibs_jobs.initialise_job(conn, + load_job_id, + command, + "load-new-phenotypes-data", + extra_meta={}), + jobs_db, + f"{app.config['UPLOAD_FOLDER']}/job_errors", + worker_manager="gn_libs.jobs.launcher") + return str(load_job) def update_phenotype_metadata(conn, metadata: dict): @@ -996,8 +1026,6 @@ def edit_upload_phenotype_data(# pylint: disable=[unused-argument] edit_file = save_file(request.files["file-upload-bulk-edit-upload"], Path(app.config["UPLOAD_FOLDER"])) - from gn_libs import jobs as gnlibs_jobs - from gn_libs import sqlite3 jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] with sqlite3.connection(jobs_db) as conn: job_id = uuid.uuid4() |