diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/qc.py | 45 | ||||
| -rw-r--r-- | scripts/validate_file.py | 65 |
2 files changed, 51 insertions, 59 deletions
diff --git a/scripts/qc.py b/scripts/qc.py index 7d41d6c..e8573a9 100644 --- a/scripts/qc.py +++ b/scripts/qc.py @@ -1,9 +1,9 @@ """Implements the command-line interface for the qc application""" import os import sys -import argparse import mimetypes from typing import Union, Callable +from argparse import ArgumentParser from functional_tools import take @@ -11,17 +11,18 @@ from quality_control.utils import make_progress_calculator from quality_control.errors import InvalidValue, DuplicateHeading from quality_control.parsing import FileType, strain_names, collect_errors +from qc_app.db_utils import database_connection + +from .cli_parser import init_cli_parser + def is_file_mime(filepath:str, mimetype:str) -> bool: """Check that `filepath` has a mimetype of `mimetype` or `text/plain`""" the_type = mimetypes.guess_type(filepath)[0] return the_type in ("text/plain", mimetype) -def cli_argument_parser(): - """Create the parser for the CLI arguments""" - parser = argparse.ArgumentParser( - prog="qc", description = ( - "Command-Line Interface program for quality control of data files")) +def add_file_validation_arguments(parser: ArgumentParser) -> ArgumentParser: + """File validation specific CLI arguments.""" parser.add_argument( "filetype", help="The type of file to check", @@ -32,14 +33,6 @@ def cli_argument_parser(): "The path to the file to be checked." "If an absolute path is not provided, then the file will be relative to" f"\t'{os.getcwd()}'")) - default_strains_file = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "etc/strains.csv") - parser.add_argument( - "-s", "--strainsfile", - help=( - "Path to the file containing allowable strains/samples. " - f"[default '{default_strains_file}']"), - default=default_strains_file) parser.add_argument( "-c", "--count", type=int, @@ -54,6 +47,14 @@ def cli_argument_parser(): default=False, action="store_true") return parser +def cli_argument_parser(): + """Create the parser for the CLI arguments""" + theparser = init_cli_parser( + "qc", + "Command-Line Interface program for quality control of data files") + theparser.add_argument("speciesid", type=int, help="ID of the species.") + return add_file_validation_arguments(theparser) + def make_progress_indicator( verbose: bool, progress_calc_fn: Callable) -> Union[Callable, None]: """Utility to display the progress""" @@ -106,26 +107,14 @@ def main(): print(f"The file '{args.filepath}' does not exist.", file=sys.stderr) return 1 - if not os.path.exists(args.strainsfile): - print(f"The file '{args.strainsfile}' does not exist.", file=sys.stderr) - return 2 - if not is_file_mime(args.filepath, "text/tab-separated-values"): print( f"The file '{args.filepath}' MUST be a tab-separated file.", file=sys.stderr) return 3 - if not is_file_mime(args.strainsfile, "text/csv"): - print( - f"The file '{args.strainsfile}' MUST be a tab-separated file.", - file=sys.stderr) - return 4 - - if args.verbose: - print(f"Parsing the strain names from '{args.strainsfile}'") - - strains = strain_names(os.path.realpath(args.strainsfile)) + with database_connection(args.databaseuri) as dbconn: + strains = strain_names(dbconn, args.speciesid) filepath = os.path.realpath(args.filepath) if args.verbose: diff --git a/scripts/validate_file.py b/scripts/validate_file.py index 9f0a561..4b4fc0c 100644 --- a/scripts/validate_file.py +++ b/scripts/validate_file.py @@ -11,16 +11,20 @@ from redis.exceptions import ConnectionError # pylint: disable=[redefined-builti from quality_control.utils import make_progress_calculator from quality_control.parsing import FileType, strain_names, collect_errors -from .qc import cli_argument_parser +from qc_app.db_utils import database_connection -def make_progress_indicator( - redis_connection: Redis, job_id: str, - progress_calc_fn: Callable) -> Callable: +from .cli_parser import init_cli_parser +from .qc import add_file_validation_arguments + + +def make_progress_indicator(redis_connection: Redis, + jobid: str, + progress_calc_fn: Callable) -> Callable: """Make function that will compute the progress and update redis""" def __indicator__(linenumber, linetext): progress = progress_calc_fn(linenumber, linetext) - redis_connection.hset(name=job_id, mapping=progress._asdict()) + redis_connection.hset(name=str(jobid), mapping=progress._asdict()) return progress @@ -32,13 +36,9 @@ def cli_args_valid(args): print(f"The file '{args.filepath}' does not exist.", file=sys.stderr) return None - if not os.path.exists(args.strainsfile): - print(f"The file '{args.strainsfile}' does not exist.", file=sys.stderr) - return None - try: - conn = Redis.from_url(args.redisurl) # pylint: disable=[unused-variable] - except ConnectionError as conn_err: # pylint: disable=[unused-variable] + _conn = Redis.from_url(args.redisuri) + except ConnectionError as _conn_err: print(traceback.format_exc(), file=sys.stderr) return None @@ -46,28 +46,31 @@ def cli_args_valid(args): def process_cli_arguments(): """Setup command-line parser""" - parser = cli_argument_parser() - parser.prog = "worker" - parser.add_argument( - "redisurl", default="redis:///", help="URL to the redis server") - parser.add_argument("job_id", help="The id of the job being processed") + parser = init_cli_parser( + "validate-file", + ("Verify that the file with the expression data conforms to " + "expectations.")) + parser.add_argument("speciesid", + type=int, + help="Species for which the data is to be processed.") + parser = add_file_validation_arguments(parser) return cli_args_valid(parser.parse_args()) -def stream_error(redis_conn, job_id, error): +def stream_error(redis_conn, jobid, error): """Update redis with the most current error(s) found""" errors = jsonpickle.decode( - redis_conn.hget(job_id, key="errors") or jsonpickle.encode(tuple())) + redis_conn.hget(str(jobid), key="errors") or jsonpickle.encode(tuple())) redis_conn.hset( - job_id, key="errors", value=jsonpickle.encode(errors + (error,))) + str(jobid), key="errors", value=jsonpickle.encode(errors + (error,))) -def make_user_aborted(redis_conn, job_id): +def make_user_aborted(redis_conn, jobid): """Mkae function that checks whether the user aborted the process""" def __aborted__(): user_aborted = bool(int( - redis_conn.hget(name=job_id, key="user_aborted") or "0")) + redis_conn.hget(name=str(jobid), key="user_aborted") or "0")) if user_aborted: - redis_conn.hset(name=job_id, key="status", value="aborted") + redis_conn.hset(name=str(jobid), key="status", value="aborted") return user_aborted return __aborted__ @@ -84,36 +87,36 @@ def main(): print("Quiting due to errors!", file=sys.stderr) return 1 - with Redis.from_url(args.redisurl) as redis_conn: + with (Redis.from_url(args.redisuri) as redis_conn, + database_connection(args.databaseuri) as dbconn): progress_calculator = make_progress_calculator( get_zipfile_size(args.filepath) if is_zipfile(args.filepath) else os.stat(args.filepath).st_size) progress_indicator = make_progress_indicator( - redis_conn, args.job_id, progress_calculator) + redis_conn, args.jobid, progress_calculator) count = args.count filepath = args.filepath filetype = ( FileType.AVERAGE if args.filetype == "average" else FileType.STANDARD_ERROR) - strains = strain_names(args.strainsfile) + strains = strain_names(dbconn, args.speciesid) + redis_conn.hset(name=str(args.jobid), key="status", value="Processing") redis_conn.hset( - name=args.job_id, key="status", value="Processing") - redis_conn.hset( - name=args.job_id, key="message", value="Collecting errors") + name=str(args.jobid), key="message", value="Collecting errors") error_count = 0 for error in collect_errors( filepath, filetype, strains, progress_indicator, - make_user_aborted(redis_conn, args.job_id)): - stream_error(redis_conn, args.job_id, error) + make_user_aborted(redis_conn, args.jobid)): + stream_error(redis_conn, args.jobid, error) if count > 0: error_count = error_count + 1 if error_count >= count: break - redis_conn.hset(name=args.job_id, key="status", value="success") + redis_conn.hset(name=str(args.jobid), key="status", value="success") return 0 |