diff options
Diffstat (limited to 'uploader/population/rqtl2.py')
-rw-r--r-- | uploader/population/rqtl2.py | 123 |
1 files changed, 0 insertions, 123 deletions
diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py index 436eca0..044cdd4 100644 --- a/uploader/population/rqtl2.py +++ b/uploader/population/rqtl2.py @@ -11,13 +11,11 @@ from typing import Union, Callable, Optional import MySQLdb as mdb from redis import Redis from MySQLdb.cursors import DictCursor -from werkzeug.utils import secure_filename from gn_libs.mysqldb import database_connection from flask import ( flash, escape, request, - jsonify, url_for, redirect, Response, @@ -191,127 +189,6 @@ def trigger_rqtl2_bundle_qc( return jobid -def chunk_name(uploadfilename: str, chunkno: int) -> str: - """Generate chunk name from original filename and chunk number""" - if uploadfilename == "": - raise ValueError("Name cannot be empty!") - if chunkno < 1: - raise ValueError("Chunk number must be greater than zero") - return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" - - -def chunks_directory(uniqueidentifier: str) -> Path: - """Compute the directory where chunks are temporarily stored.""" - if uniqueidentifier == "": - raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") - - -@rqtl2.route(("<int:species_id>/populations/<int:population_id>/rqtl2/" - "/rqtl2-bundle-chunked"), - methods=["GET"]) -@require_login -def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] - species_id: int, - population_id: int -): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that provides a way - for testing whether all the chunks have been uploaded and to assist with - resuming a failed expression-data. - """ - fileid = request.args.get("resumableIdentifier", type=str) or "" - filename = request.args.get("resumableFilename", type=str) or "" - chunk = request.args.get("resumableChunkNumber", type=int) or 0 - if not(fileid or filename or chunk): - return jsonify({ - "message": "At least one required query parameter is missing.", - "error": "BadRequest", - "statuscode": 400 - }), 400 - - if Path(chunks_directory(fileid), - chunk_name(filename, chunk)).exists(): - return "OK" - - return jsonify({ - "message": f"Chunk {chunk} was not found.", - "error": "NotFound", - "statuscode": 404 - }), 404 - - -def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: - """Merge the chunks into a single file.""" - with open(targetfile, "ab") as _target: - for chunkfile in chunkpaths: - with open(chunkfile, "rb") as _chunkdata: - _target.write(_chunkdata.read()) - - chunkfile.unlink() - return targetfile - - -@rqtl2.route(("<int:species_id>/population/<int:population_id>/rqtl2/upload/" - "/rqtl2-bundle-chunked"), - methods=["POST"]) -@require_login -def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that allows large - files to be uploaded in chunks. - - This should hopefully speed up uploads, and if done right, even enable - resumable uploads - """ - _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 - _chunk = request.form.get("resumableChunkNumber", default=1, type=int) - _uploadfilename = request.form.get( - "resumableFilename", default="", type=str) or "" - _fileid = request.form.get( - "resumableIdentifier", default="", type=str) or "" - _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid) - - if _targetfile.exists(): - return jsonify({ - "message": ( - "A file with a similar unique identifier has previously been " - "uploaded and possibly is/has being/been processed."), - "error": "BadRequest", - "statuscode": 400 - }), 400 - - try: - # save chunk data - chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) - request.files["file"].save(Path(chunks_directory(_fileid), - chunk_name(_uploadfilename, _chunk))) - - # Check whether upload is complete - chunkpaths = tuple( - Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) - for _achunk in range(1, _totalchunks+1)) - if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() - jobid = trigger_rqtl2_bundle_qc( - species_id, population_id, _targetfile, _uploadfilename) - return url_for( - "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) - except Exception as exc:# pylint: disable=[broad-except] - msg = "Error processing uploaded file chunks." - app.logger.error(msg, exc_info=True, stack_info=True) - return jsonify({ - "message": msg, - "error": type(exc).__name__, - "error-description": " ".join(str(arg) for arg in exc.args), - "error-trace": traceback.format_exception(exc) - }), 500 - - return "OK" - - @rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", methods=["GET", "POST"]) @require_login |