about summary refs log tree commit diff
path: root/uploader/population/rqtl2.py
diff options
context:
space:
mode:
Diffstat (limited to 'uploader/population/rqtl2.py')
-rw-r--r--uploader/population/rqtl2.py125
1 files changed, 1 insertions, 124 deletions
diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py
index 436eca0..97d4854 100644
--- a/uploader/population/rqtl2.py
+++ b/uploader/population/rqtl2.py
@@ -11,13 +11,11 @@ from typing import Union, Callable, Optional
 import MySQLdb as mdb
 from redis import Redis
 from MySQLdb.cursors import DictCursor
-from werkzeug.utils import secure_filename
 from gn_libs.mysqldb import database_connection
+from markupsafe import escape
 from flask import (
     flash,
-    escape,
     request,
-    jsonify,
     url_for,
     redirect,
     Response,
@@ -191,127 +189,6 @@ def trigger_rqtl2_bundle_qc(
         return jobid
 
 
-def chunk_name(uploadfilename: str, chunkno: int) -> str:
-    """Generate chunk name from original filename and chunk number"""
-    if uploadfilename == "":
-        raise ValueError("Name cannot be empty!")
-    if chunkno < 1:
-        raise ValueError("Chunk number must be greater than zero")
-    return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}"
-
-
-def chunks_directory(uniqueidentifier: str) -> Path:
-    """Compute the directory where chunks are temporarily stored."""
-    if uniqueidentifier == "":
-        raise ValueError("Unique identifier cannot be empty!")
-    return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}")
-
-
-@rqtl2.route(("<int:species_id>/populations/<int:population_id>/rqtl2/"
-              "/rqtl2-bundle-chunked"),
-             methods=["GET"])
-@require_login
-def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"]
-        species_id: int,
-        population_id: int
-):
-    """
-    Extension to the `upload_rqtl2_bundle` endpoint above that provides a way
-    for testing whether all the chunks have been uploaded and to assist with
-    resuming a failed expression-data.
-    """
-    fileid = request.args.get("resumableIdentifier", type=str) or ""
-    filename = request.args.get("resumableFilename", type=str) or ""
-    chunk = request.args.get("resumableChunkNumber", type=int) or 0
-    if not(fileid or filename or chunk):
-        return jsonify({
-            "message": "At least one required query parameter is missing.",
-            "error": "BadRequest",
-            "statuscode": 400
-        }), 400
-
-    if Path(chunks_directory(fileid),
-            chunk_name(filename, chunk)).exists():
-        return "OK"
-
-    return jsonify({
-            "message": f"Chunk {chunk} was not found.",
-            "error": "NotFound",
-            "statuscode": 404
-        }), 404
-
-
-def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
-    """Merge the chunks into a single file."""
-    with open(targetfile, "ab") as _target:
-        for chunkfile in chunkpaths:
-            with open(chunkfile, "rb") as _chunkdata:
-                _target.write(_chunkdata.read())
-
-            chunkfile.unlink()
-    return targetfile
-
-
-@rqtl2.route(("<int:species_id>/population/<int:population_id>/rqtl2/upload/"
-              "/rqtl2-bundle-chunked"),
-             methods=["POST"])
-@require_login
-def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int):
-    """
-    Extension to the `upload_rqtl2_bundle` endpoint above that allows large
-    files to be uploaded in chunks.
-
-    This should hopefully speed up uploads, and if done right, even enable
-    resumable uploads
-    """
-    _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0
-    _chunk = request.form.get("resumableChunkNumber", default=1, type=int)
-    _uploadfilename = request.form.get(
-        "resumableFilename", default="", type=str) or ""
-    _fileid = request.form.get(
-        "resumableIdentifier", default="", type=str) or ""
-    _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid)
-
-    if _targetfile.exists():
-        return jsonify({
-            "message": (
-                "A file with a similar unique identifier has previously been "
-                "uploaded and possibly is/has being/been processed."),
-            "error": "BadRequest",
-            "statuscode": 400
-        }), 400
-
-    try:
-        # save chunk data
-        chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
-        request.files["file"].save(Path(chunks_directory(_fileid),
-                                        chunk_name(_uploadfilename, _chunk)))
-
-        # Check whether upload is complete
-        chunkpaths = tuple(
-            Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
-            for _achunk in range(1, _totalchunks+1))
-        if all(_file.exists() for _file in chunkpaths):
-            # merge_files and clean up chunks
-            __merge_chunks__(_targetfile, chunkpaths)
-            chunks_directory(_fileid).rmdir()
-            jobid = trigger_rqtl2_bundle_qc(
-                species_id, population_id, _targetfile, _uploadfilename)
-            return url_for(
-                "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)
-    except Exception as exc:# pylint: disable=[broad-except]
-        msg = "Error processing uploaded file chunks."
-        app.logger.error(msg, exc_info=True, stack_info=True)
-        return jsonify({
-            "message": msg,
-            "error": type(exc).__name__,
-            "error-description": " ".join(str(arg) for arg in exc.args),
-            "error-trace": traceback.format_exception(exc)
-        }), 500
-
-    return "OK"
-
-
 @rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
              methods=["GET", "POST"])
 @require_login