diff options
Diffstat (limited to 'uploader/phenotypes')
| -rw-r--r-- | uploader/phenotypes/views.py | 111 |
1 files changed, 87 insertions, 24 deletions
diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index c4aa67a..ec4c840 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -4,6 +4,7 @@ import uuid import json import datetime from pathlib import Path +from zipfile import ZipFile from functools import wraps from logging import INFO, ERROR, DEBUG, FATAL, CRITICAL, WARNING @@ -14,6 +15,7 @@ from gn_libs.mysqldb import database_connection from flask import (flash, request, url_for, + jsonify, redirect, Blueprint, current_app as app) @@ -307,6 +309,70 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= population_id=population["Id"])) +def process_phenotypes_rqtl2_bundle(error_uri): + """Process phenotypes from the uploaded R/qtl2 bundle.""" + _redisuri = app.config["REDIS_URL"] + _sqluri = app.config["SQL_URI"] + try: + ## Handle huge files here... + phenobundle = save_file(request.files["phenotypes-bundle"], + Path(app.config["UPLOAD_FOLDER"])) + rqc.validate_bundle(phenobundle) + return phenobundle + except AssertionError as _aerr: + app.logger.debug("File upload error!", exc_info=True) + flash("Expected a zipped bundle of files with phenotypes' " + "information.", + "alert-danger") + return error_uri + except rqe.RQTLError as rqtlerr: + app.logger.debug("Bundle validation error!", exc_info=True) + flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger") + return error_uri + + +def process_phenotypes_individual_files(error_uri): + """Process the uploaded individual files.""" + form = request.form + cdata = { + "sep": form["file-separator"], + "comment.char": form["file-comment-character"], + "na.strings": form["file-na"].split(" "), + } + bundlepath = Path(app.config["UPLOAD_FOLDER"], + f"{str(uuid.uuid4()).replace('-', '')}.zip") + with ZipFile(bundlepath,mode="w") as zfile: + for rqtlkey, formkey in (("phenocovar", "phenotype-descriptions"), + ("pheno", "phenotype-data"), + ("phenose", "phenotype-se"), + ("phenonum", "phenotype-n")): + if form.get("resumable-upload", False): + # Chunked upload of large files was used + filedata = json.loads(form[formkey]) + zfile.write( + Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]), + arcname=filedata["original-name"]) + cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]] + else: + # TODO: Check this path: fix any bugs. + _sentfile = request.files[formkey] + if not bool(_sentfile): + flash(f"Expected file ('{formkey}') was not provided.", + "alert-danger") + return error_uri + + filepath = save_file( + _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False) + zfile.write( + Path(app.config["UPLOAD_FOLDER"], filepath), + arcname=filepath.name) + cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name] + + zfile.writestr("control_data.json", data=json.dumps(cdata, indent=2)) + + return bundlepath + + @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/add-phenotypes", @@ -318,6 +384,7 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= redirect_uri="species.populations.phenotypes.list_datasets") def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals] """Add one or more phenotypes to the dataset.""" + use_bundle = request.args.get("use_bundle", "").lower() == "true" add_phenos_uri = redirect(url_for( "species.populations.phenotypes.add_phenotypes", species_id=species["SpeciesId"], @@ -333,8 +400,7 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p today = datetime.date.today() return render_template( ("phenotypes/add-phenotypes-with-rqtl2-bundle.html" - if request.args.get("use_bundle", "").lower() == "true" - else "phenotypes/add-phenotypes-raw-files.html"), + if use_bundle else "phenotypes/add-phenotypes-raw-files.html"), species=species, population=population, dataset=dataset, @@ -347,23 +413,12 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p current_year=int(today.strftime("%Y")), families_with_se_and_n=( "Reference Populations (replicate average, SE, N)",), + use_bundle=use_bundle, activelink="add-phenotypes") - try: - ## Handle huge files here... - phenobundle = save_file(request.files["phenotypes-bundle"], - Path(app.config["UPLOAD_FOLDER"])) - rqc.validate_bundle(phenobundle) - except AssertionError as _aerr: - app.logger.debug("File upload error!", exc_info=True) - flash("Expected a zipped bundle of files with phenotypes' " - "information.", - "alert-danger") - return add_phenos_uri - except rqe.RQTLError as rqtlerr: - app.logger.debug("Bundle validation error!", exc_info=True) - flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger") - return add_phenos_uri + phenobundle = (process_phenotypes_rqtl2_bundle(add_phenos_uri) + if use_bundle else + process_phenotypes_individual_files(add_phenos_uri)) _jobid = uuid.uuid4() _namespace = jobs.jobsnamespace() @@ -377,7 +432,7 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]), str(population["Id"]), # str(dataset["Id"]), - str(phenobundle), + str(phenobundle), "--loglevel", { INFO: "INFO", @@ -398,12 +453,20 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p f"{app.config['UPLOAD_FOLDER']}/job_errors") app.logger.debug("JOB DETAILS: %s", _job) - - return redirect(url_for("species.populations.phenotypes.job_status", - species_id=species["SpeciesId"], - population_id=population["Id"], - dataset_id=dataset["Id"], - job_id=str(_job["jobid"]))) + jobstatusuri = url_for("species.populations.phenotypes.job_status", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + job_id=str(_job["jobid"])) + return ((jsonify({ + "redirect-to": jobstatusuri, + "statuscode": 200, + "message": ("Follow the 'redirect-to' URI to see the state " + "of the quality-control job started for your " + "uploaded files.") + }), 200) + if request.form.get("resumable-upload", False) else + redirect(jobstatusuri)) @phenotypesbp.route( |