diff options
-rw-r--r-- | MANIFEST.in | 4 | ||||
-rw-r--r-- | README.org | 8 | ||||
-rw-r--r-- | mypy.ini | 12 | ||||
-rw-r--r-- | qc_app/__init__.py | 48 | ||||
-rw-r--r-- | qc_app/templates/unhandled_exception.html | 21 | ||||
-rw-r--r-- | r_qtl/exceptions.py (renamed from r_qtl/errors.py) | 2 | ||||
-rw-r--r-- | r_qtl/fileerrors.py | 9 | ||||
-rw-r--r-- | r_qtl/r_qtl2.py | 156 | ||||
-rw-r--r-- | r_qtl/r_qtl2_qc.py | 59 | ||||
-rw-r--r-- | scripts/insert_data.py | 4 | ||||
-rw-r--r-- | scripts/insert_samples.py | 8 | ||||
-rw-r--r-- | scripts/process_rqtl2_bundle.py | 8 | ||||
-rw-r--r-- | scripts/qc.py | 2 | ||||
-rw-r--r-- | scripts/qc_on_rqtl2_bundle.py | 24 | ||||
-rw-r--r-- | scripts/qcapp_wsgi.py | 4 | ||||
-rw-r--r-- | scripts/rqtl2/entry.py | 6 | ||||
-rw-r--r-- | scripts/rqtl2/install_genotypes.py | 117 | ||||
-rw-r--r-- | scripts/validate_file.py | 4 | ||||
-rw-r--r-- | scripts/worker.py | 4 | ||||
-rw-r--r-- | tests/conftest.py | 4 | ||||
-rw-r--r-- | tests/qc_app/test_parse.py | 4 | ||||
-rw-r--r-- | uploader/__init__.py | 94 | ||||
-rw-r--r-- | uploader/authorisation.py | 21 | ||||
-rw-r--r-- | uploader/base_routes.py (renamed from qc_app/base_routes.py) | 0 | ||||
-rw-r--r-- | uploader/check_connections.py (renamed from qc_app/check_connections.py) | 2 | ||||
-rw-r--r-- | uploader/db/__init__.py (renamed from qc_app/db/__init__.py) | 0 | ||||
-rw-r--r-- | uploader/db/averaging.py (renamed from qc_app/db/averaging.py) | 0 | ||||
-rw-r--r-- | uploader/db/datasets.py (renamed from qc_app/db/datasets.py) | 0 | ||||
-rw-r--r-- | uploader/db/platforms.py (renamed from qc_app/db/platforms.py) | 0 | ||||
-rw-r--r-- | uploader/db/populations.py (renamed from qc_app/db/populations.py) | 0 | ||||
-rw-r--r-- | uploader/db/species.py (renamed from qc_app/db/species.py) | 0 | ||||
-rw-r--r-- | uploader/db/tissues.py (renamed from qc_app/db/tissues.py) | 0 | ||||
-rw-r--r-- | uploader/db_utils.py (renamed from qc_app/db_utils.py) | 7 | ||||
-rw-r--r-- | uploader/dbinsert.py (renamed from qc_app/dbinsert.py) | 34 | ||||
-rw-r--r-- | uploader/default_settings.py | 23 | ||||
-rw-r--r-- | uploader/entry.py (renamed from qc_app/entry.py) | 10 | ||||
-rw-r--r-- | uploader/errors.py (renamed from qc_app/errors.py) | 0 | ||||
-rw-r--r-- | uploader/files.py (renamed from qc_app/files.py) | 0 | ||||
-rw-r--r-- | uploader/input_validation.py (renamed from qc_app/input_validation.py) | 0 | ||||
-rw-r--r-- | uploader/jobs.py (renamed from qc_app/jobs.py) | 0 | ||||
-rw-r--r-- | uploader/monadic_requests.py | 86 | ||||
-rw-r--r-- | uploader/oauth2/__init__.py | 1 | ||||
-rw-r--r-- | uploader/oauth2/client.py | 202 | ||||
-rw-r--r-- | uploader/oauth2/jwks.py | 86 | ||||
-rw-r--r-- | uploader/oauth2/views.py | 138 | ||||
-rw-r--r-- | uploader/parse.py (renamed from qc_app/parse.py) | 9 | ||||
-rw-r--r-- | uploader/samples.py (renamed from qc_app/samples.py) | 15 | ||||
-rw-r--r-- | uploader/session.py | 91 | ||||
-rw-r--r-- | uploader/static/css/custom-bootstrap.css (renamed from qc_app/static/css/custom-bootstrap.css) | 0 | ||||
-rw-r--r-- | uploader/static/css/styles.css (renamed from qc_app/static/css/styles.css) | 0 | ||||
-rw-r--r-- | uploader/static/css/two-column-with-separator.css (renamed from qc_app/static/css/two-column-with-separator.css) | 0 | ||||
-rw-r--r-- | uploader/static/images/CITGLogo.png (renamed from qc_app/static/images/CITGLogo.png) | bin | 11962 -> 11962 bytes | |||
-rw-r--r-- | uploader/static/js/select_platform.js (renamed from qc_app/static/js/select_platform.js) | 0 | ||||
-rw-r--r-- | uploader/static/js/upload_progress.js (renamed from qc_app/static/js/upload_progress.js) | 0 | ||||
-rw-r--r-- | uploader/static/js/upload_samples.js (renamed from qc_app/static/js/upload_samples.js) | 0 | ||||
-rw-r--r-- | uploader/static/js/utils.js (renamed from qc_app/static/js/utils.js) | 0 | ||||
-rw-r--r-- | uploader/templates/base.html (renamed from qc_app/templates/base.html) | 13 | ||||
-rw-r--r-- | uploader/templates/cli-output.html (renamed from qc_app/templates/cli-output.html) | 0 | ||||
-rw-r--r-- | uploader/templates/continue_from_create_dataset.html (renamed from qc_app/templates/continue_from_create_dataset.html) | 0 | ||||
-rw-r--r-- | uploader/templates/continue_from_create_study.html (renamed from qc_app/templates/continue_from_create_study.html) | 0 | ||||
-rw-r--r-- | uploader/templates/data_review.html (renamed from qc_app/templates/data_review.html) | 0 | ||||
-rw-r--r-- | uploader/templates/dbupdate_error.html (renamed from qc_app/templates/dbupdate_error.html) | 0 | ||||
-rw-r--r-- | uploader/templates/dbupdate_hidden_fields.html (renamed from qc_app/templates/dbupdate_hidden_fields.html) | 0 | ||||
-rw-r--r-- | uploader/templates/errors_display.html (renamed from qc_app/templates/errors_display.html) | 0 | ||||
-rw-r--r-- | uploader/templates/final_confirmation.html (renamed from qc_app/templates/final_confirmation.html) | 0 | ||||
-rw-r--r-- | uploader/templates/flash_messages.html (renamed from qc_app/templates/flash_messages.html) | 0 | ||||
-rw-r--r-- | uploader/templates/http-error.html (renamed from qc_app/templates/http-error.html) | 0 | ||||
-rw-r--r-- | uploader/templates/index.html (renamed from qc_app/templates/index.html) | 3 | ||||
-rw-r--r-- | uploader/templates/insert_error.html (renamed from qc_app/templates/insert_error.html) | 0 | ||||
-rw-r--r-- | uploader/templates/insert_progress.html (renamed from qc_app/templates/insert_progress.html) | 0 | ||||
-rw-r--r-- | uploader/templates/insert_success.html (renamed from qc_app/templates/insert_success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/job_progress.html (renamed from qc_app/templates/job_progress.html) | 0 | ||||
-rw-r--r-- | uploader/templates/login.html | 32 | ||||
-rw-r--r-- | uploader/templates/no_such_job.html (renamed from qc_app/templates/no_such_job.html) | 0 | ||||
-rw-r--r-- | uploader/templates/parse_failure.html (renamed from qc_app/templates/parse_failure.html) | 0 | ||||
-rw-r--r-- | uploader/templates/parse_results.html (renamed from qc_app/templates/parse_results.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/create-geno-dataset-success.html (renamed from qc_app/templates/rqtl2/create-geno-dataset-success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/create-probe-dataset-success.html (renamed from qc_app/templates/rqtl2/create-probe-dataset-success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/create-probe-study-success.html (renamed from qc_app/templates/rqtl2/create-probe-study-success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/create-tissue-success.html (renamed from qc_app/templates/rqtl2/create-tissue-success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/index.html (renamed from qc_app/templates/rqtl2/index.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/no-such-job.html (renamed from qc_app/templates/rqtl2/no-such-job.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-job-error.html (renamed from qc_app/templates/rqtl2/rqtl2-job-error.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-job-results.html (renamed from qc_app/templates/rqtl2/rqtl2-job-results.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-job-status.html (renamed from qc_app/templates/rqtl2/rqtl2-job-status.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-qc-job-error.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-error.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-qc-job-results.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-results.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-qc-job-status.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-status.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/rqtl2-qc-job-success.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/select-geno-dataset.html (renamed from qc_app/templates/rqtl2/select-geno-dataset.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/select-population.html (renamed from qc_app/templates/rqtl2/select-population.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/select-probeset-dataset.html (renamed from qc_app/templates/rqtl2/select-probeset-dataset.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/select-probeset-study-id.html (renamed from qc_app/templates/rqtl2/select-probeset-study-id.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/select-tissue.html (renamed from qc_app/templates/rqtl2/select-tissue.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/summary-info.html (renamed from qc_app/templates/rqtl2/summary-info.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html (renamed from qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html) | 0 | ||||
-rw-r--r-- | uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html (renamed from qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html) | 0 | ||||
-rw-r--r-- | uploader/templates/samples/select-population.html (renamed from qc_app/templates/samples/select-population.html) | 0 | ||||
-rw-r--r-- | uploader/templates/samples/select-species.html (renamed from qc_app/templates/samples/select-species.html) | 0 | ||||
-rw-r--r-- | uploader/templates/samples/upload-failure.html (renamed from qc_app/templates/samples/upload-failure.html) | 0 | ||||
-rw-r--r-- | uploader/templates/samples/upload-progress.html (renamed from qc_app/templates/samples/upload-progress.html) | 0 | ||||
-rw-r--r-- | uploader/templates/samples/upload-samples.html (renamed from qc_app/templates/samples/upload-samples.html) | 0 | ||||
-rw-r--r-- | uploader/templates/samples/upload-success.html (renamed from qc_app/templates/samples/upload-success.html) | 0 | ||||
-rw-r--r-- | uploader/templates/select_dataset.html (renamed from qc_app/templates/select_dataset.html) | 0 | ||||
-rw-r--r-- | uploader/templates/select_platform.html (renamed from qc_app/templates/select_platform.html) | 0 | ||||
-rw-r--r-- | uploader/templates/select_species.html (renamed from qc_app/templates/select_species.html) | 0 | ||||
-rw-r--r-- | uploader/templates/select_study.html (renamed from qc_app/templates/select_study.html) | 0 | ||||
-rw-r--r-- | uploader/templates/stdout_output.html (renamed from qc_app/templates/stdout_output.html) | 0 | ||||
-rw-r--r-- | uploader/templates/unhandled_exception.html | 24 | ||||
-rw-r--r-- | uploader/templates/upload_progress_indicator.html (renamed from qc_app/templates/upload_progress_indicator.html) | 0 | ||||
-rw-r--r-- | uploader/templates/worker_failure.html (renamed from qc_app/templates/worker_failure.html) | 0 | ||||
-rw-r--r-- | uploader/upload/__init__.py (renamed from qc_app/upload/__init__.py) | 0 | ||||
-rw-r--r-- | uploader/upload/rqtl2.py (renamed from qc_app/upload/rqtl2.py) | 83 |
113 files changed, 1257 insertions, 225 deletions
diff --git a/MANIFEST.in b/MANIFEST.in index c515f0e..79339cd 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include README.org recursive-include etc *.py *.csv -recursive-include qc_app/static *.js *.css *.png -recursive-include qc_app/templates *.html +recursive-include uploader/static *.js *.css *.png +recursive-include uploader/templates *.html recursive-exclude tests/ *.py *.tsv *.csv
\ No newline at end of file @@ -197,7 +197,7 @@ few environment variables #+BEGIN_SRC shell export FLASK_APP=wsgi.py export FLASK_ENV=development -export QCAPP_INSTANCE_PATH=/path/to/directory/with/config.py +export UPLOADER_CONF=/path/to/directory/with/uploader/configuration.py #+END_SRC then you can run the application with #+BEGIN_SRC shell @@ -208,7 +208,7 @@ flask run To run the linter over the code base, run: #+BEGIN_SRC shell - pylint setup.py tests quality_control qc_app r_qtl scripts + pylint setup.py tests quality_control uploader r_qtl scripts #+END_SRC To check for correct type usage in the application, run: @@ -218,13 +218,13 @@ To check for correct type usage in the application, run: Run unit tests with: #+BEGIN_SRC shell - $ export QCAPP_CONF=</path/to/configuration/file.py> + $ export UPLOADER_CONF=</path/to/configuration/file.py> $ pytest -m unit_test #+END_SRC To run ALL tests (not just unit tests): #+BEGIN_SRC shell - $ export QCAPP_CONF=</path/to/configuration/file.py> + $ export UPLOADER_CONF=</path/to/configuration/file.py> $ pytest #+END_SRC @@ -28,4 +28,16 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-yaml.*] +ignore_missing_imports = True + +[mypy-pymonad.tools] +ignore_missing_imports = True + +[mypy-pymonad.either] +ignore_missing_imports = True + +[mypy-authlib.*] +ignore_missing_imports = True + +[mypy-flask_session.*] ignore_missing_imports = True
\ No newline at end of file diff --git a/qc_app/__init__.py b/qc_app/__init__.py deleted file mode 100644 index 3ee8aa0..0000000 --- a/qc_app/__init__.py +++ /dev/null @@ -1,48 +0,0 @@ -"""The Quality-Control Web Application entry point""" -import os -import logging -from pathlib import Path - -from flask import Flask, request - -from .entry import entrybp -from .upload import upload -from .parse import parsebp -from .samples import samples -from .base_routes import base -from .dbinsert import dbinsertbp -from .errors import register_error_handlers - -def override_settings_with_envvars( - app: Flask, ignore: tuple[str, ...]=tuple()) -> None: - """Override settings in `app` with those in ENVVARS""" - for setting in (key for key in app.config if key not in ignore): - app.config[setting] = os.environ.get(setting) or app.config[setting] - - -def create_app(): - """The application factory""" - app = Flask(__name__) - app.config.from_pyfile( - Path(__file__).parent.joinpath("default_settings.py")) - if "QCAPP_CONF" in os.environ: - app.config.from_envvar("QCAPP_CONF") # Override defaults with instance path - - override_settings_with_envvars(app, ignore=tuple()) - - if "QCAPP_SECRETS" in os.environ: - app.config.from_envvar("QCAPP_SECRETS") - - # setup jinja2 symbols - app.jinja_env.globals.update(request_url=lambda : request.url) - - # setup blueprints - app.register_blueprint(base, url_prefix="/") - app.register_blueprint(entrybp, url_prefix="/") - app.register_blueprint(parsebp, url_prefix="/parse") - app.register_blueprint(upload, url_prefix="/upload") - app.register_blueprint(dbinsertbp, url_prefix="/dbinsert") - app.register_blueprint(samples, url_prefix="/samples") - - register_error_handlers(app) - return app diff --git a/qc_app/templates/unhandled_exception.html b/qc_app/templates/unhandled_exception.html deleted file mode 100644 index 6e6a051..0000000 --- a/qc_app/templates/unhandled_exception.html +++ /dev/null @@ -1,21 +0,0 @@ -{%extends "base.html"%} - -{%block title%}System Error{%endblock%} - -{%block css%} -<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> -{%endblock%} - -{%block contents%} -<p> - An error has occured, and your request has been aborted. Please notify the - administrator to try and get this sorted. -</p> -<p> - Provide the following information to help the administrator figure out and fix - the issue:<br /> - <hr /><br /> - {{trace}} - <hr /><br /> -</p> -{%endblock%} diff --git a/r_qtl/errors.py b/r_qtl/exceptions.py index 417eb58..9620cf4 100644 --- a/r_qtl/errors.py +++ b/r_qtl/exceptions.py @@ -6,7 +6,7 @@ class RQTLError(Exception): class InvalidFormat(RQTLError): """Raised when the format of the file(s) is invalid.""" -class MissingFileError(InvalidFormat): +class MissingFileException(InvalidFormat): """ Raise when at least one file listed in the control file is missing from the R/qtl2 bundle. diff --git a/r_qtl/fileerrors.py b/r_qtl/fileerrors.py index e76676c..c253d71 100644 --- a/r_qtl/fileerrors.py +++ b/r_qtl/fileerrors.py @@ -1,5 +1,14 @@ """QC errors as distinguished from actual exceptions""" from collections import namedtuple +InvalidValue = namedtuple( + "InvalidValue", + ("filename", + "rowtitle", + "coltitle", + "cellvalue", + "message")) + + MissingFile = namedtuple( "MissingFile", ("controlfilekey", "filename", "message")) diff --git a/r_qtl/r_qtl2.py b/r_qtl/r_qtl2.py index 0a96e7c..9da4081 100644 --- a/r_qtl/r_qtl2.py +++ b/r_qtl/r_qtl2.py @@ -1,17 +1,18 @@ """The R/qtl2 parsing and processing code.""" import io +import os import csv import json from pathlib import Path -from zipfile import ZipFile from functools import reduce, partial +from zipfile import ZipFile, is_zipfile from typing import Union, Iterator, Iterable, Callable, Optional import yaml from functional_tools import take, chain -from r_qtl.errors import InvalidFormat, MissingFileError +from r_qtl.exceptions import InvalidFormat, MissingFileException FILE_TYPES = ( "geno", "founder_geno", "pheno", "covar", "phenocovar", "gmap", "pmap", @@ -30,7 +31,79 @@ def __special_file__(filename): return (is_macosx_special_file or is_nix_hidden_file) -def control_data(zfile: ZipFile) -> dict: +def extract(zfile: ZipFile, outputdir: Path) -> tuple[Path, ...]: + """Extract a ZipFile + + This function will extract a zipfile `zfile` to the directory `outputdir`. + + Parameters + ---------- + zfile: zipfile.ZipFile object - the zipfile to extract. + outputdir: Optional pathlib.Path object - where the extracted files go. + + Returns + ------- + A tuple of Path objects, each pointing to a member in the zipfile. + """ + outputdir.mkdir(parents=True, exist_ok=True) + return tuple(Path(zfile.extract(member, outputdir)) + for member in zfile.namelist() + if not __special_file__(member)) + + +def transpose_csv( + inpath: Path, + linesplitterfn: Callable, + linejoinerfn: Callable, + outpath: Path) -> Path: + """Transpose a file: Make its rows into columns and its columns into rows. + + This function will create a new file, `outfile`, with the same content as + the original, `infile`, except transposed i.e. The rows of `infile` are the + columns of `outfile` and the columns of `infile` are the rows of `outfile`. + + Parameters + ---------- + inpath: The CSV file to transpose. + linesplitterfn: A function to use for splitting each line into columns + linejoinerfn: A function to use to rebuild the lines + outpath: The path where the transposed data is stored + """ + def __read_by_line__(_path): + with open(_path, "r", encoding="utf8") as infile: + for line in infile: + yield line + + transposed_data= (f"{linejoinerfn(items)}\n" for items in zip(*( + linesplitterfn(line) for line in __read_by_line__(inpath)))) + + with open(outpath, "w", encoding="utf8") as outfile: + for line in transposed_data: + outfile.write(line) + + return outpath + + +def transpose_csv_with_rename(inpath: Path, + linesplitterfn: Callable, + linejoinerfn: Callable) -> Path: + """Renames input file and creates new transposed file with the original name + of the input file. + + Parameters + ---------- + inpath: Path to the input file. Should be a pathlib.Path object. + linesplitterfn: A function to use for splitting each line into columns + linejoinerfn: A function to use to rebuild the lines + """ + transposedfilepath = Path(inpath) + origbkp = inpath.parent.joinpath(f"{inpath.stem}___original{inpath.suffix}") + os.rename(inpath, origbkp) + return transpose_csv( + origbkp, linesplitterfn, linejoinerfn, transposedfilepath) + + +def __control_data_from_zipfile__(zfile: ZipFile) -> dict: """Retrieve the control file from the zip file info.""" files = tuple(filename for filename in zfile.namelist() @@ -56,6 +129,81 @@ def control_data(zfile: ZipFile) -> dict: else yaml.safe_load(zfile.read(files[0]))) } + +def __control_data_from_dirpath__(dirpath: Path): + """Load control data from a given directory path.""" + files = tuple(path for path in dirpath.iterdir() + if (not __special_file__(path.name) + and (path.suffix in (".yaml", ".json")))) + num_files = len(files) + if num_files == 0: + raise InvalidFormat("Expected a json or yaml control file.") + + if num_files > 1: + raise InvalidFormat("Found more than one possible control file.") + + with open(files[0], "r", encoding="utf8") as infile: + return { + "na.strings": ["NA"], + "comment.char": "#", + "sep": ",", + **{ + f"{key}_transposed": False for key in FILE_TYPES + }, + **(json.loads(infile.read()) + if files[0].suffix == ".json" + else yaml.safe_load(infile.read())) + } + + +def control_data(control_src: Union[Path, ZipFile]) -> dict: + """Read the R/qtl2 bundle control file. + + Parameters + ---------- + control_src: Path object of ZipFile object. + If a directory path is provided, this function will read the control + data from the control file in that directory. + It is importand that the Path be a directory and contain data from one + and only one R/qtl2 bundle. + + If a ZipFile object is provided, then the control data is read from the + control file within the zip file. We are moving away from parsing data + directly from ZipFile objects, and this is retained only until the + transition to using extracted files is complete. + + Returns + ------- + Returns a dict object with the control data that determines what the files + in the bundle are and how to parse them. + + Raises + ------ + r_qtl.exceptions.InvalidFormat + """ + def __cleanup__(cdata): + return { + **cdata, + **dict((filetype, + ([cdata[filetype]] if isinstance(cdata[filetype], str) + else cdata[filetype]) + ) for filetype in + (typ for typ in cdata.keys() if typ in FILE_TYPES)) + } + + if isinstance(control_src, ZipFile): + return __cleanup__(__control_data_from_zipfile__(control_src)) + if isinstance(control_src, Path): + if is_zipfile(control_src): + return __cleanup__( + __control_data_from_zipfile__(ZipFile(control_src))) + if control_src.is_dir(): + return __cleanup__(__control_data_from_dirpath__(control_src)) + raise InvalidFormat( + "Expects either a zipfile.ZipFile object or a pathlib.Path object " + "pointing to a directory containing the R/qtl2 bundle.") + + def replace_na_strings(cdata, val): """Replace values indicated in `na.strings` with `None`.""" return (None if val in cdata.get("na.strings", ["NA"]) else val) @@ -267,7 +415,7 @@ def file_data(zfile: ZipFile, zfile, member_key, cdata, process_transposed_value): yield row except KeyError as exc: - raise MissingFileError(*exc.args) from exc + raise MissingFileException(*exc.args) from exc def cross_information(zfile: ZipFile, cdata: dict) -> Iterator[dict]: """Load cross information where present.""" diff --git a/r_qtl/r_qtl2_qc.py b/r_qtl/r_qtl2_qc.py index be1eac4..6f7b374 100644 --- a/r_qtl/r_qtl2_qc.py +++ b/r_qtl/r_qtl2_qc.py @@ -1,10 +1,11 @@ """Quality control checks for R/qtl2 data bundles.""" -from zipfile import ZipFile +from pathlib import Path from functools import reduce, partial +from zipfile import ZipFile, is_zipfile from typing import Union, Iterator, Optional, Callable -from r_qtl import errors as rqe from r_qtl import r_qtl2 as rqtl2 +from r_qtl import exceptions as rqe from r_qtl.r_qtl2 import FILE_TYPES from r_qtl.fileerrors import MissingFile @@ -39,11 +40,10 @@ def bundle_files_list(cdata: dict) -> tuple[tuple[str, str], ...]: return fileslist -def missing_files(zfile: ZipFile) -> tuple[tuple[str, str], ...]: - """ - Retrieve a list of files listed in the control file that do not exist in the - bundle. - """ + +def __missing_from_zipfile__( + zfile: ZipFile, cdata: dict) -> tuple[tuple[str, str], ...]: + """Check for missing files from a still-compressed zip file.""" def __missing_p__(filedetails: tuple[str, str]): _cfkey, thefile = filedetails try: @@ -52,14 +52,53 @@ def missing_files(zfile: ZipFile) -> tuple[tuple[str, str], ...]: except KeyError: return True - return tuple(afile for afile in bundle_files_list(rqtl2.control_data(zfile)) + return tuple(afile for afile in bundle_files_list(cdata) if __missing_p__(afile)) + +def __missing_from_dirpath__( + dirpath: Path, cdata: dict) -> tuple[tuple[str, str], ...]: + """Check for missing files from an extracted bundle.""" + allfiles = tuple(_file.name for _file in dirpath.iterdir()) + return tuple(afile for afile in bundle_files_list(cdata) + if afile[1] not in allfiles) + + +def missing_files(bundlesrc: Union[Path, ZipFile]) -> tuple[tuple[str, str], ...]: + """ + Retrieve a list of files listed in the control file that do not exist in the + bundle. + + Parameters + ---------- + bundlesrc: Path object of ZipFile object: This is the bundle under check. + + Returns + ------- + A tuple of names listed in the control file that do not exist in the bundle. + + Raises + ------ + r_qtl.exceptions.InvalidFormat + """ + cdata = rqtl2.control_data(bundlesrc) + if isinstance(bundlesrc, ZipFile): + return __missing_from_zipfile__(bundlesrc, cdata) + if isinstance(bundlesrc, Path): + if is_zipfile(bundlesrc): + return __missing_from_zipfile__(ZipFile(bundlesrc, cdata)) + if bundlesrc.is_dir(): + return __missing_from_dirpath__(bundlesrc, cdata) + raise InvalidFormat( + "Expects either a zipfile.ZipFile object or a pathlib.Path object " + "pointing to a directory containing the R/qtl2 bundle.") + + def validate_bundle(zfile: ZipFile): """Ensure the R/qtl2 bundle is valid.""" missing = missing_files(zfile) if len(missing) > 0: - raise rqe.MissingFileError( + raise rqe.MissingFileException( "The following files do not exist in the bundle: " + ", ".join(mfile[1] for mfile in missing)) @@ -111,6 +150,6 @@ def retrieve_errors(zfile: ZipFile, filetype: str, checkers: tuple[Callable]) -> if value is not None: for checker in checkers: yield checker(lineno=lineno, field=field, value=value) - except rqe.MissingFileError: + except rqe.MissingFileException: fname = cdata.get(filetype) yield MissingFile(filetype, fname, f"Missing '{filetype}' file '{fname}'.") diff --git a/scripts/insert_data.py b/scripts/insert_data.py index 1465348..4b2e5f3 100644 --- a/scripts/insert_data.py +++ b/scripts/insert_data.py @@ -14,8 +14,8 @@ from MySQLdb.cursors import DictCursor from functional_tools import take from quality_control.file_utils import open_file -from qc_app.db_utils import database_connection -from qc_app.check_connections import check_db, check_redis +from uploader.db_utils import database_connection +from uploader.check_connections import check_db, check_redis # Set up logging stderr_handler = logging.StreamHandler(stream=sys.stderr) diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py index 8431462..87f29dc 100644 --- a/scripts/insert_samples.py +++ b/scripts/insert_samples.py @@ -7,10 +7,10 @@ import argparse import MySQLdb as mdb from redis import Redis -from qc_app.db_utils import database_connection -from qc_app.check_connections import check_db, check_redis -from qc_app.db import species_by_id, population_by_id -from qc_app.samples import ( +from uploader.db_utils import database_connection +from uploader.check_connections import check_db, check_redis +from uploader.db import species_by_id, population_by_id +from uploader.samples import ( save_samples_data, read_samples_file, cross_reference_samples) diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py index 4da3936..20cfd3b 100644 --- a/scripts/process_rqtl2_bundle.py +++ b/scripts/process_rqtl2_bundle.py @@ -13,13 +13,13 @@ from redis import Redis from functional_tools import take -import r_qtl.errors as rqe import r_qtl.r_qtl2 as rqtl2 import r_qtl.r_qtl2_qc as rqc +import r_qtl.exceptions as rqe -from qc_app import jobs -from qc_app.db_utils import database_connection -from qc_app.check_connections import check_db, check_redis +from uploader import jobs +from uploader.db_utils import database_connection +from uploader.check_connections import check_db, check_redis from scripts.cli_parser import init_cli_parser from scripts.redis_logger import setup_redis_logger diff --git a/scripts/qc.py b/scripts/qc.py index e8573a9..6de051f 100644 --- a/scripts/qc.py +++ b/scripts/qc.py @@ -11,7 +11,7 @@ from quality_control.utils import make_progress_calculator from quality_control.errors import InvalidValue, DuplicateHeading from quality_control.parsing import FileType, strain_names, collect_errors -from qc_app.db_utils import database_connection +from uploader.db_utils import database_connection from .cli_parser import init_cli_parser diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py index 40809b7..fc95d13 100644 --- a/scripts/qc_on_rqtl2_bundle.py +++ b/scripts/qc_on_rqtl2_bundle.py @@ -16,13 +16,13 @@ from redis import Redis from quality_control.errors import InvalidValue from quality_control.checks import decimal_points_error -from qc_app import jobs -from qc_app.db_utils import database_connection -from qc_app.check_connections import check_db, check_redis +from uploader import jobs +from uploader.db_utils import database_connection +from uploader.check_connections import check_db, check_redis -from r_qtl import errors as rqe from r_qtl import r_qtl2 as rqtl2 from r_qtl import r_qtl2_qc as rqc +from r_qtl import exceptions as rqe from r_qtl import fileerrors as rqfe from scripts.process_rqtl2_bundle import parse_job @@ -105,7 +105,7 @@ def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals __update_processed__(value) rconn.hset(fqjobid, f"{filetype}-linecount", count) - except rqe.MissingFileError: + except rqe.MissingFileException: fname = cdata.get(filetype) yield rqfe.MissingFile(filetype, fname, ( f"The file '{fname}' does not exist in the bundle despite it being " @@ -133,7 +133,7 @@ def qc_geno_errors(rconn, fqjobid, _dburi, _speciesid, zfile, logger) -> bool: def fetch_db_geno_samples(conn: mdb.Connection, speciesid: int) -> tuple[str, ...]: """Fetch samples/cases/individuals from the database.""" - samples = set() + samples = set()# type: ignore[var-annotated] with conn.cursor() as cursor: cursor.execute("SELECT Name, Name2 from Strain WHERE SpeciesId=%s", (speciesid,)) @@ -191,12 +191,13 @@ def check_pheno_samples( return allerrors -def qc_pheno_errors(rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool: +def qc_pheno_errors(# pylint: disable=[too-many-arguments] + rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool: """Check for errors in `pheno` file(s).""" cdata = rqtl2.control_data(zfile) if "pheno" in cdata: logger.info("Checking for errors in the 'pheno' file…") - perrs = tuple() + perrs = tuple()# type: ignore[var-annotated] with database_connection(dburi) as dbconn: perrs = check_pheno_samples( dbconn, speciesid, zfile.filename, logger) + tuple( @@ -216,7 +217,8 @@ def qc_pheno_errors(rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool: return False -def qc_phenose_errors(rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool: +def qc_phenose_errors(# pylint: disable=[too-many-arguments] + rconn, fqjobid, _dburi, _speciesid, zfile, logger) -> bool: """Check for errors in `phenose` file(s).""" cdata = rqtl2.control_data(zfile) if "phenose" in cdata: @@ -258,7 +260,9 @@ def run_qc(rconn: Redis, if qc_missing_files(rconn, fqjobid, zfile, logger): return 1 - def with_zipfile(rconn, fqjobid, dbconn, speciesid, filename, logger, func): + def with_zipfile(# pylint: disable=[too-many-arguments] + rconn, fqjobid, dbconn, speciesid, filename, logger, func + ): with ZipFile(filename, "r") as zfile: return func(rconn, fqjobid, dbconn, speciesid, zfile, logger) diff --git a/scripts/qcapp_wsgi.py b/scripts/qcapp_wsgi.py index 349c006..fe77031 100644 --- a/scripts/qcapp_wsgi.py +++ b/scripts/qcapp_wsgi.py @@ -5,8 +5,8 @@ from logging import getLogger, StreamHandler from flask import Flask -from qc_app import create_app -from qc_app.check_connections import check_db, check_redis +from uploader import create_app +from uploader.check_connections import check_db, check_redis def setup_logging(appl: Flask) -> Flask: """Setup appropriate logging paradigm depending on environment.""" diff --git a/scripts/rqtl2/entry.py b/scripts/rqtl2/entry.py index 93fc130..b7fb68e 100644 --- a/scripts/rqtl2/entry.py +++ b/scripts/rqtl2/entry.py @@ -6,9 +6,9 @@ from argparse import Namespace from redis import Redis from MySQLdb import Connection -from qc_app import jobs -from qc_app.db_utils import database_connection -from qc_app.check_connections import check_db, check_redis +from uploader import jobs +from uploader.db_utils import database_connection +from uploader.check_connections import check_db, check_redis from scripts.redis_logger import setup_redis_logger diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py index 9f8bf03..6b89142 100644 --- a/scripts/rqtl2/install_genotypes.py +++ b/scripts/rqtl2/install_genotypes.py @@ -19,10 +19,13 @@ from scripts.rqtl2.entry import build_main from scripts.rqtl2.cli_parser import add_common_arguments from scripts.cli_parser import init_cli_parser, add_global_data_arguments -def insert_markers(dbconn: mdb.Connection, - speciesid: int, - markers: tuple[str, ...], - pmapdata: Optional[Iterator[dict]]) -> int: +def insert_markers( + dbconn: mdb.Connection, + speciesid: int, + markers: tuple[str, ...], + pmapdata: Optional[Iterator[dict]], + _logger: Logger +) -> int: """Insert genotype and genotype values into the database.""" mdata = reduce(#type: ignore[var-annotated] lambda acc, row: ({#type: ignore[arg-type, return-value] @@ -48,9 +51,12 @@ def insert_markers(dbconn: mdb.Connection, } for marker in markers}.values())) return cursor.rowcount -def insert_individuals(dbconn: mdb.Connection, - speciesid: int, - individuals: tuple[str, ...]) -> int: +def insert_individuals( + dbconn: mdb.Connection, + speciesid: int, + individuals: tuple[str, ...], + _logger: Logger +) -> int: """Insert individuals/samples into the database.""" with dbconn.cursor() as cursor: cursor.executemany( @@ -61,10 +67,13 @@ def insert_individuals(dbconn: mdb.Connection, for individual in individuals)) return cursor.rowcount -def cross_reference_individuals(dbconn: mdb.Connection, - speciesid: int, - populationid: int, - individuals: tuple[str, ...]) -> int: +def cross_reference_individuals( + dbconn: mdb.Connection, + speciesid: int, + populationid: int, + individuals: tuple[str, ...], + _logger: Logger +) -> int: """Cross reference any inserted individuals.""" with dbconn.cursor(cursorclass=DictCursor) as cursor: paramstr = ", ".join(["%s"] * len(individuals)) @@ -80,11 +89,13 @@ def cross_reference_individuals(dbconn: mdb.Connection, tuple(ids)) return cursor.rowcount -def insert_genotype_data(dbconn: mdb.Connection, - speciesid: int, - genotypes: tuple[dict, ...], - individuals: tuple[str, ...]) -> tuple[ - int, tuple[dict, ...]]: +def insert_genotype_data( + dbconn: mdb.Connection, + speciesid: int, + genotypes: tuple[dict, ...], + individuals: tuple[str, ...], + _logger: Logger +) -> tuple[int, tuple[dict, ...]]: """Insert the genotype data values into the database.""" with dbconn.cursor(cursorclass=DictCursor) as cursor: paramstr = ", ".join(["%s"] * len(individuals)) @@ -120,11 +131,14 @@ def insert_genotype_data(dbconn: mdb.Connection, "markerid": row["markerid"] } for row in data) -def cross_reference_genotypes(dbconn: mdb.Connection, - speciesid: int, - datasetid: int, - dataids: tuple[dict, ...], - gmapdata: Optional[Iterator[dict]]) -> int: +def cross_reference_genotypes( + dbconn: mdb.Connection, + speciesid: int, + datasetid: int, + dataids: tuple[dict, ...], + gmapdata: Optional[Iterator[dict]], + _logger: Logger +) -> int: """Cross-reference the data to the relevant dataset.""" _rows, markers, mdata = reduce(#type: ignore[var-annotated] lambda acc, row: (#type: ignore[return-value,arg-type] @@ -140,30 +154,43 @@ def cross_reference_genotypes(dbconn: mdb.Connection, (tuple(), tuple(), {})) with dbconn.cursor(cursorclass=DictCursor) as cursor: - paramstr = ", ".join(["%s"] * len(markers)) - cursor.execute("SELECT Id, Name FROM Geno " - f"WHERE SpeciesId=%s AND Name IN ({paramstr})", - (speciesid,) + markers) - markersdict = {row["Id"]: row["Name"] for row in cursor.fetchall()} - cursor.executemany( + markersdict = {} + if len(markers) > 0: + paramstr = ", ".join(["%s"] * len(markers)) + insertparams = (speciesid,) + markers + selectquery = ("SELECT Id, Name FROM Geno " + f"WHERE SpeciesId=%s AND Name IN ({paramstr})") + _logger.debug( + "The select query was\n\t%s\n\nwith the parameters\n\t%s", + selectquery, + (speciesid,) + markers) + cursor.execute(selectquery, insertparams) + markersdict = {row["Id"]: row["Name"] for row in cursor.fetchall()} + + insertquery = ( "INSERT INTO GenoXRef(GenoFreezeId, GenoId, DataId, cM) " "VALUES(%(datasetid)s, %(markerid)s, %(dataid)s, %(pos)s) " - "ON DUPLICATE KEY UPDATE GenoFreezeId=GenoFreezeId", - tuple({ - **row, - "datasetid": datasetid, - "pos": mdata.get(markersdict.get( - row.get("markerid"), {}), {}).get("pos") - } for row in dataids)) + "ON DUPLICATE KEY UPDATE GenoFreezeId=GenoFreezeId") + insertparams = tuple({ + **row, + "datasetid": datasetid, + "pos": mdata.get(markersdict.get( + row.get("markerid"), "nosuchkey"), {}).get("pos") + } for row in dataids) + _logger.debug( + "The insert query was\n\t%s\n\nwith the parameters\n\t%s", + insertquery, insertparams) + cursor.executemany(insertquery, insertparams) return cursor.rowcount def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals] dbconn: mdb.Connection, - speciesid: int, - populationid: int, - datasetid: int, - rqtl2bundle: Path, - logger: Logger = getLogger()) -> int: + speciesid: int, + populationid: int, + datasetid: int, + rqtl2bundle: Path, + logger: Logger = getLogger(__name__) +) -> int: """Load any existing genotypes into the database.""" count = 0 with ZipFile(str(rqtl2bundle.absolute()), "r") as zfile: @@ -189,20 +216,22 @@ def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals] speciesid, tuple(key for key in batch[0].keys() if key != "id"), (rqtl2.file_data(zfile, "pmap", cdata) if "pmap" in cdata - else None)) + else None), + logger) individuals = tuple(row["id"] for row in batch) - insert_individuals(dbconn, speciesid, individuals) + insert_individuals(dbconn, speciesid, individuals, logger) cross_reference_individuals( - dbconn, speciesid, populationid, individuals) + dbconn, speciesid, populationid, individuals, logger) _num_rows, dataids = insert_genotype_data( - dbconn, speciesid, batch, individuals) + dbconn, speciesid, batch, individuals, logger) cross_reference_genotypes( dbconn, speciesid, datasetid, dataids, (rqtl2.file_data(zfile, "gmap", cdata) - if "gmap" in cdata else None)) + if "gmap" in cdata else None), + logger) count = count + len(batch) except rqtl2.InvalidFormat as exc: logger.error(str(exc)) diff --git a/scripts/validate_file.py b/scripts/validate_file.py index 0028795..a40d7e7 100644 --- a/scripts/validate_file.py +++ b/scripts/validate_file.py @@ -12,8 +12,8 @@ from redis.exceptions import ConnectionError # pylint: disable=[redefined-builti from quality_control.utils import make_progress_calculator from quality_control.parsing import FileType, strain_names, collect_errors -from qc_app import jobs -from qc_app.db_utils import database_connection +from uploader import jobs +from uploader.db_utils import database_connection from .cli_parser import init_cli_parser from .qc import add_file_validation_arguments diff --git a/scripts/worker.py b/scripts/worker.py index 0eb9ea5..91b0332 100644 --- a/scripts/worker.py +++ b/scripts/worker.py @@ -11,8 +11,8 @@ from tempfile import TemporaryDirectory from redis import Redis -from qc_app import jobs -from qc_app.check_connections import check_redis +from uploader import jobs +from uploader.check_connections import check_redis def parse_args(): "Parse the command-line arguments" diff --git a/tests/conftest.py b/tests/conftest.py index a39acf0..9012221 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,8 +11,8 @@ from redis import Redis from functional_tools import take -from qc_app import jobs, create_app -from qc_app.jobs import JOBS_PREFIX +from uploader import jobs, create_app +from uploader.jobs import JOBS_PREFIX from quality_control.errors import InvalidValue, DuplicateHeading diff --git a/tests/qc_app/test_parse.py b/tests/qc_app/test_parse.py index 3915a4d..076c47c 100644 --- a/tests/qc_app/test_parse.py +++ b/tests/qc_app/test_parse.py @@ -4,7 +4,7 @@ import sys import redis import pytest -from qc_app.jobs import job, jobsnamespace +from uploader.jobs import job, jobsnamespace from tests.conftest import uploadable_file_object @@ -24,7 +24,7 @@ def test_parse_with_existing_uploaded_file(#pylint: disable=[too-many-arguments] 1. the system redirects to the job/parse status page 2. the job is placed on redis for processing """ - monkeypatch.setattr("qc_app.jobs.uuid4", lambda : job_id) + monkeypatch.setattr("uploader.jobs.uuid4", lambda : job_id) # Upload a file speciesid = 1 filename = "no_data_errors.tsv" diff --git a/uploader/__init__.py b/uploader/__init__.py new file mode 100644 index 0000000..2d731af --- /dev/null +++ b/uploader/__init__.py @@ -0,0 +1,94 @@ +"""The Quality-Control Web Application entry point""" +import os +import sys +import logging +from pathlib import Path + +from flask import Flask, request +from flask_session import Session + +from uploader.oauth2.client import user_logged_in, authserver_authorise_uri + +from .entry import entrybp +from .upload import upload +from .parse import parsebp +from .samples import samples +from .base_routes import base +from .dbinsert import dbinsertbp +from .oauth2.views import oauth2 +from .errors import register_error_handlers + +def override_settings_with_envvars( + app: Flask, ignore: tuple[str, ...]=tuple()) -> None: + """Override settings in `app` with those in ENVVARS""" + for setting in (key for key in app.config if key not in ignore): + app.config[setting] = os.environ.get(setting) or app.config[setting] + + +def __log_gunicorn__(app: Flask) -> Flask: + """Set up logging for the WSGI environment with GUnicorn""" + logger = logging.getLogger("gunicorn.error") + app.logger.handlers = logger.handlers + app.logger.setLevel(logger.level) + return app + + +def __log_dev__(app: Flask) -> Flask: + """Set up logging for the development environment.""" + stderr_handler = logging.StreamHandler(stream=sys.stderr) + app.logger.addHandler(stderr_handler) + + root_logger = logging.getLogger() + root_logger.addHandler(stderr_handler) + root_logger.setLevel(app.config["LOG_LEVEL"]) + + return app + + +def setup_logging(app: Flask) -> Flask: + """Set up logging for the application.""" + software, *_version_and_comments = os.environ.get( + "SERVER_SOFTWARE", "").split('/') + return __log_gunicorn__(app) if bool(software) else __log_dev__(app) + + +def create_app(): + """The application factory""" + app = Flask(__name__) + app.config.from_pyfile( + Path(__file__).parent.joinpath("default_settings.py")) + if "UPLOADER_CONF" in os.environ: + app.config.from_envvar("UPLOADER_CONF") # Override defaults with instance path + + override_settings_with_envvars(app, ignore=tuple()) + + secretsfile = app.config.get("UPLOADER_SECRETS", "").strip() + if bool(secretsfile): + secretsfile = Path(secretsfile).absolute() + app.config["UPLOADER_SECRETS"] = secretsfile + if secretsfile.exists(): + # Silently ignore secrets if the file does not exist. + app.config.from_pyfile(secretsfile) + + setup_logging(app) + + # setup jinja2 symbols + app.add_template_global(lambda : request.url, name="request_url") + app.add_template_global(authserver_authorise_uri) + app.add_template_global(lambda: app.config["GN2_SERVER_URL"], + name="gn2server_uri") + app.add_template_global(user_logged_in) + + Session(app) + + # setup blueprints + app.register_blueprint(base, url_prefix="/") + app.register_blueprint(entrybp, url_prefix="/") + app.register_blueprint(parsebp, url_prefix="/parse") + app.register_blueprint(oauth2, url_prefix="/oauth2") + app.register_blueprint(upload, url_prefix="/upload") + app.register_blueprint(dbinsertbp, url_prefix="/dbinsert") + app.register_blueprint(samples, url_prefix="/samples") + + register_error_handlers(app) + return app diff --git a/uploader/authorisation.py b/uploader/authorisation.py new file mode 100644 index 0000000..8ab83f8 --- /dev/null +++ b/uploader/authorisation.py @@ -0,0 +1,21 @@ +"""Authorisation utilities.""" +from functools import wraps + +from flask import flash, redirect + +from uploader import session + +def require_login(function): + """Check that the user is logged in before executing `func`.""" + @wraps(function) + def __is_session_valid__(*args, **kwargs): + """Check that the user is logged in and their token is valid.""" + def __clear_session__(_no_token): + session.clear_session_info() + flash("You need to be logged in.", "alert-danger") + return redirect("/") + + return session.user_token().either( + __clear_session__, + lambda token: function(*args, **kwargs)) + return __is_session_valid__ diff --git a/qc_app/base_routes.py b/uploader/base_routes.py index 9daf439..9daf439 100644 --- a/qc_app/base_routes.py +++ b/uploader/base_routes.py diff --git a/qc_app/check_connections.py b/uploader/check_connections.py index ceccc32..2561e55 100644 --- a/qc_app/check_connections.py +++ b/uploader/check_connections.py @@ -5,7 +5,7 @@ import traceback import redis import MySQLdb -from qc_app.db_utils import database_connection +from uploader.db_utils import database_connection def check_redis(uri: str): "Check the redis connection" diff --git a/qc_app/db/__init__.py b/uploader/db/__init__.py index 36e93e8..36e93e8 100644 --- a/qc_app/db/__init__.py +++ b/uploader/db/__init__.py diff --git a/qc_app/db/averaging.py b/uploader/db/averaging.py index 62bbe67..62bbe67 100644 --- a/qc_app/db/averaging.py +++ b/uploader/db/averaging.py diff --git a/qc_app/db/datasets.py b/uploader/db/datasets.py index 767ec41..767ec41 100644 --- a/qc_app/db/datasets.py +++ b/uploader/db/datasets.py diff --git a/qc_app/db/platforms.py b/uploader/db/platforms.py index cb527a7..cb527a7 100644 --- a/qc_app/db/platforms.py +++ b/uploader/db/platforms.py diff --git a/qc_app/db/populations.py b/uploader/db/populations.py index 4485e52..4485e52 100644 --- a/qc_app/db/populations.py +++ b/uploader/db/populations.py diff --git a/qc_app/db/species.py b/uploader/db/species.py index 653e59b..653e59b 100644 --- a/qc_app/db/species.py +++ b/uploader/db/species.py diff --git a/qc_app/db/tissues.py b/uploader/db/tissues.py index 9fe7bab..9fe7bab 100644 --- a/qc_app/db/tissues.py +++ b/uploader/db/tissues.py diff --git a/qc_app/db_utils.py b/uploader/db_utils.py index ef26398..5b79762 100644 --- a/qc_app/db_utils.py +++ b/uploader/db_utils.py @@ -3,7 +3,7 @@ import logging import traceback import contextlib from urllib.parse import urlparse -from typing import Any, Tuple, Optional, Iterator, Callable +from typing import Any, Tuple, Iterator, Callable import MySQLdb as mdb from redis import Redis @@ -19,10 +19,9 @@ def parse_db_url(db_url) -> Tuple: @contextlib.contextmanager -def database_connection(db_url: Optional[str] = None) -> Iterator[mdb.Connection]: +def database_connection(db_url: str) -> Iterator[mdb.Connection]: """function to create db connector""" - host, user, passwd, db_name, db_port = parse_db_url( - db_url or app.config["SQL_URI"]) + host, user, passwd, db_name, db_port = parse_db_url(db_url) connection = mdb.connect( host, user, passwd, db_name, port=(db_port or 3306)) try: diff --git a/qc_app/dbinsert.py b/uploader/dbinsert.py index ef08423..559dc5e 100644 --- a/qc_app/dbinsert.py +++ b/uploader/dbinsert.py @@ -11,8 +11,9 @@ from flask import ( flash, request, url_for, Blueprint, redirect, render_template, current_app as app) -from qc_app.db_utils import with_db_connection, database_connection -from qc_app.db import species, species_by_id, populations_by_species +from uploader.authorisation import require_login +from uploader.db_utils import with_db_connection, database_connection +from uploader.db import species, species_by_id, populations_by_species from . import jobs @@ -40,7 +41,7 @@ def genechips(): return {**acc, speciesid: (chip,)} return {**acc, speciesid: acc[speciesid] + (chip,)} - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC") return reduce(__organise_by_species__, cursor.fetchall(), {}) @@ -49,7 +50,7 @@ def genechips(): def platform_by_id(genechipid:int) -> Union[dict, None]: "Retrieve the gene platform by id" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( "SELECT * FROM GeneChip WHERE GeneChipId=%s", @@ -58,7 +59,7 @@ def platform_by_id(genechipid:int) -> Union[dict, None]: def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple: "Retrieve the studies by the related species and gene platform" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: query = ( "SELECT Species.SpeciesId, ProbeFreeze.* " @@ -82,7 +83,7 @@ def organise_groups_by_family(acc:dict, group:dict) -> dict: def tissues() -> tuple: "Retrieve type (Tissue) information from the database." - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute("SELECT * FROM Tissue ORDER BY Name") return tuple(cursor.fetchall()) @@ -90,6 +91,7 @@ def tissues() -> tuple: return tuple() @dbinsertbp.route("/platform", methods=["POST"]) +@require_login def select_platform(): "Select the platform (GeneChipId) used for the data." job_id = request.form["job_id"] @@ -113,6 +115,7 @@ def select_platform(): return render_error("Unknown error") @dbinsertbp.route("/study", methods=["POST"]) +@require_login def select_study(): "View to select/create the study (ProbeFreeze) associated with the data." form = request.form @@ -142,6 +145,7 @@ def select_study(): return render_error(f"Missing data: {aserr.args[0]}") @dbinsertbp.route("/create-study", methods=["POST"]) +@require_login def create_study(): "Create a new study (ProbeFreeze)." form = request.form @@ -154,7 +158,7 @@ def create_study(): assert form.get("inbredsetid"), "group" assert form.get("tissueid"), "type/tissue" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: values = ( form["genechipid"], @@ -186,7 +190,7 @@ def create_study(): def datasets_by_study(studyid:int) -> tuple: "Retrieve datasets associated with a study with the ID `studyid`." - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s" cursor.execute(query, (studyid,)) @@ -196,7 +200,7 @@ def datasets_by_study(studyid:int) -> tuple: def averaging_methods() -> tuple: "Retrieve averaging methods from database" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute("SELECT * FROM AvgMethod") return tuple(cursor.fetchall()) @@ -205,7 +209,7 @@ def averaging_methods() -> tuple: def dataset_datascales() -> tuple: "Retrieve datascales from database" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor() as cursor: cursor.execute( 'SELECT DISTINCT DataScale FROM ProbeSetFreeze ' @@ -218,6 +222,7 @@ def dataset_datascales() -> tuple: return tuple() @dbinsertbp.route("/dataset", methods=["POST"]) +@require_login def select_dataset(): "Select the dataset to add the file contents against" form = request.form @@ -238,6 +243,7 @@ def select_dataset(): return render_error(f"Missing data: {aserr.args[0]}") @dbinsertbp.route("/create-dataset", methods=["POST"]) +@require_login def create_dataset(): "Select the dataset to add the file contents against" form = request.form @@ -255,7 +261,7 @@ def create_dataset(): assert form.get("datasetconfidentiality"), "Dataset confidentiality" assert form.get("datasetdatascale"), "Dataset Datascale" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: datasetname = form["datasetname"] cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s", @@ -293,7 +299,7 @@ def create_dataset(): def study_by_id(studyid:int) -> Union[dict, None]: "Get a study by its Id" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( "SELECT * FROM ProbeFreeze WHERE Id=%s", @@ -302,7 +308,7 @@ def study_by_id(studyid:int) -> Union[dict, None]: def dataset_by_id(datasetid:int) -> Union[dict, None]: "Retrieve a dataset by its id" - with database_connection() as conn: + with database_connection(app.config["SQL_URI"]) as conn: with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( ("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* " @@ -317,6 +323,7 @@ def selected_keys(original: dict, keys: tuple) -> dict: return {key: value for key,value in original.items() if key in keys} @dbinsertbp.route("/final-confirmation", methods=["POST"]) +@require_login def final_confirmation(): "Preview the data before triggering entry into the database" form = request.form @@ -352,6 +359,7 @@ def final_confirmation(): return render_error(f"Missing data: {aserr.args[0]}") @dbinsertbp.route("/insert-data", methods=["POST"]) +@require_login def insert_data(): "Trigger data insertion" form = request.form diff --git a/uploader/default_settings.py b/uploader/default_settings.py new file mode 100644 index 0000000..26fe665 --- /dev/null +++ b/uploader/default_settings.py @@ -0,0 +1,23 @@ +""" +The default configuration file. The values here should be overridden in the +actual configuration file used for the production and staging systems. +""" + +import os + +LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") +SECRET_KEY = b"<Please! Please! Please! Change This!>" +UPLOAD_FOLDER = "/tmp/qc_app_files" +REDIS_URL = "redis://" +JOBS_TTL_SECONDS = 1209600 # 14 days +GNQC_REDIS_PREFIX="GNQC" +SQL_URI = "" + +GN2_SERVER_URL = "https://genenetwork.org/" + +SESSION_TYPE = "redis" +SESSION_PERMANENT = True +SESSION_USE_SIGNER = True + +JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use. +JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it. diff --git a/qc_app/entry.py b/uploader/entry.py index f2db878..82034ed 100644 --- a/qc_app/entry.py +++ b/uploader/entry.py @@ -15,8 +15,10 @@ from flask import ( current_app as app, send_from_directory) -from qc_app.db import species -from qc_app.db_utils import with_db_connection +from uploader.db import species +from uploader.authorisation import require_login +from uploader.db_utils import with_db_connection +from uploader.oauth2.client import user_logged_in entrybp = Blueprint("entry", __name__) @@ -87,9 +89,10 @@ def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]: @entrybp.route("/", methods=["GET"]) def index(): """Load the landing page""" - return render_template("index.html") + return render_template("index.html" if user_logged_in() else "login.html") @entrybp.route("/upload", methods=["GET", "POST"]) +@require_login def upload_file(): """Enables uploading the files""" if request.method == "GET": @@ -122,6 +125,7 @@ def upload_file(): filetype=request.form["filetype"])) @entrybp.route("/data-review", methods=["GET"]) +@require_login def data_review(): """Provide some help on data expectations to the user.""" return render_template("data_review.html") diff --git a/qc_app/errors.py b/uploader/errors.py index 3e7c893..3e7c893 100644 --- a/qc_app/errors.py +++ b/uploader/errors.py diff --git a/qc_app/files.py b/uploader/files.py index b163612..b163612 100644 --- a/qc_app/files.py +++ b/uploader/files.py diff --git a/qc_app/input_validation.py b/uploader/input_validation.py index 9abe742..9abe742 100644 --- a/qc_app/input_validation.py +++ b/uploader/input_validation.py diff --git a/qc_app/jobs.py b/uploader/jobs.py index 21889da..21889da 100644 --- a/qc_app/jobs.py +++ b/uploader/jobs.py diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py new file mode 100644 index 0000000..aa34951 --- /dev/null +++ b/uploader/monadic_requests.py @@ -0,0 +1,86 @@ +"""Wrap requests functions with monads.""" +import traceback +from typing import Union, Optional, Callable + +import requests +from requests.models import Response +from pymonad.either import Left, Right, Either +from flask import ( + flash, + request, + redirect, + render_template, + current_app as app, + escape as flask_escape) + +# HTML Status codes indicating a successful request. +SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226) + +# Possible error(s) that can be encontered while attempting to do a request. +PossibleError = Union[Response, Exception] + + +def make_error_handler( + redirect_to: Optional[Response] = None, + cleanup_thunk: Callable = lambda *args: None +) -> Callable[[PossibleError], Response]: + """ + Build a function to gracefully handle errors encountered while doing + requests. + + :rtype: Callable + """ + redirect_to = redirect_to or redirect(request.url) + def __handler__(resp_or_exc: PossibleError) -> Response: + cleanup_thunk() + if issubclass(type(resp_or_exc), Exception): + # Is an exception! + return render_template( + "unhandled_exception.html", + trace=traceback.format_exception(resp_or_exc)) + if isinstance(resp_or_exc, Response): + flash("The authorisation server responded with " + f"({flask_escape(resp_or_exc.status_code)}, " + f"{flask_escape(resp_or_exc.reason)}) for the request to " + f"'{flask_escape(resp_or_exc.request.url)}'", + "alert-danger") + return redirect_to + + flash("Unspecified error!", "alert-danger") + app.logger.debug("Error (%s): %s", type(resp_or_exc), resp_or_exc) + return redirect_to + return __handler__ + + +def get(url, params=None, **kwargs) -> Either: + """ + A wrapper around `requests.get` function. + + Takes the same arguments as `requests.get`. + + :rtype: pymonad.either.Either + """ + try: + resp = requests.get(url, params=params, **kwargs) + if resp.status_code in SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except requests.exceptions.RequestException as exc: + return Left(exc) + + +def post(url, data=None, json=None, **kwargs) -> Either: + """ + A wrapper around `requests.post` function. + + Takes the same arguments as `requests.post`. + + :rtype: pymonad.either.Either + """ + try: + resp = requests.post(url, data=data, json=json, **kwargs) + if resp.status_code in SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except requests.exceptions.RequestException as exc: + return Left(exc) diff --git a/uploader/oauth2/__init__.py b/uploader/oauth2/__init__.py new file mode 100644 index 0000000..aaea638 --- /dev/null +++ b/uploader/oauth2/__init__.py @@ -0,0 +1 @@ +"""Package to handle OAuth2 authentication/authorisation issues.""" diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py new file mode 100644 index 0000000..a3e4ba3 --- /dev/null +++ b/uploader/oauth2/client.py @@ -0,0 +1,202 @@ +"""OAuth2 client utilities.""" +import json +from datetime import datetime, timedelta +from urllib.parse import urljoin, urlparse + +import requests +from flask import request, current_app as app + +from pymonad.either import Left, Right, Either + +from authlib.jose import jwt +from authlib.common.urls import url_decode +from authlib.jose import KeySet, JsonWebKey +from authlib.jose.errors import BadSignatureError +from authlib.integrations.requests_client import OAuth2Session + +from uploader import session +import uploader.monadic_requests as mrequests + +SCOPE = ("profile group role resource register-client user masquerade " + "introspect migrate-data") + + +def authserver_uri(): + """Return URI to authorisation server.""" + return app.config["AUTH_SERVER_URL"] + + +def oauth2_clientid(): + """Return the client id.""" + return app.config["OAUTH2_CLIENT_ID"] + + +def oauth2_clientsecret(): + """Return the client secret.""" + return app.config["OAUTH2_CLIENT_SECRET"] + + +def __make_token_validator__(keys: KeySet): + """Make a token validator function.""" + def __validator__(token: dict): + for key in keys.keys: + try: + jwt.decode(token["access_token"], key) + return Right(token) + except BadSignatureError: + pass + + return Left("INVALID-TOKEN") + + return __validator__ + + +def __validate_token__(sess_info): + """Validate that the token is really from the auth server.""" + info = sess_info + info["user"]["token"] = info["user"]["token"].then(__make_token_validator__( + KeySet([JsonWebKey.import_key(key) for key in info.get( + "auth_server_jwks", {}).get( + "jwks", {"keys": []})["keys"]]))) + return session.save_session_info(info) + + +def __update_auth_server_jwks__(sess_info): + """Updates the JWKs every 2 hours or so.""" + jwks = sess_info.get("auth_server_jwks") + if bool(jwks): + last_updated = jwks.get("last-updated") + now = datetime.now().timestamp() + if bool(last_updated) and (now - last_updated) < timedelta(hours=2).seconds: + return __validate_token__({**sess_info, "auth_server_jwks": jwks}) + + jwksuri = urljoin(authserver_uri(), "auth/public-jwks") + return __validate_token__({ + **sess_info, + "auth_server_jwks": { + "jwks": KeySet([ + JsonWebKey.import_key(key) + for key in requests.get(jwksuri).json()["jwks"]]).as_dict(), + "last-updated": datetime.now().timestamp() + } + }) + + +def oauth2_client(): + """Build the OAuth2 client for use fetching data.""" + def __update_token__(token, refresh_token=None, access_token=None):# pylint: disable=[unused-argument] + """Update the token when refreshed.""" + session.set_user_token(token) + + def __json_auth__(client, _method, uri, headers, body): + return ( + uri, + {**headers, "Content-Type": "application/json"}, + json.dumps({ + **dict(url_decode(body)), + "client_id": client.client_id, + "client_secret": client.client_secret + })) + + def __client__(token) -> OAuth2Session: + client = OAuth2Session( + oauth2_clientid(), + oauth2_clientsecret(), + scope=SCOPE, + token_endpoint=urljoin(authserver_uri(), "/auth/token"), + token_endpoint_auth_method="client_secret_post", + token=token, + update_token=__update_token__) + client.register_client_auth_method( + ("client_secret_post", __json_auth__)) + return client + + __update_auth_server_jwks__(session.session_info()) + return session.user_token().either( + lambda _notok: __client__(None), + __client__) + + +def user_logged_in(): + """Check whether the user has logged in.""" + suser = session.session_info()["user"] + return suser["logged_in"] and suser["token"].is_right() + + +def authserver_authorise_uri(): + """Build up the authorisation URI.""" + req_baseurl = urlparse(request.base_url, scheme=request.scheme) + host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/" + return urljoin( + authserver_uri(), + "auth/authorise?response_type=code" + f"&client_id={oauth2_clientid()}" + f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}") + + +def __no_token__(_err) -> Left: + """Handle situation where request is attempted with no token.""" + resp = requests.models.Response() + resp._content = json.dumps({#pylint: disable=[protected-access] + "error": "AuthenticationError", + "error-description": ("You need to authenticate to access requested " + "information.")}).encode("utf-8") + resp.status_code = 400 + return Left(resp) + + +def oauth2_get(url, **kwargs) -> Either: + """Do a get request to the authentication/authorisation server.""" + def __get__(_token) -> Either: + _uri = urljoin(authserver_uri(), url) + try: + resp = oauth2_client().get( + _uri, + **{ + **kwargs, + "headers": { + **kwargs.get("headers", {}), + "Content-Type": "application/json" + } + }) + if resp.status_code in mrequests.SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except Exception as exc:#pylint: disable=[broad-except] + app.logger.error("Error retriving data from auth server: (GET %s)", + _uri, + exc_info=True) + return Left(exc) + return session.user_token().either(__no_token__, __get__) + + +def oauth2_post(url, data=None, json=None, **kwargs):#pylint: disable=[redefined-outer-name] + """Do a POST request to the authentication/authorisation server.""" + def __post__(_token) -> Either: + _uri = urljoin(authserver_uri(), url) + _headers = ({ + **kwargs.get("headers", {}), + "Content-Type": "application/json" + } + if bool(json) else kwargs.get("headers", {})) + try: + request_data = { + **(data or {}), + **(json or {}), + "client_id": oauth2_clientid(), + "client_secret": oauth2_clientsecret() + } + resp = oauth2_client().post( + _uri, + data=(request_data if bool(data) else None), + json=(request_data if bool(json) else None), + **{**kwargs, "headers": _headers}) + if resp.status_code in mrequests.SUCCESS_CODES: + return Right(resp.json()) + return Left(resp) + except Exception as exc:#pylint: disable=[broad-except] + app.logger.error("Error retriving data from auth server: (POST %s)", + _uri, + exc_info=True) + return Left(exc) + return session.user_token().either(__no_token__, __post__) diff --git a/uploader/oauth2/jwks.py b/uploader/oauth2/jwks.py new file mode 100644 index 0000000..efd0499 --- /dev/null +++ b/uploader/oauth2/jwks.py @@ -0,0 +1,86 @@ +"""Utilities dealing with JSON Web Keys (JWK)""" +import os +from pathlib import Path +from typing import Any, Union +from datetime import datetime, timedelta + +from flask import Flask +from authlib.jose import JsonWebKey +from pymonad.either import Left, Right, Either + +def jwks_directory(app: Flask, configname: str) -> Path: + """Compute the directory where the JWKs are stored.""" + appsecretsdir = Path(app.config[configname]).parent + if appsecretsdir.exists() and appsecretsdir.is_dir(): + jwksdir = Path(appsecretsdir, "jwks/") + if not jwksdir.exists(): + jwksdir.mkdir() + return jwksdir + raise ValueError( + "The `appsecretsdir` value should be a directory that actually exists.") + + +def generate_and_save_private_key( + storagedir: Path, + kty: str = "RSA", + crv_or_size: Union[str, int] = 2048, + options: tuple[tuple[str, Any]] = (("iat", datetime.now().timestamp()),) +) -> JsonWebKey: + """Generate a private key and save to `storagedir`.""" + privatejwk = JsonWebKey.generate_key( + kty, crv_or_size, dict(options), is_private=True) + keyname = f"{privatejwk.thumbprint()}.private.pem" + with open(Path(storagedir, keyname), "wb") as pemfile: + pemfile.write(privatejwk.as_pem(is_private=True)) + + return privatejwk + + +def pem_to_jwk(filepath: Path) -> JsonWebKey: + """Parse a PEM file into a JWK object.""" + with open(filepath, "rb") as pemfile: + return JsonWebKey.import_key(pemfile.read()) + + +def __sorted_jwks_paths__(storagedir: Path) -> tuple[tuple[float, Path], ...]: + """A sorted list of the JWK file paths with their creation timestamps.""" + return tuple(sorted(((os.stat(keypath).st_ctime, keypath) + for keypath in (Path(storagedir, keyfile) + for keyfile in os.listdir(storagedir) + if keyfile.endswith(".pem"))), + key=lambda tpl: tpl[0])) + + +def list_jwks(storagedir: Path) -> tuple[JsonWebKey, ...]: + """ + List all the JWKs in a particular directory in the order they were created. + """ + return tuple(pem_to_jwk(keypath) for ctime,keypath in + __sorted_jwks_paths__(storagedir)) + + +def newest_jwk(storagedir: Path) -> Either: + """ + Return an Either monad with the newest JWK or a message if none exists. + """ + existingkeys = __sorted_jwks_paths__(storagedir) + if len(existingkeys) > 0: + return Right(pem_to_jwk(existingkeys[-1][1])) + return Left("No JWKs exist") + + +def newest_jwk_with_rotation(jwksdir: Path, keyage: int) -> JsonWebKey: + """ + Retrieve the latests JWK, creating a new one if older than `keyage` days. + """ + def newer_than_days(jwkey): + filestat = os.stat(Path( + jwksdir, f"{jwkey.as_dict()['kid']}.private.pem")) + oldesttimeallowed = (datetime.now() - timedelta(days=keyage)) + if filestat.st_ctime < (oldesttimeallowed.timestamp()): + return Left("JWK is too old!") + return jwkey + + return newest_jwk(jwksdir).then(newer_than_days).either( + lambda _errmsg: generate_and_save_private_key(jwksdir), + lambda key: key) diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py new file mode 100644 index 0000000..61037f3 --- /dev/null +++ b/uploader/oauth2/views.py @@ -0,0 +1,138 @@ +"""Views for OAuth2 related functionality.""" +import uuid +from datetime import datetime, timedelta +from urllib.parse import urljoin, urlparse, urlunparse + +from authlib.jose import jwt +from flask import ( + flash, + jsonify, + url_for, + request, + redirect, + Blueprint, + current_app as app) + +from uploader import session +from uploader import monadic_requests as mrequests +from uploader.monadic_requests import make_error_handler + +from . import jwks +from .client import ( + SCOPE, + oauth2_get, + user_logged_in, + authserver_uri, + oauth2_clientid, + oauth2_clientsecret) + +oauth2 = Blueprint("oauth2", __name__) + +@oauth2.route("/code") +def authorisation_code(): + """Receive authorisation code from auth server and use it to get token.""" + def __process_error__(resp_or_exception): + app.logger.debug("ERROR: (%s)", resp_or_exception) + flash("There was an error retrieving the authorisation token.", + "alert-danger") + return redirect("/") + + def __fail_set_user_details__(_failure): + app.logger.debug("Fetching user details fails: %s", _failure) + flash("Could not retrieve the user details", "alert-danger") + return redirect("/") + + def __success_set_user_details__(_success): + app.logger.debug("Session info: %s", _success) + return redirect("/") + + def __success__(token): + session.set_user_token(token) + return oauth2_get("auth/user/").then( + lambda usrdets: session.set_user_details({ + "user_id": uuid.UUID(usrdets["user_id"]), + "name": usrdets["name"], + "email": usrdets["email"], + "token": session.user_token(), + "logged_in": True})).either( + __fail_set_user_details__, + __success_set_user_details__) + + code = request.args.get("code", "").strip() + if not bool(code): + flash("AuthorisationError: No code was provided.", "alert-danger") + return redirect("/") + + baseurl = urlparse(request.base_url, scheme=request.scheme) + issued = datetime.now() + jwtkey = jwks.newest_jwk_with_rotation( + jwks.jwks_directory(app, "UPLOADER_SECRETS"), + int(app.config["JWKS_ROTATION_AGE_DAYS"])) + return mrequests.post( + urljoin(authserver_uri(), "auth/token"), + json={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "code": code, + "scope": SCOPE, + "redirect_uri": urljoin( + urlunparse(baseurl), + url_for("oauth2.authorisation_code")), + "assertion": jwt.encode( + header={ + "alg": "RS256", + "typ": "JWT", + "kid": jwtkey.as_dict()["kid"] + }, + payload={ + "iss": str(oauth2_clientid()), + "sub": request.args["user_id"], + "aud": urljoin(authserver_uri(),"auth/token"), + "exp": (issued + timedelta(minutes=5)).timestamp(), + "nbf": int(issued.timestamp()), + "iat": int(issued.timestamp()), + "jti": str(uuid.uuid4()) + }, + key=jwtkey).decode("utf8"), + "client_id": oauth2_clientid() + }).either(__process_error__, __success__) + +@oauth2.route("/public-jwks") +def public_jwks(): + """List the available JWKs""" + return jsonify({ + "documentation": ( + "The keys are listed in order of creation, from the oldest (first) " + "to the newest (last)."), + "jwks": tuple(key.as_dict() for key + in jwks.list_jwks(jwks.jwks_directory( + app, "UPLOADER_SECRETS"))) + }) + + +@oauth2.route("/logout", methods=["GET"]) +def logout(): + """Log out of any active sessions.""" + def __unset_session__(session_info): + _user = session_info["user"] + _user_str = f"{_user['name']} ({_user['email']})" + session.clear_session_info() + flash("Successfully logged out.", "alert-success") + return redirect("/") + + if user_logged_in(): + return session.user_token().then( + lambda _tok: mrequests.post( + urljoin(authserver_uri(), "auth/revoke"), + json={ + "token": _tok["refresh_token"], + "token_type_hint": "refresh_token", + "client_id": oauth2_clientid(), + "client_secret": oauth2_clientsecret() + })).either( + make_error_handler( + redirect_to=redirect("/"), + cleanup_thunk=lambda: __unset_session__( + session.session_info())), + lambda res: __unset_session__(session.session_info())) + flash("There is no user that is currently logged in.", "alert-info") + return redirect("/") diff --git a/qc_app/parse.py b/uploader/parse.py index d20f6f0..dea4f95 100644 --- a/qc_app/parse.py +++ b/uploader/parse.py @@ -8,9 +8,10 @@ from flask import current_app as app from quality_control.errors import InvalidValue, DuplicateHeading -from qc_app import jobs -from qc_app.dbinsert import species_by_id -from qc_app.db_utils import with_db_connection +from uploader import jobs +from uploader.dbinsert import species_by_id +from uploader.db_utils import with_db_connection +from uploader.authorisation import require_login parsebp = Blueprint("parse", __name__) @@ -23,6 +24,7 @@ def isduplicateheading(item): return isinstance(item, DuplicateHeading) @parsebp.route("/parse", methods=["GET"]) +@require_login def parse(): """Trigger file parsing""" errors = False @@ -160,6 +162,7 @@ def fail(job_id: str): return render_template("no_such_job.html", job_id=job_id) @parsebp.route("/abort", methods=["POST"]) +@require_login def abort(): """Handle user request to abort file processing""" job_id = request.form["job_id"] diff --git a/qc_app/samples.py b/uploader/samples.py index 804f262..7a80336 100644 --- a/qc_app/samples.py +++ b/uploader/samples.py @@ -20,14 +20,15 @@ from flask import ( from functional_tools import take -from qc_app import jobs -from qc_app.files import save_file -from qc_app.input_validation import is_integer_input -from qc_app.db_utils import ( +from uploader import jobs +from uploader.files import save_file +from uploader.authorisation import require_login +from uploader.input_validation import is_integer_input +from uploader.db_utils import ( with_db_connection, database_connection, with_redis_connection) -from qc_app.db import ( +from uploader.db import ( species_by_id, save_population, population_by_id, @@ -37,6 +38,7 @@ from qc_app.db import ( samples = Blueprint("samples", __name__) @samples.route("/upload/species", methods=["GET", "POST"]) +@require_login def select_species(): """Select the species.""" if request.method == "GET": @@ -58,6 +60,7 @@ def select_species(): @samples.route("/upload/species/<int:species_id>/create-population", methods=["POST"]) +@require_login def create_population(species_id: int): """Create new grouping/population.""" if not is_integer_input(species_id): @@ -100,6 +103,7 @@ def create_population(species_id: int): @samples.route("/upload/species/<int:species_id>/population", methods=["GET", "POST"]) +@require_login def select_population(species_id: int): """Select from existing groupings/populations.""" if not is_integer_input(species_id): @@ -233,6 +237,7 @@ def build_sample_upload_job(# pylint: disable=[too-many-arguments] @samples.route("/upload/species/<int:species_id>/populations/<int:population_id>/samples", methods=["GET", "POST"]) +@require_login def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] """Upload the samples.""" samples_uploads_page = redirect(url_for("samples.upload_samples", diff --git a/uploader/session.py b/uploader/session.py new file mode 100644 index 0000000..8b72bce --- /dev/null +++ b/uploader/session.py @@ -0,0 +1,91 @@ +"""Deal with user sessions""" +from uuid import UUID, uuid4 +from typing import Any, Optional, TypedDict + +from flask import request, session +from pymonad.either import Left, Right, Either + + +class UserDetails(TypedDict): + """Session information relating specifically to the user.""" + user_id: UUID + name: str + email: str + token: Either + logged_in: bool + + +class SessionInfo(TypedDict): + """All Session information we save.""" + session_id: UUID + user: UserDetails + anon_id: UUID + user_agent: str + ip_addr: str + masquerade: Optional[UserDetails] + auth_server_jwks: Optional[dict[str, Any]] + + +__SESSION_KEY__ = "GN::uploader::session_info" # Do not use this outside this module!! + + +def clear_session_info(): + """Clears the session.""" + session.pop(__SESSION_KEY__) + + +def save_session_info(sess_info: SessionInfo) -> SessionInfo: + """Save `session_info`.""" + # T0d0: if it is an existing session, verify that certain important security + # bits have not changed before saving. + # old_session_info = session.get(__SESSION_KEY__) + # if bool(old_session_info): + # if old_session_info["user_agent"] == request.headers.get("User-Agent"): + # session[__SESSION_KEY__] = sess_info + # return sess_info + # # request session verification + # return verify_session(sess_info) + # New session + session[__SESSION_KEY__] = sess_info + return sess_info + + +def session_info() -> SessionInfo: + """Retrieve the session information""" + anon_id = uuid4() + return save_session_info( + session.get(__SESSION_KEY__, { + "session_id": uuid4(), + "user": { + "user_id": anon_id, + "name": "Anonymous User", + "email": "anon@ymous.user", + "token": Left("INVALID-TOKEN"), + "logged_in": False + }, + "anon_id": anon_id, + "user_agent": request.headers.get("User-Agent"), + "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR", + request.remote_addr), + "masquerading": None + })) + + +def set_user_token(token: str) -> SessionInfo: + """Set the user's token.""" + info = session_info() + return save_session_info({ + **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc] + + +def set_user_details(userdets: UserDetails) -> SessionInfo: + """Set the user details information""" + return save_session_info({**session_info(), "user": userdets})#type: ignore[misc] + +def user_details() -> UserDetails: + """Retrieve user details.""" + return session_info()["user"] + +def user_token() -> Either: + """Retrieve the user token.""" + return session_info()["user"]["token"] diff --git a/qc_app/static/css/custom-bootstrap.css b/uploader/static/css/custom-bootstrap.css index 67f1199..67f1199 100644 --- a/qc_app/static/css/custom-bootstrap.css +++ b/uploader/static/css/custom-bootstrap.css diff --git a/qc_app/static/css/styles.css b/uploader/static/css/styles.css index a88c229..a88c229 100644 --- a/qc_app/static/css/styles.css +++ b/uploader/static/css/styles.css diff --git a/qc_app/static/css/two-column-with-separator.css b/uploader/static/css/two-column-with-separator.css index b6efd46..b6efd46 100644 --- a/qc_app/static/css/two-column-with-separator.css +++ b/uploader/static/css/two-column-with-separator.css diff --git a/qc_app/static/images/CITGLogo.png b/uploader/static/images/CITGLogo.png Binary files differindex ae99fed..ae99fed 100644 --- a/qc_app/static/images/CITGLogo.png +++ b/uploader/static/images/CITGLogo.png diff --git a/qc_app/static/js/select_platform.js b/uploader/static/js/select_platform.js index 4fdd865..4fdd865 100644 --- a/qc_app/static/js/select_platform.js +++ b/uploader/static/js/select_platform.js diff --git a/qc_app/static/js/upload_progress.js b/uploader/static/js/upload_progress.js index 9638b36..9638b36 100644 --- a/qc_app/static/js/upload_progress.js +++ b/uploader/static/js/upload_progress.js diff --git a/qc_app/static/js/upload_samples.js b/uploader/static/js/upload_samples.js index aed536f..aed536f 100644 --- a/qc_app/static/js/upload_samples.js +++ b/uploader/static/js/upload_samples.js diff --git a/qc_app/static/js/utils.js b/uploader/static/js/utils.js index 045dd47..045dd47 100644 --- a/qc_app/static/js/utils.js +++ b/uploader/static/js/utils.js diff --git a/qc_app/templates/base.html b/uploader/templates/base.html index eb5e6b7..ee60fea 100644 --- a/qc_app/templates/base.html +++ b/uploader/templates/base.html @@ -33,7 +33,18 @@ <ul class="nav navbar-nav"> <li><a href="/" style="font-weight: bold">GN Uploader</a></li> <li> - <a href="{{gnuri or 'https://genenetwork.org'}}">GeneNetwork</a> + <a href="{{gn2server_uri()}}">GeneNetwork</a> + </li> + </ul> + <ul class="nav navbar-nav" style="margin-left: 2em;"> + <li> + {%if user_logged_in()%} + <a href="{{url_for('oauth2.logout')}}" + title="Log out of the system">Log Out</a> + {%else%} + <a href="{{authserver_authorise_uri()}}" + title="Log in to the system">Log In</a> + {%endif%} </li> </ul> </div> diff --git a/qc_app/templates/cli-output.html b/uploader/templates/cli-output.html index 33fb73b..33fb73b 100644 --- a/qc_app/templates/cli-output.html +++ b/uploader/templates/cli-output.html diff --git a/qc_app/templates/continue_from_create_dataset.html b/uploader/templates/continue_from_create_dataset.html index 03bb49c..03bb49c 100644 --- a/qc_app/templates/continue_from_create_dataset.html +++ b/uploader/templates/continue_from_create_dataset.html diff --git a/qc_app/templates/continue_from_create_study.html b/uploader/templates/continue_from_create_study.html index 34e6e5e..34e6e5e 100644 --- a/qc_app/templates/continue_from_create_study.html +++ b/uploader/templates/continue_from_create_study.html diff --git a/qc_app/templates/data_review.html b/uploader/templates/data_review.html index b7528fd..b7528fd 100644 --- a/qc_app/templates/data_review.html +++ b/uploader/templates/data_review.html diff --git a/qc_app/templates/dbupdate_error.html b/uploader/templates/dbupdate_error.html index e1359d2..e1359d2 100644 --- a/qc_app/templates/dbupdate_error.html +++ b/uploader/templates/dbupdate_error.html diff --git a/qc_app/templates/dbupdate_hidden_fields.html b/uploader/templates/dbupdate_hidden_fields.html index ccbc299..ccbc299 100644 --- a/qc_app/templates/dbupdate_hidden_fields.html +++ b/uploader/templates/dbupdate_hidden_fields.html diff --git a/qc_app/templates/errors_display.html b/uploader/templates/errors_display.html index 715cfcf..715cfcf 100644 --- a/qc_app/templates/errors_display.html +++ b/uploader/templates/errors_display.html diff --git a/qc_app/templates/final_confirmation.html b/uploader/templates/final_confirmation.html index 0727fc8..0727fc8 100644 --- a/qc_app/templates/final_confirmation.html +++ b/uploader/templates/final_confirmation.html diff --git a/qc_app/templates/flash_messages.html b/uploader/templates/flash_messages.html index b7af178..b7af178 100644 --- a/qc_app/templates/flash_messages.html +++ b/uploader/templates/flash_messages.html diff --git a/qc_app/templates/http-error.html b/uploader/templates/http-error.html index 374fb86..374fb86 100644 --- a/qc_app/templates/http-error.html +++ b/uploader/templates/http-error.html diff --git a/qc_app/templates/index.html b/uploader/templates/index.html index 89d2ae9..94060b7 100644 --- a/qc_app/templates/index.html +++ b/uploader/templates/index.html @@ -1,9 +1,12 @@ {%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} {%block title%}Data Upload{%endblock%} {%block contents%} <div class="row"> + {{flash_all_messages()}} + <h1 class="heading">data upload</h1> <div class="explainer"> diff --git a/qc_app/templates/insert_error.html b/uploader/templates/insert_error.html index 5301288..5301288 100644 --- a/qc_app/templates/insert_error.html +++ b/uploader/templates/insert_error.html diff --git a/qc_app/templates/insert_progress.html b/uploader/templates/insert_progress.html index 52177d6..52177d6 100644 --- a/qc_app/templates/insert_progress.html +++ b/uploader/templates/insert_progress.html diff --git a/qc_app/templates/insert_success.html b/uploader/templates/insert_success.html index 7e1fa8d..7e1fa8d 100644 --- a/qc_app/templates/insert_success.html +++ b/uploader/templates/insert_success.html diff --git a/qc_app/templates/job_progress.html b/uploader/templates/job_progress.html index 1af0763..1af0763 100644 --- a/qc_app/templates/job_progress.html +++ b/uploader/templates/job_progress.html diff --git a/uploader/templates/login.html b/uploader/templates/login.html new file mode 100644 index 0000000..6ebf72e --- /dev/null +++ b/uploader/templates/login.html @@ -0,0 +1,32 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Data Upload{%endblock%} + +{%block contents%} +<div class="row"> + {{flash_all_messages()}} + + <h1 class="heading">log in</h1> + + <div class="explainer"> + <p> + This system enables you to upload data onto GeneNetwork. In order to do + that correctly, we need to know who you are.</p> + <p> + If you already have an account with GeneNetwork, you can simply click the + login button below, after which you can upload your data.<br /> + </p> + <a href="{{authserver_authorise_uri()}}" class="btn btn-primary" + style="display:block;width:15em;align:center;margin:1em 3em;"> + log in</a> + <p> + If you do not have an account with GeneNetwork, go to + <a href="{{gn2server_uri()}}" + title="GeneNetwork Service." + target="_blank">GeneNetwork</a> + and register for an account, then come back here to login and upload.</a> + </div> +</div> + +{%endblock%} diff --git a/qc_app/templates/no_such_job.html b/uploader/templates/no_such_job.html index 42a2d48..42a2d48 100644 --- a/qc_app/templates/no_such_job.html +++ b/uploader/templates/no_such_job.html diff --git a/qc_app/templates/parse_failure.html b/uploader/templates/parse_failure.html index 31f6be8..31f6be8 100644 --- a/qc_app/templates/parse_failure.html +++ b/uploader/templates/parse_failure.html diff --git a/qc_app/templates/parse_results.html b/uploader/templates/parse_results.html index e2bf7f0..e2bf7f0 100644 --- a/qc_app/templates/parse_results.html +++ b/uploader/templates/parse_results.html diff --git a/qc_app/templates/rqtl2/create-geno-dataset-success.html b/uploader/templates/rqtl2/create-geno-dataset-success.html index 1b50221..1b50221 100644 --- a/qc_app/templates/rqtl2/create-geno-dataset-success.html +++ b/uploader/templates/rqtl2/create-geno-dataset-success.html diff --git a/qc_app/templates/rqtl2/create-probe-dataset-success.html b/uploader/templates/rqtl2/create-probe-dataset-success.html index 790d174..790d174 100644 --- a/qc_app/templates/rqtl2/create-probe-dataset-success.html +++ b/uploader/templates/rqtl2/create-probe-dataset-success.html diff --git a/qc_app/templates/rqtl2/create-probe-study-success.html b/uploader/templates/rqtl2/create-probe-study-success.html index d0ee508..d0ee508 100644 --- a/qc_app/templates/rqtl2/create-probe-study-success.html +++ b/uploader/templates/rqtl2/create-probe-study-success.html diff --git a/qc_app/templates/rqtl2/create-tissue-success.html b/uploader/templates/rqtl2/create-tissue-success.html index 5f2c5a0..5f2c5a0 100644 --- a/qc_app/templates/rqtl2/create-tissue-success.html +++ b/uploader/templates/rqtl2/create-tissue-success.html diff --git a/qc_app/templates/rqtl2/index.html b/uploader/templates/rqtl2/index.html index f3329c2..f3329c2 100644 --- a/qc_app/templates/rqtl2/index.html +++ b/uploader/templates/rqtl2/index.html diff --git a/qc_app/templates/rqtl2/no-such-job.html b/uploader/templates/rqtl2/no-such-job.html index b17004f..b17004f 100644 --- a/qc_app/templates/rqtl2/no-such-job.html +++ b/uploader/templates/rqtl2/no-such-job.html diff --git a/qc_app/templates/rqtl2/rqtl2-job-error.html b/uploader/templates/rqtl2/rqtl2-job-error.html index 9817518..9817518 100644 --- a/qc_app/templates/rqtl2/rqtl2-job-error.html +++ b/uploader/templates/rqtl2/rqtl2-job-error.html diff --git a/qc_app/templates/rqtl2/rqtl2-job-results.html b/uploader/templates/rqtl2/rqtl2-job-results.html index 4ecd415..4ecd415 100644 --- a/qc_app/templates/rqtl2/rqtl2-job-results.html +++ b/uploader/templates/rqtl2/rqtl2-job-results.html diff --git a/qc_app/templates/rqtl2/rqtl2-job-status.html b/uploader/templates/rqtl2/rqtl2-job-status.html index e896f88..e896f88 100644 --- a/qc_app/templates/rqtl2/rqtl2-job-status.html +++ b/uploader/templates/rqtl2/rqtl2-job-status.html diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/uploader/templates/rqtl2/rqtl2-qc-job-error.html index 90e8887..90e8887 100644 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html +++ b/uploader/templates/rqtl2/rqtl2-qc-job-error.html diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-results.html b/uploader/templates/rqtl2/rqtl2-qc-job-results.html index 59bc8cd..59bc8cd 100644 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-results.html +++ b/uploader/templates/rqtl2/rqtl2-qc-job-results.html diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html b/uploader/templates/rqtl2/rqtl2-qc-job-status.html index f4a6266..f4a6266 100644 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html +++ b/uploader/templates/rqtl2/rqtl2-qc-job-status.html diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html b/uploader/templates/rqtl2/rqtl2-qc-job-success.html index 2861a04..2861a04 100644 --- a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html +++ b/uploader/templates/rqtl2/rqtl2-qc-job-success.html diff --git a/qc_app/templates/rqtl2/select-geno-dataset.html b/uploader/templates/rqtl2/select-geno-dataset.html index 873f9c3..873f9c3 100644 --- a/qc_app/templates/rqtl2/select-geno-dataset.html +++ b/uploader/templates/rqtl2/select-geno-dataset.html diff --git a/qc_app/templates/rqtl2/select-population.html b/uploader/templates/rqtl2/select-population.html index 37731f0..37731f0 100644 --- a/qc_app/templates/rqtl2/select-population.html +++ b/uploader/templates/rqtl2/select-population.html diff --git a/qc_app/templates/rqtl2/select-probeset-dataset.html b/uploader/templates/rqtl2/select-probeset-dataset.html index 26f52ed..26f52ed 100644 --- a/qc_app/templates/rqtl2/select-probeset-dataset.html +++ b/uploader/templates/rqtl2/select-probeset-dataset.html diff --git a/qc_app/templates/rqtl2/select-probeset-study-id.html b/uploader/templates/rqtl2/select-probeset-study-id.html index b9bf52e..b9bf52e 100644 --- a/qc_app/templates/rqtl2/select-probeset-study-id.html +++ b/uploader/templates/rqtl2/select-probeset-study-id.html diff --git a/qc_app/templates/rqtl2/select-tissue.html b/uploader/templates/rqtl2/select-tissue.html index 34e1758..34e1758 100644 --- a/qc_app/templates/rqtl2/select-tissue.html +++ b/uploader/templates/rqtl2/select-tissue.html diff --git a/qc_app/templates/rqtl2/summary-info.html b/uploader/templates/rqtl2/summary-info.html index 1be87fa..1be87fa 100644 --- a/qc_app/templates/rqtl2/summary-info.html +++ b/uploader/templates/rqtl2/summary-info.html diff --git a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html index 07c240f..07c240f 100644 --- a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html +++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html diff --git a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html index 93b1dc9..93b1dc9 100644 --- a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html +++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html diff --git a/qc_app/templates/samples/select-population.html b/uploader/templates/samples/select-population.html index da19ddc..da19ddc 100644 --- a/qc_app/templates/samples/select-population.html +++ b/uploader/templates/samples/select-population.html diff --git a/qc_app/templates/samples/select-species.html b/uploader/templates/samples/select-species.html index edadc61..edadc61 100644 --- a/qc_app/templates/samples/select-species.html +++ b/uploader/templates/samples/select-species.html diff --git a/qc_app/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html index 09e2ecf..09e2ecf 100644 --- a/qc_app/templates/samples/upload-failure.html +++ b/uploader/templates/samples/upload-failure.html diff --git a/qc_app/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html index 7bb02be..7bb02be 100644 --- a/qc_app/templates/samples/upload-progress.html +++ b/uploader/templates/samples/upload-progress.html diff --git a/qc_app/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html index e62de57..e62de57 100644 --- a/qc_app/templates/samples/upload-samples.html +++ b/uploader/templates/samples/upload-samples.html diff --git a/qc_app/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html index cb745c3..cb745c3 100644 --- a/qc_app/templates/samples/upload-success.html +++ b/uploader/templates/samples/upload-success.html diff --git a/qc_app/templates/select_dataset.html b/uploader/templates/select_dataset.html index 2f07de8..2f07de8 100644 --- a/qc_app/templates/select_dataset.html +++ b/uploader/templates/select_dataset.html diff --git a/qc_app/templates/select_platform.html b/uploader/templates/select_platform.html index d9bc68f..d9bc68f 100644 --- a/qc_app/templates/select_platform.html +++ b/uploader/templates/select_platform.html diff --git a/qc_app/templates/select_species.html b/uploader/templates/select_species.html index 3b1a8a9..3b1a8a9 100644 --- a/qc_app/templates/select_species.html +++ b/uploader/templates/select_species.html diff --git a/qc_app/templates/select_study.html b/uploader/templates/select_study.html index 648ad4c..648ad4c 100644 --- a/qc_app/templates/select_study.html +++ b/uploader/templates/select_study.html diff --git a/qc_app/templates/stdout_output.html b/uploader/templates/stdout_output.html index 85345a9..85345a9 100644 --- a/qc_app/templates/stdout_output.html +++ b/uploader/templates/stdout_output.html diff --git a/uploader/templates/unhandled_exception.html b/uploader/templates/unhandled_exception.html new file mode 100644 index 0000000..cfb0c0b --- /dev/null +++ b/uploader/templates/unhandled_exception.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}System Error{%endblock%} + +{%block css%} +<link rel="stylesheet" href="/static/css/two-column-with-separator.css" /> +{%endblock%} + +{%block contents%} +<div class="row"> + {{flash_all_messages()}} + <h1>Exception!</h1> + + <p>An error has occured, and your request has been aborted. Please notify the + administrator to try and get this fixed.</p> + <p>The system has failed with the following error:</p> +</div> +<div class="row"> + <pre> + {{trace}} + </pre> +</div> +{%endblock%} diff --git a/qc_app/templates/upload_progress_indicator.html b/uploader/templates/upload_progress_indicator.html index e274e83..e274e83 100644 --- a/qc_app/templates/upload_progress_indicator.html +++ b/uploader/templates/upload_progress_indicator.html diff --git a/qc_app/templates/worker_failure.html b/uploader/templates/worker_failure.html index b65b140..b65b140 100644 --- a/qc_app/templates/worker_failure.html +++ b/uploader/templates/worker_failure.html diff --git a/qc_app/upload/__init__.py b/uploader/upload/__init__.py index 5f120d4..5f120d4 100644 --- a/qc_app/upload/__init__.py +++ b/uploader/upload/__init__.py diff --git a/qc_app/upload/rqtl2.py b/uploader/upload/rqtl2.py index e691636..ff7556d 100644 --- a/qc_app/upload/rqtl2.py +++ b/uploader/upload/rqtl2.py @@ -27,20 +27,21 @@ from flask import ( from r_qtl import r_qtl2 -from qc_app import jobs -from qc_app.files import save_file, fullpath -from qc_app.dbinsert import species as all_species -from qc_app.db_utils import with_db_connection, database_connection - -from qc_app.db.platforms import platform_by_id, platforms_by_species -from qc_app.db.averaging import averaging_methods, averaging_method_by_id -from qc_app.db.tissues import all_tissues, tissue_by_id, create_new_tissue -from qc_app.db import ( +from uploader import jobs +from uploader.files import save_file, fullpath +from uploader.dbinsert import species as all_species +from uploader.db_utils import with_db_connection, database_connection + +from uploader.authorisation import require_login +from uploader.db.platforms import platform_by_id, platforms_by_species +from uploader.db.averaging import averaging_methods, averaging_method_by_id +from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue +from uploader.db import ( species_by_id, save_population, populations_by_species, population_by_species_and_id,) -from qc_app.db.datasets import ( +from uploader.db.datasets import ( geno_dataset_by_id, geno_datasets_by_species_and_population, @@ -53,8 +54,10 @@ from qc_app.db.datasets import ( rqtl2 = Blueprint("rqtl2", __name__) + @rqtl2.route("/", methods=["GET", "POST"]) @rqtl2.route("/select-species", methods=["GET", "POST"]) +@require_login def select_species(): """Select the species.""" if request.method == "GET": @@ -72,6 +75,7 @@ def select_species(): @rqtl2.route("/upload/species/<int:species_id>/select-population", methods=["GET", "POST"]) +@require_login def select_population(species_id: int): """Select/Create the population to organise data under.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -101,6 +105,7 @@ def select_population(species_id: int): @rqtl2.route("/upload/species/<int:species_id>/create-population", methods=["POST"]) +@require_login def create_population(species_id: int): """Create a new population for the given species.""" population_page = redirect(url_for("upload.rqtl2.select_population", @@ -143,6 +148,7 @@ class __RequestError__(Exception): #pylint: disable=[invalid-name] @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle"), methods=["GET", "POST"]) +@require_login def upload_rqtl2_bundle(species_id: int, population_id: int): """Allow upload of R/qtl2 bundle.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -241,6 +247,7 @@ def chunks_directory(uniqueidentifier: str) -> Path: @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle-chunked"), methods=["GET"]) +@require_login def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] species_id: int, population_id: int @@ -285,6 +292,7 @@ def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle-chunked"), methods=["POST"]) +@require_login def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): """ Extension to the `upload_rqtl2_bundle` endpoint above that allows large @@ -310,29 +318,40 @@ def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): "statuscode": 400 }), 400 - # save chunk data - chunks_directory(_fileid).mkdir(exist_ok=True) - request.files["file"].save(Path(chunks_directory(_fileid), - chunk_name(_uploadfilename, _chunk))) - - # Check whether upload is complete - chunkpaths = tuple( - Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) - for _achunk in range(1, _totalchunks+1)) - if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() - jobid = trigger_rqtl2_bundle_qc( - species_id, population_id, _targetfile, _uploadfilename) - return url_for( - "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) + try: + # save chunk data + chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) + request.files["file"].save(Path(chunks_directory(_fileid), + chunk_name(_uploadfilename, _chunk))) + + # Check whether upload is complete + chunkpaths = tuple( + Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) + for _achunk in range(1, _totalchunks+1)) + if all(_file.exists() for _file in chunkpaths): + # merge_files and clean up chunks + __merge_chunks__(_targetfile, chunkpaths) + chunks_directory(_fileid).rmdir() + jobid = trigger_rqtl2_bundle_qc( + species_id, population_id, _targetfile, _uploadfilename) + return url_for( + "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) + except Exception as exc:# pylint: disable=[broad-except] + msg = "Error processing uploaded file chunks." + app.logger.error(msg, exc_info=True, stack_info=True) + return jsonify({ + "message": msg, + "error": type(exc).__name__, + "error-description": " ".join(str(arg) for arg in exc.args), + "error-trace": traceback.format_exception(exc) + }), 500 return "OK" @rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", methods=["GET", "POST"]) +@require_login def rqtl2_bundle_qc_status(jobid: UUID): """Check the status of the QC jobs.""" with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, @@ -554,6 +573,7 @@ def with_errors(endpointthunk: Callable, *checkfns): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/select-geno-dataset"), methods=["POST"]) +@require_login def select_geno_dataset(species_id: int, population_id: int): """Select from existing geno datasets.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -592,6 +612,7 @@ def select_geno_dataset(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/create-geno-dataset"), methods=["POST"]) +@require_login def create_geno_dataset(species_id: int, population_id: int): """Create a new geno dataset.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -661,6 +682,7 @@ def create_geno_dataset(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/select-tissue"), methods=["POST"]) +@require_login def select_tissue(species_id: int, population_id: int): """Select from existing tissues.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -691,6 +713,7 @@ def select_tissue(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/create-tissue"), methods=["POST"]) +@require_login def create_tissue(species_id: int, population_id: int): """Add new tissue, organ or biological material to the system.""" form = request.form @@ -735,6 +758,7 @@ def create_tissue(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/select-probeset-study"), methods=["POST"]) +@require_login def select_probeset_study(species_id: int, population_id: int): """Select or create a probeset study.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -770,6 +794,7 @@ def select_probeset_study(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/select-probeset-dataset"), methods=["POST"]) +@require_login def select_probeset_dataset(species_id: int, population_id: int): """Select or create a probeset dataset.""" with database_connection(app.config["SQL_URI"]) as conn: @@ -810,6 +835,7 @@ def select_probeset_dataset(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/create-probeset-study"), methods=["POST"]) +@require_login def create_probeset_study(species_id: int, population_id: int): """Create a new probeset study.""" errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study" @@ -872,6 +898,7 @@ def create_probeset_study(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/create-probeset-dataset"), methods=["POST"]) +@require_login def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements] """Create a new probeset dataset.""" errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset" @@ -963,6 +990,7 @@ def create_probeset_dataset(species_id: int, population_id: int):#pylint: disabl @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/dataset-info"), methods=["POST"]) +@require_login def select_dataset_info(species_id: int, population_id: int): """ If `geno` files exist in the R/qtl2 bundle, prompt user to provide the @@ -1055,6 +1083,7 @@ def select_dataset_info(species_id: int, population_id: int): @rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>" "/rqtl2-bundle/confirm-bundle-details"), methods=["POST"]) +@require_login def confirm_bundle_details(species_id: int, population_id: int): """Confirm the details and trigger R/qtl2 bundle processing...""" redisuri = app.config["REDIS_URL"] |