aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--MANIFEST.in4
-rw-r--r--README.org8
-rw-r--r--mypy.ini12
-rw-r--r--qc_app/__init__.py48
-rw-r--r--qc_app/templates/unhandled_exception.html21
-rw-r--r--r_qtl/exceptions.py (renamed from r_qtl/errors.py)2
-rw-r--r--r_qtl/fileerrors.py9
-rw-r--r--r_qtl/r_qtl2.py156
-rw-r--r--r_qtl/r_qtl2_qc.py59
-rw-r--r--scripts/insert_data.py4
-rw-r--r--scripts/insert_samples.py8
-rw-r--r--scripts/process_rqtl2_bundle.py8
-rw-r--r--scripts/qc.py2
-rw-r--r--scripts/qc_on_rqtl2_bundle.py24
-rw-r--r--scripts/qcapp_wsgi.py4
-rw-r--r--scripts/rqtl2/entry.py6
-rw-r--r--scripts/rqtl2/install_genotypes.py117
-rw-r--r--scripts/validate_file.py4
-rw-r--r--scripts/worker.py4
-rw-r--r--tests/conftest.py4
-rw-r--r--tests/qc_app/test_parse.py4
-rw-r--r--uploader/__init__.py94
-rw-r--r--uploader/authorisation.py21
-rw-r--r--uploader/base_routes.py (renamed from qc_app/base_routes.py)0
-rw-r--r--uploader/check_connections.py (renamed from qc_app/check_connections.py)2
-rw-r--r--uploader/db/__init__.py (renamed from qc_app/db/__init__.py)0
-rw-r--r--uploader/db/averaging.py (renamed from qc_app/db/averaging.py)0
-rw-r--r--uploader/db/datasets.py (renamed from qc_app/db/datasets.py)0
-rw-r--r--uploader/db/platforms.py (renamed from qc_app/db/platforms.py)0
-rw-r--r--uploader/db/populations.py (renamed from qc_app/db/populations.py)0
-rw-r--r--uploader/db/species.py (renamed from qc_app/db/species.py)0
-rw-r--r--uploader/db/tissues.py (renamed from qc_app/db/tissues.py)0
-rw-r--r--uploader/db_utils.py (renamed from qc_app/db_utils.py)7
-rw-r--r--uploader/dbinsert.py (renamed from qc_app/dbinsert.py)34
-rw-r--r--uploader/default_settings.py23
-rw-r--r--uploader/entry.py (renamed from qc_app/entry.py)10
-rw-r--r--uploader/errors.py (renamed from qc_app/errors.py)0
-rw-r--r--uploader/files.py (renamed from qc_app/files.py)0
-rw-r--r--uploader/input_validation.py (renamed from qc_app/input_validation.py)0
-rw-r--r--uploader/jobs.py (renamed from qc_app/jobs.py)0
-rw-r--r--uploader/monadic_requests.py86
-rw-r--r--uploader/oauth2/__init__.py1
-rw-r--r--uploader/oauth2/client.py202
-rw-r--r--uploader/oauth2/jwks.py86
-rw-r--r--uploader/oauth2/views.py138
-rw-r--r--uploader/parse.py (renamed from qc_app/parse.py)9
-rw-r--r--uploader/samples.py (renamed from qc_app/samples.py)15
-rw-r--r--uploader/session.py91
-rw-r--r--uploader/static/css/custom-bootstrap.css (renamed from qc_app/static/css/custom-bootstrap.css)0
-rw-r--r--uploader/static/css/styles.css (renamed from qc_app/static/css/styles.css)0
-rw-r--r--uploader/static/css/two-column-with-separator.css (renamed from qc_app/static/css/two-column-with-separator.css)0
-rw-r--r--uploader/static/images/CITGLogo.png (renamed from qc_app/static/images/CITGLogo.png)bin11962 -> 11962 bytes
-rw-r--r--uploader/static/js/select_platform.js (renamed from qc_app/static/js/select_platform.js)0
-rw-r--r--uploader/static/js/upload_progress.js (renamed from qc_app/static/js/upload_progress.js)0
-rw-r--r--uploader/static/js/upload_samples.js (renamed from qc_app/static/js/upload_samples.js)0
-rw-r--r--uploader/static/js/utils.js (renamed from qc_app/static/js/utils.js)0
-rw-r--r--uploader/templates/base.html (renamed from qc_app/templates/base.html)13
-rw-r--r--uploader/templates/cli-output.html (renamed from qc_app/templates/cli-output.html)0
-rw-r--r--uploader/templates/continue_from_create_dataset.html (renamed from qc_app/templates/continue_from_create_dataset.html)0
-rw-r--r--uploader/templates/continue_from_create_study.html (renamed from qc_app/templates/continue_from_create_study.html)0
-rw-r--r--uploader/templates/data_review.html (renamed from qc_app/templates/data_review.html)0
-rw-r--r--uploader/templates/dbupdate_error.html (renamed from qc_app/templates/dbupdate_error.html)0
-rw-r--r--uploader/templates/dbupdate_hidden_fields.html (renamed from qc_app/templates/dbupdate_hidden_fields.html)0
-rw-r--r--uploader/templates/errors_display.html (renamed from qc_app/templates/errors_display.html)0
-rw-r--r--uploader/templates/final_confirmation.html (renamed from qc_app/templates/final_confirmation.html)0
-rw-r--r--uploader/templates/flash_messages.html (renamed from qc_app/templates/flash_messages.html)0
-rw-r--r--uploader/templates/http-error.html (renamed from qc_app/templates/http-error.html)0
-rw-r--r--uploader/templates/index.html (renamed from qc_app/templates/index.html)3
-rw-r--r--uploader/templates/insert_error.html (renamed from qc_app/templates/insert_error.html)0
-rw-r--r--uploader/templates/insert_progress.html (renamed from qc_app/templates/insert_progress.html)0
-rw-r--r--uploader/templates/insert_success.html (renamed from qc_app/templates/insert_success.html)0
-rw-r--r--uploader/templates/job_progress.html (renamed from qc_app/templates/job_progress.html)0
-rw-r--r--uploader/templates/login.html32
-rw-r--r--uploader/templates/no_such_job.html (renamed from qc_app/templates/no_such_job.html)0
-rw-r--r--uploader/templates/parse_failure.html (renamed from qc_app/templates/parse_failure.html)0
-rw-r--r--uploader/templates/parse_results.html (renamed from qc_app/templates/parse_results.html)0
-rw-r--r--uploader/templates/rqtl2/create-geno-dataset-success.html (renamed from qc_app/templates/rqtl2/create-geno-dataset-success.html)0
-rw-r--r--uploader/templates/rqtl2/create-probe-dataset-success.html (renamed from qc_app/templates/rqtl2/create-probe-dataset-success.html)0
-rw-r--r--uploader/templates/rqtl2/create-probe-study-success.html (renamed from qc_app/templates/rqtl2/create-probe-study-success.html)0
-rw-r--r--uploader/templates/rqtl2/create-tissue-success.html (renamed from qc_app/templates/rqtl2/create-tissue-success.html)0
-rw-r--r--uploader/templates/rqtl2/index.html (renamed from qc_app/templates/rqtl2/index.html)0
-rw-r--r--uploader/templates/rqtl2/no-such-job.html (renamed from qc_app/templates/rqtl2/no-such-job.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-job-error.html (renamed from qc_app/templates/rqtl2/rqtl2-job-error.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-job-results.html (renamed from qc_app/templates/rqtl2/rqtl2-job-results.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-job-status.html (renamed from qc_app/templates/rqtl2/rqtl2-job-status.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-error.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-error.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-results.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-results.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-status.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-status.html)0
-rw-r--r--uploader/templates/rqtl2/rqtl2-qc-job-success.html (renamed from qc_app/templates/rqtl2/rqtl2-qc-job-success.html)0
-rw-r--r--uploader/templates/rqtl2/select-geno-dataset.html (renamed from qc_app/templates/rqtl2/select-geno-dataset.html)0
-rw-r--r--uploader/templates/rqtl2/select-population.html (renamed from qc_app/templates/rqtl2/select-population.html)0
-rw-r--r--uploader/templates/rqtl2/select-probeset-dataset.html (renamed from qc_app/templates/rqtl2/select-probeset-dataset.html)0
-rw-r--r--uploader/templates/rqtl2/select-probeset-study-id.html (renamed from qc_app/templates/rqtl2/select-probeset-study-id.html)0
-rw-r--r--uploader/templates/rqtl2/select-tissue.html (renamed from qc_app/templates/rqtl2/select-tissue.html)0
-rw-r--r--uploader/templates/rqtl2/summary-info.html (renamed from qc_app/templates/rqtl2/summary-info.html)0
-rw-r--r--uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html (renamed from qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html)0
-rw-r--r--uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html (renamed from qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html)0
-rw-r--r--uploader/templates/samples/select-population.html (renamed from qc_app/templates/samples/select-population.html)0
-rw-r--r--uploader/templates/samples/select-species.html (renamed from qc_app/templates/samples/select-species.html)0
-rw-r--r--uploader/templates/samples/upload-failure.html (renamed from qc_app/templates/samples/upload-failure.html)0
-rw-r--r--uploader/templates/samples/upload-progress.html (renamed from qc_app/templates/samples/upload-progress.html)0
-rw-r--r--uploader/templates/samples/upload-samples.html (renamed from qc_app/templates/samples/upload-samples.html)0
-rw-r--r--uploader/templates/samples/upload-success.html (renamed from qc_app/templates/samples/upload-success.html)0
-rw-r--r--uploader/templates/select_dataset.html (renamed from qc_app/templates/select_dataset.html)0
-rw-r--r--uploader/templates/select_platform.html (renamed from qc_app/templates/select_platform.html)0
-rw-r--r--uploader/templates/select_species.html (renamed from qc_app/templates/select_species.html)0
-rw-r--r--uploader/templates/select_study.html (renamed from qc_app/templates/select_study.html)0
-rw-r--r--uploader/templates/stdout_output.html (renamed from qc_app/templates/stdout_output.html)0
-rw-r--r--uploader/templates/unhandled_exception.html24
-rw-r--r--uploader/templates/upload_progress_indicator.html (renamed from qc_app/templates/upload_progress_indicator.html)0
-rw-r--r--uploader/templates/worker_failure.html (renamed from qc_app/templates/worker_failure.html)0
-rw-r--r--uploader/upload/__init__.py (renamed from qc_app/upload/__init__.py)0
-rw-r--r--uploader/upload/rqtl2.py (renamed from qc_app/upload/rqtl2.py)83
113 files changed, 1257 insertions, 225 deletions
diff --git a/MANIFEST.in b/MANIFEST.in
index c515f0e..79339cd 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,5 +1,5 @@
include README.org
recursive-include etc *.py *.csv
-recursive-include qc_app/static *.js *.css *.png
-recursive-include qc_app/templates *.html
+recursive-include uploader/static *.js *.css *.png
+recursive-include uploader/templates *.html
recursive-exclude tests/ *.py *.tsv *.csv \ No newline at end of file
diff --git a/README.org b/README.org
index 6e9c78e..ca77653 100644
--- a/README.org
+++ b/README.org
@@ -197,7 +197,7 @@ few environment variables
#+BEGIN_SRC shell
export FLASK_APP=wsgi.py
export FLASK_ENV=development
-export QCAPP_INSTANCE_PATH=/path/to/directory/with/config.py
+export UPLOADER_CONF=/path/to/directory/with/uploader/configuration.py
#+END_SRC
then you can run the application with
#+BEGIN_SRC shell
@@ -208,7 +208,7 @@ flask run
To run the linter over the code base, run:
#+BEGIN_SRC shell
- pylint setup.py tests quality_control qc_app r_qtl scripts
+ pylint setup.py tests quality_control uploader r_qtl scripts
#+END_SRC
To check for correct type usage in the application, run:
@@ -218,13 +218,13 @@ To check for correct type usage in the application, run:
Run unit tests with:
#+BEGIN_SRC shell
- $ export QCAPP_CONF=</path/to/configuration/file.py>
+ $ export UPLOADER_CONF=</path/to/configuration/file.py>
$ pytest -m unit_test
#+END_SRC
To run ALL tests (not just unit tests):
#+BEGIN_SRC shell
- $ export QCAPP_CONF=</path/to/configuration/file.py>
+ $ export UPLOADER_CONF=</path/to/configuration/file.py>
$ pytest
#+END_SRC
diff --git a/mypy.ini b/mypy.ini
index 08e896e..6ebd850 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -28,4 +28,16 @@ ignore_missing_imports = True
ignore_missing_imports = True
[mypy-yaml.*]
+ignore_missing_imports = True
+
+[mypy-pymonad.tools]
+ignore_missing_imports = True
+
+[mypy-pymonad.either]
+ignore_missing_imports = True
+
+[mypy-authlib.*]
+ignore_missing_imports = True
+
+[mypy-flask_session.*]
ignore_missing_imports = True \ No newline at end of file
diff --git a/qc_app/__init__.py b/qc_app/__init__.py
deleted file mode 100644
index 3ee8aa0..0000000
--- a/qc_app/__init__.py
+++ /dev/null
@@ -1,48 +0,0 @@
-"""The Quality-Control Web Application entry point"""
-import os
-import logging
-from pathlib import Path
-
-from flask import Flask, request
-
-from .entry import entrybp
-from .upload import upload
-from .parse import parsebp
-from .samples import samples
-from .base_routes import base
-from .dbinsert import dbinsertbp
-from .errors import register_error_handlers
-
-def override_settings_with_envvars(
- app: Flask, ignore: tuple[str, ...]=tuple()) -> None:
- """Override settings in `app` with those in ENVVARS"""
- for setting in (key for key in app.config if key not in ignore):
- app.config[setting] = os.environ.get(setting) or app.config[setting]
-
-
-def create_app():
- """The application factory"""
- app = Flask(__name__)
- app.config.from_pyfile(
- Path(__file__).parent.joinpath("default_settings.py"))
- if "QCAPP_CONF" in os.environ:
- app.config.from_envvar("QCAPP_CONF") # Override defaults with instance path
-
- override_settings_with_envvars(app, ignore=tuple())
-
- if "QCAPP_SECRETS" in os.environ:
- app.config.from_envvar("QCAPP_SECRETS")
-
- # setup jinja2 symbols
- app.jinja_env.globals.update(request_url=lambda : request.url)
-
- # setup blueprints
- app.register_blueprint(base, url_prefix="/")
- app.register_blueprint(entrybp, url_prefix="/")
- app.register_blueprint(parsebp, url_prefix="/parse")
- app.register_blueprint(upload, url_prefix="/upload")
- app.register_blueprint(dbinsertbp, url_prefix="/dbinsert")
- app.register_blueprint(samples, url_prefix="/samples")
-
- register_error_handlers(app)
- return app
diff --git a/qc_app/templates/unhandled_exception.html b/qc_app/templates/unhandled_exception.html
deleted file mode 100644
index 6e6a051..0000000
--- a/qc_app/templates/unhandled_exception.html
+++ /dev/null
@@ -1,21 +0,0 @@
-{%extends "base.html"%}
-
-{%block title%}System Error{%endblock%}
-
-{%block css%}
-<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
-{%endblock%}
-
-{%block contents%}
-<p>
- An error has occured, and your request has been aborted. Please notify the
- administrator to try and get this sorted.
-</p>
-<p>
- Provide the following information to help the administrator figure out and fix
- the issue:<br />
- <hr /><br />
- {{trace}}
- <hr /><br />
-</p>
-{%endblock%}
diff --git a/r_qtl/errors.py b/r_qtl/exceptions.py
index 417eb58..9620cf4 100644
--- a/r_qtl/errors.py
+++ b/r_qtl/exceptions.py
@@ -6,7 +6,7 @@ class RQTLError(Exception):
class InvalidFormat(RQTLError):
"""Raised when the format of the file(s) is invalid."""
-class MissingFileError(InvalidFormat):
+class MissingFileException(InvalidFormat):
"""
Raise when at least one file listed in the control file is missing from the
R/qtl2 bundle.
diff --git a/r_qtl/fileerrors.py b/r_qtl/fileerrors.py
index e76676c..c253d71 100644
--- a/r_qtl/fileerrors.py
+++ b/r_qtl/fileerrors.py
@@ -1,5 +1,14 @@
"""QC errors as distinguished from actual exceptions"""
from collections import namedtuple
+InvalidValue = namedtuple(
+ "InvalidValue",
+ ("filename",
+ "rowtitle",
+ "coltitle",
+ "cellvalue",
+ "message"))
+
+
MissingFile = namedtuple(
"MissingFile", ("controlfilekey", "filename", "message"))
diff --git a/r_qtl/r_qtl2.py b/r_qtl/r_qtl2.py
index 0a96e7c..9da4081 100644
--- a/r_qtl/r_qtl2.py
+++ b/r_qtl/r_qtl2.py
@@ -1,17 +1,18 @@
"""The R/qtl2 parsing and processing code."""
import io
+import os
import csv
import json
from pathlib import Path
-from zipfile import ZipFile
from functools import reduce, partial
+from zipfile import ZipFile, is_zipfile
from typing import Union, Iterator, Iterable, Callable, Optional
import yaml
from functional_tools import take, chain
-from r_qtl.errors import InvalidFormat, MissingFileError
+from r_qtl.exceptions import InvalidFormat, MissingFileException
FILE_TYPES = (
"geno", "founder_geno", "pheno", "covar", "phenocovar", "gmap", "pmap",
@@ -30,7 +31,79 @@ def __special_file__(filename):
return (is_macosx_special_file or is_nix_hidden_file)
-def control_data(zfile: ZipFile) -> dict:
+def extract(zfile: ZipFile, outputdir: Path) -> tuple[Path, ...]:
+ """Extract a ZipFile
+
+ This function will extract a zipfile `zfile` to the directory `outputdir`.
+
+ Parameters
+ ----------
+ zfile: zipfile.ZipFile object - the zipfile to extract.
+ outputdir: Optional pathlib.Path object - where the extracted files go.
+
+ Returns
+ -------
+ A tuple of Path objects, each pointing to a member in the zipfile.
+ """
+ outputdir.mkdir(parents=True, exist_ok=True)
+ return tuple(Path(zfile.extract(member, outputdir))
+ for member in zfile.namelist()
+ if not __special_file__(member))
+
+
+def transpose_csv(
+ inpath: Path,
+ linesplitterfn: Callable,
+ linejoinerfn: Callable,
+ outpath: Path) -> Path:
+ """Transpose a file: Make its rows into columns and its columns into rows.
+
+ This function will create a new file, `outfile`, with the same content as
+ the original, `infile`, except transposed i.e. The rows of `infile` are the
+ columns of `outfile` and the columns of `infile` are the rows of `outfile`.
+
+ Parameters
+ ----------
+ inpath: The CSV file to transpose.
+ linesplitterfn: A function to use for splitting each line into columns
+ linejoinerfn: A function to use to rebuild the lines
+ outpath: The path where the transposed data is stored
+ """
+ def __read_by_line__(_path):
+ with open(_path, "r", encoding="utf8") as infile:
+ for line in infile:
+ yield line
+
+ transposed_data= (f"{linejoinerfn(items)}\n" for items in zip(*(
+ linesplitterfn(line) for line in __read_by_line__(inpath))))
+
+ with open(outpath, "w", encoding="utf8") as outfile:
+ for line in transposed_data:
+ outfile.write(line)
+
+ return outpath
+
+
+def transpose_csv_with_rename(inpath: Path,
+ linesplitterfn: Callable,
+ linejoinerfn: Callable) -> Path:
+ """Renames input file and creates new transposed file with the original name
+ of the input file.
+
+ Parameters
+ ----------
+ inpath: Path to the input file. Should be a pathlib.Path object.
+ linesplitterfn: A function to use for splitting each line into columns
+ linejoinerfn: A function to use to rebuild the lines
+ """
+ transposedfilepath = Path(inpath)
+ origbkp = inpath.parent.joinpath(f"{inpath.stem}___original{inpath.suffix}")
+ os.rename(inpath, origbkp)
+ return transpose_csv(
+ origbkp, linesplitterfn, linejoinerfn, transposedfilepath)
+
+
+def __control_data_from_zipfile__(zfile: ZipFile) -> dict:
"""Retrieve the control file from the zip file info."""
files = tuple(filename
for filename in zfile.namelist()
@@ -56,6 +129,81 @@ def control_data(zfile: ZipFile) -> dict:
else yaml.safe_load(zfile.read(files[0])))
}
+
+def __control_data_from_dirpath__(dirpath: Path):
+ """Load control data from a given directory path."""
+ files = tuple(path for path in dirpath.iterdir()
+ if (not __special_file__(path.name)
+ and (path.suffix in (".yaml", ".json"))))
+ num_files = len(files)
+ if num_files == 0:
+ raise InvalidFormat("Expected a json or yaml control file.")
+
+ if num_files > 1:
+ raise InvalidFormat("Found more than one possible control file.")
+
+ with open(files[0], "r", encoding="utf8") as infile:
+ return {
+ "na.strings": ["NA"],
+ "comment.char": "#",
+ "sep": ",",
+ **{
+ f"{key}_transposed": False for key in FILE_TYPES
+ },
+ **(json.loads(infile.read())
+ if files[0].suffix == ".json"
+ else yaml.safe_load(infile.read()))
+ }
+
+
+def control_data(control_src: Union[Path, ZipFile]) -> dict:
+ """Read the R/qtl2 bundle control file.
+
+ Parameters
+ ----------
+ control_src: Path object of ZipFile object.
+ If a directory path is provided, this function will read the control
+ data from the control file in that directory.
+ It is importand that the Path be a directory and contain data from one
+ and only one R/qtl2 bundle.
+
+ If a ZipFile object is provided, then the control data is read from the
+ control file within the zip file. We are moving away from parsing data
+ directly from ZipFile objects, and this is retained only until the
+ transition to using extracted files is complete.
+
+ Returns
+ -------
+ Returns a dict object with the control data that determines what the files
+ in the bundle are and how to parse them.
+
+ Raises
+ ------
+ r_qtl.exceptions.InvalidFormat
+ """
+ def __cleanup__(cdata):
+ return {
+ **cdata,
+ **dict((filetype,
+ ([cdata[filetype]] if isinstance(cdata[filetype], str)
+ else cdata[filetype])
+ ) for filetype in
+ (typ for typ in cdata.keys() if typ in FILE_TYPES))
+ }
+
+ if isinstance(control_src, ZipFile):
+ return __cleanup__(__control_data_from_zipfile__(control_src))
+ if isinstance(control_src, Path):
+ if is_zipfile(control_src):
+ return __cleanup__(
+ __control_data_from_zipfile__(ZipFile(control_src)))
+ if control_src.is_dir():
+ return __cleanup__(__control_data_from_dirpath__(control_src))
+ raise InvalidFormat(
+ "Expects either a zipfile.ZipFile object or a pathlib.Path object "
+ "pointing to a directory containing the R/qtl2 bundle.")
+
+
def replace_na_strings(cdata, val):
"""Replace values indicated in `na.strings` with `None`."""
return (None if val in cdata.get("na.strings", ["NA"]) else val)
@@ -267,7 +415,7 @@ def file_data(zfile: ZipFile,
zfile, member_key, cdata, process_transposed_value):
yield row
except KeyError as exc:
- raise MissingFileError(*exc.args) from exc
+ raise MissingFileException(*exc.args) from exc
def cross_information(zfile: ZipFile, cdata: dict) -> Iterator[dict]:
"""Load cross information where present."""
diff --git a/r_qtl/r_qtl2_qc.py b/r_qtl/r_qtl2_qc.py
index be1eac4..6f7b374 100644
--- a/r_qtl/r_qtl2_qc.py
+++ b/r_qtl/r_qtl2_qc.py
@@ -1,10 +1,11 @@
"""Quality control checks for R/qtl2 data bundles."""
-from zipfile import ZipFile
+from pathlib import Path
from functools import reduce, partial
+from zipfile import ZipFile, is_zipfile
from typing import Union, Iterator, Optional, Callable
-from r_qtl import errors as rqe
from r_qtl import r_qtl2 as rqtl2
+from r_qtl import exceptions as rqe
from r_qtl.r_qtl2 import FILE_TYPES
from r_qtl.fileerrors import MissingFile
@@ -39,11 +40,10 @@ def bundle_files_list(cdata: dict) -> tuple[tuple[str, str], ...]:
return fileslist
-def missing_files(zfile: ZipFile) -> tuple[tuple[str, str], ...]:
- """
- Retrieve a list of files listed in the control file that do not exist in the
- bundle.
- """
+
+def __missing_from_zipfile__(
+ zfile: ZipFile, cdata: dict) -> tuple[tuple[str, str], ...]:
+ """Check for missing files from a still-compressed zip file."""
def __missing_p__(filedetails: tuple[str, str]):
_cfkey, thefile = filedetails
try:
@@ -52,14 +52,53 @@ def missing_files(zfile: ZipFile) -> tuple[tuple[str, str], ...]:
except KeyError:
return True
- return tuple(afile for afile in bundle_files_list(rqtl2.control_data(zfile))
+ return tuple(afile for afile in bundle_files_list(cdata)
if __missing_p__(afile))
+
+def __missing_from_dirpath__(
+ dirpath: Path, cdata: dict) -> tuple[tuple[str, str], ...]:
+ """Check for missing files from an extracted bundle."""
+ allfiles = tuple(_file.name for _file in dirpath.iterdir())
+ return tuple(afile for afile in bundle_files_list(cdata)
+ if afile[1] not in allfiles)
+
+
+def missing_files(bundlesrc: Union[Path, ZipFile]) -> tuple[tuple[str, str], ...]:
+ """
+ Retrieve a list of files listed in the control file that do not exist in the
+ bundle.
+
+ Parameters
+ ----------
+ bundlesrc: Path object of ZipFile object: This is the bundle under check.
+
+ Returns
+ -------
+ A tuple of names listed in the control file that do not exist in the bundle.
+
+ Raises
+ ------
+ r_qtl.exceptions.InvalidFormat
+ """
+ cdata = rqtl2.control_data(bundlesrc)
+ if isinstance(bundlesrc, ZipFile):
+ return __missing_from_zipfile__(bundlesrc, cdata)
+ if isinstance(bundlesrc, Path):
+ if is_zipfile(bundlesrc):
+ return __missing_from_zipfile__(ZipFile(bundlesrc, cdata))
+ if bundlesrc.is_dir():
+ return __missing_from_dirpath__(bundlesrc, cdata)
+ raise InvalidFormat(
+ "Expects either a zipfile.ZipFile object or a pathlib.Path object "
+ "pointing to a directory containing the R/qtl2 bundle.")
+
+
def validate_bundle(zfile: ZipFile):
"""Ensure the R/qtl2 bundle is valid."""
missing = missing_files(zfile)
if len(missing) > 0:
- raise rqe.MissingFileError(
+ raise rqe.MissingFileException(
"The following files do not exist in the bundle: " +
", ".join(mfile[1] for mfile in missing))
@@ -111,6 +150,6 @@ def retrieve_errors(zfile: ZipFile, filetype: str, checkers: tuple[Callable]) ->
if value is not None:
for checker in checkers:
yield checker(lineno=lineno, field=field, value=value)
- except rqe.MissingFileError:
+ except rqe.MissingFileException:
fname = cdata.get(filetype)
yield MissingFile(filetype, fname, f"Missing '{filetype}' file '{fname}'.")
diff --git a/scripts/insert_data.py b/scripts/insert_data.py
index 1465348..4b2e5f3 100644
--- a/scripts/insert_data.py
+++ b/scripts/insert_data.py
@@ -14,8 +14,8 @@ from MySQLdb.cursors import DictCursor
from functional_tools import take
from quality_control.file_utils import open_file
-from qc_app.db_utils import database_connection
-from qc_app.check_connections import check_db, check_redis
+from uploader.db_utils import database_connection
+from uploader.check_connections import check_db, check_redis
# Set up logging
stderr_handler = logging.StreamHandler(stream=sys.stderr)
diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py
index 8431462..87f29dc 100644
--- a/scripts/insert_samples.py
+++ b/scripts/insert_samples.py
@@ -7,10 +7,10 @@ import argparse
import MySQLdb as mdb
from redis import Redis
-from qc_app.db_utils import database_connection
-from qc_app.check_connections import check_db, check_redis
-from qc_app.db import species_by_id, population_by_id
-from qc_app.samples import (
+from uploader.db_utils import database_connection
+from uploader.check_connections import check_db, check_redis
+from uploader.db import species_by_id, population_by_id
+from uploader.samples import (
save_samples_data,
read_samples_file,
cross_reference_samples)
diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py
index 4da3936..20cfd3b 100644
--- a/scripts/process_rqtl2_bundle.py
+++ b/scripts/process_rqtl2_bundle.py
@@ -13,13 +13,13 @@ from redis import Redis
from functional_tools import take
-import r_qtl.errors as rqe
import r_qtl.r_qtl2 as rqtl2
import r_qtl.r_qtl2_qc as rqc
+import r_qtl.exceptions as rqe
-from qc_app import jobs
-from qc_app.db_utils import database_connection
-from qc_app.check_connections import check_db, check_redis
+from uploader import jobs
+from uploader.db_utils import database_connection
+from uploader.check_connections import check_db, check_redis
from scripts.cli_parser import init_cli_parser
from scripts.redis_logger import setup_redis_logger
diff --git a/scripts/qc.py b/scripts/qc.py
index e8573a9..6de051f 100644
--- a/scripts/qc.py
+++ b/scripts/qc.py
@@ -11,7 +11,7 @@ from quality_control.utils import make_progress_calculator
from quality_control.errors import InvalidValue, DuplicateHeading
from quality_control.parsing import FileType, strain_names, collect_errors
-from qc_app.db_utils import database_connection
+from uploader.db_utils import database_connection
from .cli_parser import init_cli_parser
diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py
index 40809b7..fc95d13 100644
--- a/scripts/qc_on_rqtl2_bundle.py
+++ b/scripts/qc_on_rqtl2_bundle.py
@@ -16,13 +16,13 @@ from redis import Redis
from quality_control.errors import InvalidValue
from quality_control.checks import decimal_points_error
-from qc_app import jobs
-from qc_app.db_utils import database_connection
-from qc_app.check_connections import check_db, check_redis
+from uploader import jobs
+from uploader.db_utils import database_connection
+from uploader.check_connections import check_db, check_redis
-from r_qtl import errors as rqe
from r_qtl import r_qtl2 as rqtl2
from r_qtl import r_qtl2_qc as rqc
+from r_qtl import exceptions as rqe
from r_qtl import fileerrors as rqfe
from scripts.process_rqtl2_bundle import parse_job
@@ -105,7 +105,7 @@ def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals
__update_processed__(value)
rconn.hset(fqjobid, f"{filetype}-linecount", count)
- except rqe.MissingFileError:
+ except rqe.MissingFileException:
fname = cdata.get(filetype)
yield rqfe.MissingFile(filetype, fname, (
f"The file '{fname}' does not exist in the bundle despite it being "
@@ -133,7 +133,7 @@ def qc_geno_errors(rconn, fqjobid, _dburi, _speciesid, zfile, logger) -> bool:
def fetch_db_geno_samples(conn: mdb.Connection, speciesid: int) -> tuple[str, ...]:
"""Fetch samples/cases/individuals from the database."""
- samples = set()
+ samples = set()# type: ignore[var-annotated]
with conn.cursor() as cursor:
cursor.execute("SELECT Name, Name2 from Strain WHERE SpeciesId=%s",
(speciesid,))
@@ -191,12 +191,13 @@ def check_pheno_samples(
return allerrors
-def qc_pheno_errors(rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool:
+def qc_pheno_errors(# pylint: disable=[too-many-arguments]
+ rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool:
"""Check for errors in `pheno` file(s)."""
cdata = rqtl2.control_data(zfile)
if "pheno" in cdata:
logger.info("Checking for errors in the 'pheno' file…")
- perrs = tuple()
+ perrs = tuple()# type: ignore[var-annotated]
with database_connection(dburi) as dbconn:
perrs = check_pheno_samples(
dbconn, speciesid, zfile.filename, logger) + tuple(
@@ -216,7 +217,8 @@ def qc_pheno_errors(rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool:
return False
-def qc_phenose_errors(rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool:
+def qc_phenose_errors(# pylint: disable=[too-many-arguments]
+ rconn, fqjobid, _dburi, _speciesid, zfile, logger) -> bool:
"""Check for errors in `phenose` file(s)."""
cdata = rqtl2.control_data(zfile)
if "phenose" in cdata:
@@ -258,7 +260,9 @@ def run_qc(rconn: Redis,
if qc_missing_files(rconn, fqjobid, zfile, logger):
return 1
- def with_zipfile(rconn, fqjobid, dbconn, speciesid, filename, logger, func):
+ def with_zipfile(# pylint: disable=[too-many-arguments]
+ rconn, fqjobid, dbconn, speciesid, filename, logger, func
+ ):
with ZipFile(filename, "r") as zfile:
return func(rconn, fqjobid, dbconn, speciesid, zfile, logger)
diff --git a/scripts/qcapp_wsgi.py b/scripts/qcapp_wsgi.py
index 349c006..fe77031 100644
--- a/scripts/qcapp_wsgi.py
+++ b/scripts/qcapp_wsgi.py
@@ -5,8 +5,8 @@ from logging import getLogger, StreamHandler
from flask import Flask
-from qc_app import create_app
-from qc_app.check_connections import check_db, check_redis
+from uploader import create_app
+from uploader.check_connections import check_db, check_redis
def setup_logging(appl: Flask) -> Flask:
"""Setup appropriate logging paradigm depending on environment."""
diff --git a/scripts/rqtl2/entry.py b/scripts/rqtl2/entry.py
index 93fc130..b7fb68e 100644
--- a/scripts/rqtl2/entry.py
+++ b/scripts/rqtl2/entry.py
@@ -6,9 +6,9 @@ from argparse import Namespace
from redis import Redis
from MySQLdb import Connection
-from qc_app import jobs
-from qc_app.db_utils import database_connection
-from qc_app.check_connections import check_db, check_redis
+from uploader import jobs
+from uploader.db_utils import database_connection
+from uploader.check_connections import check_db, check_redis
from scripts.redis_logger import setup_redis_logger
diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py
index 9f8bf03..6b89142 100644
--- a/scripts/rqtl2/install_genotypes.py
+++ b/scripts/rqtl2/install_genotypes.py
@@ -19,10 +19,13 @@ from scripts.rqtl2.entry import build_main
from scripts.rqtl2.cli_parser import add_common_arguments
from scripts.cli_parser import init_cli_parser, add_global_data_arguments
-def insert_markers(dbconn: mdb.Connection,
- speciesid: int,
- markers: tuple[str, ...],
- pmapdata: Optional[Iterator[dict]]) -> int:
+def insert_markers(
+ dbconn: mdb.Connection,
+ speciesid: int,
+ markers: tuple[str, ...],
+ pmapdata: Optional[Iterator[dict]],
+ _logger: Logger
+) -> int:
"""Insert genotype and genotype values into the database."""
mdata = reduce(#type: ignore[var-annotated]
lambda acc, row: ({#type: ignore[arg-type, return-value]
@@ -48,9 +51,12 @@ def insert_markers(dbconn: mdb.Connection,
} for marker in markers}.values()))
return cursor.rowcount
-def insert_individuals(dbconn: mdb.Connection,
- speciesid: int,
- individuals: tuple[str, ...]) -> int:
+def insert_individuals(
+ dbconn: mdb.Connection,
+ speciesid: int,
+ individuals: tuple[str, ...],
+ _logger: Logger
+) -> int:
"""Insert individuals/samples into the database."""
with dbconn.cursor() as cursor:
cursor.executemany(
@@ -61,10 +67,13 @@ def insert_individuals(dbconn: mdb.Connection,
for individual in individuals))
return cursor.rowcount
-def cross_reference_individuals(dbconn: mdb.Connection,
- speciesid: int,
- populationid: int,
- individuals: tuple[str, ...]) -> int:
+def cross_reference_individuals(
+ dbconn: mdb.Connection,
+ speciesid: int,
+ populationid: int,
+ individuals: tuple[str, ...],
+ _logger: Logger
+) -> int:
"""Cross reference any inserted individuals."""
with dbconn.cursor(cursorclass=DictCursor) as cursor:
paramstr = ", ".join(["%s"] * len(individuals))
@@ -80,11 +89,13 @@ def cross_reference_individuals(dbconn: mdb.Connection,
tuple(ids))
return cursor.rowcount
-def insert_genotype_data(dbconn: mdb.Connection,
- speciesid: int,
- genotypes: tuple[dict, ...],
- individuals: tuple[str, ...]) -> tuple[
- int, tuple[dict, ...]]:
+def insert_genotype_data(
+ dbconn: mdb.Connection,
+ speciesid: int,
+ genotypes: tuple[dict, ...],
+ individuals: tuple[str, ...],
+ _logger: Logger
+) -> tuple[int, tuple[dict, ...]]:
"""Insert the genotype data values into the database."""
with dbconn.cursor(cursorclass=DictCursor) as cursor:
paramstr = ", ".join(["%s"] * len(individuals))
@@ -120,11 +131,14 @@ def insert_genotype_data(dbconn: mdb.Connection,
"markerid": row["markerid"]
} for row in data)
-def cross_reference_genotypes(dbconn: mdb.Connection,
- speciesid: int,
- datasetid: int,
- dataids: tuple[dict, ...],
- gmapdata: Optional[Iterator[dict]]) -> int:
+def cross_reference_genotypes(
+ dbconn: mdb.Connection,
+ speciesid: int,
+ datasetid: int,
+ dataids: tuple[dict, ...],
+ gmapdata: Optional[Iterator[dict]],
+ _logger: Logger
+) -> int:
"""Cross-reference the data to the relevant dataset."""
_rows, markers, mdata = reduce(#type: ignore[var-annotated]
lambda acc, row: (#type: ignore[return-value,arg-type]
@@ -140,30 +154,43 @@ def cross_reference_genotypes(dbconn: mdb.Connection,
(tuple(), tuple(), {}))
with dbconn.cursor(cursorclass=DictCursor) as cursor:
- paramstr = ", ".join(["%s"] * len(markers))
- cursor.execute("SELECT Id, Name FROM Geno "
- f"WHERE SpeciesId=%s AND Name IN ({paramstr})",
- (speciesid,) + markers)
- markersdict = {row["Id"]: row["Name"] for row in cursor.fetchall()}
- cursor.executemany(
+ markersdict = {}
+ if len(markers) > 0:
+ paramstr = ", ".join(["%s"] * len(markers))
+ insertparams = (speciesid,) + markers
+ selectquery = ("SELECT Id, Name FROM Geno "
+ f"WHERE SpeciesId=%s AND Name IN ({paramstr})")
+ _logger.debug(
+ "The select query was\n\t%s\n\nwith the parameters\n\t%s",
+ selectquery,
+ (speciesid,) + markers)
+ cursor.execute(selectquery, insertparams)
+ markersdict = {row["Id"]: row["Name"] for row in cursor.fetchall()}
+
+ insertquery = (
"INSERT INTO GenoXRef(GenoFreezeId, GenoId, DataId, cM) "
"VALUES(%(datasetid)s, %(markerid)s, %(dataid)s, %(pos)s) "
- "ON DUPLICATE KEY UPDATE GenoFreezeId=GenoFreezeId",
- tuple({
- **row,
- "datasetid": datasetid,
- "pos": mdata.get(markersdict.get(
- row.get("markerid"), {}), {}).get("pos")
- } for row in dataids))
+ "ON DUPLICATE KEY UPDATE GenoFreezeId=GenoFreezeId")
+ insertparams = tuple({
+ **row,
+ "datasetid": datasetid,
+ "pos": mdata.get(markersdict.get(
+ row.get("markerid"), "nosuchkey"), {}).get("pos")
+ } for row in dataids)
+ _logger.debug(
+ "The insert query was\n\t%s\n\nwith the parameters\n\t%s",
+ insertquery, insertparams)
+ cursor.executemany(insertquery, insertparams)
return cursor.rowcount
def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals]
dbconn: mdb.Connection,
- speciesid: int,
- populationid: int,
- datasetid: int,
- rqtl2bundle: Path,
- logger: Logger = getLogger()) -> int:
+ speciesid: int,
+ populationid: int,
+ datasetid: int,
+ rqtl2bundle: Path,
+ logger: Logger = getLogger(__name__)
+) -> int:
"""Load any existing genotypes into the database."""
count = 0
with ZipFile(str(rqtl2bundle.absolute()), "r") as zfile:
@@ -189,20 +216,22 @@ def install_genotypes(#pylint: disable=[too-many-arguments, too-many-locals]
speciesid,
tuple(key for key in batch[0].keys() if key != "id"),
(rqtl2.file_data(zfile, "pmap", cdata) if "pmap" in cdata
- else None))
+ else None),
+ logger)
individuals = tuple(row["id"] for row in batch)
- insert_individuals(dbconn, speciesid, individuals)
+ insert_individuals(dbconn, speciesid, individuals, logger)
cross_reference_individuals(
- dbconn, speciesid, populationid, individuals)
+ dbconn, speciesid, populationid, individuals, logger)
_num_rows, dataids = insert_genotype_data(
- dbconn, speciesid, batch, individuals)
+ dbconn, speciesid, batch, individuals, logger)
cross_reference_genotypes(
dbconn,
speciesid,
datasetid,
dataids,
(rqtl2.file_data(zfile, "gmap", cdata)
- if "gmap" in cdata else None))
+ if "gmap" in cdata else None),
+ logger)
count = count + len(batch)
except rqtl2.InvalidFormat as exc:
logger.error(str(exc))
diff --git a/scripts/validate_file.py b/scripts/validate_file.py
index 0028795..a40d7e7 100644
--- a/scripts/validate_file.py
+++ b/scripts/validate_file.py
@@ -12,8 +12,8 @@ from redis.exceptions import ConnectionError # pylint: disable=[redefined-builti
from quality_control.utils import make_progress_calculator
from quality_control.parsing import FileType, strain_names, collect_errors
-from qc_app import jobs
-from qc_app.db_utils import database_connection
+from uploader import jobs
+from uploader.db_utils import database_connection
from .cli_parser import init_cli_parser
from .qc import add_file_validation_arguments
diff --git a/scripts/worker.py b/scripts/worker.py
index 0eb9ea5..91b0332 100644
--- a/scripts/worker.py
+++ b/scripts/worker.py
@@ -11,8 +11,8 @@ from tempfile import TemporaryDirectory
from redis import Redis
-from qc_app import jobs
-from qc_app.check_connections import check_redis
+from uploader import jobs
+from uploader.check_connections import check_redis
def parse_args():
"Parse the command-line arguments"
diff --git a/tests/conftest.py b/tests/conftest.py
index a39acf0..9012221 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -11,8 +11,8 @@ from redis import Redis
from functional_tools import take
-from qc_app import jobs, create_app
-from qc_app.jobs import JOBS_PREFIX
+from uploader import jobs, create_app
+from uploader.jobs import JOBS_PREFIX
from quality_control.errors import InvalidValue, DuplicateHeading
diff --git a/tests/qc_app/test_parse.py b/tests/qc_app/test_parse.py
index 3915a4d..076c47c 100644
--- a/tests/qc_app/test_parse.py
+++ b/tests/qc_app/test_parse.py
@@ -4,7 +4,7 @@ import sys
import redis
import pytest
-from qc_app.jobs import job, jobsnamespace
+from uploader.jobs import job, jobsnamespace
from tests.conftest import uploadable_file_object
@@ -24,7 +24,7 @@ def test_parse_with_existing_uploaded_file(#pylint: disable=[too-many-arguments]
1. the system redirects to the job/parse status page
2. the job is placed on redis for processing
"""
- monkeypatch.setattr("qc_app.jobs.uuid4", lambda : job_id)
+ monkeypatch.setattr("uploader.jobs.uuid4", lambda : job_id)
# Upload a file
speciesid = 1
filename = "no_data_errors.tsv"
diff --git a/uploader/__init__.py b/uploader/__init__.py
new file mode 100644
index 0000000..2d731af
--- /dev/null
+++ b/uploader/__init__.py
@@ -0,0 +1,94 @@
+"""The Quality-Control Web Application entry point"""
+import os
+import sys
+import logging
+from pathlib import Path
+
+from flask import Flask, request
+from flask_session import Session
+
+from uploader.oauth2.client import user_logged_in, authserver_authorise_uri
+
+from .entry import entrybp
+from .upload import upload
+from .parse import parsebp
+from .samples import samples
+from .base_routes import base
+from .dbinsert import dbinsertbp
+from .oauth2.views import oauth2
+from .errors import register_error_handlers
+
+def override_settings_with_envvars(
+ app: Flask, ignore: tuple[str, ...]=tuple()) -> None:
+ """Override settings in `app` with those in ENVVARS"""
+ for setting in (key for key in app.config if key not in ignore):
+ app.config[setting] = os.environ.get(setting) or app.config[setting]
+
+
+def __log_gunicorn__(app: Flask) -> Flask:
+ """Set up logging for the WSGI environment with GUnicorn"""
+ logger = logging.getLogger("gunicorn.error")
+ app.logger.handlers = logger.handlers
+ app.logger.setLevel(logger.level)
+ return app
+
+
+def __log_dev__(app: Flask) -> Flask:
+ """Set up logging for the development environment."""
+ stderr_handler = logging.StreamHandler(stream=sys.stderr)
+ app.logger.addHandler(stderr_handler)
+
+ root_logger = logging.getLogger()
+ root_logger.addHandler(stderr_handler)
+ root_logger.setLevel(app.config["LOG_LEVEL"])
+
+ return app
+
+
+def setup_logging(app: Flask) -> Flask:
+ """Set up logging for the application."""
+ software, *_version_and_comments = os.environ.get(
+ "SERVER_SOFTWARE", "").split('/')
+ return __log_gunicorn__(app) if bool(software) else __log_dev__(app)
+
+
+def create_app():
+ """The application factory"""
+ app = Flask(__name__)
+ app.config.from_pyfile(
+ Path(__file__).parent.joinpath("default_settings.py"))
+ if "UPLOADER_CONF" in os.environ:
+ app.config.from_envvar("UPLOADER_CONF") # Override defaults with instance path
+
+ override_settings_with_envvars(app, ignore=tuple())
+
+ secretsfile = app.config.get("UPLOADER_SECRETS", "").strip()
+ if bool(secretsfile):
+ secretsfile = Path(secretsfile).absolute()
+ app.config["UPLOADER_SECRETS"] = secretsfile
+ if secretsfile.exists():
+ # Silently ignore secrets if the file does not exist.
+ app.config.from_pyfile(secretsfile)
+
+ setup_logging(app)
+
+ # setup jinja2 symbols
+ app.add_template_global(lambda : request.url, name="request_url")
+ app.add_template_global(authserver_authorise_uri)
+ app.add_template_global(lambda: app.config["GN2_SERVER_URL"],
+ name="gn2server_uri")
+ app.add_template_global(user_logged_in)
+
+ Session(app)
+
+ # setup blueprints
+ app.register_blueprint(base, url_prefix="/")
+ app.register_blueprint(entrybp, url_prefix="/")
+ app.register_blueprint(parsebp, url_prefix="/parse")
+ app.register_blueprint(oauth2, url_prefix="/oauth2")
+ app.register_blueprint(upload, url_prefix="/upload")
+ app.register_blueprint(dbinsertbp, url_prefix="/dbinsert")
+ app.register_blueprint(samples, url_prefix="/samples")
+
+ register_error_handlers(app)
+ return app
diff --git a/uploader/authorisation.py b/uploader/authorisation.py
new file mode 100644
index 0000000..8ab83f8
--- /dev/null
+++ b/uploader/authorisation.py
@@ -0,0 +1,21 @@
+"""Authorisation utilities."""
+from functools import wraps
+
+from flask import flash, redirect
+
+from uploader import session
+
+def require_login(function):
+ """Check that the user is logged in before executing `func`."""
+ @wraps(function)
+ def __is_session_valid__(*args, **kwargs):
+ """Check that the user is logged in and their token is valid."""
+ def __clear_session__(_no_token):
+ session.clear_session_info()
+ flash("You need to be logged in.", "alert-danger")
+ return redirect("/")
+
+ return session.user_token().either(
+ __clear_session__,
+ lambda token: function(*args, **kwargs))
+ return __is_session_valid__
diff --git a/qc_app/base_routes.py b/uploader/base_routes.py
index 9daf439..9daf439 100644
--- a/qc_app/base_routes.py
+++ b/uploader/base_routes.py
diff --git a/qc_app/check_connections.py b/uploader/check_connections.py
index ceccc32..2561e55 100644
--- a/qc_app/check_connections.py
+++ b/uploader/check_connections.py
@@ -5,7 +5,7 @@ import traceback
import redis
import MySQLdb
-from qc_app.db_utils import database_connection
+from uploader.db_utils import database_connection
def check_redis(uri: str):
"Check the redis connection"
diff --git a/qc_app/db/__init__.py b/uploader/db/__init__.py
index 36e93e8..36e93e8 100644
--- a/qc_app/db/__init__.py
+++ b/uploader/db/__init__.py
diff --git a/qc_app/db/averaging.py b/uploader/db/averaging.py
index 62bbe67..62bbe67 100644
--- a/qc_app/db/averaging.py
+++ b/uploader/db/averaging.py
diff --git a/qc_app/db/datasets.py b/uploader/db/datasets.py
index 767ec41..767ec41 100644
--- a/qc_app/db/datasets.py
+++ b/uploader/db/datasets.py
diff --git a/qc_app/db/platforms.py b/uploader/db/platforms.py
index cb527a7..cb527a7 100644
--- a/qc_app/db/platforms.py
+++ b/uploader/db/platforms.py
diff --git a/qc_app/db/populations.py b/uploader/db/populations.py
index 4485e52..4485e52 100644
--- a/qc_app/db/populations.py
+++ b/uploader/db/populations.py
diff --git a/qc_app/db/species.py b/uploader/db/species.py
index 653e59b..653e59b 100644
--- a/qc_app/db/species.py
+++ b/uploader/db/species.py
diff --git a/qc_app/db/tissues.py b/uploader/db/tissues.py
index 9fe7bab..9fe7bab 100644
--- a/qc_app/db/tissues.py
+++ b/uploader/db/tissues.py
diff --git a/qc_app/db_utils.py b/uploader/db_utils.py
index ef26398..5b79762 100644
--- a/qc_app/db_utils.py
+++ b/uploader/db_utils.py
@@ -3,7 +3,7 @@ import logging
import traceback
import contextlib
from urllib.parse import urlparse
-from typing import Any, Tuple, Optional, Iterator, Callable
+from typing import Any, Tuple, Iterator, Callable
import MySQLdb as mdb
from redis import Redis
@@ -19,10 +19,9 @@ def parse_db_url(db_url) -> Tuple:
@contextlib.contextmanager
-def database_connection(db_url: Optional[str] = None) -> Iterator[mdb.Connection]:
+def database_connection(db_url: str) -> Iterator[mdb.Connection]:
"""function to create db connector"""
- host, user, passwd, db_name, db_port = parse_db_url(
- db_url or app.config["SQL_URI"])
+ host, user, passwd, db_name, db_port = parse_db_url(db_url)
connection = mdb.connect(
host, user, passwd, db_name, port=(db_port or 3306))
try:
diff --git a/qc_app/dbinsert.py b/uploader/dbinsert.py
index ef08423..559dc5e 100644
--- a/qc_app/dbinsert.py
+++ b/uploader/dbinsert.py
@@ -11,8 +11,9 @@ from flask import (
flash, request, url_for, Blueprint, redirect, render_template,
current_app as app)
-from qc_app.db_utils import with_db_connection, database_connection
-from qc_app.db import species, species_by_id, populations_by_species
+from uploader.authorisation import require_login
+from uploader.db_utils import with_db_connection, database_connection
+from uploader.db import species, species_by_id, populations_by_species
from . import jobs
@@ -40,7 +41,7 @@ def genechips():
return {**acc, speciesid: (chip,)}
return {**acc, speciesid: acc[speciesid] + (chip,)}
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute("SELECT * FROM GeneChip ORDER BY GeneChipName ASC")
return reduce(__organise_by_species__, cursor.fetchall(), {})
@@ -49,7 +50,7 @@ def genechips():
def platform_by_id(genechipid:int) -> Union[dict, None]:
"Retrieve the gene platform by id"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(
"SELECT * FROM GeneChip WHERE GeneChipId=%s",
@@ -58,7 +59,7 @@ def platform_by_id(genechipid:int) -> Union[dict, None]:
def studies_by_species_and_platform(speciesid:int, genechipid:int) -> tuple:
"Retrieve the studies by the related species and gene platform"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
query = (
"SELECT Species.SpeciesId, ProbeFreeze.* "
@@ -82,7 +83,7 @@ def organise_groups_by_family(acc:dict, group:dict) -> dict:
def tissues() -> tuple:
"Retrieve type (Tissue) information from the database."
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute("SELECT * FROM Tissue ORDER BY Name")
return tuple(cursor.fetchall())
@@ -90,6 +91,7 @@ def tissues() -> tuple:
return tuple()
@dbinsertbp.route("/platform", methods=["POST"])
+@require_login
def select_platform():
"Select the platform (GeneChipId) used for the data."
job_id = request.form["job_id"]
@@ -113,6 +115,7 @@ def select_platform():
return render_error("Unknown error")
@dbinsertbp.route("/study", methods=["POST"])
+@require_login
def select_study():
"View to select/create the study (ProbeFreeze) associated with the data."
form = request.form
@@ -142,6 +145,7 @@ def select_study():
return render_error(f"Missing data: {aserr.args[0]}")
@dbinsertbp.route("/create-study", methods=["POST"])
+@require_login
def create_study():
"Create a new study (ProbeFreeze)."
form = request.form
@@ -154,7 +158,7 @@ def create_study():
assert form.get("inbredsetid"), "group"
assert form.get("tissueid"), "type/tissue"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
values = (
form["genechipid"],
@@ -186,7 +190,7 @@ def create_study():
def datasets_by_study(studyid:int) -> tuple:
"Retrieve datasets associated with a study with the ID `studyid`."
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
query = "SELECT * FROM ProbeSetFreeze WHERE ProbeFreezeId=%s"
cursor.execute(query, (studyid,))
@@ -196,7 +200,7 @@ def datasets_by_study(studyid:int) -> tuple:
def averaging_methods() -> tuple:
"Retrieve averaging methods from database"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute("SELECT * FROM AvgMethod")
return tuple(cursor.fetchall())
@@ -205,7 +209,7 @@ def averaging_methods() -> tuple:
def dataset_datascales() -> tuple:
"Retrieve datascales from database"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor() as cursor:
cursor.execute(
'SELECT DISTINCT DataScale FROM ProbeSetFreeze '
@@ -218,6 +222,7 @@ def dataset_datascales() -> tuple:
return tuple()
@dbinsertbp.route("/dataset", methods=["POST"])
+@require_login
def select_dataset():
"Select the dataset to add the file contents against"
form = request.form
@@ -238,6 +243,7 @@ def select_dataset():
return render_error(f"Missing data: {aserr.args[0]}")
@dbinsertbp.route("/create-dataset", methods=["POST"])
+@require_login
def create_dataset():
"Select the dataset to add the file contents against"
form = request.form
@@ -255,7 +261,7 @@ def create_dataset():
assert form.get("datasetconfidentiality"), "Dataset confidentiality"
assert form.get("datasetdatascale"), "Dataset Datascale"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
datasetname = form["datasetname"]
cursor.execute("SELECT * FROM ProbeSetFreeze WHERE Name=%s",
@@ -293,7 +299,7 @@ def create_dataset():
def study_by_id(studyid:int) -> Union[dict, None]:
"Get a study by its Id"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(
"SELECT * FROM ProbeFreeze WHERE Id=%s",
@@ -302,7 +308,7 @@ def study_by_id(studyid:int) -> Union[dict, None]:
def dataset_by_id(datasetid:int) -> Union[dict, None]:
"Retrieve a dataset by its id"
- with database_connection() as conn:
+ with database_connection(app.config["SQL_URI"]) as conn:
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(
("SELECT AvgMethod.Name AS AvgMethodName, ProbeSetFreeze.* "
@@ -317,6 +323,7 @@ def selected_keys(original: dict, keys: tuple) -> dict:
return {key: value for key,value in original.items() if key in keys}
@dbinsertbp.route("/final-confirmation", methods=["POST"])
+@require_login
def final_confirmation():
"Preview the data before triggering entry into the database"
form = request.form
@@ -352,6 +359,7 @@ def final_confirmation():
return render_error(f"Missing data: {aserr.args[0]}")
@dbinsertbp.route("/insert-data", methods=["POST"])
+@require_login
def insert_data():
"Trigger data insertion"
form = request.form
diff --git a/uploader/default_settings.py b/uploader/default_settings.py
new file mode 100644
index 0000000..26fe665
--- /dev/null
+++ b/uploader/default_settings.py
@@ -0,0 +1,23 @@
+"""
+The default configuration file. The values here should be overridden in the
+actual configuration file used for the production and staging systems.
+"""
+
+import os
+
+LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
+SECRET_KEY = b"<Please! Please! Please! Change This!>"
+UPLOAD_FOLDER = "/tmp/qc_app_files"
+REDIS_URL = "redis://"
+JOBS_TTL_SECONDS = 1209600 # 14 days
+GNQC_REDIS_PREFIX="GNQC"
+SQL_URI = ""
+
+GN2_SERVER_URL = "https://genenetwork.org/"
+
+SESSION_TYPE = "redis"
+SESSION_PERMANENT = True
+SESSION_USE_SIGNER = True
+
+JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use.
+JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it.
diff --git a/qc_app/entry.py b/uploader/entry.py
index f2db878..82034ed 100644
--- a/qc_app/entry.py
+++ b/uploader/entry.py
@@ -15,8 +15,10 @@ from flask import (
current_app as app,
send_from_directory)
-from qc_app.db import species
-from qc_app.db_utils import with_db_connection
+from uploader.db import species
+from uploader.authorisation import require_login
+from uploader.db_utils import with_db_connection
+from uploader.oauth2.client import user_logged_in
entrybp = Blueprint("entry", __name__)
@@ -87,9 +89,10 @@ def zip_file_errors(filepath, upload_dir) -> Tuple[str, ...]:
@entrybp.route("/", methods=["GET"])
def index():
"""Load the landing page"""
- return render_template("index.html")
+ return render_template("index.html" if user_logged_in() else "login.html")
@entrybp.route("/upload", methods=["GET", "POST"])
+@require_login
def upload_file():
"""Enables uploading the files"""
if request.method == "GET":
@@ -122,6 +125,7 @@ def upload_file():
filetype=request.form["filetype"]))
@entrybp.route("/data-review", methods=["GET"])
+@require_login
def data_review():
"""Provide some help on data expectations to the user."""
return render_template("data_review.html")
diff --git a/qc_app/errors.py b/uploader/errors.py
index 3e7c893..3e7c893 100644
--- a/qc_app/errors.py
+++ b/uploader/errors.py
diff --git a/qc_app/files.py b/uploader/files.py
index b163612..b163612 100644
--- a/qc_app/files.py
+++ b/uploader/files.py
diff --git a/qc_app/input_validation.py b/uploader/input_validation.py
index 9abe742..9abe742 100644
--- a/qc_app/input_validation.py
+++ b/uploader/input_validation.py
diff --git a/qc_app/jobs.py b/uploader/jobs.py
index 21889da..21889da 100644
--- a/qc_app/jobs.py
+++ b/uploader/jobs.py
diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py
new file mode 100644
index 0000000..aa34951
--- /dev/null
+++ b/uploader/monadic_requests.py
@@ -0,0 +1,86 @@
+"""Wrap requests functions with monads."""
+import traceback
+from typing import Union, Optional, Callable
+
+import requests
+from requests.models import Response
+from pymonad.either import Left, Right, Either
+from flask import (
+ flash,
+ request,
+ redirect,
+ render_template,
+ current_app as app,
+ escape as flask_escape)
+
+# HTML Status codes indicating a successful request.
+SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226)
+
+# Possible error(s) that can be encontered while attempting to do a request.
+PossibleError = Union[Response, Exception]
+
+
+def make_error_handler(
+ redirect_to: Optional[Response] = None,
+ cleanup_thunk: Callable = lambda *args: None
+) -> Callable[[PossibleError], Response]:
+ """
+ Build a function to gracefully handle errors encountered while doing
+ requests.
+
+ :rtype: Callable
+ """
+ redirect_to = redirect_to or redirect(request.url)
+ def __handler__(resp_or_exc: PossibleError) -> Response:
+ cleanup_thunk()
+ if issubclass(type(resp_or_exc), Exception):
+ # Is an exception!
+ return render_template(
+ "unhandled_exception.html",
+ trace=traceback.format_exception(resp_or_exc))
+ if isinstance(resp_or_exc, Response):
+ flash("The authorisation server responded with "
+ f"({flask_escape(resp_or_exc.status_code)}, "
+ f"{flask_escape(resp_or_exc.reason)}) for the request to "
+ f"'{flask_escape(resp_or_exc.request.url)}'",
+ "alert-danger")
+ return redirect_to
+
+ flash("Unspecified error!", "alert-danger")
+ app.logger.debug("Error (%s): %s", type(resp_or_exc), resp_or_exc)
+ return redirect_to
+ return __handler__
+
+
+def get(url, params=None, **kwargs) -> Either:
+ """
+ A wrapper around `requests.get` function.
+
+ Takes the same arguments as `requests.get`.
+
+ :rtype: pymonad.either.Either
+ """
+ try:
+ resp = requests.get(url, params=params, **kwargs)
+ if resp.status_code in SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except requests.exceptions.RequestException as exc:
+ return Left(exc)
+
+
+def post(url, data=None, json=None, **kwargs) -> Either:
+ """
+ A wrapper around `requests.post` function.
+
+ Takes the same arguments as `requests.post`.
+
+ :rtype: pymonad.either.Either
+ """
+ try:
+ resp = requests.post(url, data=data, json=json, **kwargs)
+ if resp.status_code in SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except requests.exceptions.RequestException as exc:
+ return Left(exc)
diff --git a/uploader/oauth2/__init__.py b/uploader/oauth2/__init__.py
new file mode 100644
index 0000000..aaea638
--- /dev/null
+++ b/uploader/oauth2/__init__.py
@@ -0,0 +1 @@
+"""Package to handle OAuth2 authentication/authorisation issues."""
diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py
new file mode 100644
index 0000000..a3e4ba3
--- /dev/null
+++ b/uploader/oauth2/client.py
@@ -0,0 +1,202 @@
+"""OAuth2 client utilities."""
+import json
+from datetime import datetime, timedelta
+from urllib.parse import urljoin, urlparse
+
+import requests
+from flask import request, current_app as app
+
+from pymonad.either import Left, Right, Either
+
+from authlib.jose import jwt
+from authlib.common.urls import url_decode
+from authlib.jose import KeySet, JsonWebKey
+from authlib.jose.errors import BadSignatureError
+from authlib.integrations.requests_client import OAuth2Session
+
+from uploader import session
+import uploader.monadic_requests as mrequests
+
+SCOPE = ("profile group role resource register-client user masquerade "
+ "introspect migrate-data")
+
+
+def authserver_uri():
+ """Return URI to authorisation server."""
+ return app.config["AUTH_SERVER_URL"]
+
+
+def oauth2_clientid():
+ """Return the client id."""
+ return app.config["OAUTH2_CLIENT_ID"]
+
+
+def oauth2_clientsecret():
+ """Return the client secret."""
+ return app.config["OAUTH2_CLIENT_SECRET"]
+
+
+def __make_token_validator__(keys: KeySet):
+ """Make a token validator function."""
+ def __validator__(token: dict):
+ for key in keys.keys:
+ try:
+ jwt.decode(token["access_token"], key)
+ return Right(token)
+ except BadSignatureError:
+ pass
+
+ return Left("INVALID-TOKEN")
+
+ return __validator__
+
+
+def __validate_token__(sess_info):
+ """Validate that the token is really from the auth server."""
+ info = sess_info
+ info["user"]["token"] = info["user"]["token"].then(__make_token_validator__(
+ KeySet([JsonWebKey.import_key(key) for key in info.get(
+ "auth_server_jwks", {}).get(
+ "jwks", {"keys": []})["keys"]])))
+ return session.save_session_info(info)
+
+
+def __update_auth_server_jwks__(sess_info):
+ """Updates the JWKs every 2 hours or so."""
+ jwks = sess_info.get("auth_server_jwks")
+ if bool(jwks):
+ last_updated = jwks.get("last-updated")
+ now = datetime.now().timestamp()
+ if bool(last_updated) and (now - last_updated) < timedelta(hours=2).seconds:
+ return __validate_token__({**sess_info, "auth_server_jwks": jwks})
+
+ jwksuri = urljoin(authserver_uri(), "auth/public-jwks")
+ return __validate_token__({
+ **sess_info,
+ "auth_server_jwks": {
+ "jwks": KeySet([
+ JsonWebKey.import_key(key)
+ for key in requests.get(jwksuri).json()["jwks"]]).as_dict(),
+ "last-updated": datetime.now().timestamp()
+ }
+ })
+
+
+def oauth2_client():
+ """Build the OAuth2 client for use fetching data."""
+ def __update_token__(token, refresh_token=None, access_token=None):# pylint: disable=[unused-argument]
+ """Update the token when refreshed."""
+ session.set_user_token(token)
+
+ def __json_auth__(client, _method, uri, headers, body):
+ return (
+ uri,
+ {**headers, "Content-Type": "application/json"},
+ json.dumps({
+ **dict(url_decode(body)),
+ "client_id": client.client_id,
+ "client_secret": client.client_secret
+ }))
+
+ def __client__(token) -> OAuth2Session:
+ client = OAuth2Session(
+ oauth2_clientid(),
+ oauth2_clientsecret(),
+ scope=SCOPE,
+ token_endpoint=urljoin(authserver_uri(), "/auth/token"),
+ token_endpoint_auth_method="client_secret_post",
+ token=token,
+ update_token=__update_token__)
+ client.register_client_auth_method(
+ ("client_secret_post", __json_auth__))
+ return client
+
+ __update_auth_server_jwks__(session.session_info())
+ return session.user_token().either(
+ lambda _notok: __client__(None),
+ __client__)
+
+
+def user_logged_in():
+ """Check whether the user has logged in."""
+ suser = session.session_info()["user"]
+ return suser["logged_in"] and suser["token"].is_right()
+
+
+def authserver_authorise_uri():
+ """Build up the authorisation URI."""
+ req_baseurl = urlparse(request.base_url, scheme=request.scheme)
+ host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/"
+ return urljoin(
+ authserver_uri(),
+ "auth/authorise?response_type=code"
+ f"&client_id={oauth2_clientid()}"
+ f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}")
+
+
+def __no_token__(_err) -> Left:
+ """Handle situation where request is attempted with no token."""
+ resp = requests.models.Response()
+ resp._content = json.dumps({#pylint: disable=[protected-access]
+ "error": "AuthenticationError",
+ "error-description": ("You need to authenticate to access requested "
+ "information.")}).encode("utf-8")
+ resp.status_code = 400
+ return Left(resp)
+
+
+def oauth2_get(url, **kwargs) -> Either:
+ """Do a get request to the authentication/authorisation server."""
+ def __get__(_token) -> Either:
+ _uri = urljoin(authserver_uri(), url)
+ try:
+ resp = oauth2_client().get(
+ _uri,
+ **{
+ **kwargs,
+ "headers": {
+ **kwargs.get("headers", {}),
+ "Content-Type": "application/json"
+ }
+ })
+ if resp.status_code in mrequests.SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except Exception as exc:#pylint: disable=[broad-except]
+ app.logger.error("Error retriving data from auth server: (GET %s)",
+ _uri,
+ exc_info=True)
+ return Left(exc)
+ return session.user_token().either(__no_token__, __get__)
+
+
+def oauth2_post(url, data=None, json=None, **kwargs):#pylint: disable=[redefined-outer-name]
+ """Do a POST request to the authentication/authorisation server."""
+ def __post__(_token) -> Either:
+ _uri = urljoin(authserver_uri(), url)
+ _headers = ({
+ **kwargs.get("headers", {}),
+ "Content-Type": "application/json"
+ }
+ if bool(json) else kwargs.get("headers", {}))
+ try:
+ request_data = {
+ **(data or {}),
+ **(json or {}),
+ "client_id": oauth2_clientid(),
+ "client_secret": oauth2_clientsecret()
+ }
+ resp = oauth2_client().post(
+ _uri,
+ data=(request_data if bool(data) else None),
+ json=(request_data if bool(json) else None),
+ **{**kwargs, "headers": _headers})
+ if resp.status_code in mrequests.SUCCESS_CODES:
+ return Right(resp.json())
+ return Left(resp)
+ except Exception as exc:#pylint: disable=[broad-except]
+ app.logger.error("Error retriving data from auth server: (POST %s)",
+ _uri,
+ exc_info=True)
+ return Left(exc)
+ return session.user_token().either(__no_token__, __post__)
diff --git a/uploader/oauth2/jwks.py b/uploader/oauth2/jwks.py
new file mode 100644
index 0000000..efd0499
--- /dev/null
+++ b/uploader/oauth2/jwks.py
@@ -0,0 +1,86 @@
+"""Utilities dealing with JSON Web Keys (JWK)"""
+import os
+from pathlib import Path
+from typing import Any, Union
+from datetime import datetime, timedelta
+
+from flask import Flask
+from authlib.jose import JsonWebKey
+from pymonad.either import Left, Right, Either
+
+def jwks_directory(app: Flask, configname: str) -> Path:
+ """Compute the directory where the JWKs are stored."""
+ appsecretsdir = Path(app.config[configname]).parent
+ if appsecretsdir.exists() and appsecretsdir.is_dir():
+ jwksdir = Path(appsecretsdir, "jwks/")
+ if not jwksdir.exists():
+ jwksdir.mkdir()
+ return jwksdir
+ raise ValueError(
+ "The `appsecretsdir` value should be a directory that actually exists.")
+
+
+def generate_and_save_private_key(
+ storagedir: Path,
+ kty: str = "RSA",
+ crv_or_size: Union[str, int] = 2048,
+ options: tuple[tuple[str, Any]] = (("iat", datetime.now().timestamp()),)
+) -> JsonWebKey:
+ """Generate a private key and save to `storagedir`."""
+ privatejwk = JsonWebKey.generate_key(
+ kty, crv_or_size, dict(options), is_private=True)
+ keyname = f"{privatejwk.thumbprint()}.private.pem"
+ with open(Path(storagedir, keyname), "wb") as pemfile:
+ pemfile.write(privatejwk.as_pem(is_private=True))
+
+ return privatejwk
+
+
+def pem_to_jwk(filepath: Path) -> JsonWebKey:
+ """Parse a PEM file into a JWK object."""
+ with open(filepath, "rb") as pemfile:
+ return JsonWebKey.import_key(pemfile.read())
+
+
+def __sorted_jwks_paths__(storagedir: Path) -> tuple[tuple[float, Path], ...]:
+ """A sorted list of the JWK file paths with their creation timestamps."""
+ return tuple(sorted(((os.stat(keypath).st_ctime, keypath)
+ for keypath in (Path(storagedir, keyfile)
+ for keyfile in os.listdir(storagedir)
+ if keyfile.endswith(".pem"))),
+ key=lambda tpl: tpl[0]))
+
+
+def list_jwks(storagedir: Path) -> tuple[JsonWebKey, ...]:
+ """
+ List all the JWKs in a particular directory in the order they were created.
+ """
+ return tuple(pem_to_jwk(keypath) for ctime,keypath in
+ __sorted_jwks_paths__(storagedir))
+
+
+def newest_jwk(storagedir: Path) -> Either:
+ """
+ Return an Either monad with the newest JWK or a message if none exists.
+ """
+ existingkeys = __sorted_jwks_paths__(storagedir)
+ if len(existingkeys) > 0:
+ return Right(pem_to_jwk(existingkeys[-1][1]))
+ return Left("No JWKs exist")
+
+
+def newest_jwk_with_rotation(jwksdir: Path, keyage: int) -> JsonWebKey:
+ """
+ Retrieve the latests JWK, creating a new one if older than `keyage` days.
+ """
+ def newer_than_days(jwkey):
+ filestat = os.stat(Path(
+ jwksdir, f"{jwkey.as_dict()['kid']}.private.pem"))
+ oldesttimeallowed = (datetime.now() - timedelta(days=keyage))
+ if filestat.st_ctime < (oldesttimeallowed.timestamp()):
+ return Left("JWK is too old!")
+ return jwkey
+
+ return newest_jwk(jwksdir).then(newer_than_days).either(
+ lambda _errmsg: generate_and_save_private_key(jwksdir),
+ lambda key: key)
diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py
new file mode 100644
index 0000000..61037f3
--- /dev/null
+++ b/uploader/oauth2/views.py
@@ -0,0 +1,138 @@
+"""Views for OAuth2 related functionality."""
+import uuid
+from datetime import datetime, timedelta
+from urllib.parse import urljoin, urlparse, urlunparse
+
+from authlib.jose import jwt
+from flask import (
+ flash,
+ jsonify,
+ url_for,
+ request,
+ redirect,
+ Blueprint,
+ current_app as app)
+
+from uploader import session
+from uploader import monadic_requests as mrequests
+from uploader.monadic_requests import make_error_handler
+
+from . import jwks
+from .client import (
+ SCOPE,
+ oauth2_get,
+ user_logged_in,
+ authserver_uri,
+ oauth2_clientid,
+ oauth2_clientsecret)
+
+oauth2 = Blueprint("oauth2", __name__)
+
+@oauth2.route("/code")
+def authorisation_code():
+ """Receive authorisation code from auth server and use it to get token."""
+ def __process_error__(resp_or_exception):
+ app.logger.debug("ERROR: (%s)", resp_or_exception)
+ flash("There was an error retrieving the authorisation token.",
+ "alert-danger")
+ return redirect("/")
+
+ def __fail_set_user_details__(_failure):
+ app.logger.debug("Fetching user details fails: %s", _failure)
+ flash("Could not retrieve the user details", "alert-danger")
+ return redirect("/")
+
+ def __success_set_user_details__(_success):
+ app.logger.debug("Session info: %s", _success)
+ return redirect("/")
+
+ def __success__(token):
+ session.set_user_token(token)
+ return oauth2_get("auth/user/").then(
+ lambda usrdets: session.set_user_details({
+ "user_id": uuid.UUID(usrdets["user_id"]),
+ "name": usrdets["name"],
+ "email": usrdets["email"],
+ "token": session.user_token(),
+ "logged_in": True})).either(
+ __fail_set_user_details__,
+ __success_set_user_details__)
+
+ code = request.args.get("code", "").strip()
+ if not bool(code):
+ flash("AuthorisationError: No code was provided.", "alert-danger")
+ return redirect("/")
+
+ baseurl = urlparse(request.base_url, scheme=request.scheme)
+ issued = datetime.now()
+ jwtkey = jwks.newest_jwk_with_rotation(
+ jwks.jwks_directory(app, "UPLOADER_SECRETS"),
+ int(app.config["JWKS_ROTATION_AGE_DAYS"]))
+ return mrequests.post(
+ urljoin(authserver_uri(), "auth/token"),
+ json={
+ "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+ "code": code,
+ "scope": SCOPE,
+ "redirect_uri": urljoin(
+ urlunparse(baseurl),
+ url_for("oauth2.authorisation_code")),
+ "assertion": jwt.encode(
+ header={
+ "alg": "RS256",
+ "typ": "JWT",
+ "kid": jwtkey.as_dict()["kid"]
+ },
+ payload={
+ "iss": str(oauth2_clientid()),
+ "sub": request.args["user_id"],
+ "aud": urljoin(authserver_uri(),"auth/token"),
+ "exp": (issued + timedelta(minutes=5)).timestamp(),
+ "nbf": int(issued.timestamp()),
+ "iat": int(issued.timestamp()),
+ "jti": str(uuid.uuid4())
+ },
+ key=jwtkey).decode("utf8"),
+ "client_id": oauth2_clientid()
+ }).either(__process_error__, __success__)
+
+@oauth2.route("/public-jwks")
+def public_jwks():
+ """List the available JWKs"""
+ return jsonify({
+ "documentation": (
+ "The keys are listed in order of creation, from the oldest (first) "
+ "to the newest (last)."),
+ "jwks": tuple(key.as_dict() for key
+ in jwks.list_jwks(jwks.jwks_directory(
+ app, "UPLOADER_SECRETS")))
+ })
+
+
+@oauth2.route("/logout", methods=["GET"])
+def logout():
+ """Log out of any active sessions."""
+ def __unset_session__(session_info):
+ _user = session_info["user"]
+ _user_str = f"{_user['name']} ({_user['email']})"
+ session.clear_session_info()
+ flash("Successfully logged out.", "alert-success")
+ return redirect("/")
+
+ if user_logged_in():
+ return session.user_token().then(
+ lambda _tok: mrequests.post(
+ urljoin(authserver_uri(), "auth/revoke"),
+ json={
+ "token": _tok["refresh_token"],
+ "token_type_hint": "refresh_token",
+ "client_id": oauth2_clientid(),
+ "client_secret": oauth2_clientsecret()
+ })).either(
+ make_error_handler(
+ redirect_to=redirect("/"),
+ cleanup_thunk=lambda: __unset_session__(
+ session.session_info())),
+ lambda res: __unset_session__(session.session_info()))
+ flash("There is no user that is currently logged in.", "alert-info")
+ return redirect("/")
diff --git a/qc_app/parse.py b/uploader/parse.py
index d20f6f0..dea4f95 100644
--- a/qc_app/parse.py
+++ b/uploader/parse.py
@@ -8,9 +8,10 @@ from flask import current_app as app
from quality_control.errors import InvalidValue, DuplicateHeading
-from qc_app import jobs
-from qc_app.dbinsert import species_by_id
-from qc_app.db_utils import with_db_connection
+from uploader import jobs
+from uploader.dbinsert import species_by_id
+from uploader.db_utils import with_db_connection
+from uploader.authorisation import require_login
parsebp = Blueprint("parse", __name__)
@@ -23,6 +24,7 @@ def isduplicateheading(item):
return isinstance(item, DuplicateHeading)
@parsebp.route("/parse", methods=["GET"])
+@require_login
def parse():
"""Trigger file parsing"""
errors = False
@@ -160,6 +162,7 @@ def fail(job_id: str):
return render_template("no_such_job.html", job_id=job_id)
@parsebp.route("/abort", methods=["POST"])
+@require_login
def abort():
"""Handle user request to abort file processing"""
job_id = request.form["job_id"]
diff --git a/qc_app/samples.py b/uploader/samples.py
index 804f262..7a80336 100644
--- a/qc_app/samples.py
+++ b/uploader/samples.py
@@ -20,14 +20,15 @@ from flask import (
from functional_tools import take
-from qc_app import jobs
-from qc_app.files import save_file
-from qc_app.input_validation import is_integer_input
-from qc_app.db_utils import (
+from uploader import jobs
+from uploader.files import save_file
+from uploader.authorisation import require_login
+from uploader.input_validation import is_integer_input
+from uploader.db_utils import (
with_db_connection,
database_connection,
with_redis_connection)
-from qc_app.db import (
+from uploader.db import (
species_by_id,
save_population,
population_by_id,
@@ -37,6 +38,7 @@ from qc_app.db import (
samples = Blueprint("samples", __name__)
@samples.route("/upload/species", methods=["GET", "POST"])
+@require_login
def select_species():
"""Select the species."""
if request.method == "GET":
@@ -58,6 +60,7 @@ def select_species():
@samples.route("/upload/species/<int:species_id>/create-population",
methods=["POST"])
+@require_login
def create_population(species_id: int):
"""Create new grouping/population."""
if not is_integer_input(species_id):
@@ -100,6 +103,7 @@ def create_population(species_id: int):
@samples.route("/upload/species/<int:species_id>/population",
methods=["GET", "POST"])
+@require_login
def select_population(species_id: int):
"""Select from existing groupings/populations."""
if not is_integer_input(species_id):
@@ -233,6 +237,7 @@ def build_sample_upload_job(# pylint: disable=[too-many-arguments]
@samples.route("/upload/species/<int:species_id>/populations/<int:population_id>/samples",
methods=["GET", "POST"])
+@require_login
def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
"""Upload the samples."""
samples_uploads_page = redirect(url_for("samples.upload_samples",
diff --git a/uploader/session.py b/uploader/session.py
new file mode 100644
index 0000000..8b72bce
--- /dev/null
+++ b/uploader/session.py
@@ -0,0 +1,91 @@
+"""Deal with user sessions"""
+from uuid import UUID, uuid4
+from typing import Any, Optional, TypedDict
+
+from flask import request, session
+from pymonad.either import Left, Right, Either
+
+
+class UserDetails(TypedDict):
+ """Session information relating specifically to the user."""
+ user_id: UUID
+ name: str
+ email: str
+ token: Either
+ logged_in: bool
+
+
+class SessionInfo(TypedDict):
+ """All Session information we save."""
+ session_id: UUID
+ user: UserDetails
+ anon_id: UUID
+ user_agent: str
+ ip_addr: str
+ masquerade: Optional[UserDetails]
+ auth_server_jwks: Optional[dict[str, Any]]
+
+
+__SESSION_KEY__ = "GN::uploader::session_info" # Do not use this outside this module!!
+
+
+def clear_session_info():
+ """Clears the session."""
+ session.pop(__SESSION_KEY__)
+
+
+def save_session_info(sess_info: SessionInfo) -> SessionInfo:
+ """Save `session_info`."""
+ # T0d0: if it is an existing session, verify that certain important security
+ # bits have not changed before saving.
+ # old_session_info = session.get(__SESSION_KEY__)
+ # if bool(old_session_info):
+ # if old_session_info["user_agent"] == request.headers.get("User-Agent"):
+ # session[__SESSION_KEY__] = sess_info
+ # return sess_info
+ # # request session verification
+ # return verify_session(sess_info)
+ # New session
+ session[__SESSION_KEY__] = sess_info
+ return sess_info
+
+
+def session_info() -> SessionInfo:
+ """Retrieve the session information"""
+ anon_id = uuid4()
+ return save_session_info(
+ session.get(__SESSION_KEY__, {
+ "session_id": uuid4(),
+ "user": {
+ "user_id": anon_id,
+ "name": "Anonymous User",
+ "email": "anon@ymous.user",
+ "token": Left("INVALID-TOKEN"),
+ "logged_in": False
+ },
+ "anon_id": anon_id,
+ "user_agent": request.headers.get("User-Agent"),
+ "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR",
+ request.remote_addr),
+ "masquerading": None
+ }))
+
+
+def set_user_token(token: str) -> SessionInfo:
+ """Set the user's token."""
+ info = session_info()
+ return save_session_info({
+ **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc]
+
+
+def set_user_details(userdets: UserDetails) -> SessionInfo:
+ """Set the user details information"""
+ return save_session_info({**session_info(), "user": userdets})#type: ignore[misc]
+
+def user_details() -> UserDetails:
+ """Retrieve user details."""
+ return session_info()["user"]
+
+def user_token() -> Either:
+ """Retrieve the user token."""
+ return session_info()["user"]["token"]
diff --git a/qc_app/static/css/custom-bootstrap.css b/uploader/static/css/custom-bootstrap.css
index 67f1199..67f1199 100644
--- a/qc_app/static/css/custom-bootstrap.css
+++ b/uploader/static/css/custom-bootstrap.css
diff --git a/qc_app/static/css/styles.css b/uploader/static/css/styles.css
index a88c229..a88c229 100644
--- a/qc_app/static/css/styles.css
+++ b/uploader/static/css/styles.css
diff --git a/qc_app/static/css/two-column-with-separator.css b/uploader/static/css/two-column-with-separator.css
index b6efd46..b6efd46 100644
--- a/qc_app/static/css/two-column-with-separator.css
+++ b/uploader/static/css/two-column-with-separator.css
diff --git a/qc_app/static/images/CITGLogo.png b/uploader/static/images/CITGLogo.png
index ae99fed..ae99fed 100644
--- a/qc_app/static/images/CITGLogo.png
+++ b/uploader/static/images/CITGLogo.png
Binary files differ
diff --git a/qc_app/static/js/select_platform.js b/uploader/static/js/select_platform.js
index 4fdd865..4fdd865 100644
--- a/qc_app/static/js/select_platform.js
+++ b/uploader/static/js/select_platform.js
diff --git a/qc_app/static/js/upload_progress.js b/uploader/static/js/upload_progress.js
index 9638b36..9638b36 100644
--- a/qc_app/static/js/upload_progress.js
+++ b/uploader/static/js/upload_progress.js
diff --git a/qc_app/static/js/upload_samples.js b/uploader/static/js/upload_samples.js
index aed536f..aed536f 100644
--- a/qc_app/static/js/upload_samples.js
+++ b/uploader/static/js/upload_samples.js
diff --git a/qc_app/static/js/utils.js b/uploader/static/js/utils.js
index 045dd47..045dd47 100644
--- a/qc_app/static/js/utils.js
+++ b/uploader/static/js/utils.js
diff --git a/qc_app/templates/base.html b/uploader/templates/base.html
index eb5e6b7..ee60fea 100644
--- a/qc_app/templates/base.html
+++ b/uploader/templates/base.html
@@ -33,7 +33,18 @@
<ul class="nav navbar-nav">
<li><a href="/" style="font-weight: bold">GN Uploader</a></li>
<li>
- <a href="{{gnuri or 'https://genenetwork.org'}}">GeneNetwork</a>
+ <a href="{{gn2server_uri()}}">GeneNetwork</a>
+ </li>
+ </ul>
+ <ul class="nav navbar-nav" style="margin-left: 2em;">
+ <li>
+ {%if user_logged_in()%}
+ <a href="{{url_for('oauth2.logout')}}"
+ title="Log out of the system">Log Out</a>
+ {%else%}
+ <a href="{{authserver_authorise_uri()}}"
+ title="Log in to the system">Log In</a>
+ {%endif%}
</li>
</ul>
</div>
diff --git a/qc_app/templates/cli-output.html b/uploader/templates/cli-output.html
index 33fb73b..33fb73b 100644
--- a/qc_app/templates/cli-output.html
+++ b/uploader/templates/cli-output.html
diff --git a/qc_app/templates/continue_from_create_dataset.html b/uploader/templates/continue_from_create_dataset.html
index 03bb49c..03bb49c 100644
--- a/qc_app/templates/continue_from_create_dataset.html
+++ b/uploader/templates/continue_from_create_dataset.html
diff --git a/qc_app/templates/continue_from_create_study.html b/uploader/templates/continue_from_create_study.html
index 34e6e5e..34e6e5e 100644
--- a/qc_app/templates/continue_from_create_study.html
+++ b/uploader/templates/continue_from_create_study.html
diff --git a/qc_app/templates/data_review.html b/uploader/templates/data_review.html
index b7528fd..b7528fd 100644
--- a/qc_app/templates/data_review.html
+++ b/uploader/templates/data_review.html
diff --git a/qc_app/templates/dbupdate_error.html b/uploader/templates/dbupdate_error.html
index e1359d2..e1359d2 100644
--- a/qc_app/templates/dbupdate_error.html
+++ b/uploader/templates/dbupdate_error.html
diff --git a/qc_app/templates/dbupdate_hidden_fields.html b/uploader/templates/dbupdate_hidden_fields.html
index ccbc299..ccbc299 100644
--- a/qc_app/templates/dbupdate_hidden_fields.html
+++ b/uploader/templates/dbupdate_hidden_fields.html
diff --git a/qc_app/templates/errors_display.html b/uploader/templates/errors_display.html
index 715cfcf..715cfcf 100644
--- a/qc_app/templates/errors_display.html
+++ b/uploader/templates/errors_display.html
diff --git a/qc_app/templates/final_confirmation.html b/uploader/templates/final_confirmation.html
index 0727fc8..0727fc8 100644
--- a/qc_app/templates/final_confirmation.html
+++ b/uploader/templates/final_confirmation.html
diff --git a/qc_app/templates/flash_messages.html b/uploader/templates/flash_messages.html
index b7af178..b7af178 100644
--- a/qc_app/templates/flash_messages.html
+++ b/uploader/templates/flash_messages.html
diff --git a/qc_app/templates/http-error.html b/uploader/templates/http-error.html
index 374fb86..374fb86 100644
--- a/qc_app/templates/http-error.html
+++ b/uploader/templates/http-error.html
diff --git a/qc_app/templates/index.html b/uploader/templates/index.html
index 89d2ae9..94060b7 100644
--- a/qc_app/templates/index.html
+++ b/uploader/templates/index.html
@@ -1,9 +1,12 @@
{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
{%block title%}Data Upload{%endblock%}
{%block contents%}
<div class="row">
+ {{flash_all_messages()}}
+
<h1 class="heading">data upload</h1>
<div class="explainer">
diff --git a/qc_app/templates/insert_error.html b/uploader/templates/insert_error.html
index 5301288..5301288 100644
--- a/qc_app/templates/insert_error.html
+++ b/uploader/templates/insert_error.html
diff --git a/qc_app/templates/insert_progress.html b/uploader/templates/insert_progress.html
index 52177d6..52177d6 100644
--- a/qc_app/templates/insert_progress.html
+++ b/uploader/templates/insert_progress.html
diff --git a/qc_app/templates/insert_success.html b/uploader/templates/insert_success.html
index 7e1fa8d..7e1fa8d 100644
--- a/qc_app/templates/insert_success.html
+++ b/uploader/templates/insert_success.html
diff --git a/qc_app/templates/job_progress.html b/uploader/templates/job_progress.html
index 1af0763..1af0763 100644
--- a/qc_app/templates/job_progress.html
+++ b/uploader/templates/job_progress.html
diff --git a/uploader/templates/login.html b/uploader/templates/login.html
new file mode 100644
index 0000000..6ebf72e
--- /dev/null
+++ b/uploader/templates/login.html
@@ -0,0 +1,32 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Data Upload{%endblock%}
+
+{%block contents%}
+<div class="row">
+ {{flash_all_messages()}}
+
+ <h1 class="heading">log in</h1>
+
+ <div class="explainer">
+ <p>
+ This system enables you to upload data onto GeneNetwork. In order to do
+ that correctly, we need to know who you are.</p>
+ <p>
+ If you already have an account with GeneNetwork, you can simply click the
+ login button below, after which you can upload your data.<br />
+ </p>
+ <a href="{{authserver_authorise_uri()}}" class="btn btn-primary"
+ style="display:block;width:15em;align:center;margin:1em 3em;">
+ log in</a>
+ <p>
+ If you do not have an account with GeneNetwork, go to
+ <a href="{{gn2server_uri()}}"
+ title="GeneNetwork Service."
+ target="_blank">GeneNetwork</a>
+ and register for an account, then come back here to login and upload.</a>
+ </div>
+</div>
+
+{%endblock%}
diff --git a/qc_app/templates/no_such_job.html b/uploader/templates/no_such_job.html
index 42a2d48..42a2d48 100644
--- a/qc_app/templates/no_such_job.html
+++ b/uploader/templates/no_such_job.html
diff --git a/qc_app/templates/parse_failure.html b/uploader/templates/parse_failure.html
index 31f6be8..31f6be8 100644
--- a/qc_app/templates/parse_failure.html
+++ b/uploader/templates/parse_failure.html
diff --git a/qc_app/templates/parse_results.html b/uploader/templates/parse_results.html
index e2bf7f0..e2bf7f0 100644
--- a/qc_app/templates/parse_results.html
+++ b/uploader/templates/parse_results.html
diff --git a/qc_app/templates/rqtl2/create-geno-dataset-success.html b/uploader/templates/rqtl2/create-geno-dataset-success.html
index 1b50221..1b50221 100644
--- a/qc_app/templates/rqtl2/create-geno-dataset-success.html
+++ b/uploader/templates/rqtl2/create-geno-dataset-success.html
diff --git a/qc_app/templates/rqtl2/create-probe-dataset-success.html b/uploader/templates/rqtl2/create-probe-dataset-success.html
index 790d174..790d174 100644
--- a/qc_app/templates/rqtl2/create-probe-dataset-success.html
+++ b/uploader/templates/rqtl2/create-probe-dataset-success.html
diff --git a/qc_app/templates/rqtl2/create-probe-study-success.html b/uploader/templates/rqtl2/create-probe-study-success.html
index d0ee508..d0ee508 100644
--- a/qc_app/templates/rqtl2/create-probe-study-success.html
+++ b/uploader/templates/rqtl2/create-probe-study-success.html
diff --git a/qc_app/templates/rqtl2/create-tissue-success.html b/uploader/templates/rqtl2/create-tissue-success.html
index 5f2c5a0..5f2c5a0 100644
--- a/qc_app/templates/rqtl2/create-tissue-success.html
+++ b/uploader/templates/rqtl2/create-tissue-success.html
diff --git a/qc_app/templates/rqtl2/index.html b/uploader/templates/rqtl2/index.html
index f3329c2..f3329c2 100644
--- a/qc_app/templates/rqtl2/index.html
+++ b/uploader/templates/rqtl2/index.html
diff --git a/qc_app/templates/rqtl2/no-such-job.html b/uploader/templates/rqtl2/no-such-job.html
index b17004f..b17004f 100644
--- a/qc_app/templates/rqtl2/no-such-job.html
+++ b/uploader/templates/rqtl2/no-such-job.html
diff --git a/qc_app/templates/rqtl2/rqtl2-job-error.html b/uploader/templates/rqtl2/rqtl2-job-error.html
index 9817518..9817518 100644
--- a/qc_app/templates/rqtl2/rqtl2-job-error.html
+++ b/uploader/templates/rqtl2/rqtl2-job-error.html
diff --git a/qc_app/templates/rqtl2/rqtl2-job-results.html b/uploader/templates/rqtl2/rqtl2-job-results.html
index 4ecd415..4ecd415 100644
--- a/qc_app/templates/rqtl2/rqtl2-job-results.html
+++ b/uploader/templates/rqtl2/rqtl2-job-results.html
diff --git a/qc_app/templates/rqtl2/rqtl2-job-status.html b/uploader/templates/rqtl2/rqtl2-job-status.html
index e896f88..e896f88 100644
--- a/qc_app/templates/rqtl2/rqtl2-job-status.html
+++ b/uploader/templates/rqtl2/rqtl2-job-status.html
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html b/uploader/templates/rqtl2/rqtl2-qc-job-error.html
index 90e8887..90e8887 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-error.html
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-error.html
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-results.html b/uploader/templates/rqtl2/rqtl2-qc-job-results.html
index 59bc8cd..59bc8cd 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-results.html
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-results.html
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html b/uploader/templates/rqtl2/rqtl2-qc-job-status.html
index f4a6266..f4a6266 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-status.html
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-status.html
diff --git a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html b/uploader/templates/rqtl2/rqtl2-qc-job-success.html
index 2861a04..2861a04 100644
--- a/qc_app/templates/rqtl2/rqtl2-qc-job-success.html
+++ b/uploader/templates/rqtl2/rqtl2-qc-job-success.html
diff --git a/qc_app/templates/rqtl2/select-geno-dataset.html b/uploader/templates/rqtl2/select-geno-dataset.html
index 873f9c3..873f9c3 100644
--- a/qc_app/templates/rqtl2/select-geno-dataset.html
+++ b/uploader/templates/rqtl2/select-geno-dataset.html
diff --git a/qc_app/templates/rqtl2/select-population.html b/uploader/templates/rqtl2/select-population.html
index 37731f0..37731f0 100644
--- a/qc_app/templates/rqtl2/select-population.html
+++ b/uploader/templates/rqtl2/select-population.html
diff --git a/qc_app/templates/rqtl2/select-probeset-dataset.html b/uploader/templates/rqtl2/select-probeset-dataset.html
index 26f52ed..26f52ed 100644
--- a/qc_app/templates/rqtl2/select-probeset-dataset.html
+++ b/uploader/templates/rqtl2/select-probeset-dataset.html
diff --git a/qc_app/templates/rqtl2/select-probeset-study-id.html b/uploader/templates/rqtl2/select-probeset-study-id.html
index b9bf52e..b9bf52e 100644
--- a/qc_app/templates/rqtl2/select-probeset-study-id.html
+++ b/uploader/templates/rqtl2/select-probeset-study-id.html
diff --git a/qc_app/templates/rqtl2/select-tissue.html b/uploader/templates/rqtl2/select-tissue.html
index 34e1758..34e1758 100644
--- a/qc_app/templates/rqtl2/select-tissue.html
+++ b/uploader/templates/rqtl2/select-tissue.html
diff --git a/qc_app/templates/rqtl2/summary-info.html b/uploader/templates/rqtl2/summary-info.html
index 1be87fa..1be87fa 100644
--- a/qc_app/templates/rqtl2/summary-info.html
+++ b/uploader/templates/rqtl2/summary-info.html
diff --git a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html
index 07c240f..07c240f 100644
--- a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-01.html
+++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-01.html
diff --git a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html
index 93b1dc9..93b1dc9 100644
--- a/qc_app/templates/rqtl2/upload-rqtl2-bundle-step-02.html
+++ b/uploader/templates/rqtl2/upload-rqtl2-bundle-step-02.html
diff --git a/qc_app/templates/samples/select-population.html b/uploader/templates/samples/select-population.html
index da19ddc..da19ddc 100644
--- a/qc_app/templates/samples/select-population.html
+++ b/uploader/templates/samples/select-population.html
diff --git a/qc_app/templates/samples/select-species.html b/uploader/templates/samples/select-species.html
index edadc61..edadc61 100644
--- a/qc_app/templates/samples/select-species.html
+++ b/uploader/templates/samples/select-species.html
diff --git a/qc_app/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html
index 09e2ecf..09e2ecf 100644
--- a/qc_app/templates/samples/upload-failure.html
+++ b/uploader/templates/samples/upload-failure.html
diff --git a/qc_app/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html
index 7bb02be..7bb02be 100644
--- a/qc_app/templates/samples/upload-progress.html
+++ b/uploader/templates/samples/upload-progress.html
diff --git a/qc_app/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html
index e62de57..e62de57 100644
--- a/qc_app/templates/samples/upload-samples.html
+++ b/uploader/templates/samples/upload-samples.html
diff --git a/qc_app/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html
index cb745c3..cb745c3 100644
--- a/qc_app/templates/samples/upload-success.html
+++ b/uploader/templates/samples/upload-success.html
diff --git a/qc_app/templates/select_dataset.html b/uploader/templates/select_dataset.html
index 2f07de8..2f07de8 100644
--- a/qc_app/templates/select_dataset.html
+++ b/uploader/templates/select_dataset.html
diff --git a/qc_app/templates/select_platform.html b/uploader/templates/select_platform.html
index d9bc68f..d9bc68f 100644
--- a/qc_app/templates/select_platform.html
+++ b/uploader/templates/select_platform.html
diff --git a/qc_app/templates/select_species.html b/uploader/templates/select_species.html
index 3b1a8a9..3b1a8a9 100644
--- a/qc_app/templates/select_species.html
+++ b/uploader/templates/select_species.html
diff --git a/qc_app/templates/select_study.html b/uploader/templates/select_study.html
index 648ad4c..648ad4c 100644
--- a/qc_app/templates/select_study.html
+++ b/uploader/templates/select_study.html
diff --git a/qc_app/templates/stdout_output.html b/uploader/templates/stdout_output.html
index 85345a9..85345a9 100644
--- a/qc_app/templates/stdout_output.html
+++ b/uploader/templates/stdout_output.html
diff --git a/uploader/templates/unhandled_exception.html b/uploader/templates/unhandled_exception.html
new file mode 100644
index 0000000..cfb0c0b
--- /dev/null
+++ b/uploader/templates/unhandled_exception.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}System Error{%endblock%}
+
+{%block css%}
+<link rel="stylesheet" href="/static/css/two-column-with-separator.css" />
+{%endblock%}
+
+{%block contents%}
+<div class="row">
+ {{flash_all_messages()}}
+ <h1>Exception!</h1>
+
+ <p>An error has occured, and your request has been aborted. Please notify the
+ administrator to try and get this fixed.</p>
+ <p>The system has failed with the following error:</p>
+</div>
+<div class="row">
+ <pre>
+ {{trace}}
+ </pre>
+</div>
+{%endblock%}
diff --git a/qc_app/templates/upload_progress_indicator.html b/uploader/templates/upload_progress_indicator.html
index e274e83..e274e83 100644
--- a/qc_app/templates/upload_progress_indicator.html
+++ b/uploader/templates/upload_progress_indicator.html
diff --git a/qc_app/templates/worker_failure.html b/uploader/templates/worker_failure.html
index b65b140..b65b140 100644
--- a/qc_app/templates/worker_failure.html
+++ b/uploader/templates/worker_failure.html
diff --git a/qc_app/upload/__init__.py b/uploader/upload/__init__.py
index 5f120d4..5f120d4 100644
--- a/qc_app/upload/__init__.py
+++ b/uploader/upload/__init__.py
diff --git a/qc_app/upload/rqtl2.py b/uploader/upload/rqtl2.py
index e691636..ff7556d 100644
--- a/qc_app/upload/rqtl2.py
+++ b/uploader/upload/rqtl2.py
@@ -27,20 +27,21 @@ from flask import (
from r_qtl import r_qtl2
-from qc_app import jobs
-from qc_app.files import save_file, fullpath
-from qc_app.dbinsert import species as all_species
-from qc_app.db_utils import with_db_connection, database_connection
-
-from qc_app.db.platforms import platform_by_id, platforms_by_species
-from qc_app.db.averaging import averaging_methods, averaging_method_by_id
-from qc_app.db.tissues import all_tissues, tissue_by_id, create_new_tissue
-from qc_app.db import (
+from uploader import jobs
+from uploader.files import save_file, fullpath
+from uploader.dbinsert import species as all_species
+from uploader.db_utils import with_db_connection, database_connection
+
+from uploader.authorisation import require_login
+from uploader.db.platforms import platform_by_id, platforms_by_species
+from uploader.db.averaging import averaging_methods, averaging_method_by_id
+from uploader.db.tissues import all_tissues, tissue_by_id, create_new_tissue
+from uploader.db import (
species_by_id,
save_population,
populations_by_species,
population_by_species_and_id,)
-from qc_app.db.datasets import (
+from uploader.db.datasets import (
geno_dataset_by_id,
geno_datasets_by_species_and_population,
@@ -53,8 +54,10 @@ from qc_app.db.datasets import (
rqtl2 = Blueprint("rqtl2", __name__)
+
@rqtl2.route("/", methods=["GET", "POST"])
@rqtl2.route("/select-species", methods=["GET", "POST"])
+@require_login
def select_species():
"""Select the species."""
if request.method == "GET":
@@ -72,6 +75,7 @@ def select_species():
@rqtl2.route("/upload/species/<int:species_id>/select-population",
methods=["GET", "POST"])
+@require_login
def select_population(species_id: int):
"""Select/Create the population to organise data under."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -101,6 +105,7 @@ def select_population(species_id: int):
@rqtl2.route("/upload/species/<int:species_id>/create-population",
methods=["POST"])
+@require_login
def create_population(species_id: int):
"""Create a new population for the given species."""
population_page = redirect(url_for("upload.rqtl2.select_population",
@@ -143,6 +148,7 @@ class __RequestError__(Exception): #pylint: disable=[invalid-name]
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle"),
methods=["GET", "POST"])
+@require_login
def upload_rqtl2_bundle(species_id: int, population_id: int):
"""Allow upload of R/qtl2 bundle."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -241,6 +247,7 @@ def chunks_directory(uniqueidentifier: str) -> Path:
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle-chunked"),
methods=["GET"])
+@require_login
def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"]
species_id: int,
population_id: int
@@ -285,6 +292,7 @@ def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle-chunked"),
methods=["POST"])
+@require_login
def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int):
"""
Extension to the `upload_rqtl2_bundle` endpoint above that allows large
@@ -310,29 +318,40 @@ def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int):
"statuscode": 400
}), 400
- # save chunk data
- chunks_directory(_fileid).mkdir(exist_ok=True)
- request.files["file"].save(Path(chunks_directory(_fileid),
- chunk_name(_uploadfilename, _chunk)))
-
- # Check whether upload is complete
- chunkpaths = tuple(
- Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
- for _achunk in range(1, _totalchunks+1))
- if all(_file.exists() for _file in chunkpaths):
- # merge_files and clean up chunks
- __merge_chunks__(_targetfile, chunkpaths)
- chunks_directory(_fileid).rmdir()
- jobid = trigger_rqtl2_bundle_qc(
- species_id, population_id, _targetfile, _uploadfilename)
- return url_for(
- "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)
+ try:
+ # save chunk data
+ chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
+ request.files["file"].save(Path(chunks_directory(_fileid),
+ chunk_name(_uploadfilename, _chunk)))
+
+ # Check whether upload is complete
+ chunkpaths = tuple(
+ Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
+ for _achunk in range(1, _totalchunks+1))
+ if all(_file.exists() for _file in chunkpaths):
+ # merge_files and clean up chunks
+ __merge_chunks__(_targetfile, chunkpaths)
+ chunks_directory(_fileid).rmdir()
+ jobid = trigger_rqtl2_bundle_qc(
+ species_id, population_id, _targetfile, _uploadfilename)
+ return url_for(
+ "upload.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)
+ except Exception as exc:# pylint: disable=[broad-except]
+ msg = "Error processing uploaded file chunks."
+ app.logger.error(msg, exc_info=True, stack_info=True)
+ return jsonify({
+ "message": msg,
+ "error": type(exc).__name__,
+ "error-description": " ".join(str(arg) for arg in exc.args),
+ "error-trace": traceback.format_exception(exc)
+ }), 500
return "OK"
@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
methods=["GET", "POST"])
+@require_login
def rqtl2_bundle_qc_status(jobid: UUID):
"""Check the status of the QC jobs."""
with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
@@ -554,6 +573,7 @@ def with_errors(endpointthunk: Callable, *checkfns):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-geno-dataset"),
methods=["POST"])
+@require_login
def select_geno_dataset(species_id: int, population_id: int):
"""Select from existing geno datasets."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -592,6 +612,7 @@ def select_geno_dataset(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-geno-dataset"),
methods=["POST"])
+@require_login
def create_geno_dataset(species_id: int, population_id: int):
"""Create a new geno dataset."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -661,6 +682,7 @@ def create_geno_dataset(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-tissue"),
methods=["POST"])
+@require_login
def select_tissue(species_id: int, population_id: int):
"""Select from existing tissues."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -691,6 +713,7 @@ def select_tissue(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-tissue"),
methods=["POST"])
+@require_login
def create_tissue(species_id: int, population_id: int):
"""Add new tissue, organ or biological material to the system."""
form = request.form
@@ -735,6 +758,7 @@ def create_tissue(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-probeset-study"),
methods=["POST"])
+@require_login
def select_probeset_study(species_id: int, population_id: int):
"""Select or create a probeset study."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -770,6 +794,7 @@ def select_probeset_study(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/select-probeset-dataset"),
methods=["POST"])
+@require_login
def select_probeset_dataset(species_id: int, population_id: int):
"""Select or create a probeset dataset."""
with database_connection(app.config["SQL_URI"]) as conn:
@@ -810,6 +835,7 @@ def select_probeset_dataset(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-probeset-study"),
methods=["POST"])
+@require_login
def create_probeset_study(species_id: int, population_id: int):
"""Create a new probeset study."""
errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-study"
@@ -872,6 +898,7 @@ def create_probeset_study(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/create-probeset-dataset"),
methods=["POST"])
+@require_login
def create_probeset_dataset(species_id: int, population_id: int):#pylint: disable=[too-many-return-statements]
"""Create a new probeset dataset."""
errorclasses = "alert-error error-rqtl2 error-rqtl2-create-probeset-dataset"
@@ -963,6 +990,7 @@ def create_probeset_dataset(species_id: int, population_id: int):#pylint: disabl
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/dataset-info"),
methods=["POST"])
+@require_login
def select_dataset_info(species_id: int, population_id: int):
"""
If `geno` files exist in the R/qtl2 bundle, prompt user to provide the
@@ -1055,6 +1083,7 @@ def select_dataset_info(species_id: int, population_id: int):
@rqtl2.route(("/upload/species/<int:species_id>/population/<int:population_id>"
"/rqtl2-bundle/confirm-bundle-details"),
methods=["POST"])
+@require_login
def confirm_bundle_details(species_id: int, population_id: int):
"""Confirm the details and trigger R/qtl2 bundle processing..."""
redisuri = app.config["REDIS_URL"]