aboutsummaryrefslogtreecommitdiff
path: root/uploader
diff options
context:
space:
mode:
Diffstat (limited to 'uploader')
-rw-r--r--uploader/__init__.py51
-rw-r--r--uploader/authorisation.py9
-rw-r--r--uploader/background_jobs.py119
-rw-r--r--uploader/base_routes.py17
-rw-r--r--uploader/default_settings.py14
-rw-r--r--uploader/files/__init__.py5
-rw-r--r--uploader/files/chunks.py32
-rw-r--r--uploader/files/functions.py (renamed from uploader/files.py)25
-rw-r--r--uploader/files/views.py157
-rw-r--r--uploader/genotypes/views.py59
-rw-r--r--uploader/jobs.py43
-rw-r--r--uploader/monadic_requests.py26
-rw-r--r--uploader/oauth2/client.py19
-rw-r--r--uploader/oauth2/views.py20
-rw-r--r--uploader/phenotypes/misc.py26
-rw-r--r--uploader/phenotypes/models.py241
-rw-r--r--uploader/phenotypes/views.py953
-rw-r--r--uploader/platforms/models.py3
-rw-r--r--uploader/platforms/views.py10
-rw-r--r--uploader/population/models.py2
-rw-r--r--uploader/population/rqtl2.py123
-rw-r--r--uploader/population/views.py26
-rw-r--r--uploader/publications/__init__.py2
-rw-r--r--uploader/publications/datatables.py52
-rw-r--r--uploader/publications/misc.py25
-rw-r--r--uploader/publications/models.py96
-rw-r--r--uploader/publications/pubmed.py103
-rw-r--r--uploader/publications/views.py107
-rw-r--r--uploader/route_utils.py42
-rw-r--r--uploader/samples/models.py10
-rw-r--r--uploader/samples/views.py95
-rw-r--r--uploader/session.py7
-rw-r--r--uploader/species/models.py8
-rw-r--r--uploader/species/views.py15
-rw-r--r--uploader/static/css/styles.css212
-rw-r--r--uploader/static/js/datatables.js69
-rw-r--r--uploader/static/js/debug.js40
-rw-r--r--uploader/static/js/files.js118
-rw-r--r--uploader/static/js/misc.js6
-rw-r--r--uploader/static/js/populations.js21
-rw-r--r--uploader/static/js/pubmed.js113
-rw-r--r--uploader/static/js/species.js20
-rw-r--r--uploader/static/js/utils.js27
-rw-r--r--uploader/templates/base.html123
-rw-r--r--uploader/templates/cli-output.html4
-rw-r--r--uploader/templates/genotypes/base.html11
-rw-r--r--uploader/templates/genotypes/index.html4
-rw-r--r--uploader/templates/genotypes/list-genotypes.html5
-rw-r--r--uploader/templates/genotypes/list-markers.html5
-rw-r--r--uploader/templates/genotypes/select-population.html16
-rw-r--r--uploader/templates/index.html134
-rw-r--r--uploader/templates/jobs/job-error.html17
-rw-r--r--uploader/templates/jobs/job-not-found.html11
-rw-r--r--uploader/templates/jobs/job-status.html24
-rw-r--r--uploader/templates/login.html7
-rw-r--r--uploader/templates/macro-step-indicator.html15
-rw-r--r--uploader/templates/phenotypes/add-phenotypes-base.html166
-rw-r--r--uploader/templates/phenotypes/add-phenotypes-raw-files.html847
-rw-r--r--uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html (renamed from uploader/templates/phenotypes/add-phenotypes.html)80
-rw-r--r--uploader/templates/phenotypes/base.html7
-rw-r--r--uploader/templates/phenotypes/bulk-edit-upload.html62
-rw-r--r--uploader/templates/phenotypes/create-dataset.html12
-rw-r--r--uploader/templates/phenotypes/edit-phenotype.html332
-rw-r--r--uploader/templates/phenotypes/index.html15
-rw-r--r--uploader/templates/phenotypes/job-status.html96
-rw-r--r--uploader/templates/phenotypes/list-datasets.html9
-rw-r--r--uploader/templates/phenotypes/load-phenotypes-success.html42
-rw-r--r--uploader/templates/phenotypes/macro-display-preview-table.html19
-rw-r--r--uploader/templates/phenotypes/macro-display-resumable-elements.html60
-rw-r--r--uploader/templates/phenotypes/review-job-data.html125
-rw-r--r--uploader/templates/phenotypes/select-population.html12
-rw-r--r--uploader/templates/phenotypes/view-dataset.html192
-rw-r--r--uploader/templates/phenotypes/view-phenotype.html83
-rw-r--r--uploader/templates/platforms/index.html4
-rw-r--r--uploader/templates/platforms/list-platforms.html2
-rw-r--r--uploader/templates/populations/base.html6
-rw-r--r--uploader/templates/populations/create-population.html14
-rw-r--r--uploader/templates/populations/index.html4
-rw-r--r--uploader/templates/populations/list-populations.html2
-rw-r--r--uploader/templates/populations/macro-display-population-card.html5
-rw-r--r--uploader/templates/populations/macro-select-population.html72
-rw-r--r--uploader/templates/populations/view-population.html26
-rw-r--r--uploader/templates/publications/base.html12
-rw-r--r--uploader/templates/publications/create-publication.html191
-rw-r--r--uploader/templates/publications/index.html92
-rw-r--r--uploader/templates/publications/view-publication.html78
-rw-r--r--uploader/templates/samples/index.html4
-rw-r--r--uploader/templates/samples/list-samples.html2
-rw-r--r--uploader/templates/samples/select-population.html23
-rw-r--r--uploader/templates/samples/upload-failure.html2
-rw-r--r--uploader/templates/species/base.html5
-rw-r--r--uploader/templates/species/create-species.html112
-rw-r--r--uploader/templates/species/list-species.html2
-rw-r--r--uploader/templates/species/macro-select-species.html83
-rw-r--r--uploader/templates/species/view-species.html6
95 files changed, 5540 insertions, 910 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py
index 9fdb383..8b49ad5 100644
--- a/uploader/__init__.py
+++ b/uploader/__init__.py
@@ -3,18 +3,33 @@ import os
import sys
import logging
from pathlib import Path
+from typing import Optional
from flask import Flask, request
+
+from cachelib import FileSystemCache
+
+from gn_libs import jobs as gnlibs_jobs
+
from flask_session import Session
+
from uploader.oauth2.client import user_logged_in, authserver_authorise_uri
from . import session
from .base_routes import base
+from .files.views import files
from .species import speciesbp
+from .publications import pubbp
from .oauth2.views import oauth2
from .expression_data import exprdatabp
from .errors import register_error_handlers
+from .background_jobs import background_jobs_bp
+
+logging.basicConfig(
+ format=("%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s "
+ "(%(thread)d:%(threadName)s): %(message)s")
+)
def override_settings_with_envvars(
app: Flask, ignore: tuple[str, ...]=tuple()) -> None:
@@ -49,10 +64,30 @@ def setup_logging(app: Flask) -> Flask:
"SERVER_SOFTWARE", "").split('/')
return __log_gunicorn__(app) if bool(software) else __log_dev__(app)
+def setup_modules_logging(app_logger):
+ """Setup module-level loggers to the same log-level as the application."""
+ loglevel = logging.getLevelName(app_logger.getEffectiveLevel())
+
+ def __setup__(logger_name):
+ _logger = logging.getLogger(logger_name)
+ _logger.setLevel(loglevel)
+
+ __setup__("uploader.publications.models")
+ __setup__("uploader.publications.datatables")
+
+
+def create_app(config: Optional[dict] = None):
+ """The application factory.
+
+ config: dict
+ Useful to override settings in the settings files and environment
+ especially in environments such as testing."""
+ if config is None:
+ config = {}
-def create_app():
- """The application factory"""
app = Flask(__name__)
+
+ ### BEGIN: Application configuration
app.config.from_pyfile(
Path(__file__).parent.joinpath("default_settings.py"))
if "UPLOADER_CONF" in os.environ:
@@ -67,8 +102,16 @@ def create_app():
if secretsfile.exists():
# Silently ignore secrets if the file does not exist.
app.config.from_pyfile(secretsfile)
+ app.config.update(config) # Override everything with passed in config
+ ### END: Application configuration
+
+ app.config["SESSION_CACHELIB"] = FileSystemCache(
+ cache_dir=Path(app.config["SESSION_FILESYSTEM_CACHE_PATH"]).absolute(),
+ threshold=int(app.config["SESSION_FILESYSTEM_CACHE_THRESHOLD"]),
+ default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"]))
setup_logging(app)
+ setup_modules_logging(app.logger)
# setup jinja2 symbols
app.add_template_global(lambda : request.url, name="request_url")
@@ -82,8 +125,12 @@ def create_app():
# setup blueprints
app.register_blueprint(base, url_prefix="/")
+ app.register_blueprint(files, url_prefix="/files")
app.register_blueprint(oauth2, url_prefix="/oauth2")
app.register_blueprint(speciesbp, url_prefix="/species")
+ app.register_blueprint(pubbp, url_prefix="/publications")
+ app.register_blueprint(background_jobs_bp, url_prefix="/background-jobs/")
register_error_handlers(app)
+ gnlibs_jobs.init_app(app)
return app
diff --git a/uploader/authorisation.py b/uploader/authorisation.py
index ee8fe97..3cf3585 100644
--- a/uploader/authorisation.py
+++ b/uploader/authorisation.py
@@ -16,13 +16,12 @@ def require_login(function):
@wraps(function)
def __is_session_valid__(*args, **kwargs):
"""Check that the user is logged in and their token is valid."""
- def __clear_session__(_no_token):
- session.clear_session_info()
- flash("You need to be logged in.", "alert-danger")
+ def __alert_needs_sign_in__(_no_token):
+ flash("You need to be signed in.", "alert alert-danger big-alert")
return redirect("/")
return session.user_token().either(
- __clear_session__,
+ __alert_needs_sign_in__,
lambda token: function(*args, **kwargs))
return __is_session_valid__
@@ -49,7 +48,7 @@ def require_token(func: Callable) -> Callable:
"""
def __invalid_token__(_whatever):
logging.debug("==========> Failure log: %s", _whatever)
- raise Exception(
+ raise Exception(# pylint: disable=[broad-exception-raised]
"You attempted to access a feature of the system that requires "
"authorisation. Unfortunately, we could not verify you have the "
"appropriate authorisation to perform the action you requested. "
diff --git a/uploader/background_jobs.py b/uploader/background_jobs.py
new file mode 100644
index 0000000..dc9f837
--- /dev/null
+++ b/uploader/background_jobs.py
@@ -0,0 +1,119 @@
+"""Generic views and utilities to handle background jobs."""
+import uuid
+import importlib
+from typing import Callable
+from functools import partial
+
+from flask import (
+ url_for,
+ redirect,
+ Response,
+ Blueprint,
+ render_template,
+ current_app as app)
+
+from gn_libs import jobs
+from gn_libs import sqlite3
+from gn_libs.jobs.jobs import JobNotFound
+
+from uploader.authorisation import require_login
+
+background_jobs_bp = Blueprint("background-jobs", __name__)
+HandlerType = Callable[[dict], Response]
+
+
+def __default_error_handler__(job: dict) -> Response:
+ return redirect(url_for("background-jobs.job_error", job_id=job["job_id"]))
+
+def register_handlers(
+ job_type: str,
+ success_handler: HandlerType,
+ # pylint: disable=[redefined-outer-name]
+ error_handler: HandlerType = __default_error_handler__
+ # pylint: disable=[redefined-outer-name]
+) -> str:
+ """Register success and error handlers for each job type."""
+ if not bool(app.config.get("background-jobs")):
+ app.config["background-jobs"] = {}
+
+ if not bool(app.config["background-jobs"].get(job_type)):
+ app.config["background-jobs"][job_type] = {
+ "success": success_handler,
+ "error": error_handler
+ }
+
+ return job_type
+
+
+def register_job_handlers(job: str):
+ """Related to register handlers above."""
+ def __load_handler__(absolute_function_path):
+ _parts = absolute_function_path.split(".")
+ app.logger.debug("THE PARTS ARE: %s", _parts)
+ assert len(_parts) > 1, f"Invalid path: {absolute_function_path}"
+ module = importlib.import_module(f".{_parts[-2]}",
+ package=".".join(_parts[0:-2]))
+ return getattr(module, _parts[-1])
+
+ metadata = job["metadata"]
+ if metadata["success_handler"]:
+ _success_handler = __load_handler__(metadata["success_handler"])
+ try:
+ _error_handler = __load_handler__(metadata["error_handler"])
+ except Exception as _exc:# pylint: disable=[broad-exception-caught]
+ _error_handler = __default_error_handler__
+ register_handlers(
+ metadata["job-type"], _success_handler, _error_handler)
+
+
+def handler(job: dict, handler_type: str) -> HandlerType:
+ """Fetch a handler for the job."""
+ _job_type = job["metadata"]["job-type"]
+ _handler = app.config.get(
+ "background-jobs", {}
+ ).get(
+ _job_type, {}
+ ).get(handler_type)
+ if bool(_handler):
+ return _handler(job)
+ raise Exception(# pylint: disable=[broad-exception-raised]
+ f"No '{handler_type}' handler registered for job type: {_job_type}")
+
+
+error_handler = partial(handler, handler_type="error")
+success_handler = partial(handler, handler_type="success")
+
+
+@background_jobs_bp.route("/status/<uuid:job_id>")
+@require_login
+def job_status(job_id: uuid.UUID):
+ """View the job status."""
+ with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn:
+ try:
+ job = jobs.job(conn, job_id, fulldetails=True)
+ status = job["metadata"]["status"]
+
+ register_job_handlers(job)
+ if status == "error":
+ return error_handler(job)
+
+ if status == "completed":
+ return success_handler(job)
+
+ return render_template("jobs/job-status.html", job=job)
+ except JobNotFound as _jnf:
+ return render_template(
+ "jobs/job-not-found.html",
+ job_id=job_id)
+
+
+@background_jobs_bp.route("/error/<uuid:job_id>")
+@require_login
+def job_error(job_id: uuid.UUID):
+ """Handle job errors in a generic manner."""
+ with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn:
+ try:
+ job = jobs.job(conn, job_id, fulldetails=True)
+ return render_template("jobs/job-error.html", job=job)
+ except JobNotFound as _jnf:
+ return render_template("jobs/job-not-found.html", job_id=job_id)
diff --git a/uploader/base_routes.py b/uploader/base_routes.py
index 742a254..74a3b90 100644
--- a/uploader/base_routes.py
+++ b/uploader/base_routes.py
@@ -35,8 +35,8 @@ def appenv():
@base.route("/bootstrap/<path:filename>")
def bootstrap(filename):
"""Fetch bootstrap files."""
- return send_from_directory(
- appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}")
+ return send_from_directory(appenv(), f"share/web/bootstrap/{filename}")
+
@base.route("/jquery/<path:filename>")
@@ -46,6 +46,19 @@ def jquery(filename):
appenv(), f"share/genenetwork2/javascript/jquery/{filename}")
+@base.route("/datatables/<path:filename>")
+def datatables(filename):
+ """Fetch DataTables files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/DataTables/{filename}")
+
+@base.route("/datatables-extensions/<path:filename>")
+def datatables_extensions(filename):
+ """Fetch DataTables files."""
+ return send_from_directory(
+ appenv(), f"share/genenetwork2/javascript/DataTablesExtensions/{filename}")
+
+
@base.route("/node-modules/<path:filename>")
def node_modules(filename):
"""Fetch node-js modules."""
diff --git a/uploader/default_settings.py b/uploader/default_settings.py
index 1acb247..1136ff8 100644
--- a/uploader/default_settings.py
+++ b/uploader/default_settings.py
@@ -2,9 +2,12 @@
The default configuration file. The values here should be overridden in the
actual configuration file used for the production and staging systems.
"""
+import hashlib
+
LOG_LEVEL = "WARNING"
SECRET_KEY = b"<Please! Please! Please! Change This!>"
UPLOAD_FOLDER = "/tmp/qc_app_files"
+TEMPORARY_DIRECTORY = "/tmp/gn-uploader-tmpdir"
REDIS_URL = "redis://"
JOBS_TTL_SECONDS = 1209600 # 14 days
GNQC_REDIS_PREFIX="gn-uploader"
@@ -12,9 +15,18 @@ SQL_URI = ""
GN2_SERVER_URL = "https://genenetwork.org/"
-SESSION_TYPE = "redis"
SESSION_PERMANENT = True
SESSION_USE_SIGNER = True
+SESSION_TYPE = "cachelib"
+## --- Settings for CacheLib session type --- ##
+## --- These are on flask-session config variables --- ##
+## --- https://cachelib.readthedocs.io/en/stable/file/ --- ##
+SESSION_FILESYSTEM_CACHE_PATH = "./flask_session"
+SESSION_FILESYSTEM_CACHE_THRESHOLD = 500
+SESSION_FILESYSTEM_CACHE_TIMEOUT = 300
+SESSION_FILESYSTEM_CACHE_MODE = 0o600
+SESSION_FILESYSTEM_CACHE_HASH_METHOD = hashlib.md5
+## --- END: Settings for CacheLib session type --- ##
JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use.
JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it.
diff --git a/uploader/files/__init__.py b/uploader/files/__init__.py
new file mode 100644
index 0000000..53c3176
--- /dev/null
+++ b/uploader/files/__init__.py
@@ -0,0 +1,5 @@
+"""General files and chunks utilities."""
+from .chunks import chunked_binary_read
+from .functions import (fullpath,
+ save_file,
+ sha256_digest_over_file)
diff --git a/uploader/files/chunks.py b/uploader/files/chunks.py
new file mode 100644
index 0000000..c4360b5
--- /dev/null
+++ b/uploader/files/chunks.py
@@ -0,0 +1,32 @@
+"""Functions dealing with chunking of files."""
+from pathlib import Path
+from typing import Iterator
+
+from flask import current_app as app
+from werkzeug.utils import secure_filename
+
+
+def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator:
+ """Read a file in binary mode in chunks."""
+ with open(filepath, "rb") as inputfile:
+ while True:
+ data = inputfile.read(chunksize)
+ if data != b"":
+ yield data
+ continue
+ break
+
+def chunk_name(uploadfilename: str, chunkno: int) -> str:
+ """Generate chunk name from original filename and chunk number"""
+ if uploadfilename == "":
+ raise ValueError("Name cannot be empty!")
+ if chunkno < 1:
+ raise ValueError("Chunk number must be greater than zero")
+ return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}"
+
+
+def chunks_directory(uniqueidentifier: str) -> Path:
+ """Compute the directory where chunks are temporarily stored."""
+ if uniqueidentifier == "":
+ raise ValueError("Unique identifier cannot be empty!")
+ return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}")
diff --git a/uploader/files.py b/uploader/files/functions.py
index d37a53e..7b9f06b 100644
--- a/uploader/files.py
+++ b/uploader/files/functions.py
@@ -1,7 +1,6 @@
"""Utilities to deal with uploaded files."""
import hashlib
from pathlib import Path
-from typing import Iterator
from datetime import datetime
from flask import current_app
@@ -9,12 +8,17 @@ from flask import current_app
from werkzeug.utils import secure_filename
from werkzeug.datastructures import FileStorage
-def save_file(fileobj: FileStorage, upload_dir: Path) -> Path:
+from .chunks import chunked_binary_read
+
+def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Path:
"""Save the uploaded file and return the path."""
assert bool(fileobj), "Invalid file object!"
- hashed_name = hashlib.sha512(
- f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8")
- ).hexdigest()
+ hashed_name = (
+ hashlib.sha512(
+ f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8")
+ ).hexdigest()
+ if hashed else
+ fileobj.filename)
filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type]
if not upload_dir.exists():
upload_dir.mkdir()
@@ -29,17 +33,6 @@ def fullpath(filename: str):
return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute()
-def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator:
- """Read a file in binary mode in chunks."""
- with open(filepath, "rb") as inputfile:
- while True:
- data = inputfile.read(chunksize)
- if data != b"":
- yield data
- continue
- break
-
-
def sha256_digest_over_file(filepath: Path) -> str:
"""Compute the sha256 digest over a file's contents."""
filehash = hashlib.sha256()
diff --git a/uploader/files/views.py b/uploader/files/views.py
new file mode 100644
index 0000000..29059c7
--- /dev/null
+++ b/uploader/files/views.py
@@ -0,0 +1,157 @@
+"""Module for generic files endpoints."""
+import time
+import random
+import traceback
+from pathlib import Path
+
+from flask import request, jsonify, Blueprint, current_app as app
+
+from .chunks import chunk_name, chunks_directory
+
+files = Blueprint("files", __name__)
+
+def target_file(fileid: str) -> Path:
+ """Compute the full path for the target file."""
+ return Path(app.config["UPLOAD_FOLDER"], fileid)
+
+
+@files.route("/upload/resumable", methods=["GET"])
+def resumable_upload_get():
+ """Used for checking whether **ALL** chunks have been uploaded."""
+ fileid = request.args.get("resumableIdentifier", type=str) or ""
+ filename = request.args.get("resumableFilename", type=str) or ""
+ chunk = request.args.get("resumableChunkNumber", type=int) or 0
+ if not(fileid or filename or chunk):
+ return jsonify({
+ "message": "At least one required query parameter is missing.",
+ "error": "BadRequest",
+ "statuscode": 400
+ }), 400
+
+ # If the complete target file exists, return 200 for all chunks.
+ _targetfile = target_file(fileid)
+ if _targetfile.exists():
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": filename,
+ "chunk": chunk,
+ "message": "The complete file already exists.",
+ "statuscode": 200
+ }), 200
+
+ if Path(chunks_directory(fileid),
+ chunk_name(filename, chunk)).exists():
+ return jsonify({
+ "chunk": chunk,
+ "message": f"Chunk {chunk} exists.",
+ "statuscode": 200
+ }), 200
+
+ return jsonify({
+ "message": f"Chunk {chunk} was not found.",
+ "error": "NotFound",
+ "statuscode": 404
+ }), 404
+
+
+def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
+ """Merge the chunks into a single file."""
+ with open(targetfile, "ab") as _target:
+ for chunkfile in chunkpaths:
+ app.logger.error("Merging chunk: %s", chunkfile)
+ with open(chunkfile, "rb") as _chunkdata:
+ _target.write(_chunkdata.read())
+
+ chunkfile.unlink() # Don't use `missing_ok=True` — chunk MUST exist
+ # If chunk does't exist, it might indicate a race condition. Handle
+ # that instead.
+ return targetfile
+
+
+@files.route("/upload/resumable", methods=["POST"])
+def resumable_upload_post():
+ """Do the actual chunks upload here."""
+ _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0
+ _chunk = request.form.get("resumableChunkNumber", default=1, type=int)
+ _uploadfilename = request.form.get(
+ "resumableFilename", default="", type=str) or ""
+ _fileid = request.form.get(
+ "resumableIdentifier", default="", type=str) or ""
+ _targetfile = target_file(_fileid)
+
+ if _targetfile.exists():
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": _uploadfilename,
+ "message": "File was uploaded successfully!",
+ "statuscode": 200
+ }), 200
+
+ try:
+ chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
+ request.files["file"].save(Path(chunks_directory(_fileid),
+ chunk_name(_uploadfilename, _chunk)))
+
+ # Check whether upload is complete
+ chunkpaths = tuple(
+ Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
+ for _achunk in range(1, _totalchunks+1))
+ if all(_file.exists() for _file in chunkpaths):
+ ### HACK: Break possible race condition ###
+ # Looks like sometimes, there are multiple threads/requests trying
+ # to merge one file, leading to race conditions and in some rare
+ # instances, actual data corruption. This hack is meant to break
+ # that race condition.
+ _delays = (
+ 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163,
+ 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233,
+ 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293)
+ _lockfile = Path(chunks_directory(_fileid), "merge.lock")
+ while True:
+ time.sleep(random.choice(_delays) / 1000)
+ if (chunks_directory(_fileid).exists()
+ and not (_lockfile.exists() and _targetfile.exists())):
+ # merge_files and clean up chunks
+ _lockfile.touch()
+ __merge_chunks__(_targetfile, chunkpaths)
+ _lockfile.unlink()
+ chunks_directory(_fileid).rmdir()
+ continue
+
+ if (_targetfile.exists()
+ and not (
+ chunks_directory(_fileid).exists()
+ and _lockfile.exists())):
+ # merge complete
+ break
+
+ # There is still a thread that's merging this file
+ continue
+ ### END: HACK: Break possible race condition ###
+
+ if _targetfile.exists():
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": _uploadfilename,
+ "message": "File was uploaded successfully!",
+ "statuscode": 200
+ }), 200
+ return jsonify({
+ "uploaded-file": _targetfile.name,
+ "original-name": _uploadfilename,
+ "message": "Uploaded file is missing!",
+ "statuscode": 404
+ }), 404
+ return jsonify({
+ "message": f"Chunk {int(_chunk)} uploaded successfully.",
+ "statuscode": 201
+ }), 201
+ except Exception as exc:# pylint: disable=[broad-except]
+ msg = "Error processing uploaded file chunks."
+ app.logger.error(msg, exc_info=True, stack_info=True)
+ return jsonify({
+ "message": msg,
+ "error": type(exc).__name__,
+ "error-description": " ".join(str(arg) for arg in exc.args),
+ "error-trace": traceback.format_exception(exc)
+ }), 500
diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py
index e6b1ba7..54c2444 100644
--- a/uploader/genotypes/views.py
+++ b/uploader/genotypes/views.py
@@ -12,12 +12,12 @@ from flask import (flash,
from uploader.ui import make_template_renderer
from uploader.oauth2.client import oauth2_post
from uploader.authorisation import require_login
+from uploader.route_utils import generic_select_population
+from uploader.datautils import safe_int, enumerate_sequence
from uploader.species.models import all_species, species_by_id
from uploader.monadic_requests import make_either_error_handler
from uploader.request_checks import with_species, with_population
-from uploader.datautils import safe_int, order_by_family, enumerate_sequence
-from uploader.population.models import (populations_by_species,
- population_by_species_and_id)
+from uploader.population.models import population_by_species_and_id
from .models import (genotype_markers,
genotype_dataset,
@@ -35,8 +35,15 @@ def index():
with database_connection(app.config["SQL_URI"]) as conn:
if not bool(request.args.get("species_id")):
return render_template("genotypes/index.html",
- species=order_by_family(all_species(conn)),
+ species=all_species(conn),
activelink="genotypes")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.genotypes.select_population"))
+
species = species_by_id(conn, request.args.get("species_id"))
if not bool(species):
flash(f"Could not find species with ID '{request.args.get('species_id')}'!",
@@ -50,28 +57,16 @@ def index():
methods=["GET"])
@require_login
@with_species(redirect_uri="species.populations.genotypes.index")
-def select_population(species: dict, species_id: int):
+def select_population(species: dict, species_id: int):# pylint: disable=[unused-argument]
"""Select the population under which the genotypes go."""
- with database_connection(app.config["SQL_URI"]) as conn:
- if not bool(request.args.get("population_id")):
- return render_template("genotypes/select-population.html",
- species=species,
- populations=order_by_family(
- populations_by_species(conn, species_id),
- order_key="FamilyOrder"),
- activelink="genotypes")
-
- population = population_by_species_and_id(
- conn, species_id, request.args.get("population_id"))
- if not bool(population):
- flash("Invalid population selected!", "alert-danger")
- return redirect(url_for(
- "species.populations.genotypes.select_population",
- species_id=species_id))
-
- return redirect(url_for("species.populations.genotypes.list_genotypes",
- species_id=species_id,
- population_id=population["Id"]))
+ return generic_select_population(
+ species,
+ "genotypes/select-population.html",
+ request.args.get("population_id") or "",
+ "species.populations.genotypes.select_population",
+ "species.populations.genotypes.list_genotypes",
+ "genotypes",
+ "Invalid population selected!")
@genotypesbp.route(
@@ -96,16 +91,24 @@ def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=
activelink="list-genotypes")
-@genotypesbp.route("/<int:species_id>/genotypes/list-markers", methods=["GET"])
+@genotypesbp.route(
+ "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers",
+ methods=["GET"])
@require_login
-@with_species(redirect_uri="species.populations.genotypes.index")
-def list_markers(species: dict, **kwargs):# pylint: disable=[unused-argument]
+@with_population(species_redirect_uri="species.populations.genotypes.index",
+ redirect_uri="species.populations.genotypes.select_population")
+def list_markers(
+ species: dict,
+ population: dict,
+ **kwargs
+):# pylint: disable=[unused-argument]
"""List a species' genetic markers."""
with database_connection(app.config["SQL_URI"]) as conn:
start_from = max(safe_int(request.args.get("start_from") or 0), 0)
count = safe_int(request.args.get("count") or 20)
return render_template("genotypes/list-markers.html",
species=species,
+ population=population,
total_markers=genotype_markers_count(
conn, species["SpeciesId"]),
start_from=start_from,
diff --git a/uploader/jobs.py b/uploader/jobs.py
index 4a3fc80..5968c03 100644
--- a/uploader/jobs.py
+++ b/uploader/jobs.py
@@ -1,6 +1,8 @@
"""Handle jobs"""
import os
import sys
+import uuid
+import json
import shlex
import subprocess
from uuid import UUID, uuid4
@@ -10,6 +12,8 @@ from typing import Union, Optional
from redis import Redis
from flask import current_app as app
+from functional_tools import take
+
JOBS_PREFIX = "jobs"
class JobNotFound(Exception):
@@ -37,7 +41,8 @@ def error_filename(jobid, error_dir):
"Compute the path of the file where errors will be dumped."
return f"{error_dir}/job_{jobid}.error"
-def initialise_job(# pylint: disable=[too-many-arguments]
+def initialise_job(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str,
ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict:
"Initialise a job 'object' and put in on redis"
@@ -50,7 +55,8 @@ def initialise_job(# pylint: disable=[too-many-arguments]
name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds))
return the_job
-def build_file_verification_job(#pylint: disable=[too-many-arguments]
+def build_file_verification_job(
+ #pylint: disable=[too-many-arguments, too-many-positional-arguments]
redis_conn: Redis,
dburi: str,
redisuri: str,
@@ -73,7 +79,8 @@ def build_file_verification_job(#pylint: disable=[too-many-arguments]
"filename": os.path.basename(filepath), "percent": 0
})
-def data_insertion_job(# pylint: disable=[too-many-arguments]
+def data_insertion_job(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
redis_conn: Redis, filepath: str, filetype: str, totallines: int,
speciesid: int, platformid: int, datasetid: int, databaseuri: str,
redisuri: str, ttl_seconds: int) -> dict:
@@ -128,3 +135,33 @@ def update_stdout_stderr(rconn: Redis,
contents = thejob.get(stream, '')
new_contents = contents + bytes_read.decode("utf-8")
rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents)
+
+
+def job_errors(
+ rconn: Redis,
+ prefix: str,
+ job_id: Union[str, uuid.UUID],
+ count: int = 100
+) -> list:
+ """Fetch job errors"""
+ return take(
+ (
+ json.loads(error)
+ for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*")
+ for error in rconn.lrange(key, 0, -1)),
+ count)
+
+
+def job_files_metadata(
+ rconn: Redis,
+ prefix: str,
+ job_id: Union[str, uuid.UUID]
+) -> dict:
+ """Get the metadata for specific job file."""
+ return {
+ key.split(":")[-1]: {
+ **rconn.hgetall(key),
+ "filetype": key.split(":")[-3]
+ }
+ for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*")
+ }
diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py
index c492df5..eda42d0 100644
--- a/uploader/monadic_requests.py
+++ b/uploader/monadic_requests.py
@@ -5,12 +5,12 @@ from typing import Union, Optional, Callable
import requests
from requests.models import Response
from pymonad.either import Left, Right, Either
+from markupsafe import escape as markupsafe_escape
from flask import (flash,
request,
redirect,
render_template,
- current_app as app,
- escape as flask_escape)
+ current_app as app)
# HTML Status codes indicating a successful request.
SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226)
@@ -39,9 +39,9 @@ def make_error_handler(
trace=traceback.format_exception(resp_or_exc))
if isinstance(resp_or_exc, Response):
flash("The authorisation server responded with "
- f"({flask_escape(resp_or_exc.status_code)}, "
- f"{flask_escape(resp_or_exc.reason)}) for the request to "
- f"'{flask_escape(resp_or_exc.request.url)}'",
+ f"({markupsafe_escape(resp_or_exc.status_code)}, "
+ f"{markupsafe_escape(resp_or_exc.reason)}) for the request to "
+ f"'{markupsafe_escape(resp_or_exc.request.url)}'",
"alert-danger")
return redirect_to
@@ -59,6 +59,11 @@ def get(url, params=None, **kwargs) -> Either:
:rtype: pymonad.either.Either
"""
+ timeout = kwargs.get("timeout")
+ kwargs = {key: val for key,val in kwargs.items() if key != "timeout"}
+ if timeout is None:
+ timeout = (9.13, 20)
+
try:
resp = requests.get(url, params=params, **kwargs)
if resp.status_code in SUCCESS_CODES:
@@ -76,6 +81,11 @@ def post(url, data=None, json=None, **kwargs) -> Either:
:rtype: pymonad.either.Either
"""
+ timeout = kwargs.get("timeout")
+ kwargs = {key: val for key,val in kwargs.items() if key != "timeout"}
+ if timeout is None:
+ timeout = (9.13, 20)
+
try:
resp = requests.post(url, data=data, json=json, **kwargs)
if resp.status_code in SUCCESS_CODES:
@@ -95,10 +105,10 @@ def make_either_error_handler(msg):
try:
_data = error.json()
except Exception as _exc:
- raise Exception(error.content) from _exc
- raise Exception(_data)
+ raise Exception(error.content) from _exc# pylint: disable=[broad-exception-raised]
+ raise Exception(_data)# pylint: disable=[broad-exception-raised]
app.logger.debug("\n\n%s\n\n", msg)
- raise Exception(error)
+ raise Exception(error)# pylint: disable=[broad-exception-raised]
return __fail__
diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py
index e7128de..12fbf80 100644
--- a/uploader/oauth2/client.py
+++ b/uploader/oauth2/client.py
@@ -1,6 +1,7 @@
"""OAuth2 client utilities."""
import json
import time
+import uuid
import random
from datetime import datetime, timedelta
from urllib.parse import urljoin, urlparse
@@ -112,7 +113,8 @@ def oauth2_client():
try:
jwt = JsonWebToken(["RS256"]).decode(
token["access_token"], key=jwk)
- return datetime.now().timestamp() > jwt["exp"]
+ if bool(jwt.get("exp")):
+ return datetime.now().timestamp() > jwt["exp"]
except BadSignatureError as _bse:
pass
@@ -145,9 +147,24 @@ def oauth2_client():
__client__)
+def fetch_user_details() -> Either:
+ """Retrieve user details from the auth server"""
+ suser = session.session_info()["user"]
+ if suser["email"] == "anon@ymous.user":
+ udets = oauth2_get("auth/user/").then(
+ lambda usrdets: session.set_user_details({
+ "user_id": uuid.UUID(usrdets["user_id"]),
+ "name": usrdets["name"],
+ "email": usrdets["email"],
+ "token": session.user_token()}))
+ return udets
+ return Right(suser)
+
+
def user_logged_in():
"""Check whether the user has logged in."""
suser = session.session_info()["user"]
+ fetch_user_details()
return suser["logged_in"] and suser["token"].is_right()
diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py
index 61037f3..db4ef61 100644
--- a/uploader/oauth2/views.py
+++ b/uploader/oauth2/views.py
@@ -24,22 +24,24 @@ from .client import (
user_logged_in,
authserver_uri,
oauth2_clientid,
+ fetch_user_details,
oauth2_clientsecret)
oauth2 = Blueprint("oauth2", __name__)
+
@oauth2.route("/code")
def authorisation_code():
"""Receive authorisation code from auth server and use it to get token."""
def __process_error__(resp_or_exception):
app.logger.debug("ERROR: (%s)", resp_or_exception)
flash("There was an error retrieving the authorisation token.",
- "alert-danger")
+ "alert alert-danger")
return redirect("/")
def __fail_set_user_details__(_failure):
app.logger.debug("Fetching user details fails: %s", _failure)
- flash("Could not retrieve the user details", "alert-danger")
+ flash("Could not retrieve the user details", "alert alert-danger")
return redirect("/")
def __success_set_user_details__(_success):
@@ -48,19 +50,13 @@ def authorisation_code():
def __success__(token):
session.set_user_token(token)
- return oauth2_get("auth/user/").then(
- lambda usrdets: session.set_user_details({
- "user_id": uuid.UUID(usrdets["user_id"]),
- "name": usrdets["name"],
- "email": usrdets["email"],
- "token": session.user_token(),
- "logged_in": True})).either(
+ return fetch_user_details().either(
__fail_set_user_details__,
__success_set_user_details__)
code = request.args.get("code", "").strip()
if not bool(code):
- flash("AuthorisationError: No code was provided.", "alert-danger")
+ flash("AuthorisationError: No code was provided.", "alert alert-danger")
return redirect("/")
baseurl = urlparse(request.base_url, scheme=request.scheme)
@@ -116,7 +112,7 @@ def logout():
_user = session_info["user"]
_user_str = f"{_user['name']} ({_user['email']})"
session.clear_session_info()
- flash("Successfully logged out.", "alert-success")
+ flash("Successfully signed out.", "alert alert-success")
return redirect("/")
if user_logged_in():
@@ -134,5 +130,5 @@ def logout():
cleanup_thunk=lambda: __unset_session__(
session.session_info())),
lambda res: __unset_session__(session.session_info()))
- flash("There is no user that is currently logged in.", "alert-info")
+ flash("There is no user that is currently logged in.", "alert alert-info")
return redirect("/")
diff --git a/uploader/phenotypes/misc.py b/uploader/phenotypes/misc.py
new file mode 100644
index 0000000..cbe3b7f
--- /dev/null
+++ b/uploader/phenotypes/misc.py
@@ -0,0 +1,26 @@
+"""Miscellaneous functions handling phenotypes and phenotypes data."""
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def phenotypes_data_differences(
+ filedata: tuple[dict, ...], dbdata: tuple[dict, ...]
+) -> tuple[dict, ...]:
+ """Compute differences between file data and db data"""
+ diff = tuple()
+ for filerow, dbrow in zip(
+ sorted(filedata, key=lambda item: (item["phenotype_id"], item["xref_id"])),
+ sorted(dbdata, key=lambda item: (item["PhenotypeId"], item["xref_id"]))):
+ for samplename, value in filerow["data"].items():
+ if value != dbrow["data"].get(samplename, {}).get("value"):
+ diff = diff + ({
+ "PhenotypeId": filerow["phenotype_id"],
+ "xref_id": filerow["xref_id"],
+ "DataId": dbrow["DataId"],
+ "StrainId": dbrow["data"].get(samplename, {}).get("StrainId"),
+ "StrainName": samplename,
+ "value": value
+ },)
+
+ return diff
diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py
index 73b1cce..c2aeebf 100644
--- a/uploader/phenotypes/models.py
+++ b/uploader/phenotypes/models.py
@@ -1,14 +1,30 @@
"""Database and utility functions for phenotypes."""
-from typing import Optional
+import logging
+import tempfile
+from pathlib import Path
from functools import reduce
from datetime import datetime
+from typing import Optional, Iterable
import MySQLdb as mdb
from MySQLdb.cursors import Cursor, DictCursor
-from flask import current_app as app
+from functional_tools import take
from gn_libs.mysqldb import debug_query
+logger = logging.getLogger(__name__)
+
+
+__PHENO_DATA_TABLES__ = {
+ "PublishData": {
+ "table": "PublishData", "valueCol": "value", "DataIdCol": "Id"},
+ "PublishSE": {
+ "table": "PublishSE", "valueCol": "error", "DataIdCol": "DataId"},
+ "NStrain": {
+ "table": "NStrain", "valueCol": "count", "DataIdCol": "DataId"}
+}
+
+
def datasets_by_population(
conn: mdb.Connection,
species_id: int,
@@ -32,10 +48,10 @@ def dataset_by_id(conn: mdb.Connection,
"""Fetch dataset details by identifier"""
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(
- "SELECT s.SpeciesId, pf.* FROM Species AS s "
- "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId "
- "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId "
- "WHERE s.Id=%s AND iset.Id=%s AND pf.Id=%s",
+ "SELECT Species.SpeciesId, PublishFreeze.* FROM Species "
+ "INNER JOIN InbredSet ON Species.Id=InbredSet.SpeciesId "
+ "INNER JOIN PublishFreeze ON InbredSet.Id=PublishFreeze.InbredSetId "
+ "WHERE Species.Id=%s AND InbredSet.Id=%s AND PublishFreeze.Id=%s",
(species_id, population_id, dataset_id))
return dict(cursor.fetchone())
@@ -54,6 +70,20 @@ def phenotypes_count(conn: mdb.Connection,
return int(cursor.fetchone()["total_phenos"])
+def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]:
+ """Retrieve the publication data for a phenotype if it exists."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT DISTINCT pxr.PhenotypeId, pub.* FROM PublishXRef AS pxr "
+ "INNER JOIN Publication as pub ON pxr.PublicationId=pub.Id "
+ "WHERE pxr.PhenotypeId=%s",
+ (phenotype_id,))
+ res = cursor.fetchone()
+ if res is None:
+ return res
+ return dict(res)
+
+
def dataset_phenotypes(conn: mdb.Connection,
population_id: int,
dataset_id: int,
@@ -61,7 +91,7 @@ def dataset_phenotypes(conn: mdb.Connection,
limit: Optional[int] = None) -> tuple[dict, ...]:
"""Fetch the actual phenotypes."""
_query = (
- "SELECT pheno.*, pxr.Id, ist.InbredSetCode FROM Phenotype AS pheno "
+ "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode FROM Phenotype AS pheno "
"INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
"INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId "
"INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id "
@@ -69,35 +99,45 @@ def dataset_phenotypes(conn: mdb.Connection,
f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "")
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(_query, (population_id, dataset_id))
- debug_query(cursor, app.logger)
+ debug_query(cursor, logger)
return tuple(dict(row) for row in cursor.fetchall())
-def __phenotype_se__(cursor: Cursor,
- species_id: int,
- population_id: int,
- dataset_id: int,
- xref_id: str) -> dict:
+def __phenotype_se__(cursor: Cursor, xref_id, dataids_and_strainids):
"""Fetch standard-error values (if they exist) for a phenotype."""
- _sequery = (
- "SELECT pxr.Id AS xref_id, pxr.DataId, str.Id AS StrainId, pse.error, nst.count "
- "FROM Phenotype AS pheno "
- "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
- "INNER JOIN PublishSE AS pse ON pxr.DataId=pse.DataId "
- "INNER JOIN NStrain AS nst ON pse.DataId=nst.DataId "
- "INNER JOIN Strain AS str ON nst.StrainId=str.Id "
- "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId "
- "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId "
- "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId "
- "WHERE (str.SpeciesId, pxr.InbredSetId, pf.Id, pxr.Id)=(%s, %s, %s, %s)")
- cursor.execute(_sequery,
- (species_id, population_id, dataset_id, xref_id))
- return {(row["DataId"], row["StrainId"]): {
- "xref_id": row["xref_id"],
- "DataId": row["DataId"],
- "error": row["error"],
- "count": row["count"]
- } for row in cursor.fetchall()}
+ paramstr = ", ".join(["(%s, %s)"] * len(dataids_and_strainids))
+ flat = tuple(item for sublist in dataids_and_strainids for item in sublist)
+ cursor.execute("SELECT * FROM PublishSE WHERE (DataId, StrainId) IN "
+ f"({paramstr})",
+ flat)
+ debug_query(cursor, logger)
+ _se = {
+ (row["DataId"], row["StrainId"]): {
+ "DataId": row["DataId"],
+ "StrainId": row["StrainId"],
+ "error": row["error"]
+ }
+ for row in cursor.fetchall()
+ }
+
+ cursor.execute("SELECT * FROM NStrain WHERE (DataId, StrainId) IN "
+ f"({paramstr})",
+ flat)
+ debug_query(cursor, logger)
+ _n = {
+ (row["DataId"], row["StrainId"]): {
+ "DataId": row["DataId"],
+ "StrainId": row["StrainId"],
+ "count": row["count"]
+ }
+ for row in cursor.fetchall()
+ }
+
+ keys = set(tuple(_se.keys()) + tuple(_n.keys()))
+ return {
+ key: {"xref_id": xref_id, **_se.get(key,{}), **_n.get(key,{})}
+ for key in keys
+ }
def __organise_by_phenotype__(pheno, row):
"""Organise disparate data rows into phenotype 'objects'."""
@@ -113,10 +153,12 @@ def __organise_by_phenotype__(pheno, row):
"Pre_publication_abbreviation": row["Pre_publication_abbreviation"],
"Post_publication_abbreviation": row["Post_publication_abbreviation"],
"xref_id": row["pxr.Id"],
+ "DataId": row["DataId"],
"data": {
**(_pheno["data"] if bool(_pheno) else {}),
(row["DataId"], row["StrainId"]): {
"DataId": row["DataId"],
+ "StrainId": row["StrainId"],
"mean": row["mean"],
"Locus": row["Locus"],
"LRS": row["LRS"],
@@ -170,11 +212,9 @@ def phenotype_by_id(
**_pheno,
"data": tuple(__merge_pheno_data_and_se__(
_pheno["data"],
- __phenotype_se__(cursor,
- species_id,
- population_id,
- dataset_id,
- xref_id)).values())
+ __phenotype_se__(
+ cursor, xref_id, tuple(_pheno["data"].keys()))
+ ).values())
}
if bool(_pheno) and len(_pheno.keys()) > 1:
raise Exception(
@@ -202,7 +242,7 @@ def phenotypes_data(conn: mdb.Connection,
f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "")
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(_query, (population_id, dataset_id))
- debug_query(cursor, app.logger)
+ debug_query(cursor, logger)
return tuple(dict(row) for row in cursor.fetchall())
@@ -229,5 +269,128 @@ def save_new_dataset(cursor: Cursor,
"%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, "
"%(users)s)",
params)
- debug_query(cursor, app.logger)
+ debug_query(cursor, logger)
return {**params, "Id": cursor.lastrowid}
+
+
+def phenotypes_data_by_ids(
+ conn: mdb.Connection,
+ inbred_pheno_xref: dict[str, int]
+) -> tuple[dict, ...]:
+ """Fetch all phenotype data, filtered by the `inbred_pheno_xref` mapping."""
+ _paramstr = ",".join(["(%s, %s, %s)"] * len(inbred_pheno_xref))
+ _query = ("SELECT "
+ "pub.PubMed_ID, pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode "
+ "FROM Publication AS pub "
+ "RIGHT JOIN PublishXRef AS pxr0 ON pub.Id=pxr0.PublicationId "
+ "INNER JOIN Phenotype AS pheno ON pxr0.PhenotypeId=pheno.id "
+ "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId "
+ "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id "
+ "INNER JOIN Strain AS str ON pd.StrainId=str.Id "
+ "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId "
+ "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId "
+ "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId "
+ f"WHERE (pxr.InbredSetId, pheno.Id, pxr.Id) IN ({_paramstr}) "
+ "ORDER BY pheno.Id")
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(_query, tuple(item for row in inbred_pheno_xref
+ for item in (row["population_id"],
+ row["phenoid"],
+ row["xref_id"])))
+ debug_query(cursor, logger)
+ return tuple(
+ reduce(__organise_by_phenotype__, cursor.fetchall(), {}).values())
+
+
+def create_new_phenotypes(conn: mdb.Connection,
+ phenotypes: Iterable[dict]) -> tuple[dict, ...]:
+ """Add entirely new phenotypes to the database."""
+ _phenos = tuple()
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ while True:
+ batch = take(phenotypes, 1000)
+ if len(batch) == 0:
+ break
+
+ cursor.executemany(
+ ("INSERT INTO "
+ "Phenotype(Pre_publication_description, Original_description, Units, Authorized_Users) "
+ "VALUES (%s, %s, %s, 'robwilliams')"),
+ tuple((row["id"], row["description"], row["units"])
+ for row in batch))
+ paramstr = ", ".join(["%s"] * len(batch))
+ cursor.execute(
+ "SELECT * FROM Phenotype WHERE Pre_publication_description IN "
+ f"({paramstr})",
+ tuple(item["id"] for item in batch))
+ _phenos = _phenos + tuple({
+ "phenotype_id": row["Id"],
+ "id": row["Pre_publication_description"],
+ "description": row["Original_description"],
+ "units": row["Units"]
+ } for row in cursor.fetchall())
+
+ return _phenos
+
+
+def save_phenotypes_data(
+ conn: mdb.Connection,
+ table: str,
+ data: Iterable[dict]
+) -> int:
+ """Save new phenotypes data into the database."""
+ _table_details = __PHENO_DATA_TABLES__[table]
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ _count = 0
+ while True:
+ batch = take(data, 100000)
+ if len(batch) == 0:
+ logger.warning("Got an empty batch. This needs investigation.")
+ break
+
+ logger.debug("Saving batch of %s items.", len(batch))
+ cursor.executemany(
+ (f"INSERT INTO {_table_details['table']}"
+ f"({_table_details['DataIdCol']}, StrainId, {_table_details['valueCol']}) "
+ "VALUES "
+ f"(%(data_id)s, %(sample_id)s, %(value)s) "),
+ tuple(batch))
+ debug_query(cursor, logger)
+ _count = _count + len(batch)
+
+
+ logger.debug("Saved a total of %s data rows", _count)
+ return _count
+
+
+def quick_save_phenotypes_data(
+ conn: mdb.Connection,
+ table: str,
+ dataitems: Iterable[dict],
+ tmpdir: Path
+) -> int:
+ """Save data items to the database, but using """
+ _table_details = __PHENO_DATA_TABLES__[table]
+ with (tempfile.NamedTemporaryFile(
+ prefix=f"{table}_data", mode="wt", dir=tmpdir) as tmpfile,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ _count = 0
+ logger.debug("Write data rows to text file.")
+ for row in dataitems:
+ tmpfile.write(
+ f'{row["data_id"]}\t{row["sample_id"]}\t{row["value"]}\n')
+ _count = _count + 1
+ tmpfile.flush()
+
+ logger.debug("Load text file into database (table: %s)",
+ _table_details["table"])
+ cursor.execute(
+ f"LOAD DATA LOCAL INFILE '{tmpfile.name}' "
+ f"INTO TABLE {_table_details['table']} "
+ "("
+ f"{_table_details['DataIdCol']}, "
+ "StrainId, "
+ f"{_table_details['valueCol']}"
+ ")")
+ debug_query(cursor, logger)
+ return _count
diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py
index b8c0e93..bc15f2d 100644
--- a/uploader/phenotypes/views.py
+++ b/uploader/phenotypes/views.py
@@ -1,37 +1,61 @@
"""Views handling ('classical') phenotypes."""
import sys
+import csv
import uuid
import json
+import logging
+import tempfile
+from typing import Any
from pathlib import Path
-from functools import wraps
+from zipfile import ZipFile
+from functools import wraps, reduce
from logging import INFO, ERROR, DEBUG, FATAL, CRITICAL, WARNING
+from urllib.parse import urljoin, urlparse, ParseResult, urlunparse, urlencode
+
+import datetime
+from datetime import timedelta
from redis import Redis
+from pymonad.either import Left
from requests.models import Response
from MySQLdb.cursors import DictCursor
+from werkzeug.utils import secure_filename
+
+from gn_libs import sqlite3
+from gn_libs import jobs as gnlibs_jobs
+from gn_libs.jobs.jobs import JobNotFound
from gn_libs.mysqldb import database_connection
+from gn_libs import monadic_requests as mrequests
+
+from authlib.jose import jwt
from flask import (flash,
request,
url_for,
+ jsonify,
redirect,
Blueprint,
- render_template,
+ send_file,
current_app as app)
# from r_qtl import r_qtl2 as rqtl2
from r_qtl import r_qtl2_qc as rqc
from r_qtl import exceptions as rqe
+
from uploader import jobs
+from uploader import session
from uploader.files import save_file#, fullpath
+from uploader.ui import make_template_renderer
from uploader.oauth2.client import oauth2_post
from uploader.authorisation import require_login
+from uploader.oauth2 import jwks, client as oauth2client
+from uploader.route_utils import generic_select_population
+from uploader.datautils import safe_int, enumerate_sequence
from uploader.species.models import all_species, species_by_id
from uploader.monadic_requests import make_either_error_handler
+from uploader.publications.models import fetch_publication_by_id
from uploader.request_checks import with_species, with_population
-from uploader.datautils import safe_int, order_by_family, enumerate_sequence
-from uploader.population.models import (populations_by_species,
- population_by_species_and_id)
+from uploader.samples.models import samples_by_species_and_population
from uploader.input_validation import (encode_errors,
decode_errors,
is_valid_representative_name)
@@ -41,9 +65,15 @@ from .models import (dataset_by_id,
phenotypes_count,
save_new_dataset,
dataset_phenotypes,
- datasets_by_population)
+ datasets_by_population,
+ phenotypes_data_by_ids,
+ phenotype_publication_data)
phenotypesbp = Blueprint("phenotypes", __name__)
+render_template = make_template_renderer("phenotypes")
+
+_FAMILIES_WITH_SE_AND_N_ = (
+ "Reference Populations (replicate average, SE, N)",)
@phenotypesbp.route("/phenotypes", methods=["GET"])
@require_login
@@ -52,10 +82,16 @@ def index():
with database_connection(app.config["SQL_URI"]) as conn:
if not bool(request.args.get("species_id")):
return render_template("phenotypes/index.html",
- species=order_by_family(all_species(conn)),
+ species=all_species(conn),
activelink="phenotypes")
- species = species_by_id(conn, request.args.get("species_id"))
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.phenotypes.select_population"))
+
+ species = species_by_id(conn, species_id)
if not bool(species):
flash("No such species!", "alert-danger")
return redirect(url_for("species.populations.phenotypes.index"))
@@ -69,27 +105,14 @@ def index():
@with_species(redirect_uri="species.populations.phenotypes.index")
def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument]
"""Select the population for your phenotypes."""
- with database_connection(app.config["SQL_URI"]) as conn:
- if not bool(request.args.get("population_id")):
- return render_template("phenotypes/select-population.html",
- species=species,
- populations=order_by_family(
- populations_by_species(
- conn, species["SpeciesId"]),
- order_key="FamilyOrder"),
- activelink="phenotypes")
-
- population = population_by_species_and_id(
- conn, species["SpeciesId"], int(request.args["population_id"]))
- if not bool(population):
- flash("No such population found!", "alert-danger")
- return redirect(url_for(
- "species.populations.phenotypes.select_population",
- species_id=species["SpeciesId"]))
-
- return redirect(url_for("species.populations.phenotypes.list_datasets",
- species_id=species["SpeciesId"],
- population_id=population["Id"]))
+ return generic_select_population(
+ species,
+ "phenotypes/select-population.html",
+ request.args.get("population_id") or "",
+ "species.populations.phenotypes.select_population",
+ "species.populations.phenotypes.list_datasets",
+ "phenotypes",
+ "No such population found!")
@@ -102,13 +125,18 @@ def select_population(species: dict, **kwargs):# pylint: disable=[unused-argumen
def list_datasets(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
"""List available phenotype datasets."""
with database_connection(app.config["SQL_URI"]) as conn:
+ datasets = datasets_by_population(
+ conn, species["SpeciesId"], population["Id"])
+ if len(datasets) == 1:
+ return redirect(url_for(
+ "species.populations.phenotypes.view_dataset",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=datasets[0]["Id"]))
return render_template("phenotypes/list-datasets.html",
species=species,
population=population,
- datasets=datasets_by_population(
- conn,
- species["SpeciesId"],
- population["Id"]),
+ datasets=datasets,
activelink="list-datasets")
@@ -182,12 +210,10 @@ def view_dataset(# pylint: disable=[unused-argument]
phenotype_count=phenotypes_count(
conn, population["Id"], dataset["Id"]),
phenotypes=enumerate_sequence(
- dataset_phenotypes(conn,
- population["Id"],
- dataset["Id"],
- offset=start_at,
- limit=count),
- start=start_at+1),
+ dataset_phenotypes(
+ conn,
+ population["Id"],
+ dataset["Id"])),
start_from=start_at,
count=count,
activelink="view-dataset")
@@ -211,16 +237,31 @@ def view_phenotype(# pylint: disable=[unused-argument]
):
"""View an individual phenotype from the dataset."""
def __render__(privileges):
+ phenotype = phenotype_by_id(conn,
+ species["SpeciesId"],
+ population["Id"],
+ dataset["Id"],
+ xref_id)
+ def __non_empty__(value) -> bool:
+ if isinstance(value, str):
+ return value.strip() != ""
+ return bool(value)
+
return render_template(
"phenotypes/view-phenotype.html",
species=species,
population=population,
dataset=dataset,
- phenotype=phenotype_by_id(conn,
- species["SpeciesId"],
- population["Id"],
- dataset["Id"],
- xref_id),
+ xref_id=xref_id,
+ phenotype=phenotype,
+ has_se=any(bool(item.get("error")) for item in phenotype["data"]),
+ publish_data={
+ key.replace("_", " "): val
+ for key,val in
+ (phenotype_publication_data(conn, phenotype["Id"]) or {}).items()
+ if (key in ("PubMed_ID", "Authors", "Title", "Journal")
+ and __non_empty__(val))
+ },
privileges=(privileges
### For demo! Do not commit this part
+ ("group:resource:edit-resource",
@@ -300,6 +341,74 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=
population_id=population["Id"]))
+def process_phenotypes_rqtl2_bundle(error_uri):
+ """Process phenotypes from the uploaded R/qtl2 bundle."""
+ _redisuri = app.config["REDIS_URL"]
+ _sqluri = app.config["SQL_URI"]
+ try:
+ ## Handle huge files here...
+ phenobundle = save_file(request.files["phenotypes-bundle"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ rqc.validate_bundle(phenobundle)
+ return phenobundle
+ except AssertionError as _aerr:
+ app.logger.debug("File upload error!", exc_info=True)
+ flash("Expected a zipped bundle of files with phenotypes' "
+ "information.",
+ "alert-danger")
+ return error_uri
+ except rqe.RQTLError as rqtlerr:
+ app.logger.debug("Bundle validation error!", exc_info=True)
+ flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger")
+ return error_uri
+
+
+def process_phenotypes_individual_files(error_uri):
+ """Process the uploaded individual files."""
+ form = request.form
+ cdata = {
+ "sep": form["file-separator"],
+ "comment.char": form["file-comment-character"],
+ "na.strings": form["file-na"].split(" "),
+ }
+ bundlepath = Path(app.config["UPLOAD_FOLDER"],
+ f"{str(uuid.uuid4()).replace('-', '')}.zip")
+ with ZipFile(bundlepath,mode="w") as zfile:
+ for rqtlkey, formkey in (("phenocovar", "phenotype-descriptions"),
+ ("pheno", "phenotype-data"),
+ ("phenose", "phenotype-se"),
+ ("phenonum", "phenotype-n")):
+ cdata[f"{rqtlkey}_transposed"] = (
+ (form.get(f"{formkey}-transposed") or "off") == "on")
+
+ if form.get("resumable-upload", False):
+ # Chunked upload of large files was used
+ filedata = json.loads(form[formkey])
+ zfile.write(
+ Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]),
+ arcname=filedata["original-name"])
+ cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]]
+ else:
+ # TODO: Check this path: fix any bugs.
+ _sentfile = request.files[formkey]
+ if not bool(_sentfile):
+ flash(f"Expected file ('{formkey}') was not provided.",
+ "alert-danger")
+ return error_uri
+
+ filepath = save_file(
+ _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False)
+ zfile.write(
+ Path(app.config["UPLOAD_FOLDER"], filepath),
+ arcname=filepath.name)
+ cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name]
+
+
+ zfile.writestr("control_data.json", data=json.dumps(cdata, indent=2))
+
+ return bundlepath
+
+
@phenotypesbp.route(
"<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
"/<int:dataset_id>/add-phenotypes",
@@ -311,6 +420,7 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=
redirect_uri="species.populations.phenotypes.list_datasets")
def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals]
"""Add one or more phenotypes to the dataset."""
+ use_bundle = request.args.get("use_bundle", "").lower() == "true"
add_phenos_uri = redirect(url_for(
"species.populations.phenotypes.add_phenotypes",
species_id=species["SpeciesId"],
@@ -323,27 +433,27 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p
# conn.cursor(cursorclass=DictCursor) as cursor
):
if request.method == "GET":
- return render_template("phenotypes/add-phenotypes.html",
- species=species,
- population=population,
- dataset=dataset,
- activelink="add-phenotypes")
+ today = datetime.date.today()
+ return render_template(
+ ("phenotypes/add-phenotypes-with-rqtl2-bundle.html"
+ if use_bundle else "phenotypes/add-phenotypes-raw-files.html"),
+ species=species,
+ population=population,
+ dataset=dataset,
+ monthnames=(
+ "January", "February", "March", "April",
+ "May", "June", "July", "August",
+ "September", "October", "November",
+ "December"),
+ current_month=today.strftime("%B"),
+ current_year=int(today.strftime("%Y")),
+ families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_,
+ use_bundle=use_bundle,
+ activelink="add-phenotypes")
- try:
- ## Handle huge files here...
- phenobundle = save_file(request.files["phenotypes-bundle"],
- Path(app.config["UPLOAD_FOLDER"]))
- rqc.validate_bundle(phenobundle)
- except AssertionError as _aerr:
- app.logger.debug("File upload error!", exc_info=True)
- flash("Expected a zipped bundle of files with phenotypes' "
- "information.",
- "alert-danger")
- return add_phenos_uri
- except rqe.RQTLError as rqtlerr:
- app.logger.debug("Bundle validation error!", exc_info=True)
- flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger")
- return add_phenos_uri
+ phenobundle = (process_phenotypes_rqtl2_bundle(add_phenos_uri)
+ if use_bundle else
+ process_phenotypes_individual_files(add_phenos_uri))
_jobid = uuid.uuid4()
_namespace = jobs.jobsnamespace()
@@ -357,33 +467,38 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p
_redisuri, _namespace, str(_jobid), str(species["SpeciesId"]),
str(population["Id"]),
# str(dataset["Id"]),
- str(phenobundle),
+ str(phenobundle),
"--loglevel",
- {
- INFO: "INFO",
- ERROR: "ERROR",
- DEBUG: "DEBUG",
- FATAL: "FATAL",
- CRITICAL: "CRITICAL",
- WARNING: "WARNING"
- }[app.logger.getEffectiveLevel()],
+ logging.getLevelName(
+ app.logger.getEffectiveLevel()
+ ).lower(),
"--redisexpiry",
str(_ttl_seconds)], "phenotype_qc", _ttl_seconds,
{"job-metadata": json.dumps({
"speciesid": species["SpeciesId"],
"populationid": population["Id"],
"datasetid": dataset["Id"],
- "bundle": str(phenobundle.absolute())})}),
+ "bundle": str(phenobundle.absolute()),
+ **({"publicationid": request.form["publication-id"]}
+ if request.form.get("publication-id") else {})})}),
_redisuri,
f"{app.config['UPLOAD_FOLDER']}/job_errors")
app.logger.debug("JOB DETAILS: %s", _job)
-
- return redirect(url_for("species.populations.phenotypes.job_status",
- species_id=species["SpeciesId"],
- population_id=population["Id"],
- dataset_id=dataset["Id"],
- job_id=str(_job["jobid"])))
+ jobstatusuri = url_for("species.populations.phenotypes.job_status",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"],
+ job_id=str(_job["jobid"]))
+ return ((jsonify({
+ "redirect-to": jobstatusuri,
+ "statuscode": 200,
+ "message": ("Follow the 'redirect-to' URI to see the state "
+ "of the quality-control job started for your "
+ "uploaded files.")
+ }), 200)
+ if request.form.get("resumable-upload", False) else
+ redirect(jobstatusuri))
@phenotypesbp.route(
@@ -395,10 +510,14 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p
species_redirect_uri="species.populations.phenotypes.index",
population_redirect_uri="species.populations.phenotypes.select_population",
redirect_uri="species.populations.phenotypes.list_datasets")
-def job_status(species: dict, population: dict, dataset: dict, job_id: uuid, **kwargs):
+def job_status(
+ species: dict,
+ population: dict,
+ dataset: dict,
+ job_id: uuid.UUID,
+ **kwargs
+):# pylint: disable=[unused-argument]
"""Retrieve current status of a particular phenotype QC job."""
- from uploader.debug import __pk__
-
with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
try:
job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id))
@@ -410,4 +529,672 @@ def job_status(species: dict, population: dict, dataset: dict, job_id: uuid, **k
dataset=dataset,
job_id=job_id,
job=job,
+ errors=jobs.job_errors(
+ rconn, jobs.jobsnamespace(), job['jobid']),
+ metadata=jobs.job_files_metadata(
+ rconn, jobs.jobsnamespace(), job['jobid']),
+ activelink="add-phenotypes")
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/job/<uuid:job_id>/review",
+ methods=["GET"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def review_job_data(
+ species: dict,
+ population: dict,
+ dataset: dict,
+ job_id: uuid.UUID,
+ **kwargs
+):# pylint: disable=[unused-argument]
+ """Review data one more time before entering it into the database."""
+ with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+ database_connection(app.config["SQL_URI"]) as conn):
+ try:
+ job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id))
+ except jobs.JobNotFound as _jnf:
+ job = None
+
+ def __metadata_by_type__(by_type, item):
+ filetype = item[1]["filetype"]
+ return {
+ **by_type,
+ filetype: (by_type.get(filetype, tuple())
+ + ({"filename": item[0], **item[1]},))
+ }
+ metadata: dict[str, Any] = reduce(
+ __metadata_by_type__,
+ (jobs.job_files_metadata(
+ rconn, jobs.jobsnamespace(), job['jobid'])
+ if job else {}).items(),
+ {})
+
+ def __desc__(filetype):
+ match filetype:
+ case "phenocovar":
+ desc = "phenotypes"
+ case "pheno":
+ desc = "phenotypes data"
+ case "phenose":
+ desc = "phenotypes standard-errors"
+ case "phenonum":
+ desc = "phenotypes samples"
+ case _:
+ desc = f"unknown file type '{filetype}'."
+
+ return desc
+
+ def __summarise__(filetype, files):
+ return {
+ "filetype": filetype,
+ "number-of-files": len(files),
+ "total-data-rows": sum(
+ int(afile["linecount"]) - 1 for afile in files),
+ "description": __desc__(filetype)
+ }
+
+ summary = {
+ filetype: __summarise__(filetype, meta)
+ for filetype,meta in metadata.items()
+ }
+ _job_metadata = json.loads(job["job-metadata"])
+ return render_template("phenotypes/review-job-data.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ job_id=job_id,
+ job=job,
+ summary=summary,
+ publication=(
+ fetch_publication_by_id(
+ conn, int(_job_metadata["publicationid"]))
+ if _job_metadata.get("publicationid")
+ else None),
activelink="add-phenotypes")
+
+
+def load_phenotypes_success_handler(job):
+ """Handle loading new phenotypes into the database successfully."""
+ return redirect(url_for(
+ "species.populations.phenotypes.load_data_success",
+ species_id=job["metadata"]["species_id"],
+ population_id=job["metadata"]["population_id"],
+ dataset_id=job["metadata"]["dataset_id"],
+ job_id=job["job_id"]))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/load-data-to-database",
+ methods=["POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def load_data_to_database(
+ species: dict,
+ population: dict,
+ dataset: dict,
+ **kwargs
+):# pylint: disable=[unused-argument]
+ """Load the data from the given QC job into the database."""
+ jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]
+ with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn,
+ sqlite3.connection(jobs_db) as conn):
+ qc_job = jobs.job(rconn, jobs.jobsnamespace(), request.form["data-qc-job-id"])
+ _meta = json.loads(qc_job["job-metadata"])
+ load_job_id = uuid.uuid4()
+ _loglevel = logging.getLevelName(app.logger.getEffectiveLevel()).lower()
+ command = [
+ sys.executable,
+ "-u",
+ "-m",
+ "scripts.load_phenotypes_to_db",
+ app.config["SQL_URI"],
+ jobs_db,
+ str(load_job_id),
+ "--log-level",
+ _loglevel
+ ]
+
+ def __handle_error__(resp):
+ return render_template("http-error.html", *resp.json())
+
+ def __handle_success__(load_job):
+ app.logger.debug("The phenotypes loading job: %s", load_job)
+ return redirect(url_for(
+ "background-jobs.job_status", job_id=load_job["job_id"]))
+
+ issued = datetime.datetime.now()
+ jwtkey = jwks.newest_jwk_with_rotation(
+ jwks.jwks_directory(app, "UPLOADER_SECRETS"),
+ int(app.config["JWKS_ROTATION_AGE_DAYS"]))
+
+ return mrequests.post(
+ urljoin(oauth2client.authserver_uri(), "auth/token"),
+ json={
+ "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer",
+ "scope": oauth2client.SCOPE,
+ "assertion": jwt.encode(
+ header={
+ "alg": "RS256",
+ "typ": "JWT",
+ "kid": jwtkey.as_dict()["kid"]
+ },
+ payload={
+ "iss": str(oauth2client.oauth2_clientid()),
+ "sub": str(session.user_details()["user_id"]),
+ "aud": urljoin(oauth2client.authserver_uri(),
+ "auth/token"),
+ # TODO: Update expiry time once fix is implemented in
+ # auth server.
+ "exp": (issued + timedelta(minutes=5)).timestamp(),
+ "nbf": int(issued.timestamp()),
+ "iat": int(issued.timestamp()),
+ "jti": str(uuid.uuid4())
+ },
+ key=jwtkey).decode("utf8"),
+ "client_id": oauth2client.oauth2_clientid()
+ }
+ ).then(
+ lambda token: gnlibs_jobs.initialise_job(
+ conn,
+ load_job_id,
+ command,
+ "load-new-phenotypes-data",
+ extra_meta={
+ "species_id": species["SpeciesId"],
+ "population_id": population["Id"],
+ "dataset_id": dataset["Id"],
+ "bundle_file": _meta["bundle"],
+ "publication_id": _meta["publicationid"],
+ "authserver": oauth2client.authserver_uri(),
+ "token": token["access_token"],
+ "success_handler": (
+ "uploader.phenotypes.views"
+ ".load_phenotypes_success_handler")
+ })
+ ).then(
+ lambda job: gnlibs_jobs.launch_job(
+ job,
+ jobs_db,
+ Path(f"{app.config['UPLOAD_FOLDER']}/job_errors"),
+ worker_manager="gn_libs.jobs.launcher",
+ loglevel=_loglevel)
+ ).either(__handle_error__, __handle_success__)
+
+
+def update_phenotype_metadata(conn, metadata: dict):
+ """Update a phenotype's basic metadata values."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Phenotype WHERE Id=%(phenotype-id)s",
+ metadata)
+ res = {
+ **{
+ _key: _val for _key,_val in {
+ key.lower().replace("_", "-"): value
+ for key, value in (cursor.fetchone() or {}).items()
+ }.items()
+ if _key in metadata.keys()
+ },
+ "phenotype-id": metadata.get("phenotype-id")
+ }
+ if res == metadata:
+ return False
+
+ cursor.execute(
+ "UPDATE Phenotype SET "
+ "Pre_publication_description=%(pre-publication-description)s, "
+ "Post_publication_description=%(post-publication-description)s, "
+ "Original_description=%(original-description)s, "
+ "Units=%(units)s, "
+ "Pre_publication_abbreviation=%(pre-publication-abbreviation)s, "
+ "Post_publication_abbreviation=%(post-publication-abbreviation)s "
+ "WHERE Id=%(phenotype-id)s",
+ metadata)
+ return cursor.rowcount
+
+
+def update_phenotype_values(conn, values):
+ """Update a phenotype's data values."""
+ with conn.cursor() as cursor:
+ cursor.executemany(
+ "UPDATE PublishData SET value=%(new)s "
+ "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in values if item["new"] is not None))
+ cursor.executemany(
+ "DELETE FROM PublishData "
+ "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in values if item["new"] is None))
+ return len(values)
+ return 0
+
+
+def update_phenotype_se(conn, serrs):
+ """Update a phenotype's standard-error values."""
+ with conn.cursor() as cursor:
+ cursor.executemany(
+ "INSERT INTO PublishSE(DataId, StrainId, error) "
+ "VALUES(%(data_id)s, %(strain_id)s, %(new)s) "
+ "ON DUPLICATE KEY UPDATE error=VALUES(error)",
+ tuple(item for item in serrs if item["new"] is not None))
+ cursor.executemany(
+ "DELETE FROM PublishSE "
+ "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in serrs if item["new"] is None))
+ return len(serrs)
+ return 0
+
+
+def update_phenotype_n(conn, counts):
+ """Update a phenotype's strain counts."""
+ with conn.cursor() as cursor:
+ cursor.executemany(
+ "INSERT INTO NStrain(DataId, StrainId, count) "
+ "VALUES(%(data_id)s, %(strain_id)s, %(new)s) "
+ "ON DUPLICATE KEY UPDATE count=VALUES(count)",
+ tuple(item for item in counts if item["new"] is not None))
+ cursor.executemany(
+ "DELETE FROM NStrain "
+ "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s",
+ tuple(item for item in counts if item["new"] is None))
+ return len(counts)
+
+ return 0
+
+
+def update_phenotype_data(conn, data: dict):
+ """Update the numeric data for a phenotype."""
+ def __organise_by_dataid_and_strainid__(acc, current):
+ _key, dataid, strainid = current[0].split("::")
+ _keysrc, _keytype = _key.split("-")
+ newkey = f"{dataid}::{strainid}"
+ newitem = acc.get(newkey, {})
+ newitem[_keysrc] = newitem.get(_keysrc, {})
+ newitem[_keysrc][_keytype] = current[1]
+ return {**acc, newkey: newitem}
+
+ def __separate_items__(acc, row):
+ key, val = row
+ return ({
+ **acc[0],
+ key: {
+ **val["value"],
+ "changed?": (not val["value"]["new"] == val["value"]["original"])
+ }
+ }, {
+ **acc[1],
+ key: {
+ **val["se"],
+ "changed?": (not val["se"]["new"] == val["se"]["original"])
+ }
+ },{
+ **acc[2],
+ key: {
+ **val["n"],
+ "changed?": (not val["n"]["new"] == val["n"]["original"])
+ }
+ })
+
+ values, serrs, counts = tuple(
+ tuple({
+ "data_id": row[0].split("::")[0],
+ "strain_id": row[0].split("::")[1],
+ "new": row[1]["new"]
+ } for row in item)
+ for item in (
+ filter(lambda val: val[1]["changed?"], item.items())# type: ignore[arg-type]
+ for item in reduce(# type: ignore[var-annotated]
+ __separate_items__,
+ reduce(__organise_by_dataid_and_strainid__,
+ data.items(),
+ {}).items(),
+ ({}, {}, {}))))
+
+ return (update_phenotype_values(conn, values),
+ update_phenotype_se(conn, serrs),
+ update_phenotype_n(conn, counts))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/phenotype/<int:xref_id>/edit",
+ methods=["GET", "POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def edit_phenotype_data(# pylint: disable=[unused-argument]
+ species: dict,
+ population: dict,
+ dataset: dict,
+ xref_id: int,
+ **kwargs
+):
+ """Edit the data for a particular phenotype."""
+ def __render__(**kwargs):
+ processed_kwargs = {
+ **kwargs,
+ "privileges": (kwargs.get("privileges", tuple())
+ ### For demo! Do not commit this part
+ + ("group:resource:edit-resource",
+ "group:resource:delete-resource",)
+ ### END: For demo! Do not commit this part
+ )
+ }
+ return render_template(
+ "phenotypes/edit-phenotype.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ xref_id=xref_id,
+ families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_,
+ **processed_kwargs,
+ activelink="edit-phenotype")
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if request.method == "GET":
+ def __fetch_phenotype__(privileges):
+ phenotype = phenotype_by_id(conn,
+ species["SpeciesId"],
+ population["Id"],
+ dataset["Id"],
+ xref_id)
+ if phenotype is None:
+ msg = ("Could not find the phenotype with cross-reference ID"
+ f" '{xref_id}' from dataset '{dataset['FullName']}' "
+ f" from the '{population['FullName']}' population of "
+ f" species '{species['FullName']}'.")
+ return Left({"privileges": privileges, "phenotype-error": msg})
+ return {"privileges": privileges, "phenotype": phenotype}
+
+ def __fetch_publication_data__(**kwargs):
+ pheno = kwargs["phenotype"]
+ return {
+ **kwargs,
+ "publication_data": phenotype_publication_data(
+ conn, pheno["Id"])
+ }
+
+ def __fail__(failure_object):
+ # process the object
+ return __render__(failure_object=failure_object)
+
+ return oauth2_post(
+ "/auth/resource/phenotypes/individual/linked-resource",
+ json={
+ "species_id": species["SpeciesId"],
+ "population_id": population["Id"],
+ "dataset_id": dataset["Id"],
+ "xref_id": xref_id
+ }
+ ).then(
+ lambda resource: tuple(
+ privilege["privilege_id"] for role in resource["roles"]
+ for privilege in role["privileges"])
+ ).then(
+ __fetch_phenotype__
+ ).then(
+ lambda args: __fetch_publication_data__(**args)
+ ).either(__fail__, lambda args: __render__(**args))
+
+ ## POST
+ _change = False
+ match request.form.get("submit", "invalid-action"):
+ case "update basic metadata":
+ _change = update_phenotype_metadata(conn, {
+ key: value.strip() if bool(value.strip()) else None
+ for key, value in request.form.items()
+ if key not in ("submit",)
+ })
+ msg = "Basic metadata was updated successfully."
+ case "update data":
+ _update = update_phenotype_data(conn, {
+ key: value.strip() if bool(value.strip()) else None
+ for key, value in request.form.items()
+ if key not in ("submit",)
+ })
+ msg = (f"{_update[0]} value rows, {_update[1]} standard-error "
+ f"rows and {_update[2]} 'N' rows were updated.")
+ _change = any(item != 0 for item in _update)
+ case "update publication":
+ flash("NOT IMPLEMENTED: Would update publication data.", "alert-success")
+ case _:
+ flash("Invalid phenotype editing action.", "alert-danger")
+
+ if _change:
+ flash(msg, "alert-success")
+ return redirect(url_for(
+ "species.populations.phenotypes.view_phenotype",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"],
+ xref_id=xref_id))
+
+ flash("No change was made by the user.", "alert-info")
+ return redirect(url_for(
+ "species.populations.phenotypes.edit_phenotype_data",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"],
+ xref_id=xref_id))
+
+
+def process_phenotype_data_for_download(pheno: dict) -> dict:
+ """Sanitise data for download."""
+ return {
+ "UniqueIdentifier": f"phId:{pheno['Id']}::xrId:{pheno['xref_id']}",
+ **{
+ key: val for key, val in pheno.items()
+ if key not in ("Id", "xref_id", "data", "Units")
+ },
+ **{
+ data_item["StrainName"]: data_item["value"]
+ for data_item in pheno.get("data", {}).values()
+ }
+ }
+
+
+BULK_EDIT_COMMON_FIELDNAMES = [
+ "UniqueIdentifier",
+ "Post_publication_description",
+ "Pre_publication_abbreviation",
+ "Pre_publication_description",
+ "Original_description",
+ "Post_publication_abbreviation",
+ "PubMed_ID"
+]
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/edit-download",
+ methods=["POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def edit_download_phenotype_data(# pylint: disable=[unused-argument]
+ species: dict,
+ population: dict,
+ dataset: dict,
+ **kwargs
+):
+ formdata = request.json
+ with database_connection(app.config["SQL_URI"]) as conn:
+ samples_list = [
+ sample["Name"] for sample in samples_by_species_and_population(
+ conn, species["SpeciesId"], population["Id"])]
+ data = (
+ process_phenotype_data_for_download(pheno)
+ for pheno in phenotypes_data_by_ids(conn, tuple({
+ "population_id": population["Id"],
+ "phenoid": row["phenotype_id"],
+ "xref_id": row["xref_id"]
+ } for row in formdata)))
+
+ with (tempfile.TemporaryDirectory(
+ prefix=app.config["TEMPORARY_DIRECTORY"]) as tmpdir):
+ filename = Path(tmpdir).joinpath("tempfile.tsv")
+ with open(filename, mode="w") as outfile:
+ outfile.write(
+ "# **DO NOT** delete the 'UniqueIdentifier' row. It is used "
+ "by the system to identify and edit the correct rows and "
+ "columns in the database.\n")
+ outfile.write(
+ "# The '…_description' fields are useful for you to figure out "
+ "what row you are working on. Changing any of this fields will "
+ "also update the database, so do be careful.\n")
+ outfile.write(
+ "# Leave a field empty to delete the value in the database.\n")
+ outfile.write(
+ "# Any line beginning with a '#' character is considered a "
+ "comment line. This line, and all the lines above it, are "
+ "all comment lines. Comment lines will be ignored.\n")
+ writer = csv.DictWriter(outfile,
+ fieldnames= (
+ BULK_EDIT_COMMON_FIELDNAMES +
+ samples_list),
+ dialect="excel-tab")
+ writer.writeheader()
+ writer.writerows(data)
+ outfile.flush()
+
+ return send_file(
+ filename,
+ mimetype="text/csv",
+ as_attachment=True,
+ download_name=secure_filename(f"{dataset['Name']}_data"))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/edit-upload",
+ methods=["GET", "POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def edit_upload_phenotype_data(# pylint: disable=[unused-argument]
+ species: dict,
+ population: dict,
+ dataset: dict,
+ **kwargs
+):
+ if request.method == "GET":
+ return render_template(
+ "phenotypes/bulk-edit-upload.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ activelink="edit-phenotype")
+
+ edit_file = save_file(request.files["file-upload-bulk-edit-upload"],
+ Path(app.config["UPLOAD_FOLDER"]))
+
+ jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]
+ with sqlite3.connection(jobs_db) as conn:
+ job_id = uuid.uuid4()
+ job_cmd = [
+ sys.executable, "-u",
+ "-m", "scripts.phenotypes_bulk_edit",
+ app.config["SQL_URI"],
+ jobs_db,
+ str(job_id),
+ "--log-level",
+ logging.getLevelName(
+ app.logger.getEffectiveLevel()
+ ).lower()
+ ]
+ app.logger.debug("Phenotype-edit, bulk-upload command: %s", job_cmd)
+ _job = gnlibs_jobs.launch_job(
+ gnlibs_jobs.initialise_job(conn,
+ job_id,
+ job_cmd,
+ "phenotype-bulk-edit",
+ extra_meta = {
+ "edit-file": str(edit_file),
+ "species-id": species["SpeciesId"],
+ "population-id": population["Id"],
+ "dataset-id": dataset["Id"]
+ }),
+ jobs_db,
+ f"{app.config['UPLOAD_FOLDER']}/job_errors",
+ worker_manager="gn_libs.jobs.launcher")
+
+
+ return redirect(url_for("background-jobs.job_status",
+ job_id=job_id,
+ job_type="phenotype-bulk-edit"))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/load-data-success/<uuid:job_id>",
+ methods=["GET"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def load_data_success(
+ species: dict,
+ population: dict,
+ dataset: dict,
+ job_id: uuid.UUID,
+ **kwargs
+):# pylint: disable=[unused-argument]
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"])
+ as jobsconn):
+ try:
+ gn2_uri = urlparse(app.config["GN2_SERVER_URL"])
+ job = gnlibs_jobs.job(jobsconn, job_id, fulldetails=True)
+ app.logger.debug("THE JOB: %s", job)
+ _xref_ids = (str(item) for item
+ in json.loads(job["metadata"].get("xref_ids", "[]")))
+ _publication = fetch_publication_by_id(
+ conn, int(job["metadata"].get("publication_id", "0")))
+ _search_terms = (item for item in
+ (str(_publication["PubMed_ID"] or ""),
+ _publication["Authors"],
+ (_publication["Title"] or ""))
+ if item != "")
+ return render_template("phenotypes/load-phenotypes-success.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ job=job,
+ search_page_uri=urlunparse(ParseResult(
+ scheme=gn2_uri.scheme,
+ netloc=gn2_uri.netloc,
+ path="/search",
+ params="",
+ query=urlencode({
+ "species": species["Name"],
+ "group": population["Name"],
+ "type": "Phenotypes",
+ "dataset": dataset["Name"],
+ "search_terms_or": (
+ # Very long URLs will cause
+ # errors.
+ " ".join(_xref_ids)
+ if len(_xref_ids) <= 100
+ else ""),
+ "search_terms_and": " ".join(
+ _search_terms).strip(),
+ "accession_id": "None",
+ "FormID": "searchResult"
+ }),
+ fragment="")))
+ except JobNotFound as jnf:
+ return render_template("jobs/job-not-found.html", job_id=job_id)
diff --git a/uploader/platforms/models.py b/uploader/platforms/models.py
index a859371..0dd9368 100644
--- a/uploader/platforms/models.py
+++ b/uploader/platforms/models.py
@@ -56,7 +56,8 @@ def platform_by_species_and_id(
return None
-def save_new_platform(# pylint: disable=[too-many-arguments]
+def save_new_platform(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
cursor: Cursor,
species_id: int,
geo_platform: str,
diff --git a/uploader/platforms/views.py b/uploader/platforms/views.py
index c20ab44..d12a9ef 100644
--- a/uploader/platforms/views.py
+++ b/uploader/platforms/views.py
@@ -12,7 +12,7 @@ from flask import (
from uploader.ui import make_template_renderer
from uploader.authorisation import require_login
from uploader.species.models import all_species, species_by_id
-from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+from uploader.datautils import safe_int, enumerate_sequence
from .models import (save_new_platform,
platforms_by_species,
@@ -29,9 +29,15 @@ def index():
if not bool(request.args.get("species_id")):
return render_template(
"platforms/index.html",
- species=order_by_family(all_species(conn)),
+ species=all_species(conn),
activelink="platforms")
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.platforms.list_platforms"))
+
species = species_by_id(conn, request.args["species_id"])
if not bool(species):
flash("No species selected.", "alert-danger")
diff --git a/uploader/population/models.py b/uploader/population/models.py
index 6dcd85e..d78a821 100644
--- a/uploader/population/models.py
+++ b/uploader/population/models.py
@@ -61,7 +61,7 @@ def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dic
**population_details,
"FamilyOrder": _families.get(
population_details["Family"],
- max(_families.values())+1)
+ max((0,) + tuple(_families.values()))+1)
}
cursor.execute(
"INSERT INTO InbredSet("
diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py
index 436eca0..044cdd4 100644
--- a/uploader/population/rqtl2.py
+++ b/uploader/population/rqtl2.py
@@ -11,13 +11,11 @@ from typing import Union, Callable, Optional
import MySQLdb as mdb
from redis import Redis
from MySQLdb.cursors import DictCursor
-from werkzeug.utils import secure_filename
from gn_libs.mysqldb import database_connection
from flask import (
flash,
escape,
request,
- jsonify,
url_for,
redirect,
Response,
@@ -191,127 +189,6 @@ def trigger_rqtl2_bundle_qc(
return jobid
-def chunk_name(uploadfilename: str, chunkno: int) -> str:
- """Generate chunk name from original filename and chunk number"""
- if uploadfilename == "":
- raise ValueError("Name cannot be empty!")
- if chunkno < 1:
- raise ValueError("Chunk number must be greater than zero")
- return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}"
-
-
-def chunks_directory(uniqueidentifier: str) -> Path:
- """Compute the directory where chunks are temporarily stored."""
- if uniqueidentifier == "":
- raise ValueError("Unique identifier cannot be empty!")
- return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}")
-
-
-@rqtl2.route(("<int:species_id>/populations/<int:population_id>/rqtl2/"
- "/rqtl2-bundle-chunked"),
- methods=["GET"])
-@require_login
-def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"]
- species_id: int,
- population_id: int
-):
- """
- Extension to the `upload_rqtl2_bundle` endpoint above that provides a way
- for testing whether all the chunks have been uploaded and to assist with
- resuming a failed expression-data.
- """
- fileid = request.args.get("resumableIdentifier", type=str) or ""
- filename = request.args.get("resumableFilename", type=str) or ""
- chunk = request.args.get("resumableChunkNumber", type=int) or 0
- if not(fileid or filename or chunk):
- return jsonify({
- "message": "At least one required query parameter is missing.",
- "error": "BadRequest",
- "statuscode": 400
- }), 400
-
- if Path(chunks_directory(fileid),
- chunk_name(filename, chunk)).exists():
- return "OK"
-
- return jsonify({
- "message": f"Chunk {chunk} was not found.",
- "error": "NotFound",
- "statuscode": 404
- }), 404
-
-
-def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path:
- """Merge the chunks into a single file."""
- with open(targetfile, "ab") as _target:
- for chunkfile in chunkpaths:
- with open(chunkfile, "rb") as _chunkdata:
- _target.write(_chunkdata.read())
-
- chunkfile.unlink()
- return targetfile
-
-
-@rqtl2.route(("<int:species_id>/population/<int:population_id>/rqtl2/upload/"
- "/rqtl2-bundle-chunked"),
- methods=["POST"])
-@require_login
-def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int):
- """
- Extension to the `upload_rqtl2_bundle` endpoint above that allows large
- files to be uploaded in chunks.
-
- This should hopefully speed up uploads, and if done right, even enable
- resumable uploads
- """
- _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0
- _chunk = request.form.get("resumableChunkNumber", default=1, type=int)
- _uploadfilename = request.form.get(
- "resumableFilename", default="", type=str) or ""
- _fileid = request.form.get(
- "resumableIdentifier", default="", type=str) or ""
- _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid)
-
- if _targetfile.exists():
- return jsonify({
- "message": (
- "A file with a similar unique identifier has previously been "
- "uploaded and possibly is/has being/been processed."),
- "error": "BadRequest",
- "statuscode": 400
- }), 400
-
- try:
- # save chunk data
- chunks_directory(_fileid).mkdir(exist_ok=True, parents=True)
- request.files["file"].save(Path(chunks_directory(_fileid),
- chunk_name(_uploadfilename, _chunk)))
-
- # Check whether upload is complete
- chunkpaths = tuple(
- Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk))
- for _achunk in range(1, _totalchunks+1))
- if all(_file.exists() for _file in chunkpaths):
- # merge_files and clean up chunks
- __merge_chunks__(_targetfile, chunkpaths)
- chunks_directory(_fileid).rmdir()
- jobid = trigger_rqtl2_bundle_qc(
- species_id, population_id, _targetfile, _uploadfilename)
- return url_for(
- "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid)
- except Exception as exc:# pylint: disable=[broad-except]
- msg = "Error processing uploaded file chunks."
- app.logger.error(msg, exc_info=True, stack_info=True)
- return jsonify({
- "message": msg,
- "error": type(exc).__name__,
- "error-description": " ".join(str(arg) for arg in exc.args),
- "error-trace": traceback.format_exception(exc)
- }), 500
-
- return "OK"
-
-
@rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>",
methods=["GET", "POST"])
@require_login
diff --git a/uploader/population/views.py b/uploader/population/views.py
index 4f985f5..270dd5f 100644
--- a/uploader/population/views.py
+++ b/uploader/population/views.py
@@ -2,6 +2,7 @@
import json
import base64
+from markupsafe import escape
from MySQLdb.cursors import DictCursor
from gn_libs.mysqldb import database_connection
from flask import (flash,
@@ -19,11 +20,9 @@ from uploader.genotypes.views import genotypesbp
from uploader.datautils import enumerate_sequence
from uploader.phenotypes.views import phenotypesbp
from uploader.expression_data.views import exprdatabp
+from uploader.species.models import all_species, species_by_id
from uploader.monadic_requests import make_either_error_handler
from uploader.input_validation import is_valid_representative_name
-from uploader.species.models import (all_species,
- species_by_id,
- order_species_by_family)
from .models import (save_population,
population_families,
@@ -48,7 +47,15 @@ def index():
if not bool(request.args.get("species_id")):
return render_template(
"populations/index.html",
- species=order_species_by_family(all_species(conn)))
+ species=all_species(conn),
+ activelink="populations")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.list_species_populations"))
+
species = species_by_id(conn, request.args.get("species_id"))
if not bool(species):
flash("Invalid species identifier provided!", "alert-danger")
@@ -101,6 +108,7 @@ def create_population(species_id: int):
{"id": "2", "value": "GEMMA"},
{"id": "3", "value": "R/qtl"},
{"id": "4", "value": "GEMMA, PLINK"}),
+ return_to=(request.args.get("return_to") or ""),
activelink="create-population",
**error_values)
@@ -151,7 +159,15 @@ def create_population(species_id: int):
})
def __flash_success__(_success):
- flash("Successfully created resource.", "alert-success")
+ flash("Successfully created population "
+ f"{escape(new_population['FullName'])}.",
+ "alert-success")
+ return_to = request.form.get("return_to") or ""
+ if return_to:
+ return redirect(url_for(
+ return_to,
+ species_id=species["SpeciesId"],
+ population_id=new_population["InbredSetId"]))
return redirect(url_for(
"species.populations.view_population",
species_id=species["SpeciesId"],
diff --git a/uploader/publications/__init__.py b/uploader/publications/__init__.py
new file mode 100644
index 0000000..7efcabb
--- /dev/null
+++ b/uploader/publications/__init__.py
@@ -0,0 +1,2 @@
+"""Package for handling publications."""
+from .views import pubbp
diff --git a/uploader/publications/datatables.py b/uploader/publications/datatables.py
new file mode 100644
index 0000000..e07fafd
--- /dev/null
+++ b/uploader/publications/datatables.py
@@ -0,0 +1,52 @@
+"""Fetch data for datatables."""
+import logging
+from typing import Optional
+
+from MySQLdb.cursors import DictCursor
+
+from gn_libs.mysqldb import Connection, debug_query
+
+logger = logging.getLogger(__name__)
+
+def fetch_publications(
+ conn: Connection,
+ search: Optional[str] = None,
+ offset: int = 0,
+ limit: int = -1
+) -> tuple[dict, int, int, int]:
+ """Fetch publications from the database."""
+ _query = "SELECT * FROM Publication"
+ _count_query = "SELECT COUNT(*) FROM Publication"
+ _params = None
+ _where_clause = ""
+ _limit_clause = ""
+ if search is not None and bool(search):
+ _where_clause = ("WHERE PubMed_ID LIKE %s "
+ "OR Authors LIKE %s "
+ "OR Title LIKE %s")
+ _params = (f"%{search}%",) * 3
+
+ if limit > 0:
+ _limit_clause = f"LIMIT {limit} OFFSET {offset}"
+
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT COUNT(*) FROM Publication")
+ _total_rows = int(cursor.fetchone()["COUNT(*)"])
+
+ cursor.execute(f"{_count_query} {_where_clause}", _params)
+ debug_query(cursor, logger)
+ _result = cursor.fetchone()
+ _total_filtered = int(_result["COUNT(*)"] if bool(_result) else 0)
+
+ cursor.execute(f"{_query} {_where_clause} {_limit_clause}", _params)
+ debug_query(cursor, logger)
+ _current_filtered = tuple(
+ {**dict(row), "index": idx}
+ for idx, row
+ in enumerate(cursor.fetchall(), start=offset+1))
+
+ return (
+ _current_filtered,
+ len(_current_filtered),
+ _total_filtered,
+ _total_rows)
diff --git a/uploader/publications/misc.py b/uploader/publications/misc.py
new file mode 100644
index 0000000..fca6f71
--- /dev/null
+++ b/uploader/publications/misc.py
@@ -0,0 +1,25 @@
+"""Miscellaneous functions dealing with publications."""
+
+
+def publications_differences(
+ filedata: tuple[dict, ...],
+ dbdata: tuple[dict, ...],
+ pubmedid2pubidmap: tuple[dict, ...]
+) -> tuple[dict, ...]:
+ """Compute the differences between file data and db data"""
+ diff = tuple()
+ for filerow, dbrow in zip(
+ sorted(filedata, key=lambda item: (
+ item["phenotype_id"], item["xref_id"])),
+ sorted(dbdata, key=lambda item: (
+ item["PhenotypeId"], item["xref_id"]))):
+ if filerow["PubMed_ID"] == dbrow["PubMed_ID"]:
+ continue
+
+ newpubmed = filerow["PubMed_ID"]
+ diff = diff + ({
+ **dbrow,
+ "PubMed_ID": newpubmed,
+ "PublicationId": pubmedid2pubidmap.get(newpubmed)},)
+
+ return diff
diff --git a/uploader/publications/models.py b/uploader/publications/models.py
new file mode 100644
index 0000000..b199991
--- /dev/null
+++ b/uploader/publications/models.py
@@ -0,0 +1,96 @@
+"""Module to handle persistence and retrieval of publication to/from MariaDB"""
+import logging
+from typing import Iterable, Optional
+
+from MySQLdb.cursors import DictCursor
+
+from gn_libs.mysqldb import Connection, debug_query
+
+logger = logging.getLogger(__name__)
+
+
+def fetch_phenotype_publications(
+ conn: Connection,
+ ids: tuple[tuple[int, int], ...]
+) -> tuple[dict, ...]:
+ """Fetch publication from database by ID."""
+ paramstr = ",".join(["(%s, %s)"] * len(ids))
+ query = (
+ "SELECT "
+ "pxr.PhenotypeId, pxr.Id AS xref_id, pxr.PublicationId, pub.PubMed_ID "
+ "FROM PublishXRef AS pxr INNER JOIN Publication AS pub "
+ "ON pxr.PublicationId=pub.Id "
+ f"WHERE (pxr.PhenotypeId, pxr.Id) IN ({paramstr})")
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(query, tuple(item for row in ids for item in row))
+ return tuple(dict(row) for row in cursor.fetchall())
+
+
+def create_new_publications(
+ conn: Connection,
+ publications: tuple[dict, ...]
+) -> tuple[dict, ...]:
+ if len(publications) > 0:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.executemany(
+ ("INSERT INTO "
+ "Publication( "
+ "PubMed_ID, Abstract, Authors, Title, Journal, Volume, Pages, "
+ "Month, Year"
+ ") "
+ "VALUES("
+ "%(pubmed_id)s, %(abstract)s, %(authors)s, %(title)s, "
+ "%(journal)s, %(volume)s, %(pages)s, %(month)s, %(year)s"
+ ") "
+ "RETURNING *"),
+ publications)
+ return tuple({
+ **row, "publication_id": row["Id"]
+ } for row in cursor.fetchall())
+ return tuple()
+
+
+def update_publications(conn: Connection , publications: tuple[dict, ...]) -> tuple[dict, ...]:
+ """Update details for multiple publications"""
+ if len(publications) > 0:
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ logger.debug("UPDATING PUBLICATIONS: %s", publications)
+ cursor.executemany(
+ ("UPDATE Publication SET "
+ "PubMed_ID=%(pubmed_id)s, Abstract=%(abstract)s, "
+ "Authors=%(authors)s, Title=%(title)s, Journal=%(journal)s, "
+ "Volume=%(volume)s, Pages=%(pages)s, Month=%(month)s, "
+ "Year=%(year)s "
+ "WHERE Id=%(publication_id)s"),
+ publications)
+ debug_query(cursor, logger)
+ return publications
+ return tuple()
+ return tuple()
+
+
+def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict:
+ """Fetch a specific publication from the database."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute("SELECT * FROM Publication WHERE Id=%s",
+ (publication_id,))
+ _res = cursor.fetchone()
+ return dict(_res) if _res else {}
+
+
+def fetch_publication_phenotypes(
+ conn: Connection, publication_id: int) -> Iterable[dict]:
+ """Fetch all phenotypes linked to this publication."""
+ with conn.cursor(cursorclass=DictCursor) as cursor:
+ cursor.execute(
+ "SELECT pxr.Id AS xref_id, pxr.PublicationId, phe.* "
+ "FROM PublishXRef AS pxr INNER JOIN Phenotype AS phe "
+ "ON pxr.PhenotypeId=phe.Id "
+ "WHERE pxr.PublicationId=%s",
+ (publication_id,))
+ while True:
+ row = cursor.fetchone()
+ if row:
+ yield row
+ else:
+ break
diff --git a/uploader/publications/pubmed.py b/uploader/publications/pubmed.py
new file mode 100644
index 0000000..ed9b652
--- /dev/null
+++ b/uploader/publications/pubmed.py
@@ -0,0 +1,103 @@
+"""Module to interact with NCBI's PubMed"""
+import logging
+
+import requests
+from lxml import etree
+
+logger = logging.getLogger(__name__)
+
+
+def __pub_date__(pubdate: etree.Element):
+ pubyear = pubdate.find("Year")
+ pubmonth = pubdate.find("Month")
+ pubday = pubdate.find("Day")
+ return {
+ "year": pubyear.text if pubyear is not None else None,
+ "month": pubmonth.text if pubmonth is not None else None,
+ "day": pubday.text if pubday is not None else None
+ }
+
+
+def __journal__(journal: etree.Element) -> dict:
+ volume = journal.find("JournalIssue/Volume")
+ issue = journal.find("JournalIssue/Issue")
+ return {
+ "volume": volume.text if volume is not None else None,
+ "issue": issue.text if issue is not None else None,
+ **__pub_date__(journal.find("JournalIssue/PubDate")),
+ "journal": journal.find("Title").text
+ }
+
+def __author__(author: etree.Element) -> str:
+ return "%s %s" % (
+ author.find("LastName").text,
+ author.find("Initials").text)
+
+
+def __pages__(pagination: etree.Element) -> str:
+ start = pagination.find("StartPage")
+ end = pagination.find("EndPage")
+ return (start.text + (
+ f"-{end.text}" if end is not None else ""
+ )) if start is not None else ""
+
+
+def __abstract__(article: etree.Element) -> str:
+ abstract = article.find("Abstract/AbstractText")
+ return abstract.text if abstract is not None else None
+
+
+def __article__(pubmed_article: etree.Element) -> dict:
+ article = pubmed_article.find("MedlineCitation/Article")
+ return {
+ "pubmed_id": int(pubmed_article.find("MedlineCitation/PMID").text),
+ "title": article.find("ArticleTitle").text,
+ **__journal__(article.find("Journal")),
+ "abstract": __abstract__(article),
+ "pages": __pages__(article.find("Pagination")),
+ "authors": ", ".join(__author__(author)
+ for author in article.findall("AuthorList/Author"))
+ }
+
+
+def __process_pubmed_publication_data__(text) -> tuple[dict, ...]:
+ """Process the data from PubMed into usable data."""
+ doc = etree.XML(text)
+ articles = doc.xpath("//PubmedArticle")
+ logger.debug("Retrieved %s publications from NCBI", len(articles))
+ return tuple(__article__(article) for article in articles)
+
+def fetch_publications(pubmed_ids: tuple[int, ...]) -> tuple[dict, ...]:
+ """Retrieve data on new publications from NCBI."""
+ # See whether we can retrieve multiple publications in one go
+ # Parse data and save to DB
+ # Return PublicationId(s) for new publication(s).
+ if len(pubmed_ids) == 0:
+ logger.debug("There are no new PubMed IDs to fetch")
+ return tuple()
+
+ logger.info("Fetching publications data for the following PubMed IDs: %s",
+ ", ".join((str(pid) for pid in pubmed_ids)))
+
+ # Should we, perhaps, pass this in from a config variable?
+ uri = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi"
+ try:
+ response = requests.get(
+ uri,
+ params={
+ "db": "pubmed",
+ "retmode": "xml",
+ "id": ",".join(str(item) for item in pubmed_ids)
+ })
+
+ if response.status_code == 200:
+ return __process_pubmed_publication_data__(response.text)
+
+ logger.error(
+ "Could not fetch the new publication from %s (status code: %s)",
+ uri,
+ response.status_code)
+ except requests.exceptions.ConnectionError:
+ logger.error("Could not find the domain %s", uri)
+
+ return tuple()
diff --git a/uploader/publications/views.py b/uploader/publications/views.py
new file mode 100644
index 0000000..0608a35
--- /dev/null
+++ b/uploader/publications/views.py
@@ -0,0 +1,107 @@
+"""Endpoints for publications"""
+import json
+
+from MySQLdb.cursors import DictCursor
+from gn_libs.mysqldb import database_connection
+from flask import (
+ flash,
+ request,
+ url_for,
+ redirect,
+ Blueprint,
+ render_template,
+ current_app as app)
+
+from uploader.authorisation import require_login
+
+from .models import (
+ fetch_publication_by_id,
+ create_new_publications,
+ fetch_publication_phenotypes)
+
+from .datatables import fetch_publications
+
+from gn_libs.debug import __pk__
+
+pubbp = Blueprint("publications", __name__)
+
+
+@pubbp.route("/", methods=["GET"])
+@require_login
+def index():
+ """Index page for publications."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return render_template("publications/index.html")
+
+
+@pubbp.route("/list", methods=["GET"])
+@require_login
+def list_publications():
+ # request breakdown:
+ # https://datatables.net/manual/server-side
+ _page = int(request.args.get("draw"))
+ _length = int(request.args.get("length") or '-1')
+ _start = int(request.args.get("start") or '0')
+ _search = request.args["search[value]"]
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ _publications, _current_rows, _totalfiltered, _totalrows = fetch_publications(
+ conn,
+ _search,
+ offset=_start,
+ limit=_length)
+
+ return json.dumps({
+ "draw": _page,
+ "recordsTotal": _totalrows,
+ "recordsFiltered": _totalfiltered,
+ "publications": _publications,
+ "status": "success"
+ })
+
+
+@pubbp.route("/view/<int:publication_id>", methods=["GET"])
+@require_login
+def view_publication(publication_id: int):
+ """View more details on a particular publication."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ return render_template(
+ "publications/view-publication.html",
+ publication=fetch_publication_by_id(conn, publication_id),
+ linked_phenotypes=tuple(fetch_publication_phenotypes(
+ conn, publication_id)))
+
+
+@pubbp.route("/create", methods=["GET", "POST"])
+@require_login
+def create_publication():
+ """Create a new publication."""
+ if(request.method == "GET"):
+ return render_template("publications/create-publication.html")
+ form = request.form
+ authors = form.get("publication-authors").encode("utf8")
+ if authors is None or authors == "":
+ flash("The publication's author(s) MUST be provided!", "alert alert-danger")
+ return redirect(url_for("publications.create", **request.args))
+
+ with database_connection(app.config["SQL_URI"]) as conn:
+ publications = create_new_publications(conn, ({
+ "pubmed_id": form.get("pubmed-id") or None,
+ "abstract": form.get("publication-abstract").encode("utf8") or None,
+ "authors": authors,
+ "title": form.get("publication-title").encode("utf8") or None,
+ "journal": form.get("publication-journal").encode("utf8") or None,
+ "volume": form.get("publication-volume").encode("utf8") or None,
+ "pages": form.get("publication-pages").encode("utf8") or None,
+ "month": (form.get("publication-month") or "").encode("utf8").capitalize() or None,
+ "year": form.get("publication-year").encode("utf8") or None
+ },))
+ flash("New publication created!", "alert alert-success")
+ return redirect(url_for(
+ request.args.get("return_to") or "publications.view_publication",
+ publication_id=publications[0]["publication_id"],
+ **request.args))
+
+ flash("Publication creation failed!", "alert alert-danger")
+ app.logger.debug("Failed to create the new publication.", exc_info=True)
+ return redirect(url_for("publications.create_publication"))
diff --git a/uploader/route_utils.py b/uploader/route_utils.py
new file mode 100644
index 0000000..ce718fb
--- /dev/null
+++ b/uploader/route_utils.py
@@ -0,0 +1,42 @@
+"""Generic routing utilities."""
+from flask import flash, url_for, redirect, render_template, current_app as app
+
+from gn_libs.mysqldb import database_connection
+
+from uploader.population.models import (populations_by_species,
+ population_by_species_and_id)
+
+def generic_select_population(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
+ species: dict,
+ template: str,
+ population_id: str,
+ back_to: str,
+ forward_to: str,
+ activelink: str,
+ error_message: str = "No such population found!"
+):
+ """Handles common flow for 'select population' step."""
+ with database_connection(app.config["SQL_URI"]) as conn:
+ if not bool(population_id):
+ return render_template(
+ template,
+ species=species,
+ populations=populations_by_species(conn, species["SpeciesId"]),
+ activelink=activelink)
+
+ if population_id == "CREATE-POPULATION":
+ return redirect(url_for(
+ "species.populations.create_population",
+ species_id=species["SpeciesId"],
+ return_to=forward_to))
+
+ population = population_by_species_and_id(
+ conn, species["SpeciesId"], int(population_id))
+ if not bool(population):
+ flash(error_message, "alert-danger")
+ return redirect(url_for(back_to, species_id=species["SpeciesId"]))
+
+ return redirect(url_for(forward_to,
+ species_id=species["SpeciesId"],
+ population_id=population["Id"]))
diff --git a/uploader/samples/models.py b/uploader/samples/models.py
index d7d5384..b419d61 100644
--- a/uploader/samples/models.py
+++ b/uploader/samples/models.py
@@ -15,11 +15,11 @@ def samples_by_species_and_population(
"""Fetch the samples by their species and population."""
with conn.cursor(cursorclass=DictCursor) as cursor:
cursor.execute(
- "SELECT iset.InbredSetId, s.* FROM InbredSet AS iset "
- "INNER JOIN StrainXRef AS sxr ON iset.InbredSetId=sxr.InbredSetId "
- "INNER JOIN Strain AS s ON sxr.StrainId=s.Id "
- "WHERE s.SpeciesId=%(species_id)s "
- "AND iset.InbredSetId=%(population_id)s",
+ "SELECT InbredSet.InbredSetId, Strain.* FROM InbredSet "
+ "INNER JOIN StrainXRef ON InbredSet.InbredSetId=StrainXRef.InbredSetId "
+ "INNER JOIN Strain ON StrainXRef.StrainId=Strain.Id "
+ "WHERE Strain.SpeciesId=%(species_id)s "
+ "AND InbredSet.InbredSetId=%(population_id)s",
{"species_id": species_id, "population_id": population_id})
return tuple(cursor.fetchall())
diff --git a/uploader/samples/views.py b/uploader/samples/views.py
index ed79101..c0adb88 100644
--- a/uploader/samples/views.py
+++ b/uploader/samples/views.py
@@ -16,16 +16,15 @@ from uploader import jobs
from uploader.files import save_file
from uploader.ui import make_template_renderer
from uploader.authorisation import require_login
-from uploader.request_checks import with_population
from uploader.input_validation import is_integer_input
-from uploader.datautils import safe_int, order_by_family, enumerate_sequence
-from uploader.population.models import population_by_id, populations_by_species
+from uploader.population.models import population_by_id
+from uploader.route_utils import generic_select_population
+from uploader.datautils import safe_int, enumerate_sequence
+from uploader.species.models import all_species, species_by_id
+from uploader.request_checks import with_species, with_population
from uploader.db_utils import (with_db_connection,
database_connection,
with_redis_connection)
-from uploader.species.models import (all_species,
- species_by_id,
- order_species_by_family)
from .models import samples_by_species_and_population
@@ -40,8 +39,15 @@ def index():
if not bool(request.args.get("species_id")):
return render_template(
"samples/index.html",
- species=order_species_by_family(all_species(conn)),
+ species=all_species(conn),
activelink="samples")
+
+ species_id = request.args.get("species_id")
+ if species_id == "CREATE-SPECIES":
+ return redirect(url_for(
+ "species.create_species",
+ return_to="species.populations.samples.select_population"))
+
species = species_by_id(conn, request.args.get("species_id"))
if not bool(species):
flash("No such species!", "alert-danger")
@@ -52,57 +58,31 @@ def index():
@samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"])
@require_login
-def select_population(species_id: int):
+@with_species(redirect_uri="species.populations.samples.index")
+def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument]
"""Select the population to use for the samples."""
- with database_connection(app.config["SQL_URI"]) as conn:
- species = species_by_id(conn, species_id)
- if not bool(species):
- flash("Invalid species!", "alert-danger")
- return redirect(url_for("species.populations.samples.index"))
-
- if not bool(request.args.get("population_id")):
- return render_template("samples/select-population.html",
- species=species,
- populations=order_by_family(
- populations_by_species(
- conn,
- species_id),
- order_key="FamilyOrder"),
- activelink="samples")
-
- population = population_by_id(conn, request.args.get("population_id"))
- if not bool(population):
- flash("Population not found!", "alert-danger")
- return redirect(url_for(
- "species.populations.samples.select_population",
- species_id=species_id))
-
- return redirect(url_for("species.populations.samples.list_samples",
- species_id=species_id,
- population_id=population["Id"]))
+ return generic_select_population(
+ species,
+ "samples/select-population.html",
+ request.args.get("population_id") or "",
+ "species.populations.samples.select_population",
+ "species.populations.samples.list_samples",
+ "samples",
+ "Population not found!")
@samplesbp.route("<int:species_id>/populations/<int:population_id>/samples")
@require_login
-def list_samples(species_id: int, population_id: int):
+@with_population(
+ species_redirect_uri="species.populations.samples.index",
+ redirect_uri="species.populations.samples.select_population")
+def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
"""
List the samples in a particular population and give the ability to upload
new ones.
"""
with database_connection(app.config["SQL_URI"]) as conn:
- species = species_by_id(conn, species_id)
- if not bool(species):
- flash("Invalid species!", "alert-danger")
- return redirect(url_for("species.populations.samples.index"))
-
- population = population_by_id(conn, population_id)
- if not bool(population):
- flash("Population not found!", "alert-danger")
- return redirect(url_for(
- "species.populations.samples.select_population",
- species_id=species_id))
-
all_samples = enumerate_sequence(samples_by_species_and_population(
- conn, species_id, population_id))
+ conn, species["SpeciesId"], population["Id"]))
total_samples = len(all_samples)
offset = max(safe_int(request.args.get("from") or 0), 0)
count = int(request.args.get("count") or 20)
@@ -241,7 +221,10 @@ def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):
if status == "error":
return redirect(url_for(
- "species.populations.samples.upload_failure", job_id=job_id))
+ "species.populations.samples.upload_failure",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ job_id=job_id))
error_filename = Path(jobs.error_filename(
job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
@@ -261,9 +244,14 @@ def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):
species=species,
population=population), 400
-@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"])
+
+@samplesbp.route("<int:species_id>/populations/<int:population_id>/"
+ "upload-samples/failure/<uuid:job_id>",
+ methods=["GET"])
@require_login
-def upload_failure(job_id: uuid.UUID):
+@with_population(species_redirect_uri="species.populations.samples.index",
+ redirect_uri="species.populations.samples.select_population")
+def upload_failure(species: dict, population: dict, job_id: uuid.UUID, **kwargs):
"""Display the errors of the samples upload failure."""
job = with_redis_connection(lambda rconn: jobs.job(
rconn, jobs.jobsnamespace(), job_id))
@@ -277,4 +265,7 @@ def upload_failure(job_id: uuid.UUID):
if stat.st_size > 0:
return render_template("worker_failure.html", job_id=job_id)
- return render_template("samples/upload-failure.html", job=job)
+ return render_template("samples/upload-failure.html",
+ species=species,
+ population=population,
+ job=job)
diff --git a/uploader/session.py b/uploader/session.py
index b538187..5af5827 100644
--- a/uploader/session.py
+++ b/uploader/session.py
@@ -77,12 +77,15 @@ def set_user_token(token: str) -> SessionInfo:
"""Set the user's token."""
info = session_info()
return save_session_info({
- **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc]
+ **info,
+ "user": {**info["user"], "token": Right(token), "logged_in": True}
+ })#type: ignore[misc]
def set_user_details(userdets: UserDetails) -> SessionInfo:
"""Set the user details information"""
- return save_session_info({**session_info(), "user": userdets})#type: ignore[misc]
+ info = session_info()
+ return save_session_info({**info, "user": {**info["user"], **userdets}})#type: ignore[misc]
def user_details() -> UserDetails:
"""Retrieve user details."""
diff --git a/uploader/species/models.py b/uploader/species/models.py
index 51f941c..db53d48 100644
--- a/uploader/species/models.py
+++ b/uploader/species/models.py
@@ -58,7 +58,8 @@ def save_species(conn: mdb.Connection,
common_name: The species' common name.
scientific_name; The species' scientific name.
"""
- genus, species_name = scientific_name.split(" ")
+ genus, *species_parts = scientific_name.split(" ")
+ species_name: str = " ".join(species_parts)
families = species_families(conn)
with conn.cursor() as cursor:
cursor.execute("SELECT MAX(OrderId) FROM Species")
@@ -68,7 +69,7 @@ def save_species(conn: mdb.Connection,
"menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})",
"scientific_name": scientific_name,
"family": family,
- "family_order": families[family],
+ "family_order": families.get(family, 999999),
"taxon_id": taxon_id,
"species_order": cursor.fetchone()[0] + 5
}
@@ -116,7 +117,8 @@ def update_species(# pylint: disable=[too-many-arguments]
species_order: The ordering of this species in relation to others
"""
with conn.cursor(cursorclass=DictCursor) as cursor:
- genus, species_name = scientific_name.split(" ")
+ genus, *species_parts = scientific_name.split(" ")
+ species_name = " ".join(species_parts)
species = {
"species_id": species_id,
"common_name": common_name,
diff --git a/uploader/species/views.py b/uploader/species/views.py
index fee5c75..cea2f68 100644
--- a/uploader/species/views.py
+++ b/uploader/species/views.py
@@ -1,4 +1,5 @@
"""Endpoints handling species."""
+from markupsafe import escape
from pymonad.either import Left, Right, Either
from gn_libs.mysqldb import database_connection
from flask import (flash,
@@ -62,6 +63,8 @@ def create_species():
if request.method == "GET":
return render_template("species/create-species.html",
families=species_families(conn),
+ return_to=(
+ request.args.get("return_to") or ""),
activelink="create-species")
error = False
@@ -79,7 +82,7 @@ def create_species():
error = True
parts = tuple(name.strip() for name in scientific_name.split(" "))
- if len(parts) != 2 or not all(bool(name) for name in parts):
+ if (len(parts) != 2 and len(parts) != 3) or not all(bool(name) for name in parts):
flash("The scientific name you provided is invalid.", "alert-danger")
error = True
@@ -113,7 +116,15 @@ def create_species():
species = save_species(
conn, common_name, scientific_name, family, taxon_id)
- flash("Species saved successfully!", "alert-success")
+ flash(
+ f"You have successfully added species "
+ f"'{escape(species['scientific_name'])} "
+ f"({escape(species['common_name'])})'.",
+ "alert-success")
+
+ return_to = request.form.get("return_to").strip()
+ if return_to:
+ return redirect(url_for(return_to, species_id=species["species_id"]))
return redirect(url_for("species.view_species", species_id=species["species_id"]))
diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css
index f482c1b..df50dec 100644
--- a/uploader/static/css/styles.css
+++ b/uploader/static/css/styles.css
@@ -1,161 +1,187 @@
+* {
+ box-sizing: border-box;
+}
+
body {
margin: 0.7em;
- box-sizing: border-box;
display: grid;
- grid-template-columns: 1fr 6fr;
- grid-template-rows: 5em 100%;
+ grid-template-columns: 2fr 8fr;
grid-gap: 20px;
- font-family: Georgia, Garamond, serif;
+ font-family: "Helvetica Neue", Helvetica, Arial, sans-serif;
font-style: normal;
+ font-size: 20px;
}
#header {
- grid-column: 1/3;
- width: 100%;
- /* background: cyan; */
- padding-top: 0.5em;
- border-radius: 0.5em;
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Define layout for the children elements */
+ display: grid;
+ grid-template-columns: 8fr 2fr;
+ /* Content styling */
background-color: #336699;
- border-color: #080808;
color: #FFFFFF;
- background-image: none;
+ border-radius: 3px;
+ min-height: 30px;
}
-#header .header {
- font-size: 2em;
- display: inline-block;
- text-align: start;
-}
+#header #header-text {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 2;
-#header .header-nav {
- display: inline-block;
- color: #FFFFFF;
+ /* Content styling */
+ padding-left: 1em;
}
-#header .header-nav li {
- border-width: 1px;
- border-color: #FFFFFF;
- vertical-align: middle;
- margin: 0.2em;
- border-style: solid;
- border-width: 2px;
- border-radius: 0.5em;
- text-align: center;
+#header #header-nav {
+ /* Place it in the parent element */
+ grid-column-start: 2;
+ grid-column-end: 3;
}
-#header .header-nav a {
+#header #header-nav .nav li a {
+ /* Content styling */
color: #FFFFFF;
- text-decoration: none;
+ background: #4477AA;
+ border: solid 5px #336699;
+ border-radius: 5px;
+ font-size: 0.7em;
+ text-align: center;
+ padding: 1px 7px;
}
#nav-sidebar {
- grid-column: 1/2;
- /* background: #e5e5ff; */
- padding-top: 0.5em;
- border-radius: 0.5em;
- font-size: 1.2em;
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 2;
}
-#main {
- grid-column: 2/3;
- width: 100%;
- /* background: gray; */
+#nav-sidebar .nav li a:hover {
border-radius: 0.5em;
}
-.pagetitle {
- padding-top: 0.5em;
- /* background: pink; */
+#nav-sidebar .nav .activemenu {
+ border-style: solid;
border-radius: 0.5em;
- /* background-color: #6699CC; */
- /* background-color: #77AADD; */
- background-color: #88BBEE;
+ border-color: #AAAAAA;
+ background-color: #EFEFEF;
}
-.pagetitle h1 {
- text-align: start;
- text-transform: capitalize;
- padding-left: 0.25em;
-}
+#main {
+ /* Place it in the parent element */
+ grid-column-start: 2;
+ grid-column-end: 3;
-.pagetitle .breadcrumb {
- background: none;
+ /* Define layout for the children elements */
+ display: grid;
+ grid-template-columns: 1fr;
+ grid-template-rows: 4em 100%;
+ grid-gap: 1em;
}
-.pagetitle .breadcrumb .active a {
- color: #333333;
+#main #pagetitle {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Content-styling */
+ border-radius: 3px;
+ background-color: #88BBEE;
}
-.pagetitle .breadcrumb a {
- color: #666666;
+#main #pagetitle .title {
+ font-size: 1.4em;
+ text-transform: capitalize;
+ padding-left: 0.5em;
}
-.main-content {
- font-size: 1.275em;
+@media screen and (max-width: 20in) {
+ #main #all-content {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Define layout for the children elements */
+ max-width: 80%;
+ }
+
+ #sidebar-content {
+ display: none;
+ }
}
-.breadcrumb {
- text-transform: capitalize;
+@media screen and (min-width: 20.1in) {
+ #main #all-content {
+ /* Place it in the parent element */
+ grid-column-start: 1;
+ grid-column-end: 3;
+
+ /* Define layout for the children elements */
+ display: grid;
+ grid-template-columns: 7fr 3fr;
+ grid-gap: 1.5em;
+ }
}
-dd {
- margin-left: 3em;
- font-size: 0.88em;
- padding-bottom: 1em;
+#main #all-content .row {
+ margin: 0 2px;
}
-input[type="submit"], .btn {
- text-transform: capitalize;
+#main #all-content #main-content {
+ background: #FFFFFF;
+ max-width: 950px;
}
-.card {
- margin-top: 0.3em;
- border-width: 1px;
- border-style: solid;
- border-radius: 0.3em;
- border-color: #AAAAAA;
- padding: 0.5em;
+#pagetitle .breadcrumb {
+ background: none;
+ text-transform: capitalize;
+ font-size: 0.75em;
}
-.activemenu {
- border-style: solid;
- border-radius: 0.5em;
- border-color: #AAAAAA;
- background-color: #EFEFEF;
+#pagetitle .breadcrumb .active a {
+ color: #333333;
}
-.danger {
- color: #A94442;
- border-color: #DCA7A7;
- background-color: #F2DEDE;
+#pagetitle .breadcrumb a {
+ color: #666666;
}
.heading {
border-bottom: solid #EEBB88;
+ text-transform: capitalize;
}
.subheading {
padding: 1em 0 0.1em 0.5em;
border-bottom: solid #88BBEE;
+ text-transform: capitalize;
+}
+
+input[type="search"] {
+ border-radius: 5px;
}
-form {
- margin-top: 0.3em;
- background: #E5E5FF;
- padding: 0.5em;
- border-radius:0.5em;
+.btn {
+ text-transform: Capitalize;
}
-form .form-control {
- background-color: #EAEAFF;
+table.dataTable thead th, table.dataTable tfoot th{
+ border-right: 1px solid white;
+ color: white;
+ background-color: #369 !important;
}
-.sidebar-content .card .card-title {
- font-size: 1.5em;
+table.dataTable tbody tr.selected td {
+ background-color: #ffee99 !important;
}
-.sidebar-content .card-text table tbody td:nth-child(1) {
- font-weight: bolder;
+.form-group {
+ margin-bottom: 2em;
+ padding-bottom: 0.2em;
+ border-bottom: solid gray 1px;
}
diff --git a/uploader/static/js/datatables.js b/uploader/static/js/datatables.js
new file mode 100644
index 0000000..82fd696
--- /dev/null
+++ b/uploader/static/js/datatables.js
@@ -0,0 +1,69 @@
+/** Handlers for events in datatables **/
+
+var addTableLength = (menuList, lengthToAdd, dataLength) => {
+ if(dataLength >= lengthToAdd) {
+ newList = structuredClone(menuList);//menuList.slice(0, menuList.length); // shallow copy
+ newList.push(lengthToAdd);
+ return newList;
+ }
+ return menuList;
+};
+
+var defaultLengthMenu = (data) => {
+ menuList = []
+ var lengths = [10, 25, 50, 100, 1000, data.length];
+ lengths.forEach((len) => {
+ menuList = addTableLength(menuList, len, data.length);
+ });
+ return menuList;
+};
+
+var buildDataTable = (tableId, data = [], columns = [], userSettings = {}) => {
+ var defaultSettings = {
+ responsive: true,
+ layout: {
+ topStart: null,
+ topEnd: null,
+ bottomStart: null,
+ bottomEnd: null,
+ },
+ select: true,
+ lengthMenu: defaultLengthMenu(data),
+ language: {
+ processing: "Processing… Please wait.",
+ loadingRecords: "Loading table data… Please wait.",
+ lengthMenu: "",
+ info: ""
+ },
+ data: data,
+ columns: columns,
+ drawCallback: (settings) => {
+ $(this[0]).find("tbody tr").each((idx, row) => {
+ var arow = $(row);
+ var checkboxOrRadio = arow.find(".chk-row-select");
+ if (checkboxOrRadio) {
+ if (arow.hasClass("selected")) {
+ checkboxOrRadio.prop("checked", true);
+ } else {
+ checkboxOrRadio.prop("checked", false);
+ }
+ }
+ });
+ }
+ }
+ var theDataTable = $(tableId).DataTable({
+ ...defaultSettings,
+ ...userSettings
+ });
+ theDataTable.on("select", (event, datatable, type, cell, originalEvent) => {
+ datatable.rows({selected: true}).nodes().each((node, index) => {
+ $(node).find(".chk-row-select").prop("checked", true)
+ });
+ });
+ theDataTable.on("deselect", (event, datatable, type, cell, originalEvent) => {
+ datatable.rows({selected: false}).nodes().each((node, index) => {
+ $(node).find(".chk-row-select").prop("checked", false)
+ });
+ });
+ return theDataTable;
+};
diff --git a/uploader/static/js/debug.js b/uploader/static/js/debug.js
new file mode 100644
index 0000000..eb01209
--- /dev/null
+++ b/uploader/static/js/debug.js
@@ -0,0 +1,40 @@
+/**
+ * The entire purpose of this function is for use to debug values inline
+ * without changing the flow of the code too much.
+ *
+ * This **MUST** be a non-arrow function to allow access to the `arguments`
+ * object.
+ *
+ * This function expects at least one argument.
+ *
+ * If more than one argument is provided, then:
+ * a) the last argument is considered the value, and will be returned
+ * b) all other arguments will be converted to string and output
+ *
+ * If only one argument is provided, it is considered the value, and will be
+ * returned.
+ *
+ * Zero arguments is an error condition.
+ **/
+function __pk__(val) {
+ /* Handle zero arguments */
+ if (arguments.length < 1) {
+ throw new Error("Invalid arguments: Expected at least one argument.");
+ }
+
+ msg = "/********** DEBUG **********/";
+ if (arguments.length > 1) {
+ msg = Array.from(
+ arguments
+ ).slice(
+ 0,
+ arguments.length - 1
+ ).map((val) => {
+ return String(val);
+ }).join("; ")
+ }
+
+ value = arguments[arguments.length - 1];
+ console.debug("/********** " + msg + " **********/", value);
+ return value;
+}
diff --git a/uploader/static/js/files.js b/uploader/static/js/files.js
new file mode 100644
index 0000000..0bde6f7
--- /dev/null
+++ b/uploader/static/js/files.js
@@ -0,0 +1,118 @@
+var readFirstNLines = (thefile, count, process_content_fns) => {
+ var reader = new FileReader();
+ if(typeof thefile !== "undefined" && thefile !== null) {
+ reader.addEventListener("load", (event) => {
+ var content = event
+ .target
+ .result
+ .split("\n")
+ .slice(0, count)
+ .map((line) => {return line.trim("\r");});
+ process_content_fns.forEach((fn) => {fn(content);});
+ });
+ reader.readAsText(thefile);
+ }
+};
+var read_first_n_lines = readFirstNLines;
+
+
+var readBinaryFile = (file) => {
+ return new Promise((resolve, reject) => {
+ var _reader = new FileReader();
+ _reader.onload = (event) => {resolve(_reader.result);};
+ _reader.readAsArrayBuffer(file);
+ });
+};
+
+
+var Uint8ArrayToHex = (arr) => {
+ var toHex = (val) => {
+ _hex = val.toString(16);
+ if(_hex.length < 2) {
+ return "0" + val;
+ }
+ return _hex;
+ };
+ _hexstr = ""
+ arr.forEach((val) => {_hexstr += toHex(val)});
+ return _hexstr
+};
+
+
+var computeFileChecksum = (file) => {
+ return readBinaryFile(file)
+ .then((content) => {
+ return window.crypto.subtle.digest(
+ "SHA-256", new Uint8Array(content));
+ }).then((digest) => {
+ return Uint8ArrayToHex(new Uint8Array(digest))
+ });
+};
+
+
+var defaultResumableHandler = (event) => {
+ throw new Error("Please provide a valid event handler!");
+};
+
+var addHandler = (resumable, handlername, handler) => {
+ if(resumable.support) {
+ resumable.on(handlername, (handler || defaultResumableHandler));
+ }
+ return resumable;
+};
+
+
+var makeResumableHandler = (handlername) => {
+ return (resumable, handler) => {
+ return addHandler(resumable, handlername, handler);
+ };
+};
+
+
+var fileSuccessHandler = makeResumableHandler("fileSuccess");
+var fileProgressHandler = makeResumableHandler("fileProgress");
+var fileAddedHandler = makeResumableHandler("fileAdded");
+var filesAddedHandler = makeResumableHandler("filesAdded");
+var filesRetryHandler = makeResumableHandler("filesRetry");
+var filesErrorHandler = makeResumableHandler("filesError");
+var uploadStartHandler = makeResumableHandler("uploadStart");
+var completeHandler = makeResumableHandler("complete");
+var progressHandler = makeResumableHandler("progress");
+var errorHandler = makeResumableHandler("error");
+
+
+var markResumableDragAndDropElement = (resumable, fileinput, droparea, browsebutton) => {
+ if(resumable.support) {
+ //Hide file input element and display drag&drop UI
+ add_class(fileinput, "visually-hidden");
+ remove_class(droparea, "visually-hidden");
+
+ // Define UI elements for browse and drag&drop
+ resumable.assignDrop(droparea);
+ resumable.assignBrowse(browsebutton);
+ }
+
+ return resumable;
+};
+
+
+var makeResumableElement = (targeturi, fileinput, droparea, uploadbutton, filetype) => {
+ var resumable = Resumable({
+ target: targeturi,
+ fileType: filetype,
+ maxFiles: 1,
+ forceChunkSize: true,
+ generateUniqueIdentifier: (file, event) => {
+ return computeFileChecksum(file).then((checksum) => {
+ var _relativePath = (file.webkitRelativePath
+ || file.relativePath
+ || file.fileName
+ || file.name);
+ return checksum + "-" + _relativePath.replace(
+ /[^a-zA-Z0-9_-]/img, "");
+ });
+ }
+ });
+
+ return resumable;
+};
diff --git a/uploader/static/js/misc.js b/uploader/static/js/misc.js
deleted file mode 100644
index cf7b39e..0000000
--- a/uploader/static/js/misc.js
+++ /dev/null
@@ -1,6 +0,0 @@
-"Miscellaneous functions and event-handlers"
-
-$(".not-implemented").click((event) => {
- event.preventDefault();
- alert("This feature is not implemented yet. Please bear with us.");
-});
diff --git a/uploader/static/js/populations.js b/uploader/static/js/populations.js
new file mode 100644
index 0000000..be1231f
--- /dev/null
+++ b/uploader/static/js/populations.js
@@ -0,0 +1,21 @@
+$(() => {
+ var populationsDataTable = buildDataTable(
+ "#tbl-select-population",
+ JSON.parse(
+ $("#tbl-select-population").attr("data-populations-list")),
+ [
+ {
+ data: (apopulation) => {
+ return `<input type="radio" name="population_id"`
+ + `id="rdo_population_id_${apopulation.InbredSetId}" `
+ + `value="${apopulation.InbredSetId}" `
+ + `class="chk-row-select">`;
+ }
+ },
+ {
+ data: (apopulation) => {
+ return `${apopulation.FullName} (${apopulation.InbredSetName})`;
+ }
+ }
+ ]);
+});
diff --git a/uploader/static/js/pubmed.js b/uploader/static/js/pubmed.js
new file mode 100644
index 0000000..9afd4c3
--- /dev/null
+++ b/uploader/static/js/pubmed.js
@@ -0,0 +1,113 @@
+var extract_details = (pubmed_id, details) => {
+ var months = {
+ "jan": "January",
+ "feb": "February",
+ "mar": "March",
+ "apr": "April",
+ "may": "May",
+ "jun": "June",
+ "jul": "July",
+ "aug": "August",
+ "sep": "September",
+ "oct": "October",
+ "nov": "November",
+ "dec": "December"
+ };
+ var _date = details[pubmed_id].pubdate.split(" ");
+ return {
+ "authors": details[pubmed_id].authors.map((authobj) => {
+ return authobj.name;
+ }),
+ "title": details[pubmed_id].title,
+ "journal": details[pubmed_id].fulljournalname,
+ "volume": details[pubmed_id].volume,
+ "pages": details[pubmed_id].pages,
+ "month": _date.length > 1 ? months[_date[1].toLowerCase()] : "jan",
+ "year": _date[0],
+ };
+};
+
+var update_publication_details = (details) => {
+ Object.entries(details).forEach((entry) => {;
+ switch(entry[0]) {
+ case "authors":
+ $("#txt-publication-authors").val(entry[1].join(", "));
+ break;
+ case "month":
+ $("#select-publication-month")
+ .children("option")
+ .each((index, child) => {
+ console.debug(entry[1].toLowerCase());
+ child.selected = child.value == entry[1].toLowerCase();
+ });
+ default:
+ $("#txt-publication-" + entry[0]).val(entry[1]);
+ break;
+ }
+ });
+};
+
+var fetch_publication_abstract = (pubmed_id, pub_details) => {
+ $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi",
+ {
+ "method": "GET",
+ "data": {
+ "db": "pubmed",
+ "id": pubmed_id,
+ "rettype": "abstract",
+ "retmode": "xml"
+ },
+ "success": (data, textStatus, jqXHR) => {
+ update_publication_details({
+ ...pub_details,
+ ...{
+ "abstract": Array.from(data
+ .getElementsByTagName(
+ "Abstract")[0]
+ .children)
+ .map((elt) => {return elt.textContent.trim();})
+ .join("\r\n")
+ }});
+ },
+ "error": (jqXHR, textStatus, errorThrown) => {},
+ "complete": (jqXHR, textStatus) => {},
+ "dataType": "xml"
+ });
+};
+
+var fetch_publication_details = (pubmed_id, complete_thunks) => {
+ error_display = $("#search-pubmed-id-error");
+ error_display.text("");
+ add_class(error_display, "visually-hidden");
+ $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi",
+ {
+ "method": "GET",
+ "data": {"db": "pubmed", "id": pubmed_id, "format": "json"},
+ "success": (data, textStatus, jqXHR) => {
+ // process and update publication details
+ hasError = (
+ Object.hasOwn(data, "error") ||
+ Object.hasOwn(data.result[pubmed_id], "error"));
+ if(hasError) {
+ error_display.text(
+ "There was an error fetching a publication with " +
+ "the given PubMed ID! The error received " +
+ "was: '" + (
+ data.error ||
+ data.result[pubmed_id].error) +
+ "'. Please check ID you provided and try " +
+ "again.");
+ remove_class(error_display, "visually-hidden");
+ } else {
+ fetch_publication_abstract(
+ pubmed_id,
+ extract_details(pubmed_id, data.result));
+ }
+ },
+ "error": (jqXHR, textStatus, errorThrown) => {},
+ "complete": () => {
+ complete_thunks.forEach((thunk) => {thunk()});
+ },
+ "dataType": "json"
+ });
+};
diff --git a/uploader/static/js/species.js b/uploader/static/js/species.js
new file mode 100644
index 0000000..9ea3017
--- /dev/null
+++ b/uploader/static/js/species.js
@@ -0,0 +1,20 @@
+$(() => {
+ var speciesDataTable = buildDataTable(
+ "#tbl-select-species",
+ JSON.parse(
+ $("#tbl-select-species").attr("data-species-list")),
+ [
+ {
+ data: (aspecies) => {
+ return `<input type="radio" name="species_id"`
+ + `id="rdo_species_id_${aspecies.SpeciesId}" `
+ + `value="${aspecies.SpeciesId}" class="chk-row-select">`;
+ }
+ },
+ {
+ data: (aspecies) => {
+ return `${aspecies.FullName} (${aspecies.SpeciesName})`;
+ }
+ }
+ ]);
+});
diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js
index 045dd47..1b31661 100644
--- a/uploader/static/js/utils.js
+++ b/uploader/static/js/utils.js
@@ -8,3 +8,30 @@ function trigger_change_event(element) {
evt = new Event("change");
element.dispatchEvent(evt);
}
+
+
+var remove_class = (element, classvalue) => {
+ new_classes = (element.attr("class") || "").split(" ").map((val) => {
+ return val.trim();
+ }).filter((val) => {
+ return ((val !== classvalue) &&
+ (val !== ""))
+ }).join(" ");
+
+ if(new_classes === "") {
+ element.removeAttr("class");
+ } else {
+ element.attr("class", new_classes);
+ }
+};
+
+
+var add_class = (element, classvalue) => {
+ remove_class(element, classvalue);
+ element.attr("class", (element.attr("class") || "") + " " + classvalue);
+};
+
+$(".not-implemented").click((event) => {
+ event.preventDefault();
+ alert("This feature is not implemented yet. Please bear with us.");
+});
diff --git a/uploader/templates/base.html b/uploader/templates/base.html
index 019aa39..3c0d0d4 100644
--- a/uploader/templates/base.html
+++ b/uploader/templates/base.html
@@ -8,14 +8,14 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
{%block extrameta%}{%endblock%}
- <title>GN Uploader: {%block title%}{%endblock%}</title>
+ <title>Data Upload and Quality Control: {%block title%}{%endblock%}</title>
<link rel="stylesheet" type="text/css"
href="{{url_for('base.bootstrap',
filename='css/bootstrap.min.css')}}" />
<link rel="stylesheet" type="text/css"
- href="{{url_for('base.bootstrap',
- filename='css/bootstrap-theme.min.css')}}" />
+ href="{{url_for('base.datatables',
+ filename='css/dataTables.bootstrap5.min.css')}}" />
<link rel="stylesheet" type="text/css" href="/static/css/styles.css" />
{%block css%}{%endblock%}
@@ -23,28 +23,32 @@
</head>
<body>
- <header id="header" class="container-fluid">
- <div class="row">
- <span class="header col-lg-9">GeneNetwork Data Quality Control and Upload</span>
- <nav class="header-nav col-lg-3">
- <ul class="nav justify-content-end">
- <li>
- {%if user_logged_in()%}
- <a href="{{url_for('oauth2.logout')}}"
- title="Log out of the system">{{user_email()}} &mdash; Log Out</a>
- {%else%}
- <a href="{{authserver_authorise_uri()}}"
- title="Log in to the system">Log In</a>
- {%endif%}
- </li>
- </ul>
- </nav>
+ <header id="header">
+ <span id="header-text">GeneNetwork</span>
+ <nav id="header-nav">
+ <ul class="nav justify-content-end">
+ <li>
+ {%if user_logged_in()%}
+ <a href="{{url_for('oauth2.logout')}}"
+ title="Log out of the system">
+ <span class="glyphicon glyphicon-user"></span>
+ {{user_email()}} Sign Out</a>
+ {%else%}
+ <a href="{{authserver_authorise_uri()}}"
+ title="Log in to the system">Sign In</a>
+ {%endif%}
+ </li>
+ </ul>
+ </nav>
</header>
- <aside id="nav-sidebar" class="container-fluid">
+ <aside id="nav-sidebar">
<ul class="nav flex-column">
<li {%if activemenu=="home"%}class="activemenu"{%endif%}>
<a href="/" >Home</a></li>
+ <li {%if activemenu=="publications"%}class="activemenu"{%endif%}>
+ <a href="{{url_for('publications.index')}}"
+ title="View and manage publications.">Publications</a></li>
<li {%if activemenu=="species"%}class="activemenu"{%endif%}>
<a href="{{url_for('species.list_species')}}"
title="View and manage species information.">Species</a></li>
@@ -70,9 +74,11 @@
<li {%if activemenu=="phenotypes"%}class="activemenu"{%endif%}>
<a href="{{url_for('species.populations.phenotypes.index')}}"
title="Upload phenotype data.">Phenotype Data</a></li>
+ <!--
<li {%if activemenu=="expression-data"%}class="activemenu"{%endif%}>
<a href="{{url_for('species.populations.expression-data.index')}}"
- title="Upload expression data.">Expression Data</a></li>
+ title="Upload expression data."
+ class="not-implemented">Expression Data</a></li>
<li {%if activemenu=="individuals"%}class="activemenu"{%endif%}>
<a href="#"
class="not-implemented"
@@ -86,47 +92,70 @@
class="not-implemented"
title="View and manage the backgroud jobs you have running">
Background Jobs</a></li>
+ -->
</ul>
</aside>
- <main id="main" class="main container-fluid">
+ <main id="main" class="main">
- <div class="pagetitle row">
- <h1>GN Uploader: {%block pagetitle%}{%endblock%}</h1>
- <nav>
- <ol class="breadcrumb">
- <li {%if activelink is not defined or activelink=="home"%}
- class="breadcrumb-item active"
- {%else%}
- class="breadcrumb-item"
- {%endif%}>
- <a href="{{url_for('base.index')}}">Home</a>
- </li>
- {%block lvl1_breadcrumbs%}{%endblock%}
- </ol>
- </nav>
+ <div id="pagetitle" class="pagetitle">
+ <span class="title">Data Upload and Quality Control: {%block pagetitle%}{%endblock%}</span>
+ <!--
+ <nav>
+ <ol class="breadcrumb">
+ <li {%if activelink is not defined or activelink=="home"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('base.index')}}">Home</a>
+ </li>
+ {%block lvl1_breadcrumbs%}{%endblock%}
+ </ol>
+ </nav>
+ -->
</div>
- <div class="row">
- <div class="container-fluid">
- <div class="col-md-8 main-content">
- {%block contents%}{%endblock%}
- </div>
- <div class="sidebar-content col-md-4">
- {%block sidebarcontents%}{%endblock%}
- </div>
+ <div id="all-content">
+ <div id="main-content">
+ {%block contents%}{%endblock%}
+ </div>
+ <div id="sidebar-content">
+ {%block sidebarcontents%}{%endblock%}
</div>
</div>
</main>
+ <!--
+ Core dependencies
+ -->
<script src="{{url_for('base.jquery',
filename='jquery.min.js')}}"></script>
<script src="{{url_for('base.bootstrap',
filename='js/bootstrap.min.js')}}"></script>
- <script type="text/javascript" src="/static/js/misc.js"></script>
- {%block javascript%}{%endblock%}
- </body>
+ <!--
+ DataTables dependencies
+ -->
+ <script type="text/javascript"
+ src="{{url_for('base.datatables',
+ filename='js/dataTables.min.js')}}"></script>
+ <script type="text/javascript"
+ src="{{url_for('base.datatables_extensions',
+ filename='scroller/js/dataTables.scroller.min.js')}}"></script>
+ <script type="text/javascript"
+ src="{{url_for('base.datatables_extensions',
+ filename='buttons/js/dataTables.buttons.min.js')}}"></script>
+ <script type="text/javascript"
+ src="{{url_for('base.datatables_extensions',
+ filename='select/js/dataTables.select.min.js')}}"></script>
+ <!--
+ local dependencies
+ -->
+ <script type="text/javascript" src="/static/js/utils.js"></script>
+ <script type="text/javascript" src="/static/js/datatables.js"></script>
+ {%block javascript%}{%endblock%}
+ </body>
</html>
diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html
index 33fb73b..64b1a9a 100644
--- a/uploader/templates/cli-output.html
+++ b/uploader/templates/cli-output.html
@@ -1,7 +1,7 @@
{%macro cli_output(job, stream)%}
-<h4>{{stream | upper}} Output</h4>
-<div class="cli-output">
+<h4 class="subheading">{{stream | upper}} Output</h4>
+<div class="cli-output" style="max-height: 10em; overflow: auto;">
<pre>{{job.get(stream, "")}}</pre>
</div>
diff --git a/uploader/templates/genotypes/base.html b/uploader/templates/genotypes/base.html
index 1b274bf..7d61312 100644
--- a/uploader/templates/genotypes/base.html
+++ b/uploader/templates/genotypes/base.html
@@ -6,7 +6,18 @@
{%else%}
class="breadcrumb-item"
{%endif%}>
+ {%if population is mapping%}
+ <a href="{{url_for('species.populations.genotypes.list_genotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">
+ {%if dataset is defined and dataset is mapping%}
+ {{dataset.Name}}
+ {%else%}
+ Genotypes
+ {%endif%}</a>
+ {%else%}
<a href="{{url_for('species.populations.genotypes.index')}}">Genotypes</a>
+ {%endif%}
</li>
{%block lvl4_breadcrumbs%}{%endblock%}
{%endblock%}
diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html
index e749f5a..b50ebc5 100644
--- a/uploader/templates/genotypes/index.html
+++ b/uploader/templates/genotypes/index.html
@@ -26,3 +26,7 @@
species)}}
</div>
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html
index e4c39eb..0f074fd 100644
--- a/uploader/templates/genotypes/list-genotypes.html
+++ b/uploader/templates/genotypes/list-genotypes.html
@@ -26,7 +26,8 @@
<p>There are a total of {{total_markers}} currently registered genetic markers
for the "{{species.FullName}}" species. You can click
<a href="{{url_for('species.populations.genotypes.list_markers',
- species_id=species.SpeciesId)}}"
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
title="View genetic markers for species '{{species.FullName}}">
this link to view the genetic markers
</a>.
@@ -70,7 +71,7 @@
{%if genocode | length < 1%}
<a href="#add-genotype-encoding"
title="Add a genotype encoding system for this population"
- class="btn btn-primary">
+ class="btn btn-primary not-implemented">
add genotype encoding
</a>
{%endif%}
diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html
index 9198b44..a705ae3 100644
--- a/uploader/templates/genotypes/list-markers.html
+++ b/uploader/templates/genotypes/list-markers.html
@@ -13,7 +13,8 @@
class="breadcrumb-item"
{%endif%}>
<a href="{{url_for('species.populations.genotypes.list_markers',
- species_id=species.SpeciesId)}}">List markers</a>
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">List markers</a>
</li>
{%endblock%}
@@ -30,6 +31,7 @@
{%if start_from > 0%}
<a href="{{url_for('species.populations.genotypes.list_markers',
species_id=species.SpeciesId,
+ population_id=population.Id,
start_from=start_from-count,
count=count)}}">
<span class="glyphicon glyphicon-backward"></span>
@@ -45,6 +47,7 @@
{%if start_from + count < total_markers%}
<a href="{{url_for('species.populations.genotypes.list_markers',
species_id=species.SpeciesId,
+ population_id=population.Id,
start_from=start_from+count,
count=count)}}">
Next
diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html
index 7c81943..acdd063 100644
--- a/uploader/templates/genotypes/select-population.html
+++ b/uploader/templates/genotypes/select-population.html
@@ -12,20 +12,14 @@
{{flash_all_messages()}}
<div class="row">
- <p>
- You have indicated that you intend to upload the genotypes for species
- '{{species.FullName}}'. We now just require the population for your
- experiment/study, and you should be good to go.
- </p>
-</div>
-
-<div class="row">
- {{select_population_form(url_for("species.populations.genotypes.select_population",
- species_id=species.SpeciesId),
- populations)}}
+ {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}}
</div>
{%endblock%}
{%block sidebarcontents%}
{{display_species_card(species)}}
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/populations.js"></script>
+{%endblock%}
diff --git a/uploader/templates/index.html b/uploader/templates/index.html
index d6f57eb..aa1414e 100644
--- a/uploader/templates/index.html
+++ b/uploader/templates/index.html
@@ -10,90 +10,98 @@
<div class="row">
{{flash_all_messages()}}
<div class="explainer">
- <p>Welcome to the <strong>GeneNetwork Data Quality Control and Upload System</strong>. This system is provided to help in uploading your data onto GeneNetwork where you can do analysis on it.</p>
+ <p>Welcome to the <strong>GeneNetwork Data Upload and Quality Control
+ System</strong>.</p>
+ <p>This tool helps you prepare and upload research data to GeneNetwork for
+ analysis.</p>
- <p>The sections below provide an overview of what features the menu items on
- the left provide to you. Please peruse the information to get a good
- big-picture understanding of what the system provides you and how to get
- the most out of it.</p>
+ <h2 class="heading">Getting Started</h2>
+ <p>The sections below explain the features of the system. Review this guide
+ to learn how to use the system.</p>
{%block extrapageinfo%}{%endblock%}
- <h2>Species</h2>
-
- <p>The GeneNetwork service provides datasets and tools for doing genetic
- studies &mdash; from
- <a href="{{gn2server_intro}}"
- target="_blank"
- title="GeneNetwork introduction — opens in a new tab.">
- its introduction</a>:
-
- <blockquote class="blockquote">
- <p>GeneNetwork is a group of linked data sets and tools used to study
- complex networks of genes, molecules, and higher order gene function
- and phenotypes. &hellip;</p>
- </blockquote>
- </p>
-
- <p>With this in mind, it follows that the data in the system is centered
- aroud a variety of species. The <strong>species section</strong> will
- list the currently available species in the system, and give you the
- ability to add new ones, if the one you want to work on does not currently
- exist on GeneNetwork</p>
-
- <h2>Populations</h2>
-
- <p>Your studies will probably focus on a particular subset of the entire
- species you are interested in &ndash; your population.</p>
- <p>Populations are a way to organise the species data so as to link data to
- specific know populations for a particular species, e.g. The BXD
- population of mice (Mus musculus)</p>
- <p>In older GeneNetwork documentation, you might run into the term
- <em>InbredSet</em>. Should you run into it, it is a term that we've
- deprecated that essentially just means the population.</p>
-
- <h2>Samples</h2>
-
- <p>These are the samples or individuals (sometimes cases) that were involved
- in the experiment, and from whom the data was derived.</p>
-
- <h2>Genotype Data</h2>
-
- <p>This section will allow you to view and upload the genetic markers for
- your species, and the genotype encodings used for your particular
- population.</p>
- <p>While, technically, genetic markers relate to the species in general, and
- not to a particular population, the data (allele information) itself
- relates to the particular population it was generated from &ndash;
- specifically, to the actual individuals used in the experiment.</p>
- <p>This is the reason why the genotype data information comes under the
- population, and will check for the prior existence of the related
- samples/individuals before attempting an upload of your data.</p>
-
- <h2>Expression Data</h2>
+ <h3 class="subheading">Species</h3>
- <p class="text-danger">
- <span class="glyphicon glyphicon-exclamation-sign"></span>
- <strong>TODO</strong>: Document this &hellip;</p>
+ <p>GeneNetwork supports genetic studies across multiple species (e.g. mice
+ [Mus musculus], human [homo sapiens], rats [Rattus norvegicus], etc.) .
+ Here you can:</p>
+ <ul>
+ <li>View all species that are currently supported</li>
+ <li>Add new species not yet in the system</li>
+ </ul>
+
+ <h3 class="subheading">Populations</h3>
+
+ <p>A "population" refers to a specific subgroup within a species that you’re
+ studying (e.g., BXD mice). Here you can:</p>
+ <ul>
+ <li>View the populations that exist for a selected species</li>
+ <li>Add new populations of study for a selected species</li>
+ </ul>
+
+ <h3 class="subheading">Samples</h3>
+
+ <p>Manage individual specimens or cases used in your experiments. These
+ include:</p>
+
+ <ul>
+ <li>Experimental subjects</li>
+ <li>Data sources (e.g., tissue samples, clinical cases)</li>
+ <li>Strain means (instead of entering multiple BXD1 individuals, for
+ example, the mean would be entered for a single BXD1 strain)</li>
+ </ul>
+
+
+ <h3 class="subheading">Genotype Data</h3>
+
+ <p>Upload and review genetic markers and allele encodings for your
+ population. Key details:</p>
+
+ <ul>
+ <li>Markers are species-level (e.g., mouse SNP databases).</li>
+ <li>Allele data is population-specific (tied to your experimental
+ samples).</li>
+ </ul>
+
+ <p><strong>Requirement</strong>: Samples must already have been registered
+ in the system before uploading genotype data.</p>
+
+ <h3 class="subheading">Phenotype Data</h3>
+
+ <p>Phenotypes are the visible traits or features of a living thing. For
+ example, phenotypes include:</p>
+
+ <ul>
+ <li>Weight</li>
+ <li>Height</li>
+ <li>Color (such as the color of fur or eyes)</li>
+ </ul>
+
+ <p>This part of the system will allow you to upload and manage the values
+ for different phenotypes from various samples in your studies.</p>
+
+ <!--
- <h2>Phenotype Data</h2>
+ <h3 class="subheading">Expression Data</h3>
<p class="text-danger">
<span class="glyphicon glyphicon-exclamation-sign"></span>
<strong>TODO</strong>: Document this &hellip;</p>
- <h2>Individual Data</h2>
+ <h3 class="subheading">Individual Data</h3>
<p class="text-danger">
<span class="glyphicon glyphicon-exclamation-sign"></span>
<strong>TODO</strong>: Document this &hellip;</p>
- <h2>RNA-Seq Data</h2>
+ <h3 class="subheading">RNA-Seq Data</h3>
<p class="text-danger">
<span class="glyphicon glyphicon-exclamation-sign"></span>
<strong>TODO</strong>: Document this &hellip;</p>
</div>
+ -->
</div>
{%endblock%}
diff --git a/uploader/templates/jobs/job-error.html b/uploader/templates/jobs/job-error.html
new file mode 100644
index 0000000..b3015fc
--- /dev/null
+++ b/uploader/templates/jobs/job-error.html
@@ -0,0 +1,17 @@
+{%extends "base.html"%}
+
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Background Jobs: Error{%endblock%}
+
+{%block pagetitle%}Background Jobs: Error{%endblock%}
+
+{%block contents%}
+
+<h1>Background Jobs: Error</h1>
+<p>Job <strong>{{job["job_id"]}}</strong> failed!</p>
+<p>The error details are in the "STDERR" section below.</p>
+
+<h2>STDERR</h2>
+<pre>{{job["stderr"]}}</pre>
+{%endblock%}
diff --git a/uploader/templates/jobs/job-not-found.html b/uploader/templates/jobs/job-not-found.html
new file mode 100644
index 0000000..a71e66f
--- /dev/null
+++ b/uploader/templates/jobs/job-not-found.html
@@ -0,0 +1,11 @@
+{%extends "base.html"%}
+
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Background Jobs{%endblock%}
+
+{%block pagetitle%}Background Jobs{%endblock%}
+
+{%block contents%}
+<p>Could not find job with ID: {{job_id}}</p>
+{%endblock%}
diff --git a/uploader/templates/jobs/job-status.html b/uploader/templates/jobs/job-status.html
new file mode 100644
index 0000000..83c02fd
--- /dev/null
+++ b/uploader/templates/jobs/job-status.html
@@ -0,0 +1,24 @@
+{%extends "base.html"%}
+
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5" />
+{%endblock%}
+
+{%block title%}Background Jobs{%endblock%}
+
+{%block pagetitle%}Background Jobs{%endblock%}
+
+{%block contents%}
+
+<p>Status: {{job["metadata"]["status"]}}</p>
+<p>Job Type: {{job["metadata"]["job-type"]}}</p>
+
+<h2>STDOUT</h2>
+<pre>{{job["stdout"]}}</pre>
+
+<h2>STDERR</h2>
+<pre>{{job["stderr"]}}</pre>
+
+{%endblock%}
diff --git a/uploader/templates/login.html b/uploader/templates/login.html
index 1f71416..e76c644 100644
--- a/uploader/templates/login.html
+++ b/uploader/templates/login.html
@@ -5,7 +5,8 @@
{%block pagetitle%}log in{%endblock%}
{%block extrapageinfo%}
-<p class="text-dark text-primary">
- You <strong>do need to be logged in</strong> to upload data onto this system.
- Please do that by clicking the "Log In" button at the top of the page.</p>
+<p class="text-dark">
+ You <strong>need to
+ <a href="{{authserver_authorise_uri()}}"
+ title="Sign in to the system">sign in</a></strong> to use this system.</p>
{%endblock%}
diff --git a/uploader/templates/macro-step-indicator.html b/uploader/templates/macro-step-indicator.html
new file mode 100644
index 0000000..ac0be77
--- /dev/null
+++ b/uploader/templates/macro-step-indicator.html
@@ -0,0 +1,15 @@
+{%macro step_indicator(step, width=100)%}
+<svg width="{{width}}" height="{{width}}" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="{{0.5*width}}"
+ cy="{{0.5*width}}"
+ r="{{0.5*width}}"
+ fill="#E5E5FF" />
+ <text x="{{0.5*width}}"
+ y="{{0.6*width}}"
+ font-size="{{0.2*width}}"
+ text-anchor="middle"
+ fill="#555555">
+ Step {{step}}
+ </text>
+</svg>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html
new file mode 100644
index 0000000..9909c20
--- /dev/null
+++ b/uploader/templates/phenotypes/add-phenotypes-base.html
@@ -0,0 +1,166 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <form id="frm-add-phenotypes"
+ method="POST"
+ enctype="multipart/form-data"
+ action="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ use_bundle=use_bundle)}}"
+ data-resumable-target="{{url_for('files.resumable_upload_post')}}">
+ <legend>Add New Phenotypes</legend>
+
+ <div class="form-text help-block">
+ {%block frm_add_phenotypes_documentation%}{%endblock%}
+ <p><strong class="text-warning">This will not update any existing phenotypes!</strong></p>
+ </div>
+
+ {%block frm_add_phenotypes_elements%}{%endblock%}
+
+ <fieldset id="fldset-publication-info">
+ <legend>Publication Information</legend>
+ <input type="hidden" name="publication-id" id="txt-publication-id" />
+ <span class="form-text text-muted">
+ Select a publication for your data. <br />
+ Can't find a publication you can use? Go ahead and
+ <a href="{{url_for(
+ 'publications.create_publication',
+ return_to='species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">create a new publication</a>.</span>
+ <table id="tbl-select-publication" class="table compact stripe">
+ <thead>
+ <tr>
+ <th>#</th>
+ <th>PubMed ID</th>
+ <th>Title</th>
+ <th>Authors</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
+ </fieldset>
+
+ <div class="form-group">
+ <input type="submit"
+ value="upload phenotypes"
+ class="btn btn-primary" />
+ </div>
+ </form>
+</div>
+
+<div class="row">
+ {%block page_documentation%}{%endblock%}
+</div>
+
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(function() {
+ var publicationsDataTable = buildDataTable(
+ "#tbl-select-publication",
+ [],
+ [
+ {data: "index"},
+ {
+ searchable: true,
+ data: (pub) => {
+ if(pub.PubMed_ID) {
+ return `<a href="https://pubmed.ncbi.nlm.nih.gov/` +
+ `${pub.PubMed_ID}/" target="_blank" ` +
+ `title="Link to publication on NCBI.">` +
+ `${pub.PubMed_ID}</a>`;
+ }
+ return "";
+ }
+ },
+ {
+ searchable: true,
+ data: (pub) => {
+ var title = "⸻";
+ if(pub.Title) {
+ title = pub.Title
+ }
+ return `<a href="/publications/view/${pub.Id}" ` +
+ `target="_blank" ` +
+ `title="Link to view publication details">` +
+ `${title}</a>`;
+ }
+ },
+ {
+ searchable: true,
+ data: (pub) => {
+ authors = pub.Authors.split(",").map(
+ (item) => {return item.trim();});
+ if(authors.length > 1) {
+ return authors[0] + ", et. al.";
+ }
+ return authors[0];
+ }
+ }
+ ],
+ {
+ serverSide: true,
+ ajax: {
+ url: "/publications/list",
+ dataSrc: "publications"
+ },
+ select: "single",
+ paging: true,
+ scrollY: 700,
+ deferRender: true,
+ scroller: true,
+ scrollCollapse: true,
+ layout: {
+ topStart: "info",
+ topEnd: "search"
+ }
+ });
+ publicationsDataTable.on("select", (event, datatable, type, indexes) => {
+ indexes.forEach((element, index, thearray) => {
+ let row = datatable.row(element).node();
+ console.debug(datatable.row(element).data());
+ $("#frm-add-phenotypes #txt-publication-id").val(
+ datatable.row(element).data().Id);
+ });
+ });
+ publicationsDataTable.on("deselect", (event, datatable, type, indexes) => {
+ indexes.forEach((element, index, thearray) => {
+ let row = datatable.row(element).node();
+ $("#frm-add-phenotypes #txt-publication-id").val(null);
+ });
+ });
+ });
+</script>
+
+{%block more_javascript%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/add-phenotypes-raw-files.html b/uploader/templates/phenotypes/add-phenotypes-raw-files.html
new file mode 100644
index 0000000..67b56e3
--- /dev/null
+++ b/uploader/templates/phenotypes/add-phenotypes-raw-files.html
@@ -0,0 +1,847 @@
+{%extends "phenotypes/add-phenotypes-base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%}
+{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
+</li>
+{%endblock%}
+
+{%block frm_add_phenotypes_documentation%}
+<p>This page will allow you to upload all the separate files that make up your
+ phenotypes. Here, you will have to upload each separate file individually. If
+ you want instead to upload all your files as a single ZIP file,
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ use_bundle=true)}}"
+ title="">click here</a>.</p>
+{%endblock%}
+
+{%block frm_add_phenotypes_elements%}
+<fieldset id="fldset-file-metadata">
+ <legend>File(s) Metadata</legend>
+ <div class="form-group">
+ <label for="txt-file-separator" class="form-label">File Separator</label>
+ <div class="input-group">
+ <input id="txt-file-separator"
+ name="file-separator"
+ type="text"
+ value="&#9;"
+ class="form-control"
+ maxlength="1" />
+ <span class="input-group-btn">
+ <button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button>
+ </span>
+ </div>
+ <span class="form-text text-muted">
+ Provide the character that separates the fields in your file(s). It should
+ be the same character for all files (if more than one is provided).<br />
+ A tab character will be assumed if you leave this field blank. See
+ <a href="#docs-file-separator"
+ title="Documentation for file-separator characters">
+ documentation for more information</a>.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-file-comment-character" class="form-label">File Comment-Character</label>
+ <div class="input-group">
+ <input id="txt-file-comment-character"
+ name="file-comment-character"
+ type="text"
+ value="#"
+ class="form-control"
+ maxlength="1" />
+ <span class="input-group-btn">
+ <button id="btn-reset-file-comment-character" class="btn btn-info">
+ Reset Default</button>
+ </span>
+ </div>
+ <span class="form-text text-muted">
+ This specifies that lines that begin with the character provided will be
+ considered comment lines and ignored in their entirety. See
+ <a href="#docs-file-comment-character"
+ title="Documentation for comment characters">
+ documentation for more information</a>.
+ </span>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-file-na" class="form-label">File "No-Value" Indicators</label>
+ <div class="input-group">
+ <input id="txt-file-na"
+ name="file-na"
+ type="text"
+ value="- NA N/A"
+ class="form-control" />
+ <span class="input-group-btn">
+ <button id="btn-reset-file-na" class="btn btn-info">Reset Default</button>
+ </span>
+ </div>
+ <span class="form-text text-muted">
+ This specifies strings in your file indicate that there is no value for a
+ particular cell (a cell is where a column and row intersect). Provide a
+ space-separated list of strings if you have more than one way of
+ indicating no values. See
+ <a href="#docs-file-na" title="Documentation for no-value fields">
+ documentation for more information</a>.</span>
+ </div>
+</fieldset>
+
+<fieldset id="fldset-files">
+ <legend>Data File(s)</legend>
+
+ <fieldset id="fldset-descriptions-file">
+ <div class="form-group">
+ <div class="form-check">
+ <input id="chk-phenotype-descriptions-transposed"
+ name="phenotype-descriptions-transposed"
+ type="checkbox"
+ class="form-check-input"
+ style="border: solid #8EABF0" />
+ <label for="chk-phenotype-descriptions-transposed"
+ class="form-check-label">
+ Description file transposed?</label>
+ </div>
+
+ <div class="non-resumable-elements">
+ <label for="finput-phenotype-descriptions" class="form-label">
+ Phenotype Descriptions</label>
+ <input id="finput-phenotype-descriptions"
+ name="phenotype-descriptions"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-desc"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the phenotype descriptions,
+ <a href="#docs-file-phenotype-description"
+ title="Documentation of the phenotype data file format.">
+ the documentation for the expected format of the file</a>.</span>
+ </div>
+ {{display_resumable_elements(
+ "resumable-phenotype-descriptions",
+ "phenotype descriptions",
+ '<p>Drag and drop the CSV file that contains the descriptions of your
+ phenotypes here.</p>
+
+ <p>The CSV file should be a matrix of
+ <strong>phenotypes × descriptions</strong> i.e. The first column
+ contains the phenotype names/identifiers whereas the first row is a list
+ of metadata fields like, "description", "units", etc.</p>
+
+ <p>If the format is transposed (i.e.
+ <strong>descriptions × phenotypes</strong>) select the checkbox above.
+ </p>
+
+ <p>Please see the
+ <a href="#docs-file-phenotype-description"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Descriptions" documentation</a> section below for more
+ information on the expected format of the file provided here.</p>')}}
+ {{display_preview_table(
+ "tbl-preview-pheno-desc", "phenotype descriptions")}}
+ </div>
+ </fieldset>
+
+
+ <fieldset id="fldset-data-file">
+ <div class="form-group">
+ <div class="form-check">
+ <input id="chk-phenotype-data-transposed"
+ name="phenotype-data-transposed"
+ type="checkbox"
+ class="form-check-input"
+ style="border: solid #8EABF0" />
+ <label for="chk-phenotype-data-transposed" class="form-check-label">
+ Data file transposed?</label>
+ </div>
+
+ <div class="non-resumable-elements">
+ <label for="finput-phenotype-data" class="form-label">Phenotype Data</label>
+ <input id="finput-phenotype-data"
+ name="phenotype-data"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-data"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the phenotype data. See
+ <a href="#docs-file-phenotype-data"
+ title="Documentation of the phenotype data file format.">
+ the documentation for the expected format of the file</a>.</span>
+ </div>
+
+ {{display_resumable_elements(
+ "resumable-phenotype-data",
+ "phenotype data",
+ '<p>Drag and drop a CSV file that contains the phenotypes numerical data
+ here. You can click the "Browse" button (below and to the right) to
+ select the file from your computer.</p>
+
+ <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>,
+ i.e. The first column contains the samples identifiers while the first
+ row is the list of phenotypes identifiers occurring in the phenotypes
+ descriptions file.</p>
+
+ <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>)
+ select the checkbox above.</p>
+ <p>Please see the
+ <a href="#docs-file-phenotype-data"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Data" documentation</a> section below for more information
+ on the expected format for the file provided here.</p>')}}
+ {{display_preview_table("tbl-preview-pheno-data", "phenotype data")}}
+ </div>
+ </fieldset>
+
+
+ {%if population.Family in families_with_se_and_n%}
+ <fieldset id="fldset-se-file">
+ <div class="form-group">
+ <div class="form-check">
+ <input id="chk-phenotype-se-transposed"
+ name="phenotype-se-transposed"
+ type="checkbox"
+ class="form-check-input"
+ style="border: solid #8EABF0" />
+ <label for="chk-phenotype-se-transposed" class="form-check-label">
+ Standard-Errors file transposed?</label>
+ </div>
+ <div class="group non-resumable-elements">
+ <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label>
+ <input id="finput-phenotype-se"
+ name="phenotype-se"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-se"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the standard errors for the phenotypes,
+ computed from the data above.</span>
+ </div>
+
+ {{display_resumable_elements(
+ "resumable-phenotype-se",
+ "standard errors",
+ '<p>Drag and drop a CSV file that contains the phenotypes standard-errors
+ data here. You can click the "Browse" button (below and to the right) to
+ select the file from your computer.</p>
+
+ <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>,
+ i.e. The first column contains the samples identifiers while the first
+ row is the list of phenotypes identifiers occurring in the phenotypes
+ descriptions file.</p>
+
+ <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>)
+ select the checkbox above.</p>
+
+ <p>Please see the
+ <a href="#docs-file-phenotype-se"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Data" documentation</a> section below for more information
+ on the expected format of the file provided here.</p>')}}
+
+ {{display_preview_table("tbl-preview-pheno-se", "standard errors")}}
+ </div>
+ </fieldset>
+
+
+ <fieldset id="fldset-n-file">
+ <div class="form-group">
+ <div class="form-check">
+ <input id="chk-phenotype-n-transposed"
+ name="phenotype-n-transposed"
+ type="checkbox"
+ class="form-check-input"
+ style="border: solid #8EABF0" />
+ <label for="chk-phenotype-n-transposed" class="form-check-label">
+ Counts file transposed?</label>
+ </div>
+ <div class="non-resumable-elements">
+ <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label>
+ <input id="finput-phenotype-n"
+ name="phenotype-n"
+ class="form-control"
+ type="file"
+ data-preview-table="tbl-preview-pheno-n"
+ required="required" />
+ <span class="form-text text-muted">
+ Provide a file that contains only the number of samples/individuals used in
+ the computation of the standard errors above.</span>
+ </div>
+
+ {{display_resumable_elements(
+ "resumable-phenotype-n",
+ "number of samples/individuals",
+ '<p>Drag and drop a CSV file that contains the samples\' phenotypes counts
+ data here. You can click the "Browse" button (below and to the right) to
+ select the file from your computer.</p>
+
+ <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>,
+ i.e. The first column contains the samples identifiers while the first
+ row is the list of phenotypes identifiers occurring in the phenotypes
+ descriptions file.</p>
+
+ <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>)
+ select the checkbox above.</p>
+
+ <p>Please see the
+ <a href="#docs-file-phenotype-se"
+ title="Documentation of the phenotype data file format.">
+ "Phenotypes Data" documentation</a> section below for more information
+ on the expected format of the file provided here.</p>')}}
+
+ {{display_preview_table("tbl-preview-pheno-n", "number of samples/individuals")}}
+ </div>
+ </fieldset>
+</fieldset>
+{%endif%}
+{%endblock%}
+
+
+{%block page_documentation%}
+<div class="row">
+ <h2 class="heading" id="docs-help">Help</h2>
+ <h3 class="subheading">Common Features</h3>
+ <p>The following are the common expectations for <strong>ALL</strong> the
+ files provided in the form above:
+ <ul>
+ <li>The file <strong>MUST</strong> be character-separated values (CSV)
+ text file</li>
+ <li>The first row in the file <strong>MUST</strong> be a heading row, and
+ will be composed of the list identifiers for all of
+ samples/individuals/cases involved in your study.</li>
+ <li>The first column of data in the file <strong>MUST</strong> be the
+ identifiers for all of the phenotypes you wish to upload.</li>
+ </ul>
+ </p>
+
+ <p>If you do not specify the separator character, then we will assume a
+ <strong>TAB</strong> character was used as your separator.</p>
+
+ <p>We also assume you might include comments lines in your files. In that
+ case, if you do not specify what character denotes that a line in your files
+ is a comment line, we will assume the <strong>#</strong> character.<br />
+ A comment <strong>MUST ALWAYS</strong> begin at the start of the line marked
+ with the comment character specified.</p>
+
+ <h3 class="subheading" id="docs-file-metadata">File Metadata</h3>
+ <p>We request some details about your files to help us parse and process the
+ files correctly. The details we collect are:</p>
+ <dl>
+ <dt id="docs-file-separator">File separator</dt>
+ <dd>The files you provide should be character-separated value (CSV) files.
+ We need to know what character you used to separate the values in your
+ file. Some common ones are the Tab character, the comma, etc.<br />
+ Providing that information makes it possible for the system to parse and
+ process your files correctly.<br>
+ <strong>NOTE:</strong> All the files you upload MUST use the same
+ separator.</dd>
+
+ <dt id="docs-file-comment-character">Comment character</dt>
+ <dd>We support use of comment lines in your files. We only support one type
+ of comment style, the <em>line comment</em>.<br />
+ This mean the comment begins at the start of the line, and the end of that
+ line indicates the end of that comment. If you have a really long comment,
+ then you need to break it across multiple lines, marking each line a
+ comment line.<br />
+ The "comment character" is the character at the start of the line that
+ indicates that the line is a line comment.</dd>
+
+ <dt id="docs-file-na">No-Value indicator(s)</dt>
+ <dd>Data in the real world is messy, and in some cases, entirely absent. You
+ need to indicate, in your files, that a particular field did not have a
+ value, and once you do that, you then need to let the system know how you
+ mark such fields. Common ways of indicating "empty values" are, leaving
+ the field blank, using a character such as '-', or using strings like
+ "NA", "N/A", "NULL", etc.<br />
+ Providing this information will help with parsing and processing such
+ no-value fields the correct way.</dd>
+ </dl>
+
+ <h3 class="subheading" id="docs-file-phenotype-description">
+ file: Phenotypes Descriptions</h3>
+ <p>The data in this file is a matrix of <em>phenotypes × metadata-fields</em>.
+ Please note we use the term "metadata-fields" above loosely, due to lack of
+ a good word for this.</p>
+ <p>The file <strong>MUST</strong> have columns in this order:
+ <dl>
+ <dt>Phenotype Identifiers</dt>
+ <dd>These are the names/identifiers for your phenotypes. These
+ names/identifiers are the same ones you will have in all the other files you are
+ uploading.</dd>
+
+ <dt>Descriptions</dt>
+ <dd>Each phenotype will need a description. Good description are necessary
+ to inform other people of what the data is about. Good description are
+ hard to construct, so we provide
+ <a href="https://info.genenetwork.org/faq.php#q-22"
+ title="How to write phenotype descriptions">
+ advice on describing your phenotypes.</a></dd>
+
+ <dt>Units</dt>
+ <dd>Each phenotype will need units for the measurements taken. If there are
+ none, then indicate the field is a no-value field.</dd>
+ </dl></p>
+ <p>You can add more columns after those three if you want to, but these 3
+ <strong>MUST</strong> be present.</p>
+ <p>The file would, for example, look like the following:</p>
+ <code>id,description,units,…<br />
+ pheno10001|Central nervous system, behavior, cognition; …|mg|…<br />
+ pheno10002|Aging, metabolism, central nervous system: …|mg|…<br />
+ ⋮<br /></code>
+
+ <p><strong>Note 01</strong>: The first usable row is the heading row.</p>
+ <p><strong>Note 02: </strong>This example demonstrates a subtle issue that
+ could make your CSV file invalid &mdash; the choice of your field separator
+ character.<br >
+ In the example above, we use the pipe character (<code>|</code>) as our
+ field separator. This is because, if we follow the advice on how to write
+ good descriptions, then we cannot use the comma as our separator &ndash; if
+ we did, then our CSV file would be invalid because the system would have no
+ way to tell the difference between the comma as a field separator, and the
+ comma as a way to separate the "general category and ontology terms".</p>
+
+ <h3 class="subheading">file: Phenotype Data, Standard Errors and/or Sample Counts</h3>
+ <span id="docs-file-phenotype-data"></span>
+ <span id="docs-file-phenotype-se"></span>
+ <span id="docs-file-phenotype-n"></span>
+ <p>The data is a matrix of <em>samples(or individuals) × phenotypes</em>, e.g.</p>
+ <code>
+ # num-cases: 2549
+ # num-phenos: 13
+ id,pheno10001,pheno10002,pheno10003,pheno10004,53.099998,…<br />
+ IND001,61.400002,49,62.5,55.099998,…<br />
+ IND002,54.099998,50.099998,53.299999,55.099998,…<br />
+ IND003,483,403,501,403,…<br />
+ IND004,49.799999,45.5,62.900002,NA,…<br />
+ ⋮<br /></code>
+
+ <p>where <code>IND001,IND002,IND003,IND004,…</code> are the
+ samples/individuals/cases in your study, and
+ <code>pheno10001,pheno10002,pheno10004,pheno10004,…</code> are the
+ identifiers for your phenotypes.</p>
+ <p>The lines beginning with the "<em>#</em>" symbol (i.e.
+ <code># num-cases: 2549</code> and <code># num-phenos: 13</code> are comment
+ lines and will be ignored</p>
+ <p>In this example, the comma (,) is used as the file separator.</p>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
+
+
+{%block more_javascript%}
+<script src="{{url_for('base.node_modules',
+ filename='resumablejs/resumable.js')}}"></script>
+<script type="text/javascript" src="/static/js/files.js"></script>
+
+<script type="text/javascript">
+ $("#btn-reset-file-separator").on("click", (event) => {
+ event.preventDefault();
+ $("#txt-file-separator").val("\t");
+ $("#txt-file-separator").trigger("change");
+ });
+ $("#btn-reset-file-comment-character").on("click", (event) => {
+ event.preventDefault();
+ $("#txt-file-comment-character").val("#");
+ $("#txt-file-comment-character").trigger("change");
+ });
+ $("#btn-reset-file-na").on("click", (event) => {
+ event.preventDefault();
+ $("#txt-file-na").val("- NA N/A");
+ $("#txt-file-na").trigger("change");
+ });
+
+ var update_preview = (table, filedata, formdata, numrows) => {
+ table.find("thead tr").remove()
+ table.find(".data-row").remove();
+ var linenum = 0;
+ var tableheader = table.find("thead");
+ var tablebody = table.find("tbody");
+ var numheadings = 0;
+ var navalues = formdata
+ .na_strings
+ .split(" ")
+ .map((v) => {return v.trim();})
+ .filter((v) => {return Boolean(v);});
+ filedata.forEach((line) => {
+ if(line.startsWith(formdata.comment_char) || linenum >= numrows) {
+ return false;
+ }
+ var row = $("<tr></tr>");
+ line.split(formdata.separator)
+ .map((field) => {
+ var value = field.trim();
+ if(navalues.includes(value)) {
+ return "⋘NUL⋙";
+ }
+ return value;
+ })
+ .filter((field) => {
+ return (field !== "" && field != undefined && field != null);
+ })
+ .forEach((field) => {
+ if(linenum == 0) {
+ numheadings += 1;
+ var tablefield = $("<th></th>");
+ tablefield.text(field);
+ row.append(tablefield);
+ } else {
+ add_class(row, "data-row");
+ var tablefield = $("<td></td>");
+ tablefield.text(field);
+ row.append(tablefield);
+ }
+ });
+
+ if(linenum == 0) {
+ tableheader.append(row);
+ } else {
+ tablebody.append(row);
+ }
+ linenum += 1;
+ });
+
+ if(table.find("tbody tr.data-row").length > 0) {
+ add_class(table.find(".data-row-template"), "visually-hidden");
+ } else {
+ remove_class(table.find(".data-row-template"), "visually-hidden");
+ }
+ };
+
+ var makePreviewUpdater = (preview_table) => {
+ return (data) => {
+ update_preview(
+ preview_table,
+ data,
+ filesMetadata(),
+ PREVIEW_ROWS);
+ };
+ };
+
+ var preview_tables_to_elements_map = {
+ "#tbl-preview-pheno-desc": "#finput-phenotype-descriptions",
+ "#tbl-preview-pheno-data": "#finput-phenotype-data",
+ "#tbl-preview-pheno-se": "#finput-phenotype-se",
+ "#tbl-preview-pheno-n": "#finput-phenotype-n"
+ };
+
+ var filesMetadata = () => {
+ return {
+ "separator": $("#txt-file-separator").val(),
+ "comment_char": $(
+ "#txt-file-comment-character").val(),
+ "na_strings": $("#txt-file-na").val()
+ }
+ };
+
+ var PREVIEW_ROWS = 5;
+
+ var handler_update_previews = (event) => {
+ Object.entries(preview_tables_to_elements_map).forEach((mapentry) => {
+ var preview_table = $(mapentry[0]);
+ var file_input = $(mapentry[1]);
+ if(file_input[0].files.length > 0) {
+ readFirstNLines(
+ file_input[0].files[0],
+ 10,
+ [makePreviewUpdater(preview_table)]);
+ }
+ });
+
+ if(typeof(resumables) !== "undefined") {
+ resumables.forEach((resumable) => {
+ if(resumable.files.length > 0) {
+ readFirstNLines(
+ resumable.files[0].file,
+ 10,
+ [makePreviewUpdater(resumable.preview_table)]);
+ }
+ });
+ }
+ };
+
+ [
+ "#txt-file-separator",
+ "#txt-file-comment-character",
+ "#txt-file-na"
+ ].forEach((elementid) => {
+ $(elementid).on("change", handler_update_previews);
+ });
+
+ [
+ "#finput-phenotype-descriptions",
+ "#finput-phenotype-data",
+ "#finput-phenotype-se",
+ "#finput-phenotype-n"
+ ].forEach((elementid) => {
+ $(elementid).on("change", (event) => {
+ readFirstNLines(
+ event.target.files[0],
+ 10,
+ [makePreviewUpdater(
+ $("#" + event.target.getAttribute("data-preview-table")))]);
+ });
+ });
+
+
+ var resumableDisplayFiles = (display_area, files) => {
+ files.forEach((file) => {
+ display_area.find(".file-display").remove();
+ var display_element = display_area
+ .find(".file-display-template")
+ .clone();
+ remove_class(display_element, "visually-hidden");
+ remove_class(display_element, "file-display-template");
+ add_class(display_element, "file-display");
+ display_element.find(".filename").text(file.name
+ || file.fileName
+ || file.relativePath
+ || file.webkitRelativePath);
+ display_element.find(".filesize").text(
+ (file.size / (1024*1024)).toFixed(2) + "MB");
+ display_element.find(".fileuniqueid").text(file.uniqueIdentifier);
+ display_element.find(".filemimetype").text(file.file.type);
+ display_area.append(display_element);
+ });
+ };
+
+
+ var indicateProgress = (resumable, progress_bar) => {
+ return () => {/*Has no event!*/
+ var progress = (resumable.progress() * 100).toFixed(2);
+ var pbar = progress_bar.find(".progress-bar");
+ remove_class(progress_bar, "visually-hidden");
+ pbar.css("width", progress+"%");
+ pbar.attr("aria-valuenow", progress);
+ pbar.text("Uploading: " + progress + "%");
+ };
+ };
+
+ var retryUpload = (retry_button, cancel_button) => {
+ retry_button.on("click", (event) => {
+ resumable.files.forEach((file) => {file.retry();});
+ add_class(retry_button, "visually-hidden");
+ remove_class(cancel_button, "visually-hidden");
+ add_class(browse_button, "visually-hidden");
+ });
+ };
+
+ var cancelUpload = (cancel_button, retry_button) => {
+ cancel_button.on("click", (event) => {
+ resumable.files.forEach((file) => {
+ if(file.isUploading()) {
+ file.abort();
+ }
+ });
+ add_class(cancel_button, "visually-hidden");
+ remove_class(retry_button, "visually-hidden");
+ remove_class(browse_button, "visually-hidden");
+ });
+ };
+
+
+ var startUpload = (browse_button, retry_button, cancel_button) => {
+ return (event) => {
+ remove_class(cancel_button, "visually-hidden");
+ add_class(retry_button, "visually-hidden");
+ add_class(browse_button, "visually-hidden");
+ };
+ };
+
+ var processForm = (form) => {
+ var formdata = new FormData(form);
+ uploaded_files.forEach((msg) => {
+ formdata.delete(msg["file-input-name"]);
+ formdata.append(msg["file-input-name"], JSON.stringify({
+ "uploaded-file": msg["uploaded-file"],
+ "original-name": msg["original-name"]
+ }));
+ });
+ formdata.append("resumable-upload", "true");
+ formdata.append("publication-id", $("#txt-publication-id").val());
+ return formdata;
+ }
+
+ var uploaded_files = new Set();
+ var submitForm = (new_file) => {
+ uploaded_files.add(new_file);
+ if(uploaded_files.size === resumables.length) {
+ var form = $("#frm-add-phenotypes");
+ if(form.length !== 1) {
+ // TODO: Handle error somehow?
+ alert("Could not find form!!!");
+ return false;
+ }
+
+ $.ajax({
+ "url": form.attr("action"),
+ "type": "POST",
+ "data": processForm(form[0]),
+ "processData": false,
+ "contentType": false,
+ "success": (data, textstatus, jqxhr) => {
+ // TODO: Redirect to endpoint that should come as part of the
+ // success/error message.
+ console.log("SUCCESS DATA: ", data);
+ console.log("SUCCESS STATUS: ", textstatus);
+ console.log("SUCCESS jqXHR: ", jqxhr);
+ window.location.assign(window.location.origin + data["redirect-to"]);
+ },
+ });
+ return false;
+ }
+ return false;
+ };
+
+ var uploadSuccess = (file_input_name) => {
+ return (file, message) => {
+ submitForm({...JSON.parse(message), "file-input-name": file_input_name});
+ };
+ };
+
+
+ var uploadError = () => {
+ return (message, file) => {
+ $("#frm-add-phenotypes input[type=submit]").removeAttr("disabled");
+ console.log("THE FILE:", file);
+ console.log("THE ERROR MESSAGE:", message);
+ };
+ };
+
+
+
+ var makeResumableObject = (form_id, file_input_id, resumable_element_id, preview_table_id) => {
+ var the_form = $("#" + form_id);
+ var file_input = $("#" + file_input_id);
+ var submit_button = the_form.find("input[type=submit]");
+ if(file_input.length != 1) {
+ return false;
+ }
+ var r = errorHandler(
+ fileSuccessHandler(
+ uploadStartHandler(
+ filesAddedHandler(
+ markResumableDragAndDropElement(
+ makeResumableElement(
+ the_form.attr("data-resumable-target"),
+ file_input.parent(),
+ $("#" + resumable_element_id),
+ submit_button,
+ ["csv", "tsv", "txt"]),
+ file_input.parent(),
+ $("#" + resumable_element_id),
+ $("#" + resumable_element_id + "-browse-button")),
+ (files) => {
+ // TODO: Also trigger preview!
+ resumableDisplayFiles(
+ $("#" + resumable_element_id + "-selected-files"), files);
+ files.forEach((file) => {
+ readFirstNLines(
+ file.file,
+ 10,
+ [makePreviewUpdater(
+ $("#" + preview_table_id))])
+ });
+ }),
+ startUpload($("#" + resumable_element_id + "-browse-button"),
+ $("#" + resumable_element_id + "-retry-button"),
+ $("#" + resumable_element_id + "-cancel-button"))),
+ uploadSuccess(file_input.attr("name"))),
+ uploadError());
+
+ /** Setup progress indicator **/
+ progressHandler(
+ r,
+ indicateProgress(r, $("#" + resumable_element_id + "-progress-bar")));
+
+ return r;
+ };
+
+ var resumables = [
+ ["frm-add-phenotypes", "finput-phenotype-descriptions", "resumable-phenotype-descriptions", "tbl-preview-pheno-desc"],
+ ["frm-add-phenotypes", "finput-phenotype-data", "resumable-phenotype-data", "tbl-preview-pheno-data"],
+ ["frm-add-phenotypes", "finput-phenotype-se", "resumable-phenotype-se", "tbl-preview-pheno-se"],
+ ["frm-add-phenotypes", "finput-phenotype-n", "resumable-phenotype-n", "tbl-preview-pheno-n"],
+ ].map((row) => {
+ r = makeResumableObject(row[0], row[1], row[2], row[3]);
+ r.preview_table = $("#" + row[3]);
+ return r;
+ }).filter((val) => {
+ return Boolean(val);
+ });
+
+ $("#frm-add-phenotypes input[type=submit]").on("click", (event) => {
+ event.preventDefault();
+ console.debug();
+ if ($("#txt-publication-id").val() == "") {
+ alert("You MUST provide a publication for the phenotypes.");
+ return false;
+ }
+ // TODO: Check all the relevant files exist
+ // TODO: Verify that files are not duplicated
+ var filenames = [];
+ var nondupfiles = [];
+ resumables.forEach((r) => {
+ var fname = r.files[0].file.name;
+ filenames.push(fname);
+ if(!nondupfiles.includes(fname)) {
+ nondupfiles.push(fname);
+ }
+ });
+
+ // Check that all files were provided
+ if(resumables.length !== filenames.length) {
+ window.alert("You MUST provide all the files requested.");
+ event.target.removeAttribute("disabled");
+ return false;
+ }
+
+ // Check that there are no duplicate files
+ var duplicates = Object.entries(filenames.reduce(
+ (acc, curr, idx, arr) => {
+ acc[curr] = (acc[curr] || 0) + 1;
+ return acc;
+ },
+ {})).filter((entry) => {return entry[1] !== 1;});
+ if(duplicates.length > 0) {
+ var msg = "The file(s):\r\n";
+ msg = msg + duplicates.reduce(
+ (msgstr, afile) => {
+ return msgstr + " • " + afile[0] + "\r\n";
+ },
+ "");
+ msg = msg + "is(are) duplicated. Please fix and try again.";
+ window.alert(msg);
+ event.target.removeAttribute("disabled");
+ return false;
+ }
+ // TODO: Check all fields
+ // Start the uploads.
+ event.target.setAttribute("disabled", "disabled");
+ resumables.forEach((r) => {r.upload();});
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/add-phenotypes.html b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html
index 196bc69..898fc0c 100644
--- a/uploader/templates/phenotypes/add-phenotypes.html
+++ b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html
@@ -1,4 +1,4 @@
-{%extends "phenotypes/base.html"%}
+{%extends "phenotypes/add-phenotypes-base.html"%}
{%from "flash_messages.html" import flash_all_messages%}
{%from "macro-table-pagination.html" import table_pagination%}
{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
@@ -16,52 +16,39 @@
<a href="{{url_for('species.populations.phenotypes.add_phenotypes',
species_id=species.SpeciesId,
population_id=population.Id,
- dataset_id=dataset.Id)}}">View Datasets</a>
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
</li>
{%endblock%}
-{%block contents%}
-{{flash_all_messages()}}
-
-<div class="row">
- <form id="frm-add-phenotypes"
- method="POST"
- enctype="multipart/form-data"
- action="{{url_for('species.populations.phenotypes.add_phenotypes',
- species_id=species.SpeciesId,
- population_id=population.Id,
- dataset_id=dataset.Id)}}">
- <legend>Add New Phenotypes</legend>
-
- <div class="form-text help-block">
- <p>Select the zip file bundle containing information on the phenotypes you
- wish to upload, then click the "Upload Phenotypes" button below to
- upload the data.</p>
- <p>See the <a href="#section-file-formats">File Formats</a> section below
- to get an understanding of what is expected of the bundle files you
- upload.</p>
- <p><strong>This will not update any existing phenotypes!</strong></p>
- </div>
-
- <div class="form-group">
- <label for="finput-phenotypes-bundle" class="form-label">
- Phenotypes Bundle</label>
- <input type="file"
- id="finput-phenotypes-bundle"
- name="phenotypes-bundle"
- accept="application/zip, .zip"
- required="required"
- class="form-control" />
- </div>
+{%block frm_add_phenotypes_documentation%}
+<p>Select the zip file bundle containing information on the phenotypes you
+ wish to upload, then click the "Upload Phenotypes" button below to
+ upload the data.</p>
+<p>If you wish to upload the files individually instead,
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ title="">click here</a>.</p>
+<p>See the <a href="#section-file-formats">File Formats</a> section below
+ to get an understanding of what is expected of the bundle files you
+ upload.</p>
+{%endblock%}
- <div class="form-group">
- <input type="submit"
- value="upload phenotypes"
- class="btn btn-primary" />
- </div>
- </form>
+{%block frm_add_phenotypes_elements%}
+<div class="form-group">
+ <label for="finput-phenotypes-bundle" class="form-label">
+ Phenotypes Bundle</label>
+ <input type="file"
+ id="finput-phenotypes-bundle"
+ name="phenotypes-bundle"
+ accept="application/zip, .zip"
+ required="required"
+ class="form-control" />
</div>
+{%endblock%}
+{%block page_documentation%}
<div class="row">
<h2 class="heading" id="section-file-formats">File Formats</h2>
<p>We accept an extended form of the
@@ -213,17 +200,6 @@
<code>pheno_transposed: True</code>, then the file will be a matrix of
<em>phenotypes × individuals</em>.</p>
</div>
-
-<div class="row text-warning">
- <h3 class="subheading">Notes for Devs (well… Fred, really.)</h3>
- <p>Use the following resources for automated retrieval of certain data</p>
- <ul>
- <li><a href="https://www.ncbi.nlm.nih.gov/pmc/tools/developers/"
- title="NCBI APIs: Retrieve articles' metadata etc.">
- NCBI APIS</a></li>
- </ul>
-</div>
-
{%endblock%}
{%block sidebarcontents%}
diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html
index 3bc5dea..adbc012 100644
--- a/uploader/templates/phenotypes/base.html
+++ b/uploader/templates/phenotypes/base.html
@@ -6,7 +6,14 @@
{%else%}
class="breadcrumb-item"
{%endif%}>
+ {%if dataset is mapping%}
+ <a href="{{url_for('species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">{{dataset.Name}}</a>
+ {%else%}
<a href="{{url_for('species.populations.phenotypes.index')}}">Phenotypes</a>
+ {%endif%}
</li>
{%block lvl4_breadcrumbs%}{%endblock%}
{%endblock%}
diff --git a/uploader/templates/phenotypes/bulk-edit-upload.html b/uploader/templates/phenotypes/bulk-edit-upload.html
new file mode 100644
index 0000000..d0f38f5
--- /dev/null
+++ b/uploader/templates/phenotypes/bulk-edit-upload.html
@@ -0,0 +1,62 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="view-dataset"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">View</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+<div class="row">
+ <p>Upload the edited file you downloaded and edited.</p>
+</div>
+
+<div class="row">
+ <form id="frm-bulk-edit-upload"
+ class="form-horizontal"
+ method="POST"
+ action="{{url_for(
+ 'species.populations.phenotypes.edit_upload_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ enctype="multipart/form-data">
+
+ <div class="form-group row">
+ <label for="file-upload-bulk-edit-upload"
+ class="form-label col-form-label col-sm-2">
+ Edited File</label>
+ <div class="col-sm-10">
+ <input id="file-upload-bulk-edit-upload"
+ name="file-upload-bulk-edit-upload"
+ class="form-control"
+ type="file"
+ accept="text/tab-separated-values"
+ required="required" />
+ </div>
+ </div>
+
+ <input type="submit" class="btn btn-primary"
+ value="upload to edit" />
+
+ </form>
+</div>
+{%endblock%}
+
+
+{%block javascript%}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html
index 93de92f..19a2b34 100644
--- a/uploader/templates/phenotypes/create-dataset.html
+++ b/uploader/templates/phenotypes/create-dataset.html
@@ -42,7 +42,7 @@
<input type="text"
name="dataset-name"
id="txt-dataset-name"
- value="{{original_formdata.get('dataset-name') or (population.InbredSetCode + 'Publish')}}"
+ value="{{original_formdata.get('dataset-name') or (population.Name + 'Publish')}}"
{%if errors["dataset-name"] is defined%}
class="form-control danger"
{%else%}
@@ -51,7 +51,7 @@
required="required" />
<small class="form-text text-muted">
<p>A short representative name for the dataset.</p>
- <p>Recommended: Use the population code and append "Publish" at the end.
+ <p>Recommended: Use the population name and append "Publish" at the end.
<br />This field will only accept names composed of
letters ('A-Za-z'), numbers (0-9), hyphens and underscores.</p>
</small>
@@ -74,8 +74,10 @@
{%endif%}
required="required" />
<small class="form-text text-muted">
- <p>A longer, descriptive name for the dataset &mdash; useful for humans.
- </p></small>
+ <p>A longer, descriptive name for the dataset. The name is meant for use
+ by humans, and therefore, it should be clear what the dataset contains
+ from the name.</p>
+ </small>
</div>
<div class="form-group">
@@ -84,7 +86,7 @@
name="dataset-shortname"
type="text"
class="form-control"
- value="{{original_formdata.get('dataset-shortname') or (population.InbredSetCode + ' Publish')}}" />
+ value="{{original_formdata.get('dataset-shortname') or (population.Name + 'Publish')}}" />
<small class="form-text text-muted">
<p>An optional, short name for the dataset. <br />
If this is not provided, it will default to the value provided for the
diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html
new file mode 100644
index 0000000..32c903f
--- /dev/null
+++ b/uploader/templates/phenotypes/edit-phenotype.html
@@ -0,0 +1,332 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="edit-phenotype"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">View Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <h2 class="heading">edit phenotype data</h2>
+ <p>The forms provided in this page help you update the data for the
+ phenotypes, and the publication information for the phenotype,
+ respectively.</p>
+</div>
+
+<div class="row">
+ <h3 class="subheading">Basic metadata</h3>
+ <form name="frm-phenotype-basic-metadata"
+ class="form-horizontal"
+ method="POST"
+ action="{{url_for(
+ 'species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">
+ <input type="hidden" name="phenotype-id" value="{{phenotype.Id}}" />
+ <div class="form-group">
+ <label for="txt-pre-publication-description"
+ class="control-label col-sm-2">Pre-Publication Description</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-pre-publication-description"
+ name="pre-publication-description"
+ class="form-control"
+ value="{{phenotype['Pre_publication_description'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-pre-publication-abbreviation"
+ class="control-label col-sm-2">Pre-Publication Abbreviation</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-pre-publication-abbreviation"
+ name="pre-publication-abbreviation"
+ class="form-control"
+ value="{{phenotype['Pre_publication_abbreviation'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-post-publication-description"
+ class="control-label col-sm-2">Post-Publication Description</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-post-publication-description"
+ name="post-publication-description"
+ class="form-control"
+ value="{{phenotype['Post_publication_description'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-post-publication-abbreviation"
+ class="control-label col-sm-2">Post-Publication Abbreviation</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-post-publication-abbreviation"
+ name="post-publication-abbreviation"
+ class="form-control"
+ value="{{phenotype['Post_publication_abbreviation'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-original-description"
+ class="control-label col-sm-2">Original Description</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-original-description"
+ name="original-description"
+ class="form-control"
+ value="{{phenotype['Original_description'] or ''}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-units"
+ class="control-label col-sm-2">units</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-units"
+ name="units"
+ class="form-control"
+ required="required"
+ value="{{phenotype['Units']}}" />
+ </div>
+ </div>
+
+ <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ name="submit"
+ class="btn btn-primary"
+ value="update basic metadata">
+ </div>
+ </div>
+ </form>
+</div>
+
+
+<div class="row">
+ <h3 class="subheading">phenotype data</h3>
+ <form id="frm-edit-phenotype-data"
+ class="form-horizontal"
+ method="POST"
+ action="{{url_for(
+ 'species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}">
+ <div style="max-height: 23.37em;overflow-y: scroll;">
+ <table class="table table-striped table-responsive table-form-table">
+ <thead style="position: sticky; top: 0;">
+ <tr>
+ <th>#</th>
+ <th>Sample</th>
+ <th>Value</th>
+ {%if population.Family in families_with_se_and_n%}
+ <th>Standard-Error</th>
+ <th>Number of Samples</th>
+ {%endif%}
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for item in phenotype.data%}
+ <tr>
+ <td>{{loop.index}}</td>
+ <td>{{item.StrainName}}</td>
+ <td>
+ <input type="text"
+ name="value-new::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.value}}"
+ class="form-control" />
+ <input type="hidden"
+ name="value-original::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.value}}" /></td>
+ {%if population.Family in families_with_se_and_n%}
+ <td>
+ <input type="text"
+ name="se-new::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.error or ''}}"
+ data-original-value="{{item.error or ''}}"
+ class="form-control" />
+ <input type="hidden"
+ name="se-original::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.error or ''}}" /></td>
+ <td>
+ <input type="text"
+ name="n-new::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.count or ''}}"
+ data-original-value="{{item.count or "-"}}"
+ class="form-control" />
+ <input type="hidden"
+ name="n-original::{{item.DataId}}::{{item.StrainId}}"
+ value="{{item.count or ''}}" /></td>
+ {%endif%}
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ </div>
+ <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ name="submit"
+ class="btn btn-primary"
+ value="update data" />
+ </div>
+ </div>
+ </form>
+</div>
+
+
+<div class="row">
+ <h3 class="subheading">publication information</h3>
+ <p>Use the form below to update the publication information for this
+ phenotype.</p>
+ <form id="frm-edit-phenotype-pub-data"
+ class="form-horizontal"
+ method="POST"
+ action="#">
+ <div class="form-group">
+ <label for="txt-pubmed-id" class="control-label col-sm-2">Pubmed ID</label>
+ <div class="col-sm-10">
+ <input id="txt-pubmed-id" name="pubmed-id" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter your publication's PubMed ID.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-authors" class="control-label col-sm-2">Authors</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-authors" name="publication-authors"
+ type="text" class="form-control" />
+ <span class="form-text text-muted">
+ Enter the authors.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-title" class="control-label col-sm-2">
+ Publication Title</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-title" name="publication-title" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter your publication's title.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-abstract" class="control-label col-sm-2">
+ Publication Abstract</label>
+ <div class="col-sm-10">
+ <textarea id="txt-publication-abstract" name="publication-abstract"
+ class="form-control" rows="10"></textarea>
+ <span class="form-text text-muted">
+ Enter the abstract for your publication.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-journal" class="control-label col-sm-2">Journal</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-journal" name="journal" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the name of the journal where your work was published.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-volume" class="control-label col-sm-2">Volume</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-volume" name="publication-volume" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the volume in the following format &hellip;</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-pages" class="control-label col-sm-2">Pages</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-pages" name="publication-pages" type="text"
+ class="form-control" />
+ <span class="form-text text-muted">
+ Enter the journal volume where your work was published.</span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="select-publication-month" class="control-label col-sm-2">
+ Publication Month</label>
+ <div class="col-sm-10">
+ <select id="select-publication-month" name="publication-month"
+ class="form-control">
+ {%for month in monthnames%}
+ <option value="{{month | lower}}"
+ {%if current_month | lower == month | lower%}
+ selected="selected"
+ {%endif%}>{{month | capitalize}}</option>
+ {%endfor%}
+ </select>
+ <span class="form-text text-muted">
+ Select the month when the work was published.
+ <span class="text-danger">
+ This cannot be before, say 1600 and cannot be in the future!</span></span>
+ </div>
+ </div>
+
+ <div class="form-group">
+ <label for="txt-publication-year" class="control-label col-sm-2">Publication Year</label>
+ <div class="col-sm-10">
+ <input id="txt-publication-year" name="publication-year" type="text"
+ class="form-control" value="{{current_year}}" />
+ <span class="form-text text-muted">
+ Enter the year your work was published.
+ <span class="text-danger">
+ This cannot be before, say 1600 and cannot be in the future!</span>
+ </span>
+ </div>
+ </div>
+ <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
+ <input type="submit"
+ name="submit"
+ class="btn btn-primary not-implemented"
+ value="update publication" />
+ </div>
+ </div>
+ </form>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/index.html b/uploader/templates/phenotypes/index.html
index 0c691e6..689c28e 100644
--- a/uploader/templates/phenotypes/index.html
+++ b/uploader/templates/phenotypes/index.html
@@ -11,16 +11,11 @@
{{flash_all_messages()}}
<div class="row">
- <p>This section deals with phenotypes that
- <span class="text-warning">
- <span class="glyphicon glyphicon-exclamation-sign"></span>
- … what are the characteristics of these phenotypes? …</span></p>
- <p>Select the species to begin the process of viewing/uploading data about
- your phenotypes</p>
+ {{select_species_form(url_for("species.populations.phenotypes.index"), species)}}
</div>
+{%endblock%}
-<div class="row">
- {{select_species_form(url_for("species.populations.phenotypes.index"),
- species)}}
-</div>
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
{%endblock%}
diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html
index d531a71..12963c1 100644
--- a/uploader/templates/phenotypes/job-status.html
+++ b/uploader/templates/phenotypes/job-status.html
@@ -30,16 +30,101 @@
{%block contents%}
{%if job%}
+<h4 class="subheading">Progress</h4>
+<div class="row" style="overflow:scroll;">
+ <p><strong>Process Status:</strong> {{job.status}}</p>
+ {%if metadata%}
+ <table class="table table-responsive">
+ <thead>
+ <tr>
+ <th>File</th>
+ <th>Status</th>
+ <th>Lines Processed</th>
+ <th>Total Errors</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ {%for file,meta in metadata.items()%}
+ <tr>
+ <td>{{file}}</td>
+ <td>{{meta.status}}</td>
+ <td>{{meta.linecount}}</td>
+ <td>{{meta["total-errors"]}}</td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%endif%}
+</div>
+
<div class="row">
- <p><strong>Status:</strong> {{job.status}}</p>
- {%if job.status in ("completed:success", "success")%}
- <p><a href="#"
- class="not-implemented btn btn-primary"
- title="Continue to process data">Continue</a>
+ {%if job.status in ("completed:success", "success")%}
+ <p>
+ {%if errors | length == 0%}
+ <a href="{{url_for('species.populations.phenotypes.review_job_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ job_id=job_id)}}"
+ class="btn btn-primary"
+ title="Continue to process data">Continue</a>
+ {%else%}
+ <span class="text-muted"
+ disabled="disabled"
+ style="border: solid 2px;border-radius: 5px;padding: 0.3em;">
+ Cannot continue due to errors. Please fix the errors first.
+ </span>
+ {%endif%}
</p>
{%endif%}
</div>
+<h4 class="subheading">Errors</h4>
+<div class="row" style="max-height: 20em; overflow: scroll;">
+ {%if errors | length == 0 %}
+ <p class="text-info">
+ <span class="glyphicon glyphicon-info-sign"></span>
+ No errors found so far
+ </p>
+ {%else%}
+ <table class="table table-responsive">
+ <thead style="position: sticky; top: 0; background: white;">
+ <tr>
+ <th>File</th>
+ <th>Row</th>
+ <th>Column</th>
+ <th>Value</th>
+ <th>Message</th>
+ </tr>
+ </thead>
+
+ <tbody style="font-size: 0.9em;">
+ {%for error in errors%}
+ <tr>
+ <td>{{error.filename}}</td>
+ <td>{{error.rowtitle}}</td>
+ <td>{{error.coltitle}}</td>
+ <td>{%if error.cellvalue | length > 25%}
+ {{error.cellvalue[0:24]}}&hellip;
+ {%else%}
+ {{error.cellvalue}}
+ {%endif%}
+ </td>
+ <td>
+ {%if error.message | length > 250 %}
+ {{error.message[0:249]}}&hellip;
+ {%else%}
+ {{error.message}}
+ {%endif%}
+ </td>
+ </tr>
+ {%endfor%}
+ </tbody>
+ </table>
+ {%endif%}
+</div>
+
<div class="row">
{{cli_output(job, "stdout")}}
</div>
@@ -47,6 +132,7 @@
<div class="row">
{{cli_output(job, "stderr")}}
</div>
+
{%else%}
<div class="row">
<h3 class="text-danger">No Such Job</h3>
diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html
index 2eaf43a..2cf2c7f 100644
--- a/uploader/templates/phenotypes/list-datasets.html
+++ b/uploader/templates/phenotypes/list-datasets.html
@@ -48,9 +48,12 @@
</tbody>
</table>
{%else%}
- <p class="text-warning">
- <span class="glyphicon glyphicon-exclamation-sign"></span>
- There is no dataset for this population!</p>
+ <p>Phenotypes need to go into a dataset. We do not currently have a dataset
+ for species <strong>'{{species["FullName"]}} ({{species["Name"]}})'</strong>
+ phenotypes.</p>
+
+ <p>Do, please, create a new dataset by clicking on the "Create Dataset" button
+ below and following the prompts/instructions.</p>
<p><a href="{{url_for('species.populations.phenotypes.create_dataset',
species_id=species.SpeciesId,
population_id=population.Id)}}"
diff --git a/uploader/templates/phenotypes/load-phenotypes-success.html b/uploader/templates/phenotypes/load-phenotypes-success.html
new file mode 100644
index 0000000..645be16
--- /dev/null
+++ b/uploader/templates/phenotypes/load-phenotypes-success.html
@@ -0,0 +1,42 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="load-phenotypes-success"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">Add Phenotypes</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+<div class="row">
+ <p>You have successfully loaded
+ <!-- maybe indicate the number of phenotypes here? -->your
+ new phenotypes into the database.</p>
+ <!-- TODO: Maybe notify user that they have sole access. -->
+ <!-- TODO: Maybe provide a link to go to GeneNetwork to view the data. -->
+ <p>View your data
+ <a href="{{search_page_uri}}"
+ target="_blank">on GeneNetwork2</a>.
+ You might need to login to GeneNetwork2 to view specific traits.</p>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
+
+
+{%block more_javascript%}{%endblock%}
diff --git a/uploader/templates/phenotypes/macro-display-preview-table.html b/uploader/templates/phenotypes/macro-display-preview-table.html
new file mode 100644
index 0000000..5a4c422
--- /dev/null
+++ b/uploader/templates/phenotypes/macro-display-preview-table.html
@@ -0,0 +1,19 @@
+{%macro display_preview_table(tableid, filetype)%}
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">{{filetype | title}}: File Preview</h5>
+ <div class="card-text" style="overflow: scroll;">
+ <table id="{{tableid}}" class="table table-condensed table-responsive">
+ <thead>
+ <tr>
+ </tr>
+ <tbody>
+ <tr>
+ <td class="data-row-template text-info"></td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/macro-display-resumable-elements.html b/uploader/templates/phenotypes/macro-display-resumable-elements.html
new file mode 100644
index 0000000..ed14ea5
--- /dev/null
+++ b/uploader/templates/phenotypes/macro-display-resumable-elements.html
@@ -0,0 +1,60 @@
+{%macro display_resumable_elements(id, title, help)%}
+<div id="{{id}}"
+ class="resumable-elements visually-hidden"
+ style="background:#D4D4EE;border-radius: 5px;;padding: 1em;border-left: solid #B2B2CC 1px;border-bottom: solid #B2B2CC 2px;margin-top:0.3em;">
+ <strong style="line-height: 1.2em;">{{title | title}}</strong>
+
+ <span class="form-text text-muted">{{help | safe}}</span>
+
+ <div id="{{id}}-selected-files"
+ class="resumable-selected-files"
+ style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;">
+ <div class="panel panel-info file-display-template visually-hidden">
+ <div class="panel-heading filename">The Filename Goes Here!</div>
+ <div class="panel-body">
+ <ul>
+ <li>
+ <strong>Name</strong>:
+ <span class="filename">the file's name</span></li>
+
+ <li><strong>Size</strong>: <span class="filesize">0 MB</span></li>
+
+ <li>
+ <strong>Unique Identifier</strong>:
+ <span class="fileuniqueid">brrr</span></li>
+
+ <li>
+ <strong>Mime</strong>:
+ <span class="filemimetype">text/csv</span></li>
+ </ul>
+ </div>
+ </div>
+ </div>
+
+ <a id="{{id}}-browse-button"
+ class="resumable-browse-button btn btn-info"
+ href="#{{id}}"
+ style="margin-left: 80%;">Browse</a>
+
+ <div id="{{id}}-progress-bar" class="progress visually-hidden">
+ <div class="progress-bar"
+ role="progress-bar"
+ aria-valuenow="60"
+ aria-valuemin="0"
+ aria-valuemax="100"
+ style="width: 0%;">
+ Uploading: 60%
+ </div>
+ </div>
+
+ <div id="{{id}}-cancel-resume-buttons">
+ <a id="{{id}}-resume-button"
+ class="resumable-resume-button btn btn-info visually-hidden"
+ href="#">resume upload</a>
+
+ <a id="{{id}}-cancel-button"
+ class="resumable-cancel-button btn btn-danger visually-hidden"
+ href="#">cancel upload</a>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html
new file mode 100644
index 0000000..859df74
--- /dev/null
+++ b/uploader/templates/phenotypes/review-job-data.html
@@ -0,0 +1,125 @@
+{%extends "phenotypes/base.html"%}
+{%from "cli-output.html" import cli_output%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block extrameta%}
+{%if not job%}
+<meta http-equiv="refresh"
+ content="20; url={{url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}" />
+{%endif%}
+{%endblock%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">View Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+
+{%if job%}
+<div class="row">
+ <h3 class="heading">Data Review</h3>
+ <p class="text-info"><strong>
+ The data has <em>NOT</em> been added/saved yet. Review the details below
+ and click "Continue" to save the data.</strong></p>
+ <p>The &#x201C;<strong>{{dataset.FullName}}</strong>&#x201D; dataset from the
+ &#x201C;<strong>{{population.FullName}}</strong>&#x201D; population of the
+ species &#x201C;<strong>{{species.SpeciesName}} ({{species.FullName}})</strong>&#x201D;
+ will be updated as follows:</p>
+
+ <ul>
+ {%if publication%}
+ <li>All {{summary.get("pheno", {}).get("total-data-rows", "0")}} phenotypes
+ are linked to the following publication:
+ <ul>
+ <li><strong>Publication Title:</strong>
+ {{publication.Title or "—"}}</li>
+ <li><strong>Author(s):</strong>
+ {{publication.Authors or "—"}}</li>
+ </ul>
+ </li>
+ {%endif%}
+ {%for ftype in ("phenocovar", "pheno", "phenose", "phenonum")%}
+ {%if summary.get(ftype, False)%}
+ <li>A total of {{summary[ftype]["number-of-files"]}} files will be processed
+ adding {%if ftype == "phenocovar"%}(possibly){%endif%}
+ {{summary[ftype]["total-data-rows"]}} new
+ {%if ftype == "phenocovar"%}
+ phenotypes
+ {%else%}
+ {{summary[ftype]["description"]}} rows
+ {%endif%}
+ to the database.
+ </li>
+ {%endif%}
+ {%endfor%}
+ </ul>
+
+ <form id="frm-review-phenotype-data"
+ method="POST"
+ action="{{url_for('species.populations.phenotypes.load_data_to_database',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">
+ <input type="hidden" name="data-qc-job-id" value="{{job.jobid}}" />
+ <input type="submit"
+ value="continue"
+ class="btn btn-primary" />
+ </form>
+</div>
+{%else%}
+<div class="row">
+ <h4 class="subheading">Invalid Job</h3>
+ <p class="text-danger">
+ Could not find a job with the ID: <strong>{{job_id}}.</p>
+ <p>You will be redirected in
+ <span id="countdown-element" class="text-info">20</span> second(s)</p>
+ <p class="text-muted">
+ <small>
+ If you are not redirected, please
+ <a href="{{url_for(
+ 'species.populations.phenotypes.view_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">click here</a> to continue
+ </small>
+ </p>
+</div>
+{%endif%}
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(document).ready(function() {
+ var countdown = 20;
+ var countdown_element = $("#countdown-element");
+ if(countdown_element.length === 1) {
+ intv = window.setInterval(function() {
+ countdown = countdown - 1;
+ countdown_element.html(countdown);
+ }, 1000);
+ }
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/select-population.html b/uploader/templates/phenotypes/select-population.html
index eafd4a7..48c19b1 100644
--- a/uploader/templates/phenotypes/select-population.html
+++ b/uploader/templates/phenotypes/select-population.html
@@ -11,18 +11,16 @@
{%block contents%}
{{flash_all_messages()}}
-<div class="row">
- <p>Select the population for your phenotypes to view and manage the phenotype
- datasets that relate to it.</p>
-</div>
<div class="row">
- {{select_population_form(url_for("species.populations.phenotypes.select_population",
- species_id=species.SpeciesId),
- populations)}}
+ {{select_population_form(url_for("species.populations.phenotypes.select_population", species_id=species.SpeciesId), species, populations)}}
</div>
{%endblock%}
{%block sidebarcontents%}
{{display_species_card(species)}}
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/populations.js"></script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html
index b136bb6..21563d6 100644
--- a/uploader/templates/phenotypes/view-dataset.html
+++ b/uploader/templates/phenotypes/view-dataset.html
@@ -16,7 +16,7 @@
<a href="{{url_for('species.populations.phenotypes.view_dataset',
species_id=species.SpeciesId,
population_id=population.Id,
- dataset_id=dataset.Id)}}">View Datasets</a>
+ dataset_id=dataset.Id)}}">View</a>
</li>
{%endblock%}
@@ -57,36 +57,26 @@
<div class="row">
<h2>Phenotype Data</h2>
- <p>This dataset has a total of {{phenotype_count}} phenotypes.</p>
+ <p>Click on any of the phenotypes in the table below to view and edit that
+ phenotype's data.</p>
+ <p>Use the search to filter through all the phenotypes and find specific
+ phenotypes of interest.</p>
+</div>
- {{table_pagination(start_from, count, phenotype_count, url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id), "phenotypes")}}
- <table class="table">
+<div class="row">
+
+ <table id="tbl-phenotypes-list" class="table compact stripe cell-border">
<thead>
<tr>
- <th>#</th>
+ <th></th>
+ <th>Index</th>
<th>Record</th>
<th>Description</th>
</tr>
</thead>
- <tbody>
- {%for pheno in phenotypes%}
- <tr>
- <td>{{pheno.sequence_number}}</td>
- <td><a href="{{url_for('species.populations.phenotypes.view_phenotype',
- species_id=species.SpeciesId,
- population_id=population.Id,
- dataset_id=dataset.Id,
- xref_id=pheno['pxr.Id'])}}"
- title="View phenotype details">
- {{pheno.InbredSetCode}}_{{pheno["pxr.Id"]}}</a></td>
- <td>{{pheno.Post_publication_description or pheno.Pre_publication_abbreviation or pheno.Original_description}}</td>
- </tr>
- {%else%}
- <tr><td colspan="5"></td></tr>
- {%endfor%}
- </tbody>
+ <tbody></tbody>
</table>
</div>
{%endblock%}
@@ -94,3 +84,161 @@
{%block sidebarcontents%}
{{display_population_card(species, population)}}
{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(function() {
+ var species_id = {{species.SpeciesId}};
+ var population_id = {{population.Id}};
+ var dataset_id = {{dataset.Id}};
+ var dataset_name = "{{dataset.Name}}";
+ var data = {{phenotypes | tojson}};
+
+ var dtPhenotypesList = buildDataTable(
+ "#tbl-phenotypes-list",
+ data,
+ [
+ {
+ data: function(pheno) {
+ return `<input type="checkbox" name="selected-phenotypes" `
+ + `id="chk-selected-phenotypes-${pheno.InbredSetCode}_${pheno.xref_id}" `
+ + `value="${pheno.InbredSetCode}_${pheno.xref_id}" `
+ + `class="chk-row-select" />`
+ }
+ },
+ {data: "sequence_number"},
+ {
+ data: function(pheno, type, set, meta) {
+ var spcs_id = {{species.SpeciesId}};
+ var pop_id = {{population.Id}};
+ var dtst_id = {{dataset.Id}};
+ return `<a href="/species/${spcs_id}` +
+ `/populations/${pop_id}` +
+ `/phenotypes/datasets/${dtst_id}` +
+ `/phenotype/${pheno.xref_id}` +
+ `" target="_blank">` +
+ `${pheno.InbredSetCode}_${pheno.xref_id}` +
+ `</a>`;
+ }
+ },
+ {
+ data: function(pheno) {
+ return (pheno.Post_publication_description ||
+ pheno.Original_description ||
+ pheno.Pre_publication_description);
+ }
+ }
+ ],
+ {
+ select: "multi+shift",
+ layout: {
+ top2: {
+ buttons: [
+ {
+ extend: "selectAll",
+ className: "btn btn-info",
+ titleAttr: "Click to select ALL records in the table."
+ },
+ {
+ extend: "selectNone",
+ className: "btn btn-info",
+ titleAttr: "Click to deselect ANY selected record(s) in the table."
+ },
+ {
+ text: "Bulk Edit (Download Data)",
+ className: "btn btn-info btn-bulk-edit",
+ titleAttr: "Click to download data for editing.",
+ action: (event, dt, node, config) => {
+ var phenoids = [];
+ var selected = dt.rows({selected: true, page: "all"}).data();
+ for(var idx = 0; idx < selected.length; idx++) {
+ phenoids.push({
+ phenotype_id: selected[idx].Id,
+ xref_id: selected[idx].xref_id
+ });
+ }
+ if(phenoids.length == 0) {
+ alert("No record selected. Nothing to do!");
+ return false;
+ }
+
+ $(".btn-bulk-edit").prop("disabled", true);
+ $(".btn-bulk-edit").addClass("d-none");
+ var spinner = $(
+ "<div id='bulk-edit-spinner' class='spinner-grow text-info'>");
+ spinner_content = $(
+ "<span class='visually-hidden'>");
+ spinner_content.html(
+ "Downloading data &hellip;");
+ spinner.append(spinner_content)
+ $(".btn-bulk-edit").parent().append(
+ spinner);
+
+ $.ajax(
+ (`/species/${species_id}/populations/` +
+ `${population_id}/phenotypes/datasets/` +
+ `${dataset_id}/edit-download`),
+ {
+ method: "POST",
+ data: JSON.stringify(phenoids),
+ xhrFields: {
+ responseType: "blob"
+ },
+ success: (data, textStatus, jqXHR) => {
+ var link = document.createElement("a");
+ uri = window.URL.createObjectURL(data);
+ link.href = uri;
+ link.download = `${dataset_name}_data.tsv`;
+
+ document.body.appendChild(link);
+ link.click();
+ window.URL.revokeObjectURL(uri);
+ link.remove();
+ },
+ error: (jQXHR, textStatus, errorThrown) => {
+ console.log("Experienced an error: ", textStatus);
+ console.log("The ERROR: ", errorThrown);
+ },
+ complete: (jqXHR, textStatus) => {
+ $("#bulk-edit-spinner").remove();
+ $(".btn-bulk-edit").removeClass(
+ "d-none");
+ $(".btn-bulk-edit").prop(
+ "disabled", false);
+ },
+ contentType: "application/json"
+ });
+ }
+ },
+ {
+ text: "Bulk Edit (Upload Data)",
+ className: "btn btn-info btn-bulk-edit",
+ titleAttr: "Click to upload edited data you got by clicking the `Bulk Edit (Download Data)` button.",
+ action: (event, dt, node, config) => {
+ window.location.assign(
+ `${window.location.protocol}//` +
+ `${window.location.host}` +
+ `/species/${species_id}` +
+ `/populations/${population_id}` +
+ `/phenotypes/datasets/${dataset_id}` +
+ `/edit-upload`)
+ }
+ }
+ ]
+ },
+ top1Start: {
+ pageLength: {
+ text: "Show _MENU_ of _TOTAL_"
+ }
+ },
+ topStart: "info",
+ top1End: null
+ },
+ rowId: function(pheno) {
+ return `${pheno.InbredSetCode}_${pheno.xref_id}`;
+ }
+ });
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html
index 99bb8e5..21ac501 100644
--- a/uploader/templates/phenotypes/view-phenotype.html
+++ b/uploader/templates/phenotypes/view-phenotype.html
@@ -16,7 +16,7 @@
species_id=species.SpeciesId,
population_id=population.Id,
dataset_id=dataset.Id,
- xref_id=xref_id)}}">View Datasets</a>
+ xref_id=xref_id)}}">View Phenotype</a>
</li>
{%endblock%}
@@ -34,51 +34,58 @@
<td>{{phenotype.Post_publication_description or phenotype.Pre_publication_abbreviation or phenotype.Original_description}}
</tr>
<tr>
- <td><strong>Cross-Reference ID</strong></td>
- <td>{{phenotype.xref_id}}</td>
- </tr>
- <tr>
- <td><strong>Collation</strong></td>
+ <td><strong>Database</strong></td>
<td>{{dataset.FullName}}</td>
</tr>
<tr>
<td><strong>Units</strong></td>
<td>{{phenotype.Units}}</td>
</tr>
+ {%for key,value in publish_data.items()%}
+ <tr>
+ <td><strong>{{key}}</strong></td>
+ <td>{{value}}</td>
+ </tr>
+ {%else%}
+ <tr>
+ <td colspan="2" class="text-muted">
+ <span class="glyphicon glyphicon-exclamation-sign"></span>
+ No publication data found.
+ </td>
+ </tr>
+ {%endfor%}
</tbody>
</table>
+ </div>
+</div>
- <form action="#edit-delete-phenotype"
- method="POST"
- id="frm-delete-phenotype">
-
- <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
- <input type="hidden" name="population_id" value="{{population.Id}}" />
- <input type="hidden" name="dataset_id" value="{{dataset.Id}}" />
- <input type="hidden" name="phenotype_id" value="{{phenotype.Id}}" />
-
- <div class="btn-group btn-group-justified">
- <div class="btn-group">
- {%if "group:resource:edit-resource" in privileges%}
- <input type="submit"
- title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values."
- class="btn btn-primary not-implemented"
- value="edit" />
- {%endif%}
- </div>
- <div class="btn-group"></div>
- <div class="btn-group">
- {%if "group:resource:delete-resource" in privileges%}
- <input type="submit"
- title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype."
- class="btn btn-danger not-implemented"
- value="delete" />
- {%endif%}
- </div>
- </div>
- </form>
+{%if "group:resource:edit-resource" in privileges
+or "group:resource:delete-resource" in privileges%}
+<div class="row">
+ <div class="btn-group btn-group-justified">
+ <div class="btn-group">
+ {%if "group:resource:edit-resource" in privileges%}
+ <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id,
+ xref_id=xref_id)}}"
+ title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values."
+ class="btn btn-primary">Edit</a>
+ {%endif%}
+ </div>
+ <div class="btn-group"></div>
+ <div class="btn-group">
+ {%if "group:resource:delete-resource" in privileges%}
+ <a href="#"
+ title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype."
+ class="btn btn-danger not-implemented"
+ disabled="disabled">delete</a>
+ {%endif%}
+ </div>
</div>
</div>
+{%endif%}
<div class="row">
<div class="panel panel-default">
@@ -90,9 +97,10 @@
<th>#</th>
<th>Sample</th>
<th>Value</th>
- <th>Symbol</th>
+ {%if has_se%}
<th>SE</th>
<th>N</th>
+ {%endif%}
</tr>
</thead>
@@ -102,9 +110,10 @@
<td>{{loop.index}}</td>
<td>{{item.StrainName}}</td>
<td>{{item.value}}</td>
- <td>{{item.Symbol or "-"}}</td>
+ {%if has_se%}
<td>{{item.error or "-"}}</td>
<td>{{item.count or "-"}}</td>
+ {%endif%}
</tr>
{%endfor%}
</tbody>
diff --git a/uploader/templates/platforms/index.html b/uploader/templates/platforms/index.html
index 35b6464..555b444 100644
--- a/uploader/templates/platforms/index.html
+++ b/uploader/templates/platforms/index.html
@@ -19,3 +19,7 @@
{{select_species_form(url_for("species.platforms.index"), species)}}
</div>
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/platforms/list-platforms.html b/uploader/templates/platforms/list-platforms.html
index 718dd1d..a6bcfdc 100644
--- a/uploader/templates/platforms/list-platforms.html
+++ b/uploader/templates/platforms/list-platforms.html
@@ -58,7 +58,7 @@
<table class="table">
<thead>
<tr>
- <th>#</th>
+ <th></th>
<th>Platform Name</th>
<th><a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}"
title="Gene Expression Omnibus: Platforms section"
diff --git a/uploader/templates/populations/base.html b/uploader/templates/populations/base.html
index d763fc1..9db8083 100644
--- a/uploader/templates/populations/base.html
+++ b/uploader/templates/populations/base.html
@@ -6,7 +6,13 @@
{%else%}
class="breadcrumb-item"
{%endif%}>
+ {%if population is mapping%}
+ <a href="{{url_for('species.populations.view_population',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">{{population.Name}}</a>
+ {%else%}
<a href="{{url_for('species.populations.index')}}">Populations</a>
+ {%endif%}
</li>
{%block lvl3_breadcrumbs%}{%endblock%}
{%endblock%}
diff --git a/uploader/templates/populations/create-population.html b/uploader/templates/populations/create-population.html
index b05ce37..c0c4f45 100644
--- a/uploader/templates/populations/create-population.html
+++ b/uploader/templates/populations/create-population.html
@@ -37,12 +37,15 @@
<div class="row">
<form method="POST"
action="{{url_for('species.populations.create_population',
- species_id=species.SpeciesId)}}">
+ species_id=species.SpeciesId,
+ return_to=return_to)}}">
<legend>Create Population</legend>
{{flash_all_messages()}}
+ <input type="hidden" name="return_to" value="{{return_to}}">
+
<div {%if errors.population_fullname%}
class="form-group has-error"
{%else%}
@@ -107,9 +110,12 @@
value="{{error_values.population_code or ''}}"
class="form-control" />
<small class="form-text text-muted">
- <p class="text-danger">
- <span class="glyphicon glyphicon-exclamation-sign"></span>
- What is this field is for? Confirm with Arthur and the rest.
+ <p class="form-text text-muted">
+ This is a 3-character code for your population, that is prepended to
+ the phenotype identifiers. e.g. For the "BXD Family" population, the
+ code is "BXD" and therefore, the phenotype identifiers for the
+ population look like the following examples: <em>BXD_10148</em>,
+ <em>BXD_10180</em>, <em>BXD_10197</em>, etc.
</p>
</small>
</div>
diff --git a/uploader/templates/populations/index.html b/uploader/templates/populations/index.html
index 4354e02..d2bee77 100644
--- a/uploader/templates/populations/index.html
+++ b/uploader/templates/populations/index.html
@@ -22,3 +22,7 @@
{{select_species_form(url_for("species.populations.index"), species)}}
</div>
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/populations/list-populations.html b/uploader/templates/populations/list-populations.html
index 7c7145f..f780e94 100644
--- a/uploader/templates/populations/list-populations.html
+++ b/uploader/templates/populations/list-populations.html
@@ -51,7 +51,7 @@
<caption>Populations for {{species.FullName}}</caption>
<thead>
<tr>
- <th>#</th>
+ <th></th>
<th>Name</th>
<th>Full Name</th>
<th>Description</th>
diff --git a/uploader/templates/populations/macro-display-population-card.html b/uploader/templates/populations/macro-display-population-card.html
index 79f7925..16b477f 100644
--- a/uploader/templates/populations/macro-display-population-card.html
+++ b/uploader/templates/populations/macro-display-population-card.html
@@ -33,11 +33,6 @@
<td>Family</td>
<td>{{population.Family}}</td>
</tr>
-
- <tr>
- <td>Description</td>
- <td>{{(population.Description or "")[0:500]}}&hellip;</td>
- </tr>
</tbody>
</table>
</div>
diff --git a/uploader/templates/populations/macro-select-population.html b/uploader/templates/populations/macro-select-population.html
index af4fd3a..14b0510 100644
--- a/uploader/templates/populations/macro-select-population.html
+++ b/uploader/templates/populations/macro-select-population.html
@@ -1,30 +1,52 @@
-{%macro select_population_form(form_action, populations)%}
-<form method="GET" action="{{form_action}}">
- <legend>Select Population</legend>
-
- <div class="form-group">
- <label for="select-population" class="form-label">Select Population</label>
- <select id="select-population"
- name="population_id"
- class="form-control"
- required="required">
- <option value="">Select Population</option>
- {%for family in populations%}
- <optgroup {%if family[0][1] is not none%}
- label="{{family[0][1]}}"
- {%else%}
- label="Undefined"
- {%endif%}>
- {%for population in family[1]%}
- <option value="{{population.Id}}">{{population.FullName}}</option>
- {%endfor%}
- </optgroup>
- {%endfor%}
- </select>
+{%from "macro-step-indicator.html" import step_indicator%}
+
+{%macro select_population_form(form_action, species, populations)%}
+<form method="GET" action="{{form_action}}" class="form-horizontal">
+
+ <h2>{{step_indicator("2")}} What population do you want to work with?</h2>
+
+ {%if populations | length != 0%}
+
+ <p class="form-text">Search for, and select the population from the table
+ below and click "Continue"</p>
+
+ <div class="radio">
+ <label class="control-label" for="rdo-cant-find-population">
+ <input type="radio" id="rdo-cant-find-population"
+ name="population_id" value="CREATE-POPULATION" />
+ I cannot find the population I want &mdash; create it!
+ </label>
+ </div>
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit" value="continue" class="btn btn-primary" />
+ </div>
+
+ <div style="margin-top:3em;">
+ <table id="tbl-select-population" class="table compact stripe"
+ data-populations-list='{{populations | tojson}}'>
+ <thead>
+ <tr>
+ <th></th>
+ <th>Population</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
</div>
- <div class="form-group">
- <input type="submit" value="Select" class="btn btn-primary" />
+ {%else%}
+ <p class="form-text">
+ There are no populations currently defined for {{species['FullName']}}
+ ({{species['SpeciesName']}}).<br />
+ Click "Continue" to create the first!</p>
+ <input type="hidden" name="population_id" value="CREATE-POPULATION" />
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit" value="continue" class="btn btn-primary" />
</div>
+ {%endif%}
+
</form>
{%endmacro%}
diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html
index 1e2964e..b23caeb 100644
--- a/uploader/templates/populations/view-population.html
+++ b/uploader/templates/populations/view-population.html
@@ -15,7 +15,7 @@
{%endif%}>
<a href="{{url_for('species.populations.view_population',
species_id=species.SpeciesId,
- population_id=population.InbredSetId)}}">view population</a>
+ population_id=population.InbredSetId)}}">view</a>
</li>
{%endblock%}
@@ -62,29 +62,35 @@
<nav class="nav">
<ul>
<li>
- <a href="{{url_for('species.populations.genotypes.list_genotypes',
+ <a href="{{url_for('species.populations.samples.list_samples',
species_id=species.SpeciesId,
population_id=population.Id)}}"
- title="Upload genotypes for {{species.FullName}}">Upload Genotypes</a>
+ title="Manage samples: Add new or delete existing.">
+ manage samples</a>
</li>
<li>
- <a href="{{url_for('species.populations.samples.list_samples',
+ <a href="{{url_for('species.populations.genotypes.list_genotypes',
species_id=species.SpeciesId,
population_id=population.Id)}}"
- title="Manage samples: Add new or delete existing.">
- manage samples</a>
+ title="Manage genotypes for {{species.FullName}}">Manage Genotypes</a>
</li>
<li>
- <a href="#" title="Upload expression data">upload expression data</a>
+ <a href="{{url_for('species.populations.phenotypes.list_datasets',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ title="Manage phenotype data.">manage phenotype data</a>
</li>
<li>
- <a href="#" title="Upload phenotype data">upload phenotype data</a>
+ <a href="#" title="Manage expression data"
+ class="not-implemented">manage expression data</a>
</li>
<li>
- <a href="#" title="Upload individual data">upload individual data</a>
+ <a href="#" title="Manage individual data"
+ class="not-implemented">manage individual data</a>
</li>
<li>
- <a href="#" title="Upload RNA-Seq data">upload RNA-Seq data</a>
+ <a href="#" title="Manage RNA-Seq data"
+ class="not-implemented">manage RNA-Seq data</a>
</li>
</ul>
</nav>
diff --git a/uploader/templates/publications/base.html b/uploader/templates/publications/base.html
new file mode 100644
index 0000000..db80bfa
--- /dev/null
+++ b/uploader/templates/publications/base.html
@@ -0,0 +1,12 @@
+{%extends "base.html"%}
+
+{%block lvl1_breadcrumbs%}
+<li {%if activelink=="publications"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('publications.index')}}">Publications</a>
+</li>
+{%block lvl2_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html
new file mode 100644
index 0000000..3f828a9
--- /dev/null
+++ b/uploader/templates/publications/create-publication.html
@@ -0,0 +1,191 @@
+{%extends "publications/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}View Publication{%endblock%}
+
+{%block pagetitle%}View Publication{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <form id="frm-create-publication"
+ method="POST"
+ action="{{url_for('publications.create_publication', **request.args)}}"
+ class="form-horizontal">
+
+ <div class="row mb-3">
+ <label for="txt-pubmed-id" class="col-sm-2 col-form-label">
+ PubMed ID</label>
+ <div class="col-sm-10">
+ <div class="input-group">
+ <input type="text"
+ id="txt-pubmed-id"
+ name="pubmed-id"
+ class="form-control"/>
+ <div class="input-group-text">
+ <button class="btn btn-outline-primary"
+ id="btn-search-pubmed-id">search</button>
+ </div>
+ </div>
+ <span id="search-pubmed-id-error"
+ class="form-text text-muted text-danger visually-hidden">
+ </span>
+ <span class="form-text text-muted">This is the publication's ID on
+ <a href="https://pubmed.ncbi.nlm.nih.gov/"
+ title="Link to NCBI's PubMed service">NCBI's Pubmed Service</a>
+ </span>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <label for="txt-publication-title" class="col-sm-2 col-form-label">
+ Title</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-publication-title"
+ name="publication-title"
+ class="form-control" />
+ <span class="form-text text-muted">Provide the publication's title here.</span>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <label for="txt-publication-authors" class="col-sm-2 col-form-label">
+ Authors</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-publication-authors"
+ name="publication-authors"
+ required="required"
+ class="form-control" />
+ <span class="form-text text-muted">
+ A publication <strong>MUST</strong> have an author. You <em>must</em>
+ provide a value for the authors field.
+ </span>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <label for="txt-publication-journal" class="col-sm-2 col-form-label">
+ Journal</label>
+ <div class="col-sm-10">
+ <input type="text"
+ id="txt-publication-journal"
+ name="publication-journal"
+ class="form-control" />
+ <span class="form-text text-muted">Provide the name journal where the
+ publication was done, here.</span>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <label for="select-publication-month"
+ class="col-sm-2 col-form-label">
+ Month</label>
+ <div class="col-sm-4">
+ <select class="form-control"
+ id="select-publication-month"
+ name="publication-month">
+ <option value="">Select a month</option>
+ <option value="january">January</option>
+ <option value="february">February</option>
+ <option value="march">March</option>
+ <option value="april">April</option>
+ <option value="may">May</option>
+ <option value="june">June</option>
+ <option value="july">July</option>
+ <option value="august">August</option>
+ <option value="september">September</option>
+ <option value="october">October</option>
+ <option value="november">November</option>
+ <option value="december">December</option>
+ </select>
+ <span class="form-text text-muted">Month of publication</span>
+ </div>
+
+ <label for="txt-publication-year"
+ class="col-sm-2 col-form-label">
+ Year</label>
+ <div class="col-sm-4">
+ <input type="number"
+ id="txt-publication-year"
+ name="publication-year"
+ class="form-control"
+ min="1960" />
+ <span class="form-text text-muted">Year of publication</span>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <label for="txt-publication-volume"
+ class="col-sm-2 col-form-label">
+ Volume</label>
+ <div class="col-sm-4">
+ <input type="text"
+ id="txt-publication-volume"
+ name="publication-volume"
+ class="form-control">
+ <span class="form-text text-muted">Journal volume</span>
+ </div>
+
+ <label for="txt-publication-pages"
+ class="col-sm-2 col-form-label">
+ Pages</label>
+ <div class="col-sm-4">
+ <input type="text"
+ id="txt-publication-pages"
+ name="publication-pages"
+ class="form-control" />
+ <span class="form-text text-muted">Journal pages for the publication</span>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <label for="txt-abstract" class="col-sm-2 col-form-label">Abstract</label>
+ <div class="col-sm-10">
+ <textarea id="txt-publication-abstract"
+ name="publication-abstract"
+ class="form-control"
+ rows="7"></textarea>
+ </div>
+ </div>
+
+ <div class="row mb-3">
+ <div class="col-sm-2"></div>
+ <div class="col-sm-8">
+ <input type="submit" class="btn btn-primary" value="Add" />
+ <input type="reset" class="btn btn-danger" />
+ </div>
+ </div>
+
+</form>
+</div>
+
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/pubmed.js"></script>
+<script type="text/javascript">
+ $(function() {
+ $("#btn-search-pubmed-id").on("click", (event) => {
+ event.preventDefault();
+ var search_button = event.target;
+ var pubmed_id = $("#txt-pubmed-id").val().trim();
+ remove_class($("#txt-pubmed-id").parent(), "has-error");
+ if(pubmed_id == "") {
+ add_class($("#txt-pubmed-id").parent(), "has-error");
+ return false;
+ }
+
+ search_button.disabled = true;
+ // Fetch publication details
+ fetch_publication_details(pubmed_id,
+ [() => {search_button.disabled = false;}]);
+ return false;
+ });
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/publications/index.html b/uploader/templates/publications/index.html
new file mode 100644
index 0000000..f846d54
--- /dev/null
+++ b/uploader/templates/publications/index.html
@@ -0,0 +1,92 @@
+{%extends "publications/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}Publications{%endblock%}
+
+{%block pagetitle%}Publications{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row" style="padding-bottom: 1em;">
+ <a href="{{url_for('publications.create_publication')}}"
+ class="btn btn-primary">
+ add new publication</a>
+</div>
+
+<div class="row">
+ <table id="tbl-list-publications" class="table compact stripe">
+ <thead>
+ <tr>
+ <th>#</th>
+ <th>PubMed ID</th>
+ <th>Title</th>
+ <th>Authors</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
+</div>
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(function() {
+ var publicationsDataTable = buildDataTable(
+ "#tbl-list-publications",
+ [],
+ [
+ {data: "index"},
+ {
+ data: (pub) => {
+ if(pub.PubMed_ID) {
+ return `<a href="https://pubmed.ncbi.nlm.nih.gov/` +
+ `${pub.PubMed_ID}/" target="_blank" ` +
+ `title="Link to publication on NCBI.">` +
+ `${pub.PubMed_ID}</a>`;
+ }
+ return "";
+ }
+ },
+ {
+ data: (pub) => {
+ var title = "⸻";
+ if(pub.Title) {
+ title = pub.Title
+ }
+ return `<a href="/publications/view/${pub.Id}" ` +
+ `target="_blank" ` +
+ `title="Link to view publication details">` +
+ `${title}</a>`;
+ }
+ },
+ {
+ data: (pub) => {
+ authors = pub.Authors.split(",").map(
+ (item) => {return item.trim();});
+ if(authors.length > 1) {
+ return authors[0] + ", et. al.";
+ }
+ return authors[0];
+ }
+ }
+ ],
+ {
+ ajax: {
+ url: "/publications/list",
+ dataSrc: "publications"
+ },
+ scrollY: 700,
+ paging: false,
+ deferRender: true,
+ layout: {
+ topStart: "info",
+ topEnd: "search"
+ }
+ });
+ });
+</script>
+{%endblock%}
diff --git a/uploader/templates/publications/view-publication.html b/uploader/templates/publications/view-publication.html
new file mode 100644
index 0000000..388547a
--- /dev/null
+++ b/uploader/templates/publications/view-publication.html
@@ -0,0 +1,78 @@
+{%extends "publications/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+
+{%block title%}View Publication{%endblock%}
+
+{%block pagetitle%}View Publication{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <table class="table">
+ <tr>
+ <th>PubMed</th>
+ <td>
+ {%if publication.PubMed_ID%}
+ <a href="https://pubmed.ncbi.nlm.nih.gov/{{publication.PubMed_ID}}/"
+ target="_blank">{{publication.PubMed_ID}}</a>
+ {%else%}
+ —
+ {%endif%}
+ </td>
+ </tr>
+ <tr>
+ <th>Title</th>
+ <td>{{publication.Title or "—"}}</td>
+ </tr>
+ <tr>
+ <th>Authors</th>
+ <td>{{publication.Authors or "—"}}</td>
+ </tr>
+ <tr>
+ <th>Journal</th>
+ <td>{{publication.Journal or "—"}}</td>
+ </tr>
+ <tr>
+ <th>Published</th>
+ <td>{{publication.Month or ""}} {{publication.Year or "—"}}</td>
+ </tr>
+ <tr>
+ <th>Volume</th>
+ <td>{{publication.Volume or "—"}}</td>
+ </tr>
+ <tr>
+ <th>Pages</th>
+ <td>{{publication.Pages or "—"}}</td>
+ </tr>
+ <tr>
+ <th>Abstract</th>
+ <td>
+ {%for line in (publication.Abstract or "—").replace("\r\n", "<br />").replace("\n", "<br />").split("<br />")%}
+ <p>{{line}}</p>
+ {%endfor%}
+ </td>
+ </tr>
+ </table>
+</div>
+
+<div class="row">
+ <form id="frm-edit-delete-publication" method="POST" action="#">
+ <input type="hidden" name="publication_id" value="{{publication.Id}}" />
+ <div class="form-group">
+ <input type="submit" value="edit" class="btn btn-primary not-implemented" />
+ {%if linked_phenotypes | length == 0%}
+ <input type="submit" value="delete" class="btn btn-danger not-implemented" />
+ {%endif%}
+ </div>
+ </form>
+</div>
+{%endblock%}
+
+
+{%block javascript%}
+<script type="text/javascript">
+ $(function() {});
+</script>
+{%endblock%}
diff --git a/uploader/templates/samples/index.html b/uploader/templates/samples/index.html
index ee4a63e..ee98734 100644
--- a/uploader/templates/samples/index.html
+++ b/uploader/templates/samples/index.html
@@ -17,3 +17,7 @@
{{select_species_form(url_for("species.populations.samples.index"), species)}}
</div>
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/species.js"></script>
+{%endblock%}
diff --git a/uploader/templates/samples/list-samples.html b/uploader/templates/samples/list-samples.html
index 13e5cec..185e784 100644
--- a/uploader/templates/samples/list-samples.html
+++ b/uploader/templates/samples/list-samples.html
@@ -73,7 +73,7 @@
<table class="table">
<thead>
<tr>
- <th>#</th>
+ <th></th>
<th>Name</th>
<th>Auxilliary Name</th>
<th>Symbol</th>
diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html
index f437780..1cc7573 100644
--- a/uploader/templates/samples/select-population.html
+++ b/uploader/templates/samples/select-population.html
@@ -12,28 +12,15 @@
{{flash_all_messages()}}
<div class="row">
- <p>You have selected "{{species.FullName}}" as the species that your data relates to.</p>
- <p>Next, we need information regarding the population your data relates to. Do please select the population from the existing ones below</p>
-</div>
-
-<div class="row">
{{select_population_form(
- url_for("species.populations.samples.select_population", species_id=species.SpeciesId),
- populations)}}
-</div>
-
-<div class="row">
- <p>
- If you cannot find the population your data relates to in the drop-down
- above, you might want to
- <a href="{{url_for('species.populations.create_population',
- species_id=species.SpeciesId)}}"
- title="Create a new population for species '{{species.FullName}},">
- add a new population to GeneNetwork</a>
- instead.
+ url_for("species.populations.samples.select_population", species_id=species.SpeciesId), species, populations)}}
</div>
{%endblock%}
{%block sidebarcontents%}
{{display_species_card(species)}}
{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/populations.js"></script>
+{%endblock%}
diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html
index 458ab55..2cf8053 100644
--- a/uploader/templates/samples/upload-failure.html
+++ b/uploader/templates/samples/upload-failure.html
@@ -15,7 +15,7 @@
<h3>Debugging Information</h3>
<ul>
- <li><strong>job id</strong>: {{job.job_id}}</li>
+ <li><strong>job id</strong>: {{job.jobid}}</li>
<li><strong>status</strong>: {{job.status}}</li>
<li><strong>job type</strong>: {{job["job-type"]}}</li>
</ul>
diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html
index 04391db..f64f72b 100644
--- a/uploader/templates/species/base.html
+++ b/uploader/templates/species/base.html
@@ -6,7 +6,12 @@
{%else%}
class="breadcrumb-item"
{%endif%}>
+ {%if species is mapping%}
+ <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}">
+ {{species.Name}}</a>
+ {%else%}
<a href="{{url_for('species.list_species')}}">Species</a>
+ {%endif%}
</li>
{%block lvl2_breadcrumbs%}{%endblock%}
{%endblock%}
diff --git a/uploader/templates/species/create-species.html b/uploader/templates/species/create-species.html
index 0d0bedf..138dbaa 100644
--- a/uploader/templates/species/create-species.html
+++ b/uploader/templates/species/create-species.html
@@ -19,72 +19,88 @@
<div class="row">
<form id="frm-create-species"
method="POST"
- action="{{url_for('species.create_species')}}">
+ action="{{url_for('species.create_species', return_to=return_to)}}"
+ class="form-horizontal">
<legend>Create Species</legend>
{{flash_all_messages()}}
+ <input type="hidden" name="return_to" value="{{return_to}}">
+
<div class="form-group">
- <label for="txt-taxonomy-id" class="form-label">
+ <label for="txt-taxonomy-id" class="control-label col-sm-2">
Taxonomy ID</label>
- <div class="input-group">
- <input id="txt-taxonomy-id"
- name="species_taxonomy_id"
- type="text"
- class="form-control" />
- <span class="input-group-btn">
- <button id="btn-search-taxonid" class="btn btn-info">Search</button>
- </span>
+ <div class="col-sm-10">
+ <div class="input-group">
+ <input id="txt-taxonomy-id"
+ name="species_taxonomy_id"
+ type="text"
+ class="form-control" />
+ <span class="input-group-btn">
+ <button id="btn-search-taxonid" class="btn btn-info">Search</button>
+ </span>
+ </div>
+ <small class="form-text text-small text-muted">
+ Use
+ <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/"
+ title="NCBI's Taxonomy Browser homepage"
+ target="_blank">
+ NCBI's Taxonomy Browser homepage</a> to search for the species you
+ want. If the species exists on NCBI, they will have a Taxonomy ID. Copy
+ that Taxonomy ID to this field, and click "Search" to auto-fill the
+ details.<br />
+ This field is optional.</small>
</div>
- <small class="form-text text-small text-muted">Provide the taxonomy ID for
- your species that can be used to link to external sites like NCBI. Enter
- the taxonomy ID and click "Search" to auto-fill the form with data.
- <br />
- While it is recommended to provide a value for this field, doing so is
- optional.
- </small>
</div>
<div class="form-group">
- <label for="txt-species-name" class="form-label">Common Name</label>
- <input id="txt-species-name"
- name="common_name"
- type="text"
- class="form-control"
- required="required" />
- <small class="form-text text-muted">Provide the common, possibly
- non-scientific name for the species here, e.g. Human, Mouse, etc.</small>
+ <label for="txt-species-name" class="control-label col-sm-2">Common Name</label>
+ <div class="col-sm-10">
+ <input id="txt-species-name"
+ name="common_name"
+ type="text"
+ class="form-control"
+ required="required" />
+ <small class="form-text text-muted">This is the day-to-day term used by
+ laymen, e.g. Mouse (instead of Mus musculus), round worm (instead of
+ Ascaris lumbricoides), etc.<br />
+ For species without this, just enter the scientific name.
+ </small>
+ </div>
</div>
<div class="form-group">
- <label for="txt-species-scientific" class="form-label">
+ <label for="txt-species-scientific" class="control-label col-sm-2">
Scientific Name</label>
- <input id="txt-species-scientific"
- name="scientific_name"
- type="text"
- class="form-control"
- required="required" />
- <small class="form-text text-muted">Provide the scientific name for the
- species you are creating, e.g. Homo sapiens, Mus musculus, etc.</small>
+ <div class="col-sm-10">
+ <input id="txt-species-scientific"
+ name="scientific_name"
+ type="text"
+ class="form-control"
+ required="required" />
+ <small class="form-text text-muted">This is the scientific name for the
+ species e.g. Homo sapiens, Mus musculus, etc.</small>
+ </div>
</div>
<div class="form-group">
- <label for="select-species-family" class="form-label">Family</label>
- <select id="select-species-family"
- name="species_family"
- required="required"
- class="form-control">
- <option value="">Please select a grouping</option>
- {%for family in families%}
- <option value="{{family}}">{{family}}</option>
- {%endfor%}
- </select>
- <small class="form-text text-muted">
- This is a generic grouping for the species that determines under which
- grouping the species appears in the GeneNetwork menus</small>
+ <label for="select-species-family" class="control-label col-sm-2">Family</label>
+ <div class="col-sm-10">
+ <select id="select-species-family"
+ name="species_family"
+ required="required"
+ class="form-control">
+ <option value="ungrouped">I do not know what to pick</option>
+ {%for family in families%}
+ <option value="{{family}}">{{family}}</option>
+ {%endfor%}
+ </select>
+ <small class="form-text text-muted">
+ This is a rough grouping of the species.</small>
+ </div>
</div>
- <div class="form-group">
+ <div class="col-sm-offset-2 col-sm-10">
<input type="submit"
value="create new species"
class="btn btn-primary" />
@@ -113,7 +129,7 @@
}
msg = (
"Request to '${uri}' failed with message '${textStatus}'. "
- + "Please try again later, or fill the details manually.");
+ + "Please try again later, or fill the details manually.");
alert(msg);
console.error(msg, data, textStatus);
return false;
diff --git a/uploader/templates/species/list-species.html b/uploader/templates/species/list-species.html
index 85c9d40..64084b0 100644
--- a/uploader/templates/species/list-species.html
+++ b/uploader/templates/species/list-species.html
@@ -29,7 +29,7 @@
<caption>Available Species</caption>
<thead>
<tr>
- <th>#</td>
+ <th></td>
<th title="A common, layman's name for the species.">Common Name</th>
<th title="The scientific name for the species">Organism Name</th>
<th title="An identifier for the species in the NCBI taxonomy database">
diff --git a/uploader/templates/species/macro-select-species.html b/uploader/templates/species/macro-select-species.html
index dd086c0..3714ae4 100644
--- a/uploader/templates/species/macro-select-species.html
+++ b/uploader/templates/species/macro-select-species.html
@@ -1,36 +1,59 @@
+{%from "macro-step-indicator.html" import step_indicator%}
+
{%macro select_species_form(form_action, species)%}
-{%if species | length > 0%}
-<form method="GET" action="{{form_action}}">
- <div class="form-group">
- <label for="select-species" class="form-label">Species</label>
- <select id="select-species"
- name="species_id"
- class="form-control"
- required="required">
- <option value="">Select Species</option>
- {%for group in species%}
- {{group}}
- <optgroup {%if group[0][1] is not none%}
- label="{{group[0][1].capitalize()}}"
- {%else%}
- label="Undefined"
- {%endif%}>
- {%for aspecies in group[1]%}
- <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option>
- {%endfor%}
- </optgroup>
- {%endfor%}
- </select>
+<form method="GET" action="{{form_action}}" class="form-horizontal">
+
+ <h2>{{step_indicator("1")}} What species do you want to work with?</h2>
+
+ {%if species | length != 0%}
+
+ <p class="form-text">Search for, and select the species from the table below
+ and click "Continue"</p>
+
+ <div class="radio">
+ <label for="rdo-cant-find-species"
+ style="font-weight: 1;">
+ <input id="rdo-cant-find-species" type="radio" name="species_id"
+ value="CREATE-SPECIES" />
+ I could not find the species I want (create it).
+ </label>
</div>
- <div class="form-group">
- <input type="submit" value="Select" class="btn btn-primary" />
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit"
+ class="btn btn-primary"
+ value="continue" />
</div>
+
+ <div style="margin-top:3em;">
+ <table id="tbl-select-species" class="table compact stripe"
+ data-species-list='{{species | tojson}}'>
+ <div class="">
+ <thead>
+ <tr>
+ <th></th>
+ <th>Species Name</th>
+ </tr>
+ </thead>
+
+ <tbody></tbody>
+ </table>
+ </div>
+
+ {%else%}
+
+ <label class="control-label" for="rdo-cant-find-species">
+ <input id="rdo-cant-find-species" type="radio" name="species_id"
+ value="CREATE-SPECIES" />
+ There are no species to select from. Create the first one.</label>
+
+ <div class="col-sm-offset-10 col-sm-2">
+ <input type="submit"
+ class="btn btn-primary col-sm-offset-1"
+ value="continue" />
+ </div>
+
+ {%endif%}
+
</form>
-{%else%}
-<p class="text-danger">
- <span class="glyphicon glyphicon-exclamation-mark"></span>
- We could not find species to select from!
-</p>
-{%endif%}
{%endmacro%}
diff --git a/uploader/templates/species/view-species.html b/uploader/templates/species/view-species.html
index b01864d..2d02f7e 100644
--- a/uploader/templates/species/view-species.html
+++ b/uploader/templates/species/view-species.html
@@ -45,6 +45,12 @@
title="Create/Edit populations for {{species.FullName}}">
Manage populations</a>
</li>
+ <li>
+ <a href="{{url_for('species.platforms.list_platforms',
+ species_id=species.SpeciesId)}}"
+ title="Create/Edit sequencing platforms for {{species.FullName}}">
+ Manage sequencing platforms</a>
+ </li>
</ol>