diff options
Diffstat (limited to 'uploader')
109 files changed, 6228 insertions, 992 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py index 9fdb383..8b49ad5 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -3,18 +3,33 @@ import os import sys import logging from pathlib import Path +from typing import Optional from flask import Flask, request + +from cachelib import FileSystemCache + +from gn_libs import jobs as gnlibs_jobs + from flask_session import Session + from uploader.oauth2.client import user_logged_in, authserver_authorise_uri from . import session from .base_routes import base +from .files.views import files from .species import speciesbp +from .publications import pubbp from .oauth2.views import oauth2 from .expression_data import exprdatabp from .errors import register_error_handlers +from .background_jobs import background_jobs_bp + +logging.basicConfig( + format=("%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s " + "(%(thread)d:%(threadName)s): %(message)s") +) def override_settings_with_envvars( app: Flask, ignore: tuple[str, ...]=tuple()) -> None: @@ -49,10 +64,30 @@ def setup_logging(app: Flask) -> Flask: "SERVER_SOFTWARE", "").split('/') return __log_gunicorn__(app) if bool(software) else __log_dev__(app) +def setup_modules_logging(app_logger): + """Setup module-level loggers to the same log-level as the application.""" + loglevel = logging.getLevelName(app_logger.getEffectiveLevel()) + + def __setup__(logger_name): + _logger = logging.getLogger(logger_name) + _logger.setLevel(loglevel) + + __setup__("uploader.publications.models") + __setup__("uploader.publications.datatables") + + +def create_app(config: Optional[dict] = None): + """The application factory. + + config: dict + Useful to override settings in the settings files and environment + especially in environments such as testing.""" + if config is None: + config = {} -def create_app(): - """The application factory""" app = Flask(__name__) + + ### BEGIN: Application configuration app.config.from_pyfile( Path(__file__).parent.joinpath("default_settings.py")) if "UPLOADER_CONF" in os.environ: @@ -67,8 +102,16 @@ def create_app(): if secretsfile.exists(): # Silently ignore secrets if the file does not exist. app.config.from_pyfile(secretsfile) + app.config.update(config) # Override everything with passed in config + ### END: Application configuration + + app.config["SESSION_CACHELIB"] = FileSystemCache( + cache_dir=Path(app.config["SESSION_FILESYSTEM_CACHE_PATH"]).absolute(), + threshold=int(app.config["SESSION_FILESYSTEM_CACHE_THRESHOLD"]), + default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) + setup_modules_logging(app.logger) # setup jinja2 symbols app.add_template_global(lambda : request.url, name="request_url") @@ -82,8 +125,12 @@ def create_app(): # setup blueprints app.register_blueprint(base, url_prefix="/") + app.register_blueprint(files, url_prefix="/files") app.register_blueprint(oauth2, url_prefix="/oauth2") app.register_blueprint(speciesbp, url_prefix="/species") + app.register_blueprint(pubbp, url_prefix="/publications") + app.register_blueprint(background_jobs_bp, url_prefix="/background-jobs/") register_error_handlers(app) + gnlibs_jobs.init_app(app) return app diff --git a/uploader/authorisation.py b/uploader/authorisation.py index ee8fe97..3cf3585 100644 --- a/uploader/authorisation.py +++ b/uploader/authorisation.py @@ -16,13 +16,12 @@ def require_login(function): @wraps(function) def __is_session_valid__(*args, **kwargs): """Check that the user is logged in and their token is valid.""" - def __clear_session__(_no_token): - session.clear_session_info() - flash("You need to be logged in.", "alert-danger") + def __alert_needs_sign_in__(_no_token): + flash("You need to be signed in.", "alert alert-danger big-alert") return redirect("/") return session.user_token().either( - __clear_session__, + __alert_needs_sign_in__, lambda token: function(*args, **kwargs)) return __is_session_valid__ @@ -49,7 +48,7 @@ def require_token(func: Callable) -> Callable: """ def __invalid_token__(_whatever): logging.debug("==========> Failure log: %s", _whatever) - raise Exception( + raise Exception(# pylint: disable=[broad-exception-raised] "You attempted to access a feature of the system that requires " "authorisation. Unfortunately, we could not verify you have the " "appropriate authorisation to perform the action you requested. " diff --git a/uploader/background_jobs.py b/uploader/background_jobs.py new file mode 100644 index 0000000..dc9f837 --- /dev/null +++ b/uploader/background_jobs.py @@ -0,0 +1,119 @@ +"""Generic views and utilities to handle background jobs.""" +import uuid +import importlib +from typing import Callable +from functools import partial + +from flask import ( + url_for, + redirect, + Response, + Blueprint, + render_template, + current_app as app) + +from gn_libs import jobs +from gn_libs import sqlite3 +from gn_libs.jobs.jobs import JobNotFound + +from uploader.authorisation import require_login + +background_jobs_bp = Blueprint("background-jobs", __name__) +HandlerType = Callable[[dict], Response] + + +def __default_error_handler__(job: dict) -> Response: + return redirect(url_for("background-jobs.job_error", job_id=job["job_id"])) + +def register_handlers( + job_type: str, + success_handler: HandlerType, + # pylint: disable=[redefined-outer-name] + error_handler: HandlerType = __default_error_handler__ + # pylint: disable=[redefined-outer-name] +) -> str: + """Register success and error handlers for each job type.""" + if not bool(app.config.get("background-jobs")): + app.config["background-jobs"] = {} + + if not bool(app.config["background-jobs"].get(job_type)): + app.config["background-jobs"][job_type] = { + "success": success_handler, + "error": error_handler + } + + return job_type + + +def register_job_handlers(job: str): + """Related to register handlers above.""" + def __load_handler__(absolute_function_path): + _parts = absolute_function_path.split(".") + app.logger.debug("THE PARTS ARE: %s", _parts) + assert len(_parts) > 1, f"Invalid path: {absolute_function_path}" + module = importlib.import_module(f".{_parts[-2]}", + package=".".join(_parts[0:-2])) + return getattr(module, _parts[-1]) + + metadata = job["metadata"] + if metadata["success_handler"]: + _success_handler = __load_handler__(metadata["success_handler"]) + try: + _error_handler = __load_handler__(metadata["error_handler"]) + except Exception as _exc:# pylint: disable=[broad-exception-caught] + _error_handler = __default_error_handler__ + register_handlers( + metadata["job-type"], _success_handler, _error_handler) + + +def handler(job: dict, handler_type: str) -> HandlerType: + """Fetch a handler for the job.""" + _job_type = job["metadata"]["job-type"] + _handler = app.config.get( + "background-jobs", {} + ).get( + _job_type, {} + ).get(handler_type) + if bool(_handler): + return _handler(job) + raise Exception(# pylint: disable=[broad-exception-raised] + f"No '{handler_type}' handler registered for job type: {_job_type}") + + +error_handler = partial(handler, handler_type="error") +success_handler = partial(handler, handler_type="success") + + +@background_jobs_bp.route("/status/<uuid:job_id>") +@require_login +def job_status(job_id: uuid.UUID): + """View the job status.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + + register_job_handlers(job) + if status == "error": + return error_handler(job) + + if status == "completed": + return success_handler(job) + + return render_template("jobs/job-status.html", job=job) + except JobNotFound as _jnf: + return render_template( + "jobs/job-not-found.html", + job_id=job_id) + + +@background_jobs_bp.route("/error/<uuid:job_id>") +@require_login +def job_error(job_id: uuid.UUID): + """Handle job errors in a generic manner.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + return render_template("jobs/job-error.html", job=job) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) diff --git a/uploader/base_routes.py b/uploader/base_routes.py index 742a254..74a3b90 100644 --- a/uploader/base_routes.py +++ b/uploader/base_routes.py @@ -35,8 +35,8 @@ def appenv(): @base.route("/bootstrap/<path:filename>") def bootstrap(filename): """Fetch bootstrap files.""" - return send_from_directory( - appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}") + return send_from_directory(appenv(), f"share/web/bootstrap/{filename}") + @base.route("/jquery/<path:filename>") @@ -46,6 +46,19 @@ def jquery(filename): appenv(), f"share/genenetwork2/javascript/jquery/{filename}") +@base.route("/datatables/<path:filename>") +def datatables(filename): + """Fetch DataTables files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/DataTables/{filename}") + +@base.route("/datatables-extensions/<path:filename>") +def datatables_extensions(filename): + """Fetch DataTables files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/DataTablesExtensions/{filename}") + + @base.route("/node-modules/<path:filename>") def node_modules(filename): """Fetch node-js modules.""" diff --git a/uploader/check_connections.py b/uploader/check_connections.py index 2561e55..c9b9aa3 100644 --- a/uploader/check_connections.py +++ b/uploader/check_connections.py @@ -4,8 +4,7 @@ import traceback import redis import MySQLdb - -from uploader.db_utils import database_connection +from gn_libs.mysqldb import database_connection def check_redis(uri: str): "Check the redis connection" diff --git a/uploader/db/datasets.py b/uploader/db/datasets.py index 767ec41..4b263f5 100644 --- a/uploader/db/datasets.py +++ b/uploader/db/datasets.py @@ -53,7 +53,7 @@ def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]: _study = cursor.fetchone() return dict(_study) if bool(_study) else None -def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments] +def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments, too-many-positional-arguments] populationid: int, platformid: int, tissueid: int, @@ -87,7 +87,7 @@ def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-argume (studyid, studyid)) return {**studydata, "studyid": studyid} -def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments] +def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments, too-many-positional-arguments] studyid: int, averageid: int, datasetname: str, diff --git a/uploader/db_utils.py b/uploader/db_utils.py index d31e2c2..d9d521e 100644 --- a/uploader/db_utils.py +++ b/uploader/db_utils.py @@ -1,54 +1,20 @@ """module contains all db related stuff""" -import logging -import traceback -import contextlib -from urllib.parse import urlparse -from typing import Any, Tuple, Iterator, Callable +from typing import Any, Callable import MySQLdb as mdb from redis import Redis -from MySQLdb.cursors import Cursor from flask import current_app as app +from gn_libs.mysqldb import database_connection -def parse_db_url(db_url) -> Tuple: - """ - Parse SQL_URI configuration variable. - """ - parsed_db = urlparse(db_url) - return (parsed_db.hostname, parsed_db.username, - parsed_db.password, parsed_db.path[1:], parsed_db.port) - - -@contextlib.contextmanager -def database_connection(db_url: str) -> Iterator[mdb.Connection]: - """function to create db connector""" - host, user, passwd, db_name, db_port = parse_db_url(db_url) - connection = mdb.connect( - host, user, passwd, db_name, port=(db_port or 3306)) - try: - yield connection - connection.commit() - except mdb.Error as _mdb_err: - logging.error(traceback.format_exc()) - connection.rollback() - finally: - connection.close() def with_db_connection(func: Callable[[mdb.Connection], Any]) -> Any: """Call `func` with a MySQDdb database connection.""" with database_connection(app.config["SQL_URI"]) as conn: return func(conn) + def with_redis_connection(func: Callable[[Redis], Any]) -> Any: """Call `func` with a redis connection.""" redisuri = app.config["REDIS_URL"] with Redis.from_url(redisuri, decode_responses=True) as rconn: return func(rconn) - - -def debug_query(cursor: Cursor): - """Debug the actual query run with MySQLdb""" - for attr in ("_executed", "statement", "_last_executed"): - if hasattr(cursor, attr): - logging.debug("MySQLdb QUERY: %s", getattr(cursor, attr)) - break diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 26fe665..c5986ab 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -3,21 +3,29 @@ The default configuration file. The values here should be overridden in the actual configuration file used for the production and staging systems. """ -import os - -LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") +LOG_LEVEL = "WARNING" SECRET_KEY = b"<Please! Please! Please! Change This!>" UPLOAD_FOLDER = "/tmp/qc_app_files" +TEMPORARY_DIRECTORY = "/tmp/gn-uploader-tmpdir" REDIS_URL = "redis://" JOBS_TTL_SECONDS = 1209600 # 14 days -GNQC_REDIS_PREFIX="GNQC" +GNQC_REDIS_PREFIX="gn-uploader" SQL_URI = "" GN2_SERVER_URL = "https://genenetwork.org/" -SESSION_TYPE = "redis" SESSION_PERMANENT = True SESSION_USE_SIGNER = True +SESSION_TYPE = "cachelib" +## --- Settings for CacheLib session type --- ## +## --- These are on flask-session config variables --- ## +## --- https://cachelib.readthedocs.io/en/stable/file/ --- ## +SESSION_FILESYSTEM_CACHE_PATH = "./flask_session" +SESSION_FILESYSTEM_CACHE_THRESHOLD = 500 +SESSION_FILESYSTEM_CACHE_TIMEOUT = 300 +SESSION_FILESYSTEM_CACHE_MODE = 0o600 +SESSION_FILESYSTEM_CACHE_HASH_METHOD = None # default: hashlib.md5 +## --- END: Settings for CacheLib session type --- ## JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use. JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it. diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py index 32ca359..6d8ce80 100644 --- a/uploader/expression_data/dbinsert.py +++ b/uploader/expression_data/dbinsert.py @@ -7,16 +7,17 @@ from datetime import datetime from redis import Redis from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection from flask import ( flash, request, url_for, Blueprint, redirect, render_template, current_app as app) from uploader import jobs from uploader.authorisation import require_login +from uploader.db_utils import with_db_connection from uploader.population.models import populations_by_species from uploader.species.models import all_species, species_by_id from uploader.platforms.models import platform_by_species_and_id -from uploader.db_utils import with_db_connection, database_connection dbinsertbp = Blueprint("dbinsert", __name__) diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py index bbe6538..7629f3e 100644 --- a/uploader/expression_data/views.py +++ b/uploader/expression_data/views.py @@ -8,6 +8,7 @@ from zipfile import ZipFile, is_zipfile import jsonpickle from redis import Redis from werkzeug.utils import secure_filename +from gn_libs.mysqldb import database_connection from flask import (flash, request, url_for, @@ -21,8 +22,8 @@ from uploader import jobs from uploader.datautils import order_by_family from uploader.ui import make_template_renderer from uploader.authorisation import require_login +from uploader.db_utils import with_db_connection from uploader.species.models import all_species, species_by_id -from uploader.db_utils import with_db_connection, database_connection from uploader.population.models import (populations_by_species, population_by_species_and_id) diff --git a/uploader/files/__init__.py b/uploader/files/__init__.py new file mode 100644 index 0000000..53c3176 --- /dev/null +++ b/uploader/files/__init__.py @@ -0,0 +1,5 @@ +"""General files and chunks utilities.""" +from .chunks import chunked_binary_read +from .functions import (fullpath, + save_file, + sha256_digest_over_file) diff --git a/uploader/files/chunks.py b/uploader/files/chunks.py new file mode 100644 index 0000000..c4360b5 --- /dev/null +++ b/uploader/files/chunks.py @@ -0,0 +1,32 @@ +"""Functions dealing with chunking of files.""" +from pathlib import Path +from typing import Iterator + +from flask import current_app as app +from werkzeug.utils import secure_filename + + +def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator: + """Read a file in binary mode in chunks.""" + with open(filepath, "rb") as inputfile: + while True: + data = inputfile.read(chunksize) + if data != b"": + yield data + continue + break + +def chunk_name(uploadfilename: str, chunkno: int) -> str: + """Generate chunk name from original filename and chunk number""" + if uploadfilename == "": + raise ValueError("Name cannot be empty!") + if chunkno < 1: + raise ValueError("Chunk number must be greater than zero") + return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" + + +def chunks_directory(uniqueidentifier: str) -> Path: + """Compute the directory where chunks are temporarily stored.""" + if uniqueidentifier == "": + raise ValueError("Unique identifier cannot be empty!") + return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") diff --git a/uploader/files.py b/uploader/files/functions.py index b163612..7b9f06b 100644 --- a/uploader/files.py +++ b/uploader/files/functions.py @@ -2,17 +2,23 @@ import hashlib from pathlib import Path from datetime import datetime + from flask import current_app from werkzeug.utils import secure_filename from werkzeug.datastructures import FileStorage -def save_file(fileobj: FileStorage, upload_dir: Path) -> Path: +from .chunks import chunked_binary_read + +def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Path: """Save the uploaded file and return the path.""" assert bool(fileobj), "Invalid file object!" - hashed_name = hashlib.sha512( - f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8") - ).hexdigest() + hashed_name = ( + hashlib.sha512( + f"{fileobj.filename}::{datetime.now().isoformat()}".encode("utf8") + ).hexdigest() + if hashed else + fileobj.filename) filename = Path(secure_filename(hashed_name)) # type: ignore[arg-type] if not upload_dir.exists(): upload_dir.mkdir() @@ -21,6 +27,16 @@ def save_file(fileobj: FileStorage, upload_dir: Path) -> Path: fileobj.save(filepath) return filepath + def fullpath(filename: str): """Get a file's full path. This makes use of `flask.current_app`.""" return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute() + + +def sha256_digest_over_file(filepath: Path) -> str: + """Compute the sha256 digest over a file's contents.""" + filehash = hashlib.sha256() + for chunk in chunked_binary_read(filepath): + filehash.update(chunk) + + return filehash.hexdigest() diff --git a/uploader/files/views.py b/uploader/files/views.py new file mode 100644 index 0000000..29059c7 --- /dev/null +++ b/uploader/files/views.py @@ -0,0 +1,157 @@ +"""Module for generic files endpoints.""" +import time +import random +import traceback +from pathlib import Path + +from flask import request, jsonify, Blueprint, current_app as app + +from .chunks import chunk_name, chunks_directory + +files = Blueprint("files", __name__) + +def target_file(fileid: str) -> Path: + """Compute the full path for the target file.""" + return Path(app.config["UPLOAD_FOLDER"], fileid) + + +@files.route("/upload/resumable", methods=["GET"]) +def resumable_upload_get(): + """Used for checking whether **ALL** chunks have been uploaded.""" + fileid = request.args.get("resumableIdentifier", type=str) or "" + filename = request.args.get("resumableFilename", type=str) or "" + chunk = request.args.get("resumableChunkNumber", type=int) or 0 + if not(fileid or filename or chunk): + return jsonify({ + "message": "At least one required query parameter is missing.", + "error": "BadRequest", + "statuscode": 400 + }), 400 + + # If the complete target file exists, return 200 for all chunks. + _targetfile = target_file(fileid) + if _targetfile.exists(): + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": filename, + "chunk": chunk, + "message": "The complete file already exists.", + "statuscode": 200 + }), 200 + + if Path(chunks_directory(fileid), + chunk_name(filename, chunk)).exists(): + return jsonify({ + "chunk": chunk, + "message": f"Chunk {chunk} exists.", + "statuscode": 200 + }), 200 + + return jsonify({ + "message": f"Chunk {chunk} was not found.", + "error": "NotFound", + "statuscode": 404 + }), 404 + + +def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: + """Merge the chunks into a single file.""" + with open(targetfile, "ab") as _target: + for chunkfile in chunkpaths: + app.logger.error("Merging chunk: %s", chunkfile) + with open(chunkfile, "rb") as _chunkdata: + _target.write(_chunkdata.read()) + + chunkfile.unlink() # Don't use `missing_ok=True` — chunk MUST exist + # If chunk does't exist, it might indicate a race condition. Handle + # that instead. + return targetfile + + +@files.route("/upload/resumable", methods=["POST"]) +def resumable_upload_post(): + """Do the actual chunks upload here.""" + _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 + _chunk = request.form.get("resumableChunkNumber", default=1, type=int) + _uploadfilename = request.form.get( + "resumableFilename", default="", type=str) or "" + _fileid = request.form.get( + "resumableIdentifier", default="", type=str) or "" + _targetfile = target_file(_fileid) + + if _targetfile.exists(): + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": _uploadfilename, + "message": "File was uploaded successfully!", + "statuscode": 200 + }), 200 + + try: + chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) + request.files["file"].save(Path(chunks_directory(_fileid), + chunk_name(_uploadfilename, _chunk))) + + # Check whether upload is complete + chunkpaths = tuple( + Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) + for _achunk in range(1, _totalchunks+1)) + if all(_file.exists() for _file in chunkpaths): + ### HACK: Break possible race condition ### + # Looks like sometimes, there are multiple threads/requests trying + # to merge one file, leading to race conditions and in some rare + # instances, actual data corruption. This hack is meant to break + # that race condition. + _delays = ( + 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, + 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, + 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293) + _lockfile = Path(chunks_directory(_fileid), "merge.lock") + while True: + time.sleep(random.choice(_delays) / 1000) + if (chunks_directory(_fileid).exists() + and not (_lockfile.exists() and _targetfile.exists())): + # merge_files and clean up chunks + _lockfile.touch() + __merge_chunks__(_targetfile, chunkpaths) + _lockfile.unlink() + chunks_directory(_fileid).rmdir() + continue + + if (_targetfile.exists() + and not ( + chunks_directory(_fileid).exists() + and _lockfile.exists())): + # merge complete + break + + # There is still a thread that's merging this file + continue + ### END: HACK: Break possible race condition ### + + if _targetfile.exists(): + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": _uploadfilename, + "message": "File was uploaded successfully!", + "statuscode": 200 + }), 200 + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": _uploadfilename, + "message": "Uploaded file is missing!", + "statuscode": 404 + }), 404 + return jsonify({ + "message": f"Chunk {int(_chunk)} uploaded successfully.", + "statuscode": 201 + }), 201 + except Exception as exc:# pylint: disable=[broad-except] + msg = "Error processing uploaded file chunks." + app.logger.error(msg, exc_info=True, stack_info=True) + return jsonify({ + "message": msg, + "error": type(exc).__name__, + "error-description": " ".join(str(arg) for arg in exc.args), + "error-trace": traceback.format_exception(exc) + }), 500 diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py index 44c98b1..4c3e634 100644 --- a/uploader/genotypes/models.py +++ b/uploader/genotypes/models.py @@ -4,8 +4,9 @@ from datetime import datetime import MySQLdb as mdb from MySQLdb.cursors import Cursor, DictCursor +from flask import current_app as app -from uploader.db_utils import debug_query +from gn_libs.mysqldb import debug_query def genocode_by_population( conn: mdb.Connection, population_id: int) -> tuple[dict, ...]: @@ -38,7 +39,7 @@ def genotype_markers( with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute(_query, (species_id,)) - debug_query(cursor) + debug_query(cursor, app.logger) return tuple(dict(row) for row in cursor.fetchall()) @@ -64,7 +65,7 @@ def genotype_dataset( with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute(_query, _params) - debug_query(cursor) + debug_query(cursor, app.logger) result = cursor.fetchone() if bool(result): return dict(result) diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index 0821eca..54c2444 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -1,5 +1,6 @@ """Views for the genotypes.""" from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection from flask import (flash, request, url_for, @@ -11,13 +12,12 @@ from flask import (flash, from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post from uploader.authorisation import require_login -from uploader.db_utils import database_connection +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler from uploader.request_checks import with_species, with_population -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import (populations_by_species, - population_by_species_and_id) +from uploader.population.models import population_by_species_and_id from .models import (genotype_markers, genotype_dataset, @@ -35,8 +35,15 @@ def index(): with database_connection(app.config["SQL_URI"]) as conn: if not bool(request.args.get("species_id")): return render_template("genotypes/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="genotypes") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.genotypes.select_population")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash(f"Could not find species with ID '{request.args.get('species_id')}'!", @@ -50,28 +57,16 @@ def index(): methods=["GET"]) @require_login @with_species(redirect_uri="species.populations.genotypes.index") -def select_population(species: dict, species_id: int): +def select_population(species: dict, species_id: int):# pylint: disable=[unused-argument] """Select the population under which the genotypes go.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("population_id")): - return render_template("genotypes/select-population.html", - species=species, - populations=order_by_family( - populations_by_species(conn, species_id), - order_key="FamilyOrder"), - activelink="genotypes") - - population = population_by_species_and_id( - conn, species_id, request.args.get("population_id")) - if not bool(population): - flash("Invalid population selected!", "alert-danger") - return redirect(url_for( - "species.populations.genotypes.select_population", - species_id=species_id)) - - return redirect(url_for("species.populations.genotypes.list_genotypes", - species_id=species_id, - population_id=population["Id"])) + return generic_select_population( + species, + "genotypes/select-population.html", + request.args.get("population_id") or "", + "species.populations.genotypes.select_population", + "species.populations.genotypes.list_genotypes", + "genotypes", + "Invalid population selected!") @genotypesbp.route( @@ -96,16 +91,24 @@ def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable= activelink="list-genotypes") -@genotypesbp.route("/<int:species_id>/genotypes/list-markers", methods=["GET"]) +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers", + methods=["GET"]) @require_login -@with_species(redirect_uri="species.populations.genotypes.index") -def list_markers(species: dict, **kwargs):# pylint: disable=[unused-argument] +@with_population(species_redirect_uri="species.populations.genotypes.index", + redirect_uri="species.populations.genotypes.select_population") +def list_markers( + species: dict, + population: dict, + **kwargs +):# pylint: disable=[unused-argument] """List a species' genetic markers.""" with database_connection(app.config["SQL_URI"]) as conn: start_from = max(safe_int(request.args.get("start_from") or 0), 0) count = safe_int(request.args.get("count") or 20) return render_template("genotypes/list-markers.html", species=species, + population=population, total_markers=genotype_markers_count( conn, species["SpeciesId"]), start_from=start_from, diff --git a/uploader/input_validation.py b/uploader/input_validation.py index 9abe742..627c69e 100644 --- a/uploader/input_validation.py +++ b/uploader/input_validation.py @@ -1,14 +1,19 @@ """Input validation utilities""" +import re +import json +import base64 from typing import Any def is_empty_string(value: str) -> bool: """Check whether as string is empty""" return (isinstance(value, str) and value.strip() == "") + def is_empty_input(value: Any) -> bool: """Check whether user provided an empty value.""" return (value is None or is_empty_string(value)) + def is_integer_input(value: Any) -> bool: """ Check whether user provided a value that can be parsed into an integer. @@ -25,3 +30,42 @@ def is_integer_input(value: Any) -> bool: __is_int__(value, 10) or __is_int__(value, 8) or __is_int__(value, 16)))) + + +def is_valid_representative_name(repr_name: str) -> bool: + """ + Check whether the given representative name is a valid according to our rules. + + Parameters + ---------- + repr_name: a string of characters. + + Checks For + ---------- + * The name MUST start with an alphabet [a-zA-Z] + * The name MUST end with an alphabet [a-zA-Z] or number [0-9] + * The name MUST be composed of alphabets [a-zA-Z], numbers [0-9], + underscores (_) and/or hyphens (-). + + Returns + ------- + Boolean indicating whether or not the name is valid. + """ + pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]$") + return bool(pattern.match(repr_name)) + + +def encode_errors(errors: tuple[tuple[str, str], ...], form) -> bytes: + """Encode form errors into base64 string.""" + return base64.b64encode( + json.dumps({ + "errors": dict(errors), + "original_formdata": dict(form) + }).encode("utf8")) + + +def decode_errors(errorstr) -> dict[str, dict]: + """Decode errors from base64 string""" + if not bool(errorstr): + return {"errors": {}, "original_formdata": {}} + return json.loads(base64.b64decode(errorstr.encode("utf8")).decode("utf8")) diff --git a/uploader/jobs.py b/uploader/jobs.py index 21889da..5968c03 100644 --- a/uploader/jobs.py +++ b/uploader/jobs.py @@ -1,6 +1,8 @@ """Handle jobs""" import os import sys +import uuid +import json import shlex import subprocess from uuid import UUID, uuid4 @@ -10,7 +12,9 @@ from typing import Union, Optional from redis import Redis from flask import current_app as app -JOBS_PREFIX = "JOBS" +from functional_tools import take + +JOBS_PREFIX = "jobs" class JobNotFound(Exception): """Raised if we try to retrieve a non-existent job.""" @@ -37,7 +41,8 @@ def error_filename(jobid, error_dir): "Compute the path of the file where errors will be dumped." return f"{error_dir}/job_{jobid}.error" -def initialise_job(# pylint: disable=[too-many-arguments] +def initialise_job( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str, ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict: "Initialise a job 'object' and put in on redis" @@ -50,7 +55,8 @@ def initialise_job(# pylint: disable=[too-many-arguments] name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds)) return the_job -def build_file_verification_job(#pylint: disable=[too-many-arguments] +def build_file_verification_job( + #pylint: disable=[too-many-arguments, too-many-positional-arguments] redis_conn: Redis, dburi: str, redisuri: str, @@ -73,7 +79,8 @@ def build_file_verification_job(#pylint: disable=[too-many-arguments] "filename": os.path.basename(filepath), "percent": 0 }) -def data_insertion_job(# pylint: disable=[too-many-arguments] +def data_insertion_job( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] redis_conn: Redis, filepath: str, filetype: str, totallines: int, speciesid: int, platformid: int, datasetid: int, databaseuri: str, redisuri: str, ttl_seconds: int) -> dict: @@ -128,3 +135,33 @@ def update_stdout_stderr(rconn: Redis, contents = thejob.get(stream, '') new_contents = contents + bytes_read.decode("utf-8") rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents) + + +def job_errors( + rconn: Redis, + prefix: str, + job_id: Union[str, uuid.UUID], + count: int = 100 +) -> list: + """Fetch job errors""" + return take( + ( + json.loads(error) + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*") + for error in rconn.lrange(key, 0, -1)), + count) + + +def job_files_metadata( + rconn: Redis, + prefix: str, + job_id: Union[str, uuid.UUID] +) -> dict: + """Get the metadata for specific job file.""" + return { + key.split(":")[-1]: { + **rconn.hgetall(key), + "filetype": key.split(":")[-3] + } + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*") + } diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py index c492df5..eda42d0 100644 --- a/uploader/monadic_requests.py +++ b/uploader/monadic_requests.py @@ -5,12 +5,12 @@ from typing import Union, Optional, Callable import requests from requests.models import Response from pymonad.either import Left, Right, Either +from markupsafe import escape as markupsafe_escape from flask import (flash, request, redirect, render_template, - current_app as app, - escape as flask_escape) + current_app as app) # HTML Status codes indicating a successful request. SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226) @@ -39,9 +39,9 @@ def make_error_handler( trace=traceback.format_exception(resp_or_exc)) if isinstance(resp_or_exc, Response): flash("The authorisation server responded with " - f"({flask_escape(resp_or_exc.status_code)}, " - f"{flask_escape(resp_or_exc.reason)}) for the request to " - f"'{flask_escape(resp_or_exc.request.url)}'", + f"({markupsafe_escape(resp_or_exc.status_code)}, " + f"{markupsafe_escape(resp_or_exc.reason)}) for the request to " + f"'{markupsafe_escape(resp_or_exc.request.url)}'", "alert-danger") return redirect_to @@ -59,6 +59,11 @@ def get(url, params=None, **kwargs) -> Either: :rtype: pymonad.either.Either """ + timeout = kwargs.get("timeout") + kwargs = {key: val for key,val in kwargs.items() if key != "timeout"} + if timeout is None: + timeout = (9.13, 20) + try: resp = requests.get(url, params=params, **kwargs) if resp.status_code in SUCCESS_CODES: @@ -76,6 +81,11 @@ def post(url, data=None, json=None, **kwargs) -> Either: :rtype: pymonad.either.Either """ + timeout = kwargs.get("timeout") + kwargs = {key: val for key,val in kwargs.items() if key != "timeout"} + if timeout is None: + timeout = (9.13, 20) + try: resp = requests.post(url, data=data, json=json, **kwargs) if resp.status_code in SUCCESS_CODES: @@ -95,10 +105,10 @@ def make_either_error_handler(msg): try: _data = error.json() except Exception as _exc: - raise Exception(error.content) from _exc - raise Exception(_data) + raise Exception(error.content) from _exc# pylint: disable=[broad-exception-raised] + raise Exception(_data)# pylint: disable=[broad-exception-raised] app.logger.debug("\n\n%s\n\n", msg) - raise Exception(error) + raise Exception(error)# pylint: disable=[broad-exception-raised] return __fail__ diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py index e7128de..b94a044 100644 --- a/uploader/oauth2/client.py +++ b/uploader/oauth2/client.py @@ -1,6 +1,7 @@ """OAuth2 client utilities.""" import json import time +import uuid import random from datetime import datetime, timedelta from urllib.parse import urljoin, urlparse @@ -42,7 +43,8 @@ def __fetch_auth_server_jwks__() -> KeySet: return KeySet([ JsonWebKey.import_key(key) for key in requests.get( - urljoin(authserver_uri(), "auth/public-jwks") + urljoin(authserver_uri(), "auth/public-jwks"), + timeout=(9.13, 20) ).json()["jwks"]]) @@ -112,7 +114,8 @@ def oauth2_client(): try: jwt = JsonWebToken(["RS256"]).decode( token["access_token"], key=jwk) - return datetime.now().timestamp() > jwt["exp"] + if bool(jwt.get("exp")): + return datetime.now().timestamp() > jwt["exp"] except BadSignatureError as _bse: pass @@ -145,9 +148,24 @@ def oauth2_client(): __client__) +def fetch_user_details() -> Either: + """Retrieve user details from the auth server""" + suser = session.session_info()["user"] + if suser["email"] == "anon@ymous.user": + udets = oauth2_get("auth/user/").then( + lambda usrdets: session.set_user_details({ + "user_id": uuid.UUID(usrdets["user_id"]), + "name": usrdets["name"], + "email": usrdets["email"], + "token": session.user_token()})) + return udets + return Right(suser) + + def user_logged_in(): """Check whether the user has logged in.""" suser = session.session_info()["user"] + fetch_user_details() return suser["logged_in"] and suser["token"].is_right() diff --git a/uploader/oauth2/tokens.py b/uploader/oauth2/tokens.py new file mode 100644 index 0000000..eb650f6 --- /dev/null +++ b/uploader/oauth2/tokens.py @@ -0,0 +1,47 @@ +"""Utilities for dealing with tokens.""" +import uuid +from typing import Union +from urllib.parse import urljoin +from datetime import datetime, timedelta + +from authlib.jose import jwt +from flask import current_app as app + +from uploader import monadic_requests as mrequests + +from . import jwks +from .client import (SCOPE, authserver_uri, oauth2_clientid) + + +def request_token(token_uri: str, user_id: Union[uuid.UUID, str], **kwargs): + """Request token from the auth server.""" + issued = datetime.now() + jwtkey = jwks.newest_jwk_with_rotation( + jwks.jwks_directory(app, "UPLOADER_SECRETS"), + int(app.config["JWKS_ROTATION_AGE_DAYS"])) + _mins2expiry = kwargs.get("minutes_to_expiry", 5) + return mrequests.post( + token_uri, + json={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "scope": kwargs.get("scope", SCOPE), + "assertion": jwt.encode( + header={ + "alg": "RS256", + "typ": "JWT", + "kid": jwtkey.as_dict()["kid"] + }, + payload={ + "iss": str(oauth2_clientid()), + "sub": str(user_id), + "aud": urljoin(authserver_uri(), "auth/token"), + "exp": (issued + timedelta(minutes=_mins2expiry)).timestamp(), + "nbf": int(issued.timestamp()), + "iat": int(issued.timestamp()), + "jti": str(uuid.uuid4()) + }, + key=jwtkey).decode("utf8"), + "client_id": oauth2_clientid(), + **kwargs.get("extra_params", {}) + } + ) diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py index 61037f3..1ee4257 100644 --- a/uploader/oauth2/views.py +++ b/uploader/oauth2/views.py @@ -1,9 +1,6 @@ """Views for OAuth2 related functionality.""" -import uuid -from datetime import datetime, timedelta from urllib.parse import urljoin, urlparse, urlunparse -from authlib.jose import jwt from flask import ( flash, jsonify, @@ -18,28 +15,29 @@ from uploader import monadic_requests as mrequests from uploader.monadic_requests import make_error_handler from . import jwks +from .tokens import request_token from .client import ( - SCOPE, - oauth2_get, user_logged_in, authserver_uri, oauth2_clientid, + fetch_user_details, oauth2_clientsecret) oauth2 = Blueprint("oauth2", __name__) + @oauth2.route("/code") def authorisation_code(): """Receive authorisation code from auth server and use it to get token.""" - def __process_error__(resp_or_exception): - app.logger.debug("ERROR: (%s)", resp_or_exception) + def __process_error__(error_response): + app.logger.debug("ERROR: (%s)", error_response.content) flash("There was an error retrieving the authorisation token.", - "alert-danger") + "alert alert-danger") return redirect("/") def __fail_set_user_details__(_failure): app.logger.debug("Fetching user details fails: %s", _failure) - flash("Could not retrieve the user details", "alert-danger") + flash("Could not retrieve the user details", "alert alert-danger") return redirect("/") def __success_set_user_details__(_success): @@ -48,52 +46,24 @@ def authorisation_code(): def __success__(token): session.set_user_token(token) - return oauth2_get("auth/user/").then( - lambda usrdets: session.set_user_details({ - "user_id": uuid.UUID(usrdets["user_id"]), - "name": usrdets["name"], - "email": usrdets["email"], - "token": session.user_token(), - "logged_in": True})).either( + return fetch_user_details().either( __fail_set_user_details__, __success_set_user_details__) code = request.args.get("code", "").strip() if not bool(code): - flash("AuthorisationError: No code was provided.", "alert-danger") + flash("AuthorisationError: No code was provided.", "alert alert-danger") return redirect("/") baseurl = urlparse(request.base_url, scheme=request.scheme) - issued = datetime.now() - jwtkey = jwks.newest_jwk_with_rotation( - jwks.jwks_directory(app, "UPLOADER_SECRETS"), - int(app.config["JWKS_ROTATION_AGE_DAYS"])) - return mrequests.post( - urljoin(authserver_uri(), "auth/token"), - json={ - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + return request_token( + token_uri=urljoin(authserver_uri(), "auth/token"), + user_id=request.args["user_id"], + extra_params={ "code": code, - "scope": SCOPE, "redirect_uri": urljoin( urlunparse(baseurl), url_for("oauth2.authorisation_code")), - "assertion": jwt.encode( - header={ - "alg": "RS256", - "typ": "JWT", - "kid": jwtkey.as_dict()["kid"] - }, - payload={ - "iss": str(oauth2_clientid()), - "sub": request.args["user_id"], - "aud": urljoin(authserver_uri(),"auth/token"), - "exp": (issued + timedelta(minutes=5)).timestamp(), - "nbf": int(issued.timestamp()), - "iat": int(issued.timestamp()), - "jti": str(uuid.uuid4()) - }, - key=jwtkey).decode("utf8"), - "client_id": oauth2_clientid() }).either(__process_error__, __success__) @oauth2.route("/public-jwks") @@ -116,7 +86,7 @@ def logout(): _user = session_info["user"] _user_str = f"{_user['name']} ({_user['email']})" session.clear_session_info() - flash("Successfully logged out.", "alert-success") + flash("Successfully signed out.", "alert alert-success") return redirect("/") if user_logged_in(): @@ -134,5 +104,5 @@ def logout(): cleanup_thunk=lambda: __unset_session__( session.session_info())), lambda res: __unset_session__(session.session_info())) - flash("There is no user that is currently logged in.", "alert-info") + flash("There is no user that is currently logged in.", "alert alert-info") return redirect("/") diff --git a/uploader/phenotypes/misc.py b/uploader/phenotypes/misc.py new file mode 100644 index 0000000..cbe3b7f --- /dev/null +++ b/uploader/phenotypes/misc.py @@ -0,0 +1,26 @@ +"""Miscellaneous functions handling phenotypes and phenotypes data.""" +import logging + +logger = logging.getLogger(__name__) + + +def phenotypes_data_differences( + filedata: tuple[dict, ...], dbdata: tuple[dict, ...] +) -> tuple[dict, ...]: + """Compute differences between file data and db data""" + diff = tuple() + for filerow, dbrow in zip( + sorted(filedata, key=lambda item: (item["phenotype_id"], item["xref_id"])), + sorted(dbdata, key=lambda item: (item["PhenotypeId"], item["xref_id"]))): + for samplename, value in filerow["data"].items(): + if value != dbrow["data"].get(samplename, {}).get("value"): + diff = diff + ({ + "PhenotypeId": filerow["phenotype_id"], + "xref_id": filerow["xref_id"], + "DataId": dbrow["DataId"], + "StrainId": dbrow["data"].get(samplename, {}).get("StrainId"), + "StrainName": samplename, + "value": value + },) + + return diff diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index eb5a189..b30f624 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -1,11 +1,30 @@ """Database and utility functions for phenotypes.""" -from typing import Optional +import logging +import tempfile +from pathlib import Path from functools import reduce +from datetime import datetime +from typing import Optional, Iterable import MySQLdb as mdb from MySQLdb.cursors import Cursor, DictCursor -from uploader.db_utils import debug_query +from gn_libs.mysqldb import debug_query + +from functional_tools import take + +logger = logging.getLogger(__name__) + + +__PHENO_DATA_TABLES__ = { + "PublishData": { + "table": "PublishData", "valueCol": "value", "DataIdCol": "Id"}, + "PublishSE": { + "table": "PublishSE", "valueCol": "error", "DataIdCol": "DataId"}, + "NStrain": { + "table": "NStrain", "valueCol": "count", "DataIdCol": "DataId"} +} + def datasets_by_population( conn: mdb.Connection, @@ -30,10 +49,10 @@ def dataset_by_id(conn: mdb.Connection, """Fetch dataset details by identifier""" with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( - "SELECT s.SpeciesId, pf.* FROM Species AS s " - "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId " - "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId " - "WHERE s.Id=%s AND iset.Id=%s AND pf.Id=%s", + "SELECT Species.SpeciesId, PublishFreeze.* FROM Species " + "INNER JOIN InbredSet ON Species.Id=InbredSet.SpeciesId " + "INNER JOIN PublishFreeze ON InbredSet.Id=PublishFreeze.InbredSetId " + "WHERE Species.Id=%s AND InbredSet.Id=%s AND PublishFreeze.Id=%s", (species_id, population_id, dataset_id)) return dict(cursor.fetchone()) @@ -52,6 +71,20 @@ def phenotypes_count(conn: mdb.Connection, return int(cursor.fetchone()["total_phenos"]) +def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]: + """Retrieve the publication data for a phenotype if it exists.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT DISTINCT pxr.PhenotypeId, pub.* FROM PublishXRef AS pxr " + "INNER JOIN Publication as pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.PhenotypeId=%s", + (phenotype_id,)) + res = cursor.fetchone() + if res is None: + return res + return dict(res) + + def dataset_phenotypes(conn: mdb.Connection, population_id: int, dataset_id: int, @@ -59,7 +92,8 @@ def dataset_phenotypes(conn: mdb.Connection, limit: Optional[int] = None) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" _query = ( - "SELECT pheno.*, pxr.Id, ist.InbredSetCode FROM Phenotype AS pheno " + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode " + "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " @@ -67,30 +101,45 @@ def dataset_phenotypes(conn: mdb.Connection, f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute(_query, (population_id, dataset_id)) - debug_query(cursor) + debug_query(cursor, logger) return tuple(dict(row) for row in cursor.fetchall()) -def __phenotype_se__(cursor: Cursor, - species_id: int, - population_id: int, - dataset_id: int, - xref_id: str) -> dict: +def __phenotype_se__(cursor: Cursor, xref_id, dataids_and_strainids): """Fetch standard-error values (if they exist) for a phenotype.""" - _sequery = ( - "SELECT pxr.Id AS xref_id, pxr.DataId, pse.error, nst.count " - "FROM Phenotype AS pheno " - "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " - "INNER JOIN PublishSE AS pse ON pxr.DataId=pse.DataId " - "INNER JOIN NStrain AS nst ON pse.DataId=nst.DataId " - "INNER JOIN Strain AS str ON nst.StrainId=str.Id " - "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId " - "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId " - "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId " - "WHERE (str.SpeciesId, pxr.InbredSetId, pf.Id, pxr.Id)=(%s, %s, %s, %s)") - cursor.execute(_sequery, - (species_id, population_id, dataset_id, xref_id)) - return {row["xref_id"]: dict(row) for row in cursor.fetchall()} + paramstr = ", ".join(["(%s, %s)"] * len(dataids_and_strainids)) + flat = tuple(item for sublist in dataids_and_strainids for item in sublist) + cursor.execute("SELECT * FROM PublishSE WHERE (DataId, StrainId) IN " + f"({paramstr})", + flat) + debug_query(cursor, logger) + _se = { + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "error": row["error"] + } + for row in cursor.fetchall() + } + + cursor.execute("SELECT * FROM NStrain WHERE (DataId, StrainId) IN " + f"({paramstr})", + flat) + debug_query(cursor, logger) + _n = { + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "count": row["count"] + } + for row in cursor.fetchall() + } + + keys = set(tuple(_se.keys()) + tuple(_n.keys())) + return { + key: {"xref_id": xref_id, **_se.get(key,{}), **_n.get(key,{})} + for key in keys + } def __organise_by_phenotype__(pheno, row): """Organise disparate data rows into phenotype 'objects'.""" @@ -105,12 +154,13 @@ def __organise_by_phenotype__(pheno, row): "Units": row["Units"], "Pre_publication_abbreviation": row["Pre_publication_abbreviation"], "Post_publication_abbreviation": row["Post_publication_abbreviation"], + "xref_id": row["pxr.Id"], + "DataId": row["DataId"], "data": { - #TOD0: organise these by DataId and StrainId **(_pheno["data"] if bool(_pheno) else {}), - row["pxr.Id"]: { - "xref_id": row["pxr.Id"], + (row["DataId"], row["StrainId"]): { "DataId": row["DataId"], + "StrainId": row["StrainId"], "mean": row["mean"], "Locus": row["Locus"], "LRS": row["LRS"], @@ -164,14 +214,12 @@ def phenotype_by_id( **_pheno, "data": tuple(__merge_pheno_data_and_se__( _pheno["data"], - __phenotype_se__(cursor, - species_id, - population_id, - dataset_id, - xref_id)).items()) + __phenotype_se__( + cursor, xref_id, tuple(_pheno["data"].keys())) + ).values()) } if bool(_pheno) and len(_pheno.keys()) > 1: - raise Exception( + raise Exception(# pylint: disable=[broad-exception-raised] "We found more than one phenotype with the same identifier!") return None @@ -196,5 +244,266 @@ def phenotypes_data(conn: mdb.Connection, f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute(_query, (population_id, dataset_id)) - debug_query(cursor) + debug_query(cursor, logger) return tuple(dict(row) for row in cursor.fetchall()) + + +def save_new_dataset(cursor: Cursor, + population_id: int, + dataset_name: str, + dataset_fullname: str, + dataset_shortname: str) -> dict: + """Create a new phenotype dataset.""" + params = { + "population_id": population_id, + "dataset_name": dataset_name, + "dataset_fullname": dataset_fullname, + "dataset_shortname": dataset_shortname, + "created": datetime.now().date().isoformat(), + "public": 2, + "confidentiality": 0, + "users": None + } + cursor.execute( + "INSERT INTO PublishFreeze(Name, FullName, ShortName, CreateTime, " + "public, InbredSetId, confidentiality, AuthorisedUsers) " + "VALUES(%(dataset_name)s, %(dataset_fullname)s, %(dataset_shortname)s, " + "%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, " + "%(users)s)", + params) + debug_query(cursor, logger) + return {**params, "Id": cursor.lastrowid} + + +def phenotypes_data_by_ids( + conn: mdb.Connection, + inbred_pheno_xref: dict[str, int] +) -> tuple[dict, ...]: + """Fetch all phenotype data, filtered by the `inbred_pheno_xref` mapping.""" + _paramstr = ",".join(["(%s, %s, %s)"] * len(inbred_pheno_xref)) + _query = ("SELECT " + "pub.PubMed_ID, pheno.*, pxr.*, pd.*, str.*, iset.InbredSetCode " + "FROM Publication AS pub " + "RIGHT JOIN PublishXRef AS pxr0 ON pub.Id=pxr0.PublicationId " + "INNER JOIN Phenotype AS pheno ON pxr0.PhenotypeId=pheno.id " + "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " + "INNER JOIN PublishData AS pd ON pxr.DataId=pd.Id " + "INNER JOIN Strain AS str ON pd.StrainId=str.Id " + "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId " + "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId " + "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId " + f"WHERE (pxr.InbredSetId, pheno.Id, pxr.Id) IN ({_paramstr}) " + "ORDER BY pheno.Id") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_query, tuple(item for row in inbred_pheno_xref + for item in (row["population_id"], + row["phenoid"], + row["xref_id"]))) + debug_query(cursor, logger) + return tuple( + reduce(__organise_by_phenotype__, cursor.fetchall(), {}).values()) + + +def __pre_process_phenotype_data__(row): + _desc = row.get("description", "") + _pre_pub_desc = row.get("pre_publication_description", _desc) + _orig_desc = row.get("original_description", _desc) + _post_pub_desc = row.get("post_publication_description", _orig_desc) + _pre_pub_abbr = row.get("pre_publication_abbreviation", row["id"]) + _post_pub_abbr = row.get("post_publication_abbreviation", _pre_pub_abbr) + return { + "pre_publication_description": _pre_pub_desc, + "post_publication_description": _post_pub_desc, + "original_description": _orig_desc, + "units": row["units"], + "pre_publication_abbreviation": _pre_pub_abbr, + "post_publication_abbreviation": _post_pub_abbr + } + + +def create_new_phenotypes(conn: mdb.Connection, + population_id: int, + publication_id: int, + phenotypes: Iterable[dict]) -> tuple[dict, ...]: + """Add entirely new phenotypes to the database. WARNING: Not thread-safe.""" + _phenos = tuple() + with conn.cursor(cursorclass=DictCursor) as cursor: + def make_next_id(idcol, table): + cursor.execute(f"SELECT MAX({idcol}) AS last_id FROM {table}") + _last_id = int(cursor.fetchone()["last_id"]) + def __next_id__(): + _next_id = _last_id + 1 + while True: + yield _next_id + _next_id = _next_id + 1 + + return __next_id__ + + ### Bottleneck: Everything below makes this function not ### + ### thread-safe because we have to retrieve the last IDs from ### + ### the database and increment those to compute the next IDs. ### + ### This is an unfortunate result from the current schema that ### + ### has a cross-reference table that requires that a phenotype ### + ### be linked to an existing publication, and have data IDs to ### + ### link to that phenotype's data. ### + ### The fact that the IDs are sequential also compounds the ### + ### bottleneck. ### + ### + ### For extra safety, ensure the following tables are locked ### + ### for `WRITE`: ### + ### - PublishXRef ### + ### - Phenotype ### + ### - PublishXRef ### + __next_xref_id = make_next_id("Id", "PublishXRef")() + __next_pheno_id__ = make_next_id("Id", "Phenotype")() + __next_data_id__ = make_next_id("DataId", "PublishXRef")() + + def __build_params_and_prepubabbrevs__(acc, row): + processed = __pre_process_phenotype_data__(row) + return ( + acc[0] + ({ + **processed, + "population_id": population_id, + "publication_id": publication_id, + "phenotype_id": next(__next_pheno_id__), + "xref_id": next(__next_xref_id), + "data_id": next(__next_data_id__) + },), + acc[1] + (processed["pre_publication_abbreviation"],)) + while True: + batch = take(phenotypes, 1000) + if len(batch) == 0: + break + + params, abbrevs = reduce(__build_params_and_prepubabbrevs__, + batch, + (tuple(), tuple())) + # Check for uniqueness for all "Pre_publication_description" values + abbrevs_paramsstr = ", ".join(["%s"] * len(abbrevs)) + _query = ("SELECT PublishXRef.PhenotypeId, Phenotype.* " + "FROM PublishXRef " + "INNER JOIN Phenotype " + "ON PublishXRef.PhenotypeId=Phenotype.Id " + "WHERE PublishXRef.InbredSetId=%s " + "AND Phenotype.Pre_publication_abbreviation IN " + f"({abbrevs_paramsstr})") + cursor.execute(_query, + ((population_id,) + abbrevs)) + existing = tuple(row["Pre_publication_abbreviation"] + for row in cursor.fetchall()) + if len(existing) > 0: + # Narrow this exception, perhaps? + raise Exception( + "Found already existing phenotypes with the following " + "'Pre-publication abbreviations':\n\t" + "\n\t".join(f"* {item}" for item in existing)) + + cursor.executemany( + ( + "INSERT INTO " + "Phenotype(" + "Id, " + "Pre_publication_description, " + "Post_publication_description, " + "Original_description, " + "Units, " + "Pre_publication_abbreviation, " + "Post_publication_abbreviation, " + "Authorized_Users" + ")" + "VALUES (" + "%(phenotype_id)s, " + "%(pre_publication_description)s, " + "%(post_publication_description)s, " + "%(original_description)s, " + "%(units)s, " + "%(pre_publication_abbreviation)s, " + "%(post_publication_abbreviation)s, " + "'robwilliams'" + ")"), + params) + _comments = f"Created at {datetime.now().isoformat()}" + cursor.executemany( + ("INSERT INTO PublishXRef(" + "Id, " + "InbredSetId, " + "PhenotypeId, " + "PublicationId, " + "DataId, " + "comments" + ")" + "VALUES(" + "%(xref_id)s, " + "%(population_id)s, " + "%(phenotype_id)s, " + "%(publication_id)s, " + "%(data_id)s, " + f"'{_comments}'" + ")"), + params) + _phenos = _phenos + params + + return _phenos + + +def save_phenotypes_data( + conn: mdb.Connection, + table: str, + data: Iterable[dict] +) -> int: + """Save new phenotypes data into the database.""" + _table_details = __PHENO_DATA_TABLES__[table] + with conn.cursor(cursorclass=DictCursor) as cursor: + _count = 0 + while True: + batch = take(data, 100000) + if len(batch) == 0: + logger.warning("Got an empty batch. This needs investigation.") + break + + logger.debug("Saving batch of %s items.", len(batch)) + cursor.executemany( + (f"INSERT INTO {_table_details['table']}" + f"({_table_details['DataIdCol']}, StrainId, {_table_details['valueCol']}) " + "VALUES " + f"(%(data_id)s, %(sample_id)s, %(value)s) "), + tuple(batch)) + debug_query(cursor, logger) + _count = _count + len(batch) + + + logger.debug("Saved a total of %s data rows", _count) + return _count + + +def quick_save_phenotypes_data( + conn: mdb.Connection, + table: str, + dataitems: Iterable[dict], + tmpdir: Path +) -> int: + """Save data items to the database, but using """ + _table_details = __PHENO_DATA_TABLES__[table] + with (tempfile.NamedTemporaryFile( + prefix=f"{table}_data", mode="wt", dir=tmpdir) as tmpfile, + conn.cursor(cursorclass=DictCursor) as cursor): + _count = 0 + logger.debug("Write data rows to text file.") + for row in dataitems: + tmpfile.write( + f'{row["data_id"]}\t{row["sample_id"]}\t{row["value"]}\n') + _count = _count + 1 + tmpfile.flush() + + logger.debug("Load text file into database (table: %s)", + _table_details["table"]) + cursor.execute( + f"LOAD DATA LOCAL INFILE '{tmpfile.name}' " + f"INTO TABLE {_table_details['table']} " + "(" + f"{_table_details['DataIdCol']}, " + "StrainId, " + f"{_table_details['valueCol']}" + ")") + debug_query(cursor, logger) + return _count diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index 7fc3b08..556b5ff 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -1,31 +1,69 @@ -"""Views handling ('classical') phenotypes.""" -from functools import wraps +"""Views handling ('classical') phenotypes."""# pylint: disable=[too-many-lines] +import sys +import uuid +import json +import logging +from typing import Any +from pathlib import Path +from zipfile import ZipFile +from functools import wraps, reduce +from urllib.parse import urljoin, urlparse, ParseResult, urlunparse, urlencode + +import datetime + +from redis import Redis +from pymonad.either import Left +from requests.models import Response +from MySQLdb.cursors import DictCursor + +from gn_libs import sqlite3 +from gn_libs import jobs as gnlibs_jobs +from gn_libs.jobs.jobs import JobNotFound +from gn_libs.mysqldb import database_connection from flask import (flash, request, url_for, + jsonify, redirect, Blueprint, - render_template, current_app as app) +from r_qtl import r_qtl2_qc as rqc +from r_qtl import exceptions as rqe + + +from uploader import jobs +from uploader import session +from uploader.files import save_file#, fullpath +from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post +from uploader.oauth2.tokens import request_token from uploader.authorisation import require_login -from uploader.db_utils import database_connection +from uploader.oauth2 import client as oauth2client +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler +from uploader.publications.models import fetch_publication_by_id from uploader.request_checks import with_species, with_population -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import (populations_by_species, - population_by_species_and_id) +from uploader.input_validation import (encode_errors, + decode_errors, + is_valid_representative_name) from .models import (dataset_by_id, phenotype_by_id, phenotypes_count, + save_new_dataset, dataset_phenotypes, - datasets_by_population) + datasets_by_population, + phenotype_publication_data) phenotypesbp = Blueprint("phenotypes", __name__) +render_template = make_template_renderer("phenotypes") + +_FAMILIES_WITH_SE_AND_N_ = ( + "Reference Populations (replicate average, SE, N)",) @phenotypesbp.route("/phenotypes", methods=["GET"]) @require_login @@ -34,10 +72,16 @@ def index(): with database_connection(app.config["SQL_URI"]) as conn: if not bool(request.args.get("species_id")): return render_template("phenotypes/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="phenotypes") - species = species_by_id(conn, request.args.get("species_id")) + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.phenotypes.select_population")) + + species = species_by_id(conn, species_id) if not bool(species): flash("No such species!", "alert-danger") return redirect(url_for("species.populations.phenotypes.index")) @@ -51,27 +95,14 @@ def index(): @with_species(redirect_uri="species.populations.phenotypes.index") def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population for your phenotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("population_id")): - return render_template("phenotypes/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, species["SpeciesId"]), - order_key="FamilyOrder"), - activelink="phenotypes") - - population = population_by_species_and_id( - conn, species["SpeciesId"], int(request.args["population_id"])) - if not bool(population): - flash("No such population found!", "alert-danger") - return redirect(url_for( - "species.populations.phenotypes.select_population", - species_id=species["SpeciesId"])) - - return redirect(url_for("species.populations.phenotypes.list_datasets", - species_id=species["SpeciesId"], - population_id=population["Id"])) + return generic_select_population( + species, + "phenotypes/select-population.html", + request.args.get("population_id") or "", + "species.populations.phenotypes.select_population", + "species.populations.phenotypes.list_datasets", + "phenotypes", + "No such population found!") @@ -84,13 +115,18 @@ def select_population(species: dict, **kwargs):# pylint: disable=[unused-argumen def list_datasets(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """List available phenotype datasets.""" with database_connection(app.config["SQL_URI"]) as conn: + datasets = datasets_by_population( + conn, species["SpeciesId"], population["Id"]) + if len(datasets) == 1: + return redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=datasets[0]["Id"])) return render_template("phenotypes/list-datasets.html", species=species, population=population, - datasets=datasets_by_population( - conn, - species["SpeciesId"], - population["Id"]), + datasets=datasets, activelink="list-datasets") @@ -164,12 +200,12 @@ def view_dataset(# pylint: disable=[unused-argument] phenotype_count=phenotypes_count( conn, population["Id"], dataset["Id"]), phenotypes=enumerate_sequence( - dataset_phenotypes(conn, - population["Id"], - dataset["Id"], - offset=start_at, - limit=count), - start=start_at+1), + dataset_phenotypes( + conn, + population["Id"], + dataset["Id"])), + start_from=start_at, + count=count, activelink="view-dataset") @@ -190,6 +226,41 @@ def view_phenotype(# pylint: disable=[unused-argument] **kwargs ): """View an individual phenotype from the dataset.""" + def __render__(privileges): + phenotype = phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id) + def __non_empty__(value) -> bool: + if isinstance(value, str): + return value.strip() != "" + return bool(value) + + return render_template( + "phenotypes/view-phenotype.html", + species=species, + population=population, + dataset=dataset, + xref_id=xref_id, + phenotype=phenotype, + has_se=any(bool(item.get("error")) for item in phenotype["data"]), + publish_data={ + key.replace("_", " "): val + for key,val in + (phenotype_publication_data(conn, phenotype["Id"]) or {}).items() + if (key in ("PubMed_ID", "Authors", "Title", "Journal") + and __non_empty__(val)) + }, + privileges=privileges, + activelink="view-phenotype") + + def __fail__(error): + if isinstance(error, Response) and error.json() == "No linked resource!": + return __render__(tuple()) + return make_either_error_handler( + "There was an error fetching the roles and privileges.")(error) + with database_connection(app.config["SQL_URI"]) as conn: return oauth2_post( "/auth/resource/phenotypes/individual/linked-resource", @@ -200,19 +271,739 @@ def view_phenotype(# pylint: disable=[unused-argument] "xref_id": xref_id } ).then( - lambda resource: render_template( - "phenotypes/view-phenotype.html", + lambda resource: tuple( + privilege["privilege_id"] for role in resource["roles"] + for privilege in role["privileges"]) + ).then(__render__).either(__fail__, lambda resp: resp) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets/create", + methods=["GET", "POST"]) +@require_login +@with_population( + species_redirect_uri="species.populations.phenotypes.index", + redirect_uri="species.populations.phenotypes.select_population") +def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """Create a new phenotype dataset.""" + with (database_connection(app.config["SQL_URI"]) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + if request.method == "GET": + return render_template("phenotypes/create-dataset.html", + activelink="create-dataset", + species=species, + population=population, + **decode_errors( + request.args.get("error_values", ""))) + + form = request.form + _errors: tuple[tuple[str, str], ...] = tuple() + if not is_valid_representative_name( + (form.get("dataset-name") or "").strip()): + _errors = _errors + (("dataset-name", "Invalid dataset name."),) + + if not bool((form.get("dataset-fullname") or "").strip()): + _errors = _errors + (("dataset-fullname", + "You must provide a value for 'Full Name'."),) + + if bool(_errors) > 0: + return redirect(url_for( + "species.populations.phenotypes.create_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + error_values=encode_errors(_errors, form))) + + dataset_shortname = ( + form["dataset-shortname"] or form["dataset-name"]).strip() + _pheno_dataset = save_new_dataset( + cursor, + population["Id"], + form["dataset-name"].strip(), + form["dataset-fullname"].strip(), + dataset_shortname) + return redirect(url_for("species.populations.phenotypes.list_datasets", + species_id=species["SpeciesId"], + population_id=population["Id"])) + + +def process_phenotypes_rqtl2_bundle(error_uri): + """Process phenotypes from the uploaded R/qtl2 bundle.""" + _redisuri = app.config["REDIS_URL"] + _sqluri = app.config["SQL_URI"] + try: + ## Handle huge files here... + phenobundle = save_file(request.files["phenotypes-bundle"], + Path(app.config["UPLOAD_FOLDER"])) + rqc.validate_bundle(phenobundle) + return phenobundle + except AssertionError as _aerr: + app.logger.debug("File upload error!", exc_info=True) + flash("Expected a zipped bundle of files with phenotypes' " + "information.", + "alert-danger") + return error_uri + except rqe.RQTLError as rqtlerr: + app.logger.debug("Bundle validation error!", exc_info=True) + flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger") + return error_uri + + +def process_phenotypes_individual_files(error_uri): + """Process the uploaded individual files.""" + form = request.form + cdata = { + "sep": form["file-separator"], + "comment.char": form["file-comment-character"], + "na.strings": form["file-na"].split(" "), + } + bundlepath = Path(app.config["UPLOAD_FOLDER"], + f"{str(uuid.uuid4()).replace('-', '')}.zip") + with ZipFile(bundlepath,mode="w") as zfile: + for rqtlkey, formkey, _type in ( + ("phenocovar", "phenotype-descriptions", "mandatory"), + ("pheno", "phenotype-data", "mandatory"), + ("phenose", "phenotype-se", "optional"), + ("phenonum", "phenotype-n", "optional")): + if _type == "optional" and not bool(form.get(formkey)): + continue # skip if an optional key does not exist. + + cdata[f"{rqtlkey}_transposed"] = ( + (form.get(f"{formkey}-transposed") or "off") == "on") + + if form.get("resumable-upload", False): + # Chunked upload of large files was used + filedata = json.loads(form[formkey]) + zfile.write( + Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]), + arcname=filedata["original-name"]) + cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]] + else: + # T0DO: Check this path: fix any bugs. + _sentfile = request.files[formkey] + if not bool(_sentfile): + flash(f"Expected file ('{formkey}') was not provided.", + "alert-danger") + return error_uri + + filepath = save_file( + _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False) + zfile.write( + Path(app.config["UPLOAD_FOLDER"], filepath), + arcname=filepath.name) + cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name] + + + zfile.writestr("control_data.json", data=json.dumps(cdata, indent=2)) + + return bundlepath + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/add-phenotypes", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals] + """Add one or more phenotypes to the dataset.""" + use_bundle = request.args.get("use_bundle", "").lower() == "true" + add_phenos_uri = redirect(url_for( + "species.populations.phenotypes.add_phenotypes", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + _redisuri = app.config["REDIS_URL"] + _sqluri = app.config["SQL_URI"] + with Redis.from_url(_redisuri, decode_responses=True) as rconn: + if request.method == "GET": + today = datetime.date.today() + return render_template( + ("phenotypes/add-phenotypes-with-rqtl2-bundle.html" + if use_bundle else "phenotypes/add-phenotypes-raw-files.html"), species=species, population=population, dataset=dataset, - phenotype=phenotype_by_id(conn, - species["SpeciesId"], - population["Id"], - dataset["Id"], - xref_id), - resource=resource, - activelink="view-phenotype") - ).either( - make_either_error_handler( - "There was an error fetching the roles and privileges."), - lambda resp: resp) + monthnames=( + "January", "February", "March", "April", + "May", "June", "July", "August", + "September", "October", "November", + "December"), + current_month=today.strftime("%B"), + current_year=int(today.strftime("%Y")), + families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_, + use_bundle=use_bundle, + activelink="add-phenotypes") + + phenobundle = (process_phenotypes_rqtl2_bundle(add_phenos_uri) + if use_bundle else + process_phenotypes_individual_files(add_phenos_uri)) + + _jobid = uuid.uuid4() + _namespace = jobs.jobsnamespace() + _ttl_seconds = app.config["JOBS_TTL_SECONDS"] + _job = jobs.launch_job( + jobs.initialise_job( + rconn, + _namespace, + str(_jobid), + [sys.executable, "-m", "scripts.rqtl2.phenotypes_qc", _sqluri, + _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]), + str(population["Id"]), + str(phenobundle), + "--loglevel", + logging.getLevelName( + app.logger.getEffectiveLevel() + ).lower(), + "--redisexpiry", + str(_ttl_seconds)], "phenotype_qc", _ttl_seconds, + {"job-metadata": json.dumps({ + "speciesid": species["SpeciesId"], + "populationid": population["Id"], + "datasetid": dataset["Id"], + "bundle": str(phenobundle.absolute()), + **({"publicationid": request.form["publication-id"]} + if request.form.get("publication-id") else {})})}), + _redisuri, + f"{app.config['UPLOAD_FOLDER']}/job_errors") + + app.logger.debug("JOB DETAILS: %s", _job) + jobstatusuri = url_for("species.populations.phenotypes.job_status", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + job_id=str(_job["jobid"])) + return ((jsonify({ + "redirect-to": jobstatusuri, + "statuscode": 200, + "message": ("Follow the 'redirect-to' URI to see the state " + "of the quality-control job started for your " + "uploaded files.") + }), 200) + if request.form.get("resumable-upload", False) else + redirect(jobstatusuri)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def job_status( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Retrieve current status of a particular phenotype QC job.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + except jobs.JobNotFound as _jnf: + job = None + return render_template("phenotypes/job-status.html", + species=species, + population=population, + dataset=dataset, + job_id=job_id, + job=job, + errors=jobs.job_errors( + rconn, jobs.jobsnamespace(), job['jobid']), + metadata=jobs.job_files_metadata( + rconn, jobs.jobsnamespace(), job['jobid']), + activelink="add-phenotypes") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>/review", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def review_job_data( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Review data one more time before entering it into the database.""" + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as conn): + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + except jobs.JobNotFound as _jnf: + job = None + + def __metadata_by_type__(by_type, item): + filetype = item[1]["filetype"] + return { + **by_type, + filetype: (by_type.get(filetype, tuple()) + + ({"filename": item[0], **item[1]},)) + } + metadata: dict[str, Any] = reduce( + __metadata_by_type__, + (jobs.job_files_metadata( + rconn, jobs.jobsnamespace(), job['jobid']) + if job else {}).items(), + {}) + + def __desc__(filetype): + match filetype: + case "phenocovar": + desc = "phenotypes" + case "pheno": + desc = "phenotypes data" + case "phenose": + desc = "phenotypes standard-errors" + case "phenonum": + desc = "phenotypes samples" + case _: + desc = f"unknown file type '{filetype}'." + + return desc + + def __summarise__(filetype, files): + return { + "filetype": filetype, + "number-of-files": len(files), + "total-data-rows": sum( + int(afile["linecount"]) - 1 for afile in files), + "description": __desc__(filetype) + } + + summary = { + filetype: __summarise__(filetype, meta) + for filetype,meta in metadata.items() + } + _job_metadata = json.loads(job["job-metadata"]) + return render_template("phenotypes/review-job-data.html", + species=species, + population=population, + dataset=dataset, + job_id=job_id, + job=job, + summary=summary, + publication=( + fetch_publication_by_id( + conn, int(_job_metadata["publicationid"])) + if _job_metadata.get("publicationid") + else None), + activelink="add-phenotypes") + + +def load_phenotypes_success_handler(job): + """Handle loading new phenotypes into the database successfully.""" + return redirect(url_for( + "species.populations.phenotypes.load_data_success", + species_id=job["metadata"]["species_id"], + population_id=job["metadata"]["population_id"], + dataset_id=job["metadata"]["dataset_id"], + job_id=job["job_id"])) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/load-data-to-database", + methods=["POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def load_data_to_database( + species: dict, + population: dict, + dataset: dict, + **kwargs +):# pylint: disable=[unused-argument] + """Load the data from the given QC job into the database.""" + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + sqlite3.connection(_jobs_db) as conn): + # T0DO: Maybe break the connection between the jobs here, pass: + # - the bundle name (rebuild the full path here.) + # - publication details, where separate + # - details about the files: e.g. total lines, etc + qc_job = jobs.job(rconn, jobs.jobsnamespace(), request.form["data-qc-job-id"]) + _meta = json.loads(qc_job["job-metadata"]) + _load_job_id = uuid.uuid4() + _loglevel = logging.getLevelName(app.logger.getEffectiveLevel()).lower() + command = [ + sys.executable, + "-u", + "-m", + "scripts.load_phenotypes_to_db", + app.config["SQL_URI"], + _jobs_db, + str(_load_job_id), + "--log-level", + _loglevel + ] + + def __handle_error__(resp): + return render_template("http-error.html", *resp.json()) + + def __handle_success__(load_job): + app.logger.debug("The phenotypes loading job: %s", load_job) + return redirect(url_for( + "background-jobs.job_status", job_id=load_job["job_id"])) + + + return request_token( + token_uri=urljoin(oauth2client.authserver_uri(), "auth/token"), + user_id=session.user_details()["user_id"] + ).then( + lambda token: gnlibs_jobs.initialise_job( + conn, + _load_job_id, + command, + "load-new-phenotypes-data", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "bundle_file": _meta["bundle"], + "publication_id": _meta["publicationid"], + "authserver": oauth2client.authserver_uri(), + "token": token["access_token"], + "success_handler": ( + "uploader.phenotypes.views" + ".load_phenotypes_success_handler") + }) + ).then( + lambda job: gnlibs_jobs.launch_job( + job, + _jobs_db, + Path(f"{app.config['UPLOAD_FOLDER']}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + ).either(__handle_error__, __handle_success__) + + +def update_phenotype_metadata(conn, metadata: dict): + """Update a phenotype's basic metadata values.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Phenotype WHERE Id=%(phenotype-id)s", + metadata) + res = { + **{ + _key: _val for _key,_val in { + key.lower().replace("_", "-"): value + for key, value in (cursor.fetchone() or {}).items() + }.items() + if _key in metadata.keys() + }, + "phenotype-id": metadata.get("phenotype-id") + } + if res == metadata: + return False + + cursor.execute( + "UPDATE Phenotype SET " + "Pre_publication_description=%(pre-publication-description)s, " + "Post_publication_description=%(post-publication-description)s, " + "Original_description=%(original-description)s, " + "Units=%(units)s, " + "Pre_publication_abbreviation=%(pre-publication-abbreviation)s, " + "Post_publication_abbreviation=%(post-publication-abbreviation)s " + "WHERE Id=%(phenotype-id)s", + metadata) + return cursor.rowcount + + +def update_phenotype_values(conn, values): + """Update a phenotype's data values.""" + with conn.cursor() as cursor: + cursor.executemany( + "UPDATE PublishData SET value=%(new)s " + "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in values if item["new"] is not None)) + cursor.executemany( + "DELETE FROM PublishData " + "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in values if item["new"] is None)) + return len(values) + return 0 + + +def update_phenotype_se(conn, serrs): + """Update a phenotype's standard-error values.""" + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO PublishSE(DataId, StrainId, error) " + "VALUES(%(data_id)s, %(strain_id)s, %(new)s) " + "ON DUPLICATE KEY UPDATE error=VALUES(error)", + tuple(item for item in serrs if item["new"] is not None)) + cursor.executemany( + "DELETE FROM PublishSE " + "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in serrs if item["new"] is None)) + return len(serrs) + return 0 + + +def update_phenotype_n(conn, counts): + """Update a phenotype's strain counts.""" + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO NStrain(DataId, StrainId, count) " + "VALUES(%(data_id)s, %(strain_id)s, %(new)s) " + "ON DUPLICATE KEY UPDATE count=VALUES(count)", + tuple(item for item in counts if item["new"] is not None)) + cursor.executemany( + "DELETE FROM NStrain " + "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in counts if item["new"] is None)) + return len(counts) + + return 0 + + +def update_phenotype_data(conn, data: dict): + """Update the numeric data for a phenotype.""" + def __organise_by_dataid_and_strainid__(acc, current): + _key, dataid, strainid = current[0].split("::") + _keysrc, _keytype = _key.split("-") + newkey = f"{dataid}::{strainid}" + newitem = acc.get(newkey, {}) + newitem[_keysrc] = newitem.get(_keysrc, {}) + newitem[_keysrc][_keytype] = current[1] + return {**acc, newkey: newitem} + + def __separate_items__(acc, row): + key, val = row + return ({ + **acc[0], + key: { + **val["value"], + "changed?": (not val["value"]["new"] == val["value"]["original"]) + } + }, { + **acc[1], + key: { + **val["se"], + "changed?": (not val["se"]["new"] == val["se"]["original"]) + } + },{ + **acc[2], + key: { + **val["n"], + "changed?": (not val["n"]["new"] == val["n"]["original"]) + } + }) + + values, serrs, counts = tuple( + tuple({ + "data_id": row[0].split("::")[0], + "strain_id": row[0].split("::")[1], + "new": row[1]["new"] + } for row in item) + for item in ( + filter(lambda val: val[1]["changed?"], item.items())# type: ignore[arg-type] + for item in reduce(# type: ignore[var-annotated] + __separate_items__, + reduce(__organise_by_dataid_and_strainid__, + data.items(), + {}).items(), + ({}, {}, {})))) + + return (update_phenotype_values(conn, values), + update_phenotype_se(conn, serrs), + update_phenotype_n(conn, counts)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/phenotype/<int:xref_id>/edit", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def edit_phenotype_data(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + xref_id: int, + **kwargs +): + """Edit the data for a particular phenotype.""" + def __render__(**kwargs): + processed_kwargs = { + **kwargs, + "privileges": kwargs.get("privileges", tuple()) + } + return render_template( + "phenotypes/edit-phenotype.html", + species=species, + population=population, + dataset=dataset, + xref_id=xref_id, + families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_, + **processed_kwargs, + activelink="edit-phenotype") + + with database_connection(app.config["SQL_URI"]) as conn: + if request.method == "GET": + def __fetch_phenotype__(privileges): + phenotype = phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id) + if phenotype is None: + msg = ("Could not find the phenotype with cross-reference ID" + f" '{xref_id}' from dataset '{dataset['FullName']}' " + f" from the '{population['FullName']}' population of " + f" species '{species['FullName']}'.") + return Left({"privileges": privileges, "phenotype-error": msg}) + return {"privileges": privileges, "phenotype": phenotype} + + def __fetch_publication_data__(**kwargs): + pheno = kwargs["phenotype"] + return { + **kwargs, + "publication_data": phenotype_publication_data( + conn, pheno["Id"]) + } + + def __fail__(failure_object): + # process the object + return __render__(failure_object=failure_object) + + return oauth2_post( + "/auth/resource/phenotypes/individual/linked-resource", + json={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "xref_id": xref_id + } + ).then( + lambda resource: tuple( + privilege["privilege_id"] for role in resource["roles"] + for privilege in role["privileges"]) + ).then( + __fetch_phenotype__ + ).then( + lambda args: __fetch_publication_data__(**args) + ).either(__fail__, lambda args: __render__(**args)) + + ## POST + _change = False + match request.form.get("submit", "invalid-action"): + case "update basic metadata": + _change = update_phenotype_metadata(conn, { + key: value.strip() if bool(value.strip()) else None + for key, value in request.form.items() + if key not in ("submit",) + }) + msg = "Basic metadata was updated successfully." + case "update data": + _update = update_phenotype_data(conn, { + key: value.strip() if bool(value.strip()) else None + for key, value in request.form.items() + if key not in ("submit",) + }) + msg = (f"{_update[0]} value rows, {_update[1]} standard-error " + f"rows and {_update[2]} 'N' rows were updated.") + _change = any(item != 0 for item in _update) + case "update publication": + flash("NOT IMPLEMENTED: Would update publication data.", "alert-success") + case _: + flash("Invalid phenotype editing action.", "alert-danger") + + if _change: + flash(msg, "alert-success") + return redirect(url_for( + "species.populations.phenotypes.view_phenotype", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id)) + + flash("No change was made by the user.", "alert-info") + return redirect(url_for( + "species.populations.phenotypes.edit_phenotype_data", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/load-data-success/<uuid:job_id>", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets") +def load_data_success( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Display success page if loading data to database was successful.""" + with (database_connection(app.config["SQL_URI"]) as conn, + sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) + as jobsconn): + try: + gn2_uri = urlparse(app.config["GN2_SERVER_URL"]) + job = gnlibs_jobs.job(jobsconn, job_id, fulldetails=True) + app.logger.debug("THE JOB: %s", job) + _xref_ids = tuple( + str(item) for item + in json.loads(job["metadata"].get("xref_ids", "[]"))) + _publication = fetch_publication_by_id( + conn, int(job["metadata"].get("publication_id", "0"))) + _search_terms = (item for item in + (str(_publication["PubMed_ID"] or ""), + _publication["Authors"], + (_publication["Title"] or "")) + if item != "") + return render_template("phenotypes/load-phenotypes-success.html", + species=species, + population=population, + dataset=dataset, + job=job, + search_page_uri=urlunparse(ParseResult( + scheme=gn2_uri.scheme, + netloc=gn2_uri.netloc, + path="/search", + params="", + query=urlencode({ + "species": species["Name"], + "group": population["Name"], + "type": "Phenotypes", + "dataset": dataset["Name"], + "search_terms_or": ( + # Very long URLs will cause + # errors. + " ".join(_xref_ids) + if len(_xref_ids) <= 100 + else ""), + "search_terms_and": " ".join( + _search_terms).strip(), + "accession_id": "None", + "FormID": "searchResult" + }), + fragment=""))) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) diff --git a/uploader/platforms/models.py b/uploader/platforms/models.py index a859371..0dd9368 100644 --- a/uploader/platforms/models.py +++ b/uploader/platforms/models.py @@ -56,7 +56,8 @@ def platform_by_species_and_id( return None -def save_new_platform(# pylint: disable=[too-many-arguments] +def save_new_platform( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] cursor: Cursor, species_id: int, geo_platform: str, diff --git a/uploader/platforms/views.py b/uploader/platforms/views.py index 2d61b6a..d12a9ef 100644 --- a/uploader/platforms/views.py +++ b/uploader/platforms/views.py @@ -1,5 +1,6 @@ """The endpoints for the platforms""" from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection from flask import ( flash, request, @@ -10,9 +11,8 @@ from flask import ( from uploader.ui import make_template_renderer from uploader.authorisation import require_login -from uploader.db_utils import database_connection from uploader.species.models import all_species, species_by_id -from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.datautils import safe_int, enumerate_sequence from .models import (save_new_platform, platforms_by_species, @@ -29,9 +29,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "platforms/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="platforms") + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.platforms.list_platforms")) + species = species_by_id(conn, request.args["species_id"]) if not bool(species): flash("No species selected.", "alert-danger") diff --git a/uploader/population/models.py b/uploader/population/models.py index 6dcd85e..4d95065 100644 --- a/uploader/population/models.py +++ b/uploader/population/models.py @@ -26,13 +26,23 @@ def populations_by_species(conn: mdb.Connection, speciesid) -> tuple: return tuple() +__GENERIC_POPULATION_FAMILIES__ = ( + "Reference Populations (replicate average, SE, N)", + "Crosses and Heterogeneous Stock (individuals)", + "Groups Without Genotypes") -def population_families(conn) -> tuple: +def population_families(conn, species_id: int) -> tuple[str]: """Fetch the families under which populations are grouped.""" with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(__GENERIC_POPULATION_FAMILIES__)) cursor.execute( - "SELECT DISTINCT(Family) FROM InbredSet WHERE Family IS NOT NULL") - return tuple(row["Family"] for row in cursor.fetchall()) + "SELECT DISTINCT(Family) FROM InbredSet " + "WHERE SpeciesId=%s " + "AND Family IS NOT NULL " + f"AND Family NOT IN ({paramstr})", + (species_id, *__GENERIC_POPULATION_FAMILIES__)) + return __GENERIC_POPULATION_FAMILIES__ + tuple( + row["Family"] for row in cursor.fetchall()) def population_genetic_types(conn) -> tuple: @@ -47,9 +57,11 @@ def population_genetic_types(conn) -> tuple: def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dict: """Save the population details to the db.""" cursor.execute("SELECT DISTINCT(Family), FamilyOrder FROM InbredSet " - "WHERE Family IS NOT NULL AND Family != '' " + "WHERE SpeciesId=%s " + "AND Family IS NOT NULL AND Family != '' " "AND FamilyOrder IS NOT NULL " - "ORDER BY FamilyOrder ASC") + "ORDER BY FamilyOrder ASC", + (population_details["SpeciesId"],)) _families = { row["Family"]: int(row["FamilyOrder"]) for row in cursor.fetchall() @@ -61,7 +73,7 @@ def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dic **population_details, "FamilyOrder": _families.get( population_details["Family"], - max(_families.values())+1) + max((0,) + tuple(_families.values()))+1) } cursor.execute( "INSERT INTO InbredSet(" diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py index 9968bd6..97d4854 100644 --- a/uploader/population/rqtl2.py +++ b/uploader/population/rqtl2.py @@ -11,12 +11,11 @@ from typing import Union, Callable, Optional import MySQLdb as mdb from redis import Redis from MySQLdb.cursors import DictCursor -from werkzeug.utils import secure_filename +from gn_libs.mysqldb import database_connection +from markupsafe import escape from flask import ( flash, - escape, request, - jsonify, url_for, redirect, Response, @@ -29,7 +28,7 @@ from r_qtl import r_qtl2 from uploader import jobs from uploader.files import save_file, fullpath from uploader.species.models import all_species -from uploader.db_utils import with_db_connection, database_connection +from uploader.db_utils import with_db_connection from uploader.authorisation import require_login from uploader.platforms.models import platform_by_id, platforms_by_species @@ -190,127 +189,6 @@ def trigger_rqtl2_bundle_qc( return jobid -def chunk_name(uploadfilename: str, chunkno: int) -> str: - """Generate chunk name from original filename and chunk number""" - if uploadfilename == "": - raise ValueError("Name cannot be empty!") - if chunkno < 1: - raise ValueError("Chunk number must be greater than zero") - return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" - - -def chunks_directory(uniqueidentifier: str) -> Path: - """Compute the directory where chunks are temporarily stored.""" - if uniqueidentifier == "": - raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") - - -@rqtl2.route(("<int:species_id>/populations/<int:population_id>/rqtl2/" - "/rqtl2-bundle-chunked"), - methods=["GET"]) -@require_login -def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] - species_id: int, - population_id: int -): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that provides a way - for testing whether all the chunks have been uploaded and to assist with - resuming a failed expression-data. - """ - fileid = request.args.get("resumableIdentifier", type=str) or "" - filename = request.args.get("resumableFilename", type=str) or "" - chunk = request.args.get("resumableChunkNumber", type=int) or 0 - if not(fileid or filename or chunk): - return jsonify({ - "message": "At least one required query parameter is missing.", - "error": "BadRequest", - "statuscode": 400 - }), 400 - - if Path(chunks_directory(fileid), - chunk_name(filename, chunk)).exists(): - return "OK" - - return jsonify({ - "message": f"Chunk {chunk} was not found.", - "error": "NotFound", - "statuscode": 404 - }), 404 - - -def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: - """Merge the chunks into a single file.""" - with open(targetfile, "ab") as _target: - for chunkfile in chunkpaths: - with open(chunkfile, "rb") as _chunkdata: - _target.write(_chunkdata.read()) - - chunkfile.unlink() - return targetfile - - -@rqtl2.route(("<int:species_id>/population/<int:population_id>/rqtl2/upload/" - "/rqtl2-bundle-chunked"), - methods=["POST"]) -@require_login -def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that allows large - files to be uploaded in chunks. - - This should hopefully speed up uploads, and if done right, even enable - resumable uploads - """ - _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 - _chunk = request.form.get("resumableChunkNumber", default=1, type=int) - _uploadfilename = request.form.get( - "resumableFilename", default="", type=str) or "" - _fileid = request.form.get( - "resumableIdentifier", default="", type=str) or "" - _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid) - - if _targetfile.exists(): - return jsonify({ - "message": ( - "A file with a similar unique identifier has previously been " - "uploaded and possibly is/has being/been processed."), - "error": "BadRequest", - "statuscode": 400 - }), 400 - - try: - # save chunk data - chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) - request.files["file"].save(Path(chunks_directory(_fileid), - chunk_name(_uploadfilename, _chunk))) - - # Check whether upload is complete - chunkpaths = tuple( - Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) - for _achunk in range(1, _totalchunks+1)) - if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() - jobid = trigger_rqtl2_bundle_qc( - species_id, population_id, _targetfile, _uploadfilename) - return url_for( - "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) - except Exception as exc:# pylint: disable=[broad-except] - msg = "Error processing uploaded file chunks." - app.logger.error(msg, exc_info=True, stack_info=True) - return jsonify({ - "message": msg, - "error": type(exc).__name__, - "error-description": " ".join(str(arg) for arg in exc.args), - "error-trace": traceback.format_exception(exc) - }), 500 - - return "OK" - - @rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", methods=["GET", "POST"]) @require_login diff --git a/uploader/population/views.py b/uploader/population/views.py index 3638453..87a33d9 100644 --- a/uploader/population/views.py +++ b/uploader/population/views.py @@ -1,9 +1,10 @@ """Views dealing with populations/inbredsets""" -import re import json import base64 +from markupsafe import escape from MySQLdb.cursors import DictCursor +from gn_libs.mysqldb import database_connection from flask import (flash, request, url_for, @@ -16,14 +17,12 @@ from uploader.oauth2.client import oauth2_post from uploader.ui import make_template_renderer from uploader.authorisation import require_login from uploader.genotypes.views import genotypesbp -from uploader.db_utils import database_connection from uploader.datautils import enumerate_sequence from uploader.phenotypes.views import phenotypesbp from uploader.expression_data.views import exprdatabp +from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) +from uploader.input_validation import is_valid_representative_name from .models import (save_population, population_families, @@ -48,7 +47,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "populations/index.html", - species=order_species_by_family(all_species(conn))) + species=all_species(conn), + activelink="populations") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.list_species_populations")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("Invalid species identifier provided!", "alert-danger") @@ -73,29 +80,6 @@ def list_species_populations(species_id: int): activelink="list-populations") -def valid_population_name(population_name: str) -> bool: - """ - Check whether the given name is a valid population name. - - Parameters - ---------- - population_name: a string of characters. - - Checks For - ---------- - * The name MUST start with an alphabet [a-zA-Z] - * The name MUST end with an alphabet [a-zA-Z] or number [0-9] - * The name MUST be composed of alphabets [a-zA-Z], numbers [0-9], - underscores (_) and/or hyphens (-). - - Returns - ------- - Boolean indicating whether or not the name is valid. - """ - pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]$") - return bool(pattern.match(population_name)) - - @popbp.route("/<int:species_id>/populations/create", methods=["GET", "POST"]) @require_login def create_population(species_id: int): @@ -116,7 +100,7 @@ def create_population(species_id: int): return render_template( "populations/create-population.html", species=species, - families = population_families(conn), + families = population_families(conn, species["SpeciesId"]), genetic_types = population_genetic_types(conn), mapping_methods=( {"id": "0", "value": "No mapping support"}, @@ -124,6 +108,7 @@ def create_population(species_id: int): {"id": "2", "value": "GEMMA"}, {"id": "3", "value": "R/qtl"}, {"id": "4", "value": "GEMMA, PLINK"}), + return_to=(request.args.get("return_to") or ""), activelink="create-population", **error_values) @@ -139,7 +124,7 @@ def create_population(species_id: int): errors = errors + (("population_name", "You must provide a name for the population!"),) - if not valid_population_name(population_name): + if not is_valid_representative_name(population_name): errors = errors + (( "population_name", "The population name can only contain letters, numbers, " @@ -168,13 +153,21 @@ def create_population(species_id: int): "FullName": population_fullname, "InbredSetCode": request.form.get("population_code") or None, "Description": request.form.get("population_description") or None, - "Family": request.form.get("population_family") or None, + "Family": request.form.get("population_family").strip() or None, "MappingMethodId": request.form.get("population_mapping_method_id"), "GeneticType": request.form.get("population_genetic_type") or None }) def __flash_success__(_success): - flash("Successfully created resource.", "alert-success") + flash("Successfully created population " + f"{escape(new_population['FullName'])}.", + "alert-success") + return_to = request.form.get("return_to") or "" + if return_to: + return redirect(url_for( + return_to, + species_id=species["SpeciesId"], + population_id=new_population["InbredSetId"])) return redirect(url_for( "species.populations.view_population", species_id=species["SpeciesId"], diff --git a/uploader/publications/__init__.py b/uploader/publications/__init__.py new file mode 100644 index 0000000..7efcabb --- /dev/null +++ b/uploader/publications/__init__.py @@ -0,0 +1,2 @@ +"""Package for handling publications.""" +from .views import pubbp diff --git a/uploader/publications/datatables.py b/uploader/publications/datatables.py new file mode 100644 index 0000000..e07fafd --- /dev/null +++ b/uploader/publications/datatables.py @@ -0,0 +1,52 @@ +"""Fetch data for datatables.""" +import logging +from typing import Optional + +from MySQLdb.cursors import DictCursor + +from gn_libs.mysqldb import Connection, debug_query + +logger = logging.getLogger(__name__) + +def fetch_publications( + conn: Connection, + search: Optional[str] = None, + offset: int = 0, + limit: int = -1 +) -> tuple[dict, int, int, int]: + """Fetch publications from the database.""" + _query = "SELECT * FROM Publication" + _count_query = "SELECT COUNT(*) FROM Publication" + _params = None + _where_clause = "" + _limit_clause = "" + if search is not None and bool(search): + _where_clause = ("WHERE PubMed_ID LIKE %s " + "OR Authors LIKE %s " + "OR Title LIKE %s") + _params = (f"%{search}%",) * 3 + + if limit > 0: + _limit_clause = f"LIMIT {limit} OFFSET {offset}" + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT COUNT(*) FROM Publication") + _total_rows = int(cursor.fetchone()["COUNT(*)"]) + + cursor.execute(f"{_count_query} {_where_clause}", _params) + debug_query(cursor, logger) + _result = cursor.fetchone() + _total_filtered = int(_result["COUNT(*)"] if bool(_result) else 0) + + cursor.execute(f"{_query} {_where_clause} {_limit_clause}", _params) + debug_query(cursor, logger) + _current_filtered = tuple( + {**dict(row), "index": idx} + for idx, row + in enumerate(cursor.fetchall(), start=offset+1)) + + return ( + _current_filtered, + len(_current_filtered), + _total_filtered, + _total_rows) diff --git a/uploader/publications/misc.py b/uploader/publications/misc.py new file mode 100644 index 0000000..fca6f71 --- /dev/null +++ b/uploader/publications/misc.py @@ -0,0 +1,25 @@ +"""Miscellaneous functions dealing with publications.""" + + +def publications_differences( + filedata: tuple[dict, ...], + dbdata: tuple[dict, ...], + pubmedid2pubidmap: tuple[dict, ...] +) -> tuple[dict, ...]: + """Compute the differences between file data and db data""" + diff = tuple() + for filerow, dbrow in zip( + sorted(filedata, key=lambda item: ( + item["phenotype_id"], item["xref_id"])), + sorted(dbdata, key=lambda item: ( + item["PhenotypeId"], item["xref_id"]))): + if filerow["PubMed_ID"] == dbrow["PubMed_ID"]: + continue + + newpubmed = filerow["PubMed_ID"] + diff = diff + ({ + **dbrow, + "PubMed_ID": newpubmed, + "PublicationId": pubmedid2pubidmap.get(newpubmed)},) + + return diff diff --git a/uploader/publications/models.py b/uploader/publications/models.py new file mode 100644 index 0000000..f83be58 --- /dev/null +++ b/uploader/publications/models.py @@ -0,0 +1,98 @@ +"""Module to handle persistence and retrieval of publication to/from MariaDB""" +import logging +from typing import Iterable + +from MySQLdb.cursors import DictCursor + +from gn_libs.mysqldb import Connection, debug_query + +logger = logging.getLogger(__name__) + + +def fetch_phenotype_publications( + conn: Connection, + ids: tuple[tuple[int, int], ...] +) -> tuple[dict, ...]: + """Fetch publication from database by ID.""" + paramstr = ",".join(["(%s, %s)"] * len(ids)) + query = ( + "SELECT " + "pxr.PhenotypeId, pxr.Id AS xref_id, pxr.PublicationId, pub.PubMed_ID " + "FROM PublishXRef AS pxr INNER JOIN Publication AS pub " + "ON pxr.PublicationId=pub.Id " + f"WHERE (pxr.PhenotypeId, pxr.Id) IN ({paramstr})") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(query, tuple(item for row in ids for item in row)) + return tuple(dict(row) for row in cursor.fetchall()) + + +def create_new_publications( + conn: Connection, + publications: tuple[dict, ...] +) -> tuple[dict, ...]: + """Create new publications in the database.""" + if len(publications) > 0: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.executemany( + ("INSERT INTO " + "Publication( " + "PubMed_ID, Abstract, Authors, Title, Journal, Volume, Pages, " + "Month, Year" + ") " + "VALUES(" + "%(pubmed_id)s, %(abstract)s, %(authors)s, %(title)s, " + "%(journal)s, %(volume)s, %(pages)s, %(month)s, %(year)s" + ") " + "RETURNING *"), + publications) + return tuple({ + **row, "publication_id": row["Id"] + } for row in cursor.fetchall()) + + return tuple() + + +def update_publications(conn: Connection , publications: tuple[dict, ...]) -> tuple[dict, ...]: + """Update details for multiple publications""" + if len(publications) > 0: + with conn.cursor(cursorclass=DictCursor) as cursor: + logger.debug("UPDATING PUBLICATIONS: %s", publications) + cursor.executemany( + ("UPDATE Publication SET " + "PubMed_ID=%(pubmed_id)s, Abstract=%(abstract)s, " + "Authors=%(authors)s, Title=%(title)s, Journal=%(journal)s, " + "Volume=%(volume)s, Pages=%(pages)s, Month=%(month)s, " + "Year=%(year)s " + "WHERE Id=%(publication_id)s"), + publications) + debug_query(cursor, logger) + return publications + return tuple() + return tuple() + + +def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict: + """Fetch a specific publication from the database.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Publication WHERE Id=%s", + (publication_id,)) + _res = cursor.fetchone() + return dict(_res) if _res else {} + + +def fetch_publication_phenotypes( + conn: Connection, publication_id: int) -> Iterable[dict]: + """Fetch all phenotypes linked to this publication.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT pxr.Id AS xref_id, pxr.PublicationId, phe.* " + "FROM PublishXRef AS pxr INNER JOIN Phenotype AS phe " + "ON pxr.PhenotypeId=phe.Id " + "WHERE pxr.PublicationId=%s", + (publication_id,)) + while True: + row = cursor.fetchone() + if row: + yield row + else: + break diff --git a/uploader/publications/pubmed.py b/uploader/publications/pubmed.py new file mode 100644 index 0000000..2531c4a --- /dev/null +++ b/uploader/publications/pubmed.py @@ -0,0 +1,102 @@ +"""Module to interact with NCBI's PubMed""" +import logging + +import requests +from lxml import etree + +logger = logging.getLogger(__name__) + + +def __pub_date__(pubdate: etree.Element): + pubyear = pubdate.find("Year") + pubmonth = pubdate.find("Month") + pubday = pubdate.find("Day") + return { + "year": pubyear.text if pubyear is not None else None, + "month": pubmonth.text if pubmonth is not None else None, + "day": pubday.text if pubday is not None else None + } + + +def __journal__(journal: etree.Element) -> dict: + volume = journal.find("JournalIssue/Volume") + issue = journal.find("JournalIssue/Issue") + return { + "volume": volume.text if volume is not None else None, + "issue": issue.text if issue is not None else None, + **__pub_date__(journal.find("JournalIssue/PubDate")), + "journal": journal.find("Title").text + } + +def __author__(author: etree.Element) -> str: + return f'{author.find("LastName").text} {author.find("Initials").text}' + + +def __pages__(pagination: etree.Element) -> str: + start = pagination.find("StartPage") + end = pagination.find("EndPage") + return (start.text + ( + f"-{end.text}" if end is not None else "" + )) if start is not None else "" + + +def __abstract__(article: etree.Element) -> str: + abstract = article.find("Abstract/AbstractText") + return abstract.text if abstract is not None else None + + +def __article__(pubmed_article: etree.Element) -> dict: + article = pubmed_article.find("MedlineCitation/Article") + return { + "pubmed_id": int(pubmed_article.find("MedlineCitation/PMID").text), + "title": article.find("ArticleTitle").text, + **__journal__(article.find("Journal")), + "abstract": __abstract__(article), + "pages": __pages__(article.find("Pagination")), + "authors": ", ".join(__author__(author) + for author in article.findall("AuthorList/Author")) + } + + +def __process_pubmed_publication_data__(text) -> tuple[dict, ...]: + """Process the data from PubMed into usable data.""" + doc = etree.XML(text) + articles = doc.xpath("//PubmedArticle") + logger.debug("Retrieved %s publications from NCBI", len(articles)) + return tuple(__article__(article) for article in articles) + +def fetch_publications(pubmed_ids: tuple[int, ...]) -> tuple[dict, ...]: + """Retrieve data on new publications from NCBI.""" + # See whether we can retrieve multiple publications in one go + # Parse data and save to DB + # Return PublicationId(s) for new publication(s). + if len(pubmed_ids) == 0: + logger.debug("There are no new PubMed IDs to fetch") + return tuple() + + logger.info("Fetching publications data for the following PubMed IDs: %s", + ", ".join((str(pid) for pid in pubmed_ids))) + + # Should we, perhaps, pass this in from a config variable? + uri = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + try: + response = requests.get( + uri, + params={ + "db": "pubmed", + "retmode": "xml", + "id": ",".join(str(item) for item in pubmed_ids) + }, + timeout=(9.13, 20)) + + if response.status_code == 200: + return __process_pubmed_publication_data__(response.text) + + logger.error( + "Could not fetch the new publication from %s (status code: %s)", + uri, + response.status_code) + except requests.exceptions.ConnectionError: + logger.error("Could not find the domain %s", uri) + + return tuple() diff --git a/uploader/publications/views.py b/uploader/publications/views.py new file mode 100644 index 0000000..a88f754 --- /dev/null +++ b/uploader/publications/views.py @@ -0,0 +1,104 @@ +"""Endpoints for publications""" +import json + +from gn_libs.mysqldb import database_connection +from flask import ( + flash, + request, + url_for, + redirect, + Blueprint, + render_template, + current_app as app) + +from uploader.authorisation import require_login + +from .models import ( + fetch_publication_by_id, + create_new_publications, + fetch_publication_phenotypes) + +from .datatables import fetch_publications + +pubbp = Blueprint("publications", __name__) + + +@pubbp.route("/", methods=["GET"]) +@require_login +def index(): + """Index page for publications.""" + return render_template("publications/index.html") + + +@pubbp.route("/list", methods=["GET"]) +@require_login +def list_publications(): + """Fetch publications that fulfill a specific search, or all of them, if + there is no search term.""" + # request breakdown: + # https://datatables.net/manual/server-side + _page = int(request.args.get("draw")) + _length = int(request.args.get("length") or '-1') + _start = int(request.args.get("start") or '0') + _search = request.args["search[value]"] + with database_connection(app.config["SQL_URI"]) as conn: + _publications, _current_rows, _totalfiltered, _totalrows = fetch_publications( + conn, + _search, + offset=_start, + limit=_length) + + return json.dumps({ + "draw": _page, + "recordsTotal": _totalrows, + "recordsFiltered": _totalfiltered, + "publications": _publications, + "status": "success" + }) + + +@pubbp.route("/view/<int:publication_id>", methods=["GET"]) +@require_login +def view_publication(publication_id: int): + """View more details on a particular publication.""" + with database_connection(app.config["SQL_URI"]) as conn: + return render_template( + "publications/view-publication.html", + publication=fetch_publication_by_id(conn, publication_id), + linked_phenotypes=tuple(fetch_publication_phenotypes( + conn, publication_id))) + + +@pubbp.route("/create", methods=["GET", "POST"]) +@require_login +def create_publication(): + """Create a new publication.""" + if request.method == "GET": + return render_template("publications/create-publication.html") + form = request.form + authors = form.get("publication-authors").encode("utf8") + if authors is None or authors == "": + flash("The publication's author(s) MUST be provided!", "alert alert-danger") + return redirect(url_for("publications.create", **request.args)) + + with database_connection(app.config["SQL_URI"]) as conn: + publications = create_new_publications(conn, ({ + "pubmed_id": form.get("pubmed-id") or None, + "abstract": form.get("publication-abstract").encode("utf8") or None, + "authors": authors, + "title": form.get("publication-title").encode("utf8") or None, + "journal": form.get("publication-journal").encode("utf8") or None, + "volume": form.get("publication-volume").encode("utf8") or None, + "pages": form.get("publication-pages").encode("utf8") or None, + "month": (form.get("publication-month") or "").encode("utf8").capitalize() or None, + "year": form.get("publication-year").encode("utf8") or None + },)) + flash("New publication created!", "alert alert-success") + return redirect(url_for( + request.args.get("return_to") or "publications.view_publication", + publication_id=publications[0]["publication_id"], + **request.args)) + + flash("Publication creation failed!", "alert alert-danger") + app.logger.debug("Failed to create the new publication.", exc_info=True) + return redirect(url_for("publications.create_publication")) diff --git a/uploader/request_checks.py b/uploader/request_checks.py index a24b2f7..f1d8027 100644 --- a/uploader/request_checks.py +++ b/uploader/request_checks.py @@ -4,10 +4,10 @@ These are useful for reusability, and hence maintainability of the code. """ from functools import wraps +from gn_libs.mysqldb import database_connection from flask import flash, url_for, redirect, current_app as app from uploader.species.models import species_by_id -from uploader.db_utils import database_connection from uploader.population.models import population_by_species_and_id def with_species(redirect_uri: str): diff --git a/uploader/route_utils.py b/uploader/route_utils.py new file mode 100644 index 0000000..ce718fb --- /dev/null +++ b/uploader/route_utils.py @@ -0,0 +1,42 @@ +"""Generic routing utilities.""" +from flask import flash, url_for, redirect, render_template, current_app as app + +from gn_libs.mysqldb import database_connection + +from uploader.population.models import (populations_by_species, + population_by_species_and_id) + +def generic_select_population( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] + species: dict, + template: str, + population_id: str, + back_to: str, + forward_to: str, + activelink: str, + error_message: str = "No such population found!" +): + """Handles common flow for 'select population' step.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(population_id): + return render_template( + template, + species=species, + populations=populations_by_species(conn, species["SpeciesId"]), + activelink=activelink) + + if population_id == "CREATE-POPULATION": + return redirect(url_for( + "species.populations.create_population", + species_id=species["SpeciesId"], + return_to=forward_to)) + + population = population_by_species_and_id( + conn, species["SpeciesId"], int(population_id)) + if not bool(population): + flash(error_message, "alert-danger") + return redirect(url_for(back_to, species_id=species["SpeciesId"])) + + return redirect(url_for(forward_to, + species_id=species["SpeciesId"], + population_id=population["Id"])) diff --git a/uploader/samples/__init__.py b/uploader/samples/__init__.py new file mode 100644 index 0000000..1bd6d2d --- /dev/null +++ b/uploader/samples/__init__.py @@ -0,0 +1 @@ +"""Samples package. Handle samples uploads and editing.""" diff --git a/uploader/samples/models.py b/uploader/samples/models.py index d7d5384..1e9293f 100644 --- a/uploader/samples/models.py +++ b/uploader/samples/models.py @@ -15,11 +15,11 @@ def samples_by_species_and_population( """Fetch the samples by their species and population.""" with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( - "SELECT iset.InbredSetId, s.* FROM InbredSet AS iset " - "INNER JOIN StrainXRef AS sxr ON iset.InbredSetId=sxr.InbredSetId " - "INNER JOIN Strain AS s ON sxr.StrainId=s.Id " - "WHERE s.SpeciesId=%(species_id)s " - "AND iset.InbredSetId=%(population_id)s", + "SELECT InbredSet.InbredSetId, Strain.* FROM InbredSet " + "INNER JOIN StrainXRef ON InbredSet.InbredSetId=StrainXRef.InbredSetId " + "INNER JOIN Strain ON StrainXRef.StrainId=Strain.Id " + "WHERE Strain.SpeciesId=%(species_id)s " + "AND InbredSet.InbredSetId=%(population_id)s", {"species_id": species_id, "population_id": population_id}) return tuple(cursor.fetchall()) @@ -34,8 +34,7 @@ def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs else ("Name", "Name2", "Symbol", "Alias")), delimiter=separator, quotechar=kwargs.get("quotechar", '"')) - for row in reader: - yield row + yield from reader def save_samples_data(conn: mdb.Connection, diff --git a/uploader/samples/views.py b/uploader/samples/views.py index 9ba1df8..4705a96 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -3,11 +3,8 @@ import os import sys import uuid from pathlib import Path -from typing import Iterator -import MySQLdb as mdb from redis import Redis -from MySQLdb.cursors import DictCursor from flask import (flash, request, url_for, @@ -20,18 +17,14 @@ from uploader.files import save_file from uploader.ui import make_template_renderer from uploader.authorisation import require_login from uploader.input_validation import is_integer_input -from uploader.datautils import order_by_family, enumerate_sequence -from uploader.db_utils import ( - with_db_connection, - database_connection, - with_redis_connection) -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) -from uploader.population.models import(save_population, - population_by_id, - populations_by_species, - population_by_species_and_id) +from uploader.population.models import population_by_id +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence +from uploader.species.models import all_species, species_by_id +from uploader.request_checks import with_species, with_population +from uploader.db_utils import (with_db_connection, + database_connection, + with_redis_connection) from .models import samples_by_species_and_population @@ -46,8 +39,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "samples/index.html", - species=order_species_by_family(all_species(conn)), + species=all_species(conn), activelink="samples") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.samples.select_population")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("No such species!", "alert-danger") @@ -58,61 +58,33 @@ def index(): @samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"]) @require_login -def select_population(species_id: int): +@with_species(redirect_uri="species.populations.samples.index") +def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population to use for the samples.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - if not bool(request.args.get("population_id")): - return render_template("samples/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, - species_id), - order_key="FamilyOrder"), - activelink="samples") - - population = population_by_id(conn, request.args.get("population_id")) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - - return redirect(url_for("species.populations.samples.list_samples", - species_id=species_id, - population_id=population["Id"])) + return generic_select_population( + species, + "samples/select-population.html", + request.args.get("population_id") or "", + "species.populations.samples.select_population", + "species.populations.samples.list_samples", + "samples", + "Population not found!") @samplesbp.route("<int:species_id>/populations/<int:population_id>/samples") @require_login -def list_samples(species_id: int, population_id: int): +@with_population( + species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """ List the samples in a particular population and give the ability to upload new ones. """ with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - population = population_by_id(conn, population_id) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - all_samples = enumerate_sequence(samples_by_species_and_population( - conn, species_id, population_id)) + conn, species["SpeciesId"], population["Id"])) total_samples = len(all_samples) - offset = int(request.args.get("from") or 0) - if offset < 0: - offset = 0 + offset = max(safe_int(request.args.get("from") or 0), 0) count = int(request.args.get("count") or 20) return render_template("samples/list-samples.html", species=species, @@ -124,7 +96,7 @@ def list_samples(species_id: int, population_id: int): activelink="list-samples") -def build_sample_upload_job(# pylint: disable=[too-many-arguments] +def build_sample_upload_job(# pylint: disable=[too-many-arguments, too-many-positional-arguments] speciesid: int, populationid: int, samplesfile: Path, @@ -187,7 +159,7 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma "alert-error") return samples_uploads_page - firstlineheading = (request.form.get("first_line_heading") == "on") + firstlineheading = request.form.get("first_line_heading") == "on" separator = request.form.get("separator", ",") if separator == "other": @@ -200,7 +172,7 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma redisuri = app.config["REDIS_URL"] with Redis.from_url(redisuri, decode_responses=True) as rconn: - #TODO: Add a QC step here — what do we check? + #T0DO: Add a QC step here — what do we check? # 1. Does any sample in the uploaded file exist within the database? # If yes, what is/are its/their species and population? # 2. If yes 1. above, provide error with notes on which species and @@ -233,57 +205,53 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma "upload-samples/status/<uuid:job_id>", methods=["GET"]) @require_login -def upload_status(species_id: int, population_id: int, job_id: uuid.UUID): +@with_population(species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] """Check on the status of a samples upload job.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("You must provide a valid species.", "alert-danger") - return redirect(url_for("species.populations.samples.index")) + job = with_redis_connection(lambda rconn: jobs.job( + rconn, jobs.jobsnamespace(), job_id)) + if job: + status = job["status"] + if status == "success": + return render_template("samples/upload-success.html", + job=job, + species=species, + population=population,) - population = population_by_species_and_id( - conn, species_id, population_id) - if not bool(population): - flash("You must provide a valid population.", "alert-danger") + if status == "error": return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if job: - status = job["status"] - if status == "success": - return render_template("samples/upload-success.html", - job=job, - species=species, - population=population,) - - if status == "error": + "species.populations.samples.upload_failure", + species_id=species["SpeciesId"], + population_id=population["Id"], + job_id=job_id)) + + error_filename = Path(jobs.error_filename( + job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) + if error_filename.exists(): + stat = os.stat(error_filename) + if stat.st_size > 0: return redirect(url_for( - "species.populations.samples.upload_failure", job_id=job_id)) + "samples.upload_failure", job_id=job_id)) - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for( - "samples.upload_failure", job_id=job_id)) + return render_template("samples/upload-progress.html", + species=species, + population=population, + job=job) # maybe also handle this? - return render_template("samples/upload-progress.html", - species=species, - population=population, - job=job) # maybe also handle this? + return render_template("no_such_job.html", + job_id=job_id, + species=species, + population=population), 400 - return render_template("no_such_job.html", - job_id=job_id, - species=species, - population=population), 400 -@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"]) +@samplesbp.route("<int:species_id>/populations/<int:population_id>/" + "upload-samples/failure/<uuid:job_id>", + methods=["GET"]) @require_login -def upload_failure(job_id: uuid.UUID): +@with_population(species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def upload_failure(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] """Display the errors of the samples upload failure.""" job = with_redis_connection(lambda rconn: jobs.job( rconn, jobs.jobsnamespace(), job_id)) @@ -297,4 +265,7 @@ def upload_failure(job_id: uuid.UUID): if stat.st_size > 0: return render_template("worker_failure.html", job_id=job_id) - return render_template("samples/upload-failure.html", job=job) + return render_template("samples/upload-failure.html", + species=species, + population=population, + job=job) diff --git a/uploader/session.py b/uploader/session.py index b538187..5af5827 100644 --- a/uploader/session.py +++ b/uploader/session.py @@ -77,12 +77,15 @@ def set_user_token(token: str) -> SessionInfo: """Set the user's token.""" info = session_info() return save_session_info({ - **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc] + **info, + "user": {**info["user"], "token": Right(token), "logged_in": True} + })#type: ignore[misc] def set_user_details(userdets: UserDetails) -> SessionInfo: """Set the user details information""" - return save_session_info({**session_info(), "user": userdets})#type: ignore[misc] + info = session_info() + return save_session_info({**info, "user": {**info["user"], **userdets}})#type: ignore[misc] def user_details() -> UserDetails: """Retrieve user details.""" diff --git a/uploader/species/models.py b/uploader/species/models.py index 51f941c..acfa51e 100644 --- a/uploader/species/models.py +++ b/uploader/species/models.py @@ -58,7 +58,8 @@ def save_species(conn: mdb.Connection, common_name: The species' common name. scientific_name; The species' scientific name. """ - genus, species_name = scientific_name.split(" ") + genus, *species_parts = scientific_name.split(" ") + species_name: str = " ".join(species_parts) families = species_families(conn) with conn.cursor() as cursor: cursor.execute("SELECT MAX(OrderId) FROM Species") @@ -68,7 +69,7 @@ def save_species(conn: mdb.Connection, "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})", "scientific_name": scientific_name, "family": family, - "family_order": families[family], + "family_order": families.get(family, 999999), "taxon_id": taxon_id, "species_order": cursor.fetchone()[0] + 5 } @@ -91,7 +92,7 @@ def save_species(conn: mdb.Connection, } -def update_species(# pylint: disable=[too-many-arguments] +def update_species(# pylint: disable=[too-many-arguments, too-many-positional-arguments] conn: mdb.Connection, species_id: int, common_name: str, @@ -116,7 +117,8 @@ def update_species(# pylint: disable=[too-many-arguments] species_order: The ordering of this species in relation to others """ with conn.cursor(cursorclass=DictCursor) as cursor: - genus, species_name = scientific_name.split(" ") + genus, *species_parts = scientific_name.split(" ") + species_name = " ".join(species_parts) species = { "species_id": species_id, "common_name": common_name, diff --git a/uploader/species/views.py b/uploader/species/views.py index 10715a5..cea2f68 100644 --- a/uploader/species/views.py +++ b/uploader/species/views.py @@ -1,5 +1,7 @@ """Endpoints handling species.""" +from markupsafe import escape from pymonad.either import Left, Right, Either +from gn_libs.mysqldb import database_connection from flask import (flash, request, url_for, @@ -10,7 +12,6 @@ from flask import (flash, from uploader.population import popbp from uploader.platforms import platformsbp from uploader.ui import make_template_renderer -from uploader.db_utils import database_connection from uploader.oauth2.client import oauth2_get, oauth2_post from uploader.authorisation import require_login, require_token from uploader.datautils import order_by_family, enumerate_sequence @@ -62,6 +63,8 @@ def create_species(): if request.method == "GET": return render_template("species/create-species.html", families=species_families(conn), + return_to=( + request.args.get("return_to") or ""), activelink="create-species") error = False @@ -79,7 +82,7 @@ def create_species(): error = True parts = tuple(name.strip() for name in scientific_name.split(" ")) - if len(parts) != 2 or not all(bool(name) for name in parts): + if (len(parts) != 2 and len(parts) != 3) or not all(bool(name) for name in parts): flash("The scientific name you provided is invalid.", "alert-danger") error = True @@ -113,7 +116,15 @@ def create_species(): species = save_species( conn, common_name, scientific_name, family, taxon_id) - flash("Species saved successfully!", "alert-success") + flash( + f"You have successfully added species " + f"'{escape(species['scientific_name'])} " + f"({escape(species['common_name'])})'.", + "alert-success") + + return_to = request.form.get("return_to").strip() + if return_to: + return redirect(url_for(return_to, species_id=species["species_id"])) return redirect(url_for("species.view_species", species_id=species["species_id"])) diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css index 574f53e..df50dec 100644 --- a/uploader/static/css/styles.css +++ b/uploader/static/css/styles.css @@ -1,127 +1,187 @@ +* { + box-sizing: border-box; +} + body { margin: 0.7em; - box-sizing: border-box; display: grid; - grid-template-columns: 1fr 6fr; - grid-template-rows: 5em 100%; + grid-template-columns: 2fr 8fr; grid-gap: 20px; - font-family: Georgia, Garamond, serif; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; font-style: normal; + font-size: 20px; } #header { - grid-column: 1/3; - width: 100%; - /* background: cyan; */ - padding-top: 0.5em; - border-radius: 0.5em; + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 8fr 2fr; + /* Content styling */ background-color: #336699; - border-color: #080808; color: #FFFFFF; - background-image: none; + border-radius: 3px; + min-height: 30px; } -#header .header { - font-size: 2em; - display: inline-block; - text-align: start; -} +#header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; -#header .header-nav { - display: inline-block; - color: #FFFFFF; + /* Content styling */ + padding-left: 1em; } -#header .header-nav li { - border-width: 1px; - border-color: #FFFFFF; - vertical-align: middle; - margin: 0.2em; - border-style: solid; - border-width: 2px; - border-radius: 0.5em; - text-align: center; +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; } -#header .header-nav a { +#header #header-nav .nav li a { + /* Content styling */ color: #FFFFFF; - text-decoration: none; + background: #4477AA; + border: solid 5px #336699; + border-radius: 5px; + font-size: 0.7em; + text-align: center; + padding: 1px 7px; } #nav-sidebar { - grid-column: 1/2; - /* background: #e5e5ff; */ - padding-top: 0.5em; - border-radius: 0.5em; - font-size: 1.2em; + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; } -#main { - grid-column: 2/3; - width: 100%; - /* background: gray; */ +#nav-sidebar .nav li a:hover { border-radius: 0.5em; } -.pagetitle { - padding-top: 0.5em; - /* background: pink; */ +#nav-sidebar .nav .activemenu { + border-style: solid; border-radius: 0.5em; - /* background-color: #6699CC; */ - /* background-color: #77AADD; */ + border-color: #AAAAAA; + background-color: #EFEFEF; +} + +#main { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 1fr; + grid-template-rows: 4em 100%; + grid-gap: 1em; +} + +#main #pagetitle { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Content-styling */ + border-radius: 3px; background-color: #88BBEE; } -.pagetitle h1 { - text-align: start; +#main #pagetitle .title { + font-size: 1.4em; text-transform: capitalize; - padding-left: 0.25em; + padding-left: 0.5em; +} + +@media screen and (max-width: 20in) { + #main #all-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + max-width: 80%; + } + + #sidebar-content { + display: none; + } +} + +@media screen and (min-width: 20.1in) { + #main #all-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 7fr 3fr; + grid-gap: 1.5em; + } +} + +#main #all-content .row { + margin: 0 2px; +} + +#main #all-content #main-content { + background: #FFFFFF; + max-width: 950px; } -.pagetitle .breadcrumb { +#pagetitle .breadcrumb { background: none; + text-transform: capitalize; + font-size: 0.75em; } -.pagetitle .breadcrumb .active a { +#pagetitle .breadcrumb .active a { color: #333333; } -.pagetitle .breadcrumb a { +#pagetitle .breadcrumb a { color: #666666; } -.main-content { - font-size: 1.275em; +.heading { + border-bottom: solid #EEBB88; + text-transform: capitalize; } -.breadcrumb { +.subheading { + padding: 1em 0 0.1em 0.5em; + border-bottom: solid #88BBEE; text-transform: capitalize; } -dd { - margin-left: 3em; - font-size: 0.88em; - padding-bottom: 1em; +input[type="search"] { + border-radius: 5px; } -input[type="submit"], .btn { - text-transform: capitalize; +.btn { + text-transform: Capitalize; } -.card { - margin-top: 0.3em; - border-width: 1px; - border-style: solid; - border-radius: 0.3em; - border-color: #AAAAAA; - padding: 0.5em; +table.dataTable thead th, table.dataTable tfoot th{ + border-right: 1px solid white; + color: white; + background-color: #369 !important; } -.activemenu { - border-style: solid; - border-radius: 0.5em; - border-color: #AAAAAA; - background-color: #EFEFEF; +table.dataTable tbody tr.selected td { + background-color: #ffee99 !important; +} + +.form-group { + margin-bottom: 2em; + padding-bottom: 0.2em; + border-bottom: solid gray 1px; } diff --git a/uploader/static/js/datatables.js b/uploader/static/js/datatables.js new file mode 100644 index 0000000..82fd696 --- /dev/null +++ b/uploader/static/js/datatables.js @@ -0,0 +1,69 @@ +/** Handlers for events in datatables **/ + +var addTableLength = (menuList, lengthToAdd, dataLength) => { + if(dataLength >= lengthToAdd) { + newList = structuredClone(menuList);//menuList.slice(0, menuList.length); // shallow copy + newList.push(lengthToAdd); + return newList; + } + return menuList; +}; + +var defaultLengthMenu = (data) => { + menuList = [] + var lengths = [10, 25, 50, 100, 1000, data.length]; + lengths.forEach((len) => { + menuList = addTableLength(menuList, len, data.length); + }); + return menuList; +}; + +var buildDataTable = (tableId, data = [], columns = [], userSettings = {}) => { + var defaultSettings = { + responsive: true, + layout: { + topStart: null, + topEnd: null, + bottomStart: null, + bottomEnd: null, + }, + select: true, + lengthMenu: defaultLengthMenu(data), + language: { + processing: "Processing… Please wait.", + loadingRecords: "Loading table data… Please wait.", + lengthMenu: "", + info: "" + }, + data: data, + columns: columns, + drawCallback: (settings) => { + $(this[0]).find("tbody tr").each((idx, row) => { + var arow = $(row); + var checkboxOrRadio = arow.find(".chk-row-select"); + if (checkboxOrRadio) { + if (arow.hasClass("selected")) { + checkboxOrRadio.prop("checked", true); + } else { + checkboxOrRadio.prop("checked", false); + } + } + }); + } + } + var theDataTable = $(tableId).DataTable({ + ...defaultSettings, + ...userSettings + }); + theDataTable.on("select", (event, datatable, type, cell, originalEvent) => { + datatable.rows({selected: true}).nodes().each((node, index) => { + $(node).find(".chk-row-select").prop("checked", true) + }); + }); + theDataTable.on("deselect", (event, datatable, type, cell, originalEvent) => { + datatable.rows({selected: false}).nodes().each((node, index) => { + $(node).find(".chk-row-select").prop("checked", false) + }); + }); + return theDataTable; +}; diff --git a/uploader/static/js/debug.js b/uploader/static/js/debug.js new file mode 100644 index 0000000..eb01209 --- /dev/null +++ b/uploader/static/js/debug.js @@ -0,0 +1,40 @@ +/** + * The entire purpose of this function is for use to debug values inline + * without changing the flow of the code too much. + * + * This **MUST** be a non-arrow function to allow access to the `arguments` + * object. + * + * This function expects at least one argument. + * + * If more than one argument is provided, then: + * a) the last argument is considered the value, and will be returned + * b) all other arguments will be converted to string and output + * + * If only one argument is provided, it is considered the value, and will be + * returned. + * + * Zero arguments is an error condition. + **/ +function __pk__(val) { + /* Handle zero arguments */ + if (arguments.length < 1) { + throw new Error("Invalid arguments: Expected at least one argument."); + } + + msg = "/********** DEBUG **********/"; + if (arguments.length > 1) { + msg = Array.from( + arguments + ).slice( + 0, + arguments.length - 1 + ).map((val) => { + return String(val); + }).join("; ") + } + + value = arguments[arguments.length - 1]; + console.debug("/********** " + msg + " **********/", value); + return value; +} diff --git a/uploader/static/js/files.js b/uploader/static/js/files.js new file mode 100644 index 0000000..0bde6f7 --- /dev/null +++ b/uploader/static/js/files.js @@ -0,0 +1,118 @@ +var readFirstNLines = (thefile, count, process_content_fns) => { + var reader = new FileReader(); + if(typeof thefile !== "undefined" && thefile !== null) { + reader.addEventListener("load", (event) => { + var content = event + .target + .result + .split("\n") + .slice(0, count) + .map((line) => {return line.trim("\r");}); + process_content_fns.forEach((fn) => {fn(content);}); + }); + reader.readAsText(thefile); + } +}; +var read_first_n_lines = readFirstNLines; + + +var readBinaryFile = (file) => { + return new Promise((resolve, reject) => { + var _reader = new FileReader(); + _reader.onload = (event) => {resolve(_reader.result);}; + _reader.readAsArrayBuffer(file); + }); +}; + + +var Uint8ArrayToHex = (arr) => { + var toHex = (val) => { + _hex = val.toString(16); + if(_hex.length < 2) { + return "0" + val; + } + return _hex; + }; + _hexstr = "" + arr.forEach((val) => {_hexstr += toHex(val)}); + return _hexstr +}; + + +var computeFileChecksum = (file) => { + return readBinaryFile(file) + .then((content) => { + return window.crypto.subtle.digest( + "SHA-256", new Uint8Array(content)); + }).then((digest) => { + return Uint8ArrayToHex(new Uint8Array(digest)) + }); +}; + + +var defaultResumableHandler = (event) => { + throw new Error("Please provide a valid event handler!"); +}; + +var addHandler = (resumable, handlername, handler) => { + if(resumable.support) { + resumable.on(handlername, (handler || defaultResumableHandler)); + } + return resumable; +}; + + +var makeResumableHandler = (handlername) => { + return (resumable, handler) => { + return addHandler(resumable, handlername, handler); + }; +}; + + +var fileSuccessHandler = makeResumableHandler("fileSuccess"); +var fileProgressHandler = makeResumableHandler("fileProgress"); +var fileAddedHandler = makeResumableHandler("fileAdded"); +var filesAddedHandler = makeResumableHandler("filesAdded"); +var filesRetryHandler = makeResumableHandler("filesRetry"); +var filesErrorHandler = makeResumableHandler("filesError"); +var uploadStartHandler = makeResumableHandler("uploadStart"); +var completeHandler = makeResumableHandler("complete"); +var progressHandler = makeResumableHandler("progress"); +var errorHandler = makeResumableHandler("error"); + + +var markResumableDragAndDropElement = (resumable, fileinput, droparea, browsebutton) => { + if(resumable.support) { + //Hide file input element and display drag&drop UI + add_class(fileinput, "visually-hidden"); + remove_class(droparea, "visually-hidden"); + + // Define UI elements for browse and drag&drop + resumable.assignDrop(droparea); + resumable.assignBrowse(browsebutton); + } + + return resumable; +}; + + +var makeResumableElement = (targeturi, fileinput, droparea, uploadbutton, filetype) => { + var resumable = Resumable({ + target: targeturi, + fileType: filetype, + maxFiles: 1, + forceChunkSize: true, + generateUniqueIdentifier: (file, event) => { + return computeFileChecksum(file).then((checksum) => { + var _relativePath = (file.webkitRelativePath + || file.relativePath + || file.fileName + || file.name); + return checksum + "-" + _relativePath.replace( + /[^a-zA-Z0-9_-]/img, ""); + }); + } + }); + + return resumable; +}; diff --git a/uploader/static/js/misc.js b/uploader/static/js/misc.js deleted file mode 100644 index cf7b39e..0000000 --- a/uploader/static/js/misc.js +++ /dev/null @@ -1,6 +0,0 @@ -"Miscellaneous functions and event-handlers" - -$(".not-implemented").click((event) => { - event.preventDefault(); - alert("This feature is not implemented yet. Please bear with us."); -}); diff --git a/uploader/static/js/populations.js b/uploader/static/js/populations.js new file mode 100644 index 0000000..89ededa --- /dev/null +++ b/uploader/static/js/populations.js @@ -0,0 +1,36 @@ +$(() => { + var populationsDataTable = buildDataTable( + "#tbl-select-population", + JSON.parse( + $("#tbl-select-population").attr("data-populations-list")), + [ + { + data: (apopulation) => { + return `<input type="radio" name="population_id"` + + `id="rdo_population_id_${apopulation.InbredSetId}" ` + + `value="${apopulation.InbredSetId}" ` + + `class="chk-row-select">`; + } + }, + { + searchable: true, + data: (apopulation) => { + return `${apopulation.FullName} (${apopulation.InbredSetName})`; + } + } + ], + { + select: "single", + paging: true, + scrollY: 700, + deferRender: true, + scroller: true, + scrollCollapse: true, + layout: { + topStart: "info", + topEnd: "search", + bottomStart: "pageLength", + bottomEnd: false + } + }); +}); diff --git a/uploader/static/js/pubmed.js b/uploader/static/js/pubmed.js new file mode 100644 index 0000000..9afd4c3 --- /dev/null +++ b/uploader/static/js/pubmed.js @@ -0,0 +1,113 @@ +var extract_details = (pubmed_id, details) => { + var months = { + "jan": "January", + "feb": "February", + "mar": "March", + "apr": "April", + "may": "May", + "jun": "June", + "jul": "July", + "aug": "August", + "sep": "September", + "oct": "October", + "nov": "November", + "dec": "December" + }; + var _date = details[pubmed_id].pubdate.split(" "); + return { + "authors": details[pubmed_id].authors.map((authobj) => { + return authobj.name; + }), + "title": details[pubmed_id].title, + "journal": details[pubmed_id].fulljournalname, + "volume": details[pubmed_id].volume, + "pages": details[pubmed_id].pages, + "month": _date.length > 1 ? months[_date[1].toLowerCase()] : "jan", + "year": _date[0], + }; +}; + +var update_publication_details = (details) => { + Object.entries(details).forEach((entry) => {; + switch(entry[0]) { + case "authors": + $("#txt-publication-authors").val(entry[1].join(", ")); + break; + case "month": + $("#select-publication-month") + .children("option") + .each((index, child) => { + console.debug(entry[1].toLowerCase()); + child.selected = child.value == entry[1].toLowerCase(); + }); + default: + $("#txt-publication-" + entry[0]).val(entry[1]); + break; + } + }); +}; + +var fetch_publication_abstract = (pubmed_id, pub_details) => { + $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi", + { + "method": "GET", + "data": { + "db": "pubmed", + "id": pubmed_id, + "rettype": "abstract", + "retmode": "xml" + }, + "success": (data, textStatus, jqXHR) => { + update_publication_details({ + ...pub_details, + ...{ + "abstract": Array.from(data + .getElementsByTagName( + "Abstract")[0] + .children) + .map((elt) => {return elt.textContent.trim();}) + .join("\r\n") + }}); + }, + "error": (jqXHR, textStatus, errorThrown) => {}, + "complete": (jqXHR, textStatus) => {}, + "dataType": "xml" + }); +}; + +var fetch_publication_details = (pubmed_id, complete_thunks) => { + error_display = $("#search-pubmed-id-error"); + error_display.text(""); + add_class(error_display, "visually-hidden"); + $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", + { + "method": "GET", + "data": {"db": "pubmed", "id": pubmed_id, "format": "json"}, + "success": (data, textStatus, jqXHR) => { + // process and update publication details + hasError = ( + Object.hasOwn(data, "error") || + Object.hasOwn(data.result[pubmed_id], "error")); + if(hasError) { + error_display.text( + "There was an error fetching a publication with " + + "the given PubMed ID! The error received " + + "was: '" + ( + data.error || + data.result[pubmed_id].error) + + "'. Please check ID you provided and try " + + "again."); + remove_class(error_display, "visually-hidden"); + } else { + fetch_publication_abstract( + pubmed_id, + extract_details(pubmed_id, data.result)); + } + }, + "error": (jqXHR, textStatus, errorThrown) => {}, + "complete": () => { + complete_thunks.forEach((thunk) => {thunk()}); + }, + "dataType": "json" + }); +}; diff --git a/uploader/static/js/species.js b/uploader/static/js/species.js new file mode 100644 index 0000000..d42e081 --- /dev/null +++ b/uploader/static/js/species.js @@ -0,0 +1,34 @@ +$(() => { + var speciesDataTable = buildDataTable( + "#tbl-select-species", + JSON.parse( + $("#tbl-select-species").attr("data-species-list")), + [ + { + data: (aspecies) => { + return `<input type="radio" name="species_id"` + + `id="rdo_species_id_${aspecies.SpeciesId}" ` + + `value="${aspecies.SpeciesId}" class="chk-row-select">`; + } + }, + { + data: (aspecies) => { + return `${aspecies.FullName} (${aspecies.SpeciesName})`; + } + } + ], + { + select: "single", + paging: true, + scrollY: 700, + deferRender: true, + scroller: true, + scrollCollapse: true, + layout: { + topStart: "info", + topEnd: "search", + bottomStart: "pageLength", + bottomEnd: false + } + }); +}); diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js index 045dd47..1b31661 100644 --- a/uploader/static/js/utils.js +++ b/uploader/static/js/utils.js @@ -8,3 +8,30 @@ function trigger_change_event(element) { evt = new Event("change"); element.dispatchEvent(evt); } + + +var remove_class = (element, classvalue) => { + new_classes = (element.attr("class") || "").split(" ").map((val) => { + return val.trim(); + }).filter((val) => { + return ((val !== classvalue) && + (val !== "")) + }).join(" "); + + if(new_classes === "") { + element.removeAttr("class"); + } else { + element.attr("class", new_classes); + } +}; + + +var add_class = (element, classvalue) => { + remove_class(element, classvalue); + element.attr("class", (element.attr("class") || "") + " " + classvalue); +}; + +$(".not-implemented").click((event) => { + event.preventDefault(); + alert("This feature is not implemented yet. Please bear with us."); +}); diff --git a/uploader/templates/base.html b/uploader/templates/base.html index 019aa39..3c0d0d4 100644 --- a/uploader/templates/base.html +++ b/uploader/templates/base.html @@ -8,14 +8,14 @@ <meta name="viewport" content="width=device-width, initial-scale=1.0" /> {%block extrameta%}{%endblock%} - <title>GN Uploader: {%block title%}{%endblock%}</title> + <title>Data Upload and Quality Control: {%block title%}{%endblock%}</title> <link rel="stylesheet" type="text/css" href="{{url_for('base.bootstrap', filename='css/bootstrap.min.css')}}" /> <link rel="stylesheet" type="text/css" - href="{{url_for('base.bootstrap', - filename='css/bootstrap-theme.min.css')}}" /> + href="{{url_for('base.datatables', + filename='css/dataTables.bootstrap5.min.css')}}" /> <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> {%block css%}{%endblock%} @@ -23,28 +23,32 @@ </head> <body> - <header id="header" class="container-fluid"> - <div class="row"> - <span class="header col-lg-9">GeneNetwork Data Quality Control and Upload</span> - <nav class="header-nav col-lg-3"> - <ul class="nav justify-content-end"> - <li> - {%if user_logged_in()%} - <a href="{{url_for('oauth2.logout')}}" - title="Log out of the system">{{user_email()}} — Log Out</a> - {%else%} - <a href="{{authserver_authorise_uri()}}" - title="Log in to the system">Log In</a> - {%endif%} - </li> - </ul> - </nav> + <header id="header"> + <span id="header-text">GeneNetwork</span> + <nav id="header-nav"> + <ul class="nav justify-content-end"> + <li> + {%if user_logged_in()%} + <a href="{{url_for('oauth2.logout')}}" + title="Log out of the system"> + <span class="glyphicon glyphicon-user"></span> + {{user_email()}} Sign Out</a> + {%else%} + <a href="{{authserver_authorise_uri()}}" + title="Log in to the system">Sign In</a> + {%endif%} + </li> + </ul> + </nav> </header> - <aside id="nav-sidebar" class="container-fluid"> + <aside id="nav-sidebar"> <ul class="nav flex-column"> <li {%if activemenu=="home"%}class="activemenu"{%endif%}> <a href="/" >Home</a></li> + <li {%if activemenu=="publications"%}class="activemenu"{%endif%}> + <a href="{{url_for('publications.index')}}" + title="View and manage publications.">Publications</a></li> <li {%if activemenu=="species"%}class="activemenu"{%endif%}> <a href="{{url_for('species.list_species')}}" title="View and manage species information.">Species</a></li> @@ -70,9 +74,11 @@ <li {%if activemenu=="phenotypes"%}class="activemenu"{%endif%}> <a href="{{url_for('species.populations.phenotypes.index')}}" title="Upload phenotype data.">Phenotype Data</a></li> + <!-- <li {%if activemenu=="expression-data"%}class="activemenu"{%endif%}> <a href="{{url_for('species.populations.expression-data.index')}}" - title="Upload expression data.">Expression Data</a></li> + title="Upload expression data." + class="not-implemented">Expression Data</a></li> <li {%if activemenu=="individuals"%}class="activemenu"{%endif%}> <a href="#" class="not-implemented" @@ -86,47 +92,70 @@ class="not-implemented" title="View and manage the backgroud jobs you have running"> Background Jobs</a></li> + --> </ul> </aside> - <main id="main" class="main container-fluid"> + <main id="main" class="main"> - <div class="pagetitle row"> - <h1>GN Uploader: {%block pagetitle%}{%endblock%}</h1> - <nav> - <ol class="breadcrumb"> - <li {%if activelink is not defined or activelink=="home"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('base.index')}}">Home</a> - </li> - {%block lvl1_breadcrumbs%}{%endblock%} - </ol> - </nav> + <div id="pagetitle" class="pagetitle"> + <span class="title">Data Upload and Quality Control: {%block pagetitle%}{%endblock%}</span> + <!-- + <nav> + <ol class="breadcrumb"> + <li {%if activelink is not defined or activelink=="home"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('base.index')}}">Home</a> + </li> + {%block lvl1_breadcrumbs%}{%endblock%} + </ol> + </nav> + --> </div> - <div class="row"> - <div class="container-fluid"> - <div class="col-md-8 main-content"> - {%block contents%}{%endblock%} - </div> - <div class="sidebar-content col-md-4"> - {%block sidebarcontents%}{%endblock%} - </div> + <div id="all-content"> + <div id="main-content"> + {%block contents%}{%endblock%} + </div> + <div id="sidebar-content"> + {%block sidebarcontents%}{%endblock%} </div> </div> </main> + <!-- + Core dependencies + --> <script src="{{url_for('base.jquery', filename='jquery.min.js')}}"></script> <script src="{{url_for('base.bootstrap', filename='js/bootstrap.min.js')}}"></script> - <script type="text/javascript" src="/static/js/misc.js"></script> - {%block javascript%}{%endblock%} - </body> + <!-- + DataTables dependencies + --> + <script type="text/javascript" + src="{{url_for('base.datatables', + filename='js/dataTables.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='scroller/js/dataTables.scroller.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='buttons/js/dataTables.buttons.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='select/js/dataTables.select.min.js')}}"></script> + <!-- + local dependencies + --> + <script type="text/javascript" src="/static/js/utils.js"></script> + <script type="text/javascript" src="/static/js/datatables.js"></script> + {%block javascript%}{%endblock%} + </body> </html> diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html index 33fb73b..64b1a9a 100644 --- a/uploader/templates/cli-output.html +++ b/uploader/templates/cli-output.html @@ -1,7 +1,7 @@ {%macro cli_output(job, stream)%} -<h4>{{stream | upper}} Output</h4> -<div class="cli-output"> +<h4 class="subheading">{{stream | upper}} Output</h4> +<div class="cli-output" style="max-height: 10em; overflow: auto;"> <pre>{{job.get(stream, "")}}</pre> </div> diff --git a/uploader/templates/genotypes/base.html b/uploader/templates/genotypes/base.html index 1b274bf..7d61312 100644 --- a/uploader/templates/genotypes/base.html +++ b/uploader/templates/genotypes/base.html @@ -6,7 +6,18 @@ {%else%} class="breadcrumb-item" {%endif%}> + {%if population is mapping%} + <a href="{{url_for('species.populations.genotypes.list_genotypes', + species_id=species.SpeciesId, + population_id=population.Id)}}"> + {%if dataset is defined and dataset is mapping%} + {{dataset.Name}} + {%else%} + Genotypes + {%endif%}</a> + {%else%} <a href="{{url_for('species.populations.genotypes.index')}}">Genotypes</a> + {%endif%} </li> {%block lvl4_breadcrumbs%}{%endblock%} {%endblock%} diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html index e749f5a..b50ebc5 100644 --- a/uploader/templates/genotypes/index.html +++ b/uploader/templates/genotypes/index.html @@ -26,3 +26,7 @@ species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html index e4c39eb..0f074fd 100644 --- a/uploader/templates/genotypes/list-genotypes.html +++ b/uploader/templates/genotypes/list-genotypes.html @@ -26,7 +26,8 @@ <p>There are a total of {{total_markers}} currently registered genetic markers for the "{{species.FullName}}" species. You can click <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId)}}" + species_id=species.SpeciesId, + population_id=population.Id)}}" title="View genetic markers for species '{{species.FullName}}"> this link to view the genetic markers </a>. @@ -70,7 +71,7 @@ {%if genocode | length < 1%} <a href="#add-genotype-encoding" title="Add a genotype encoding system for this population" - class="btn btn-primary"> + class="btn btn-primary not-implemented"> add genotype encoding </a> {%endif%} diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html index 9198b44..a705ae3 100644 --- a/uploader/templates/genotypes/list-markers.html +++ b/uploader/templates/genotypes/list-markers.html @@ -13,7 +13,8 @@ class="breadcrumb-item" {%endif%}> <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId)}}">List markers</a> + species_id=species.SpeciesId, + population_id=population.Id)}}">List markers</a> </li> {%endblock%} @@ -30,6 +31,7 @@ {%if start_from > 0%} <a href="{{url_for('species.populations.genotypes.list_markers', species_id=species.SpeciesId, + population_id=population.Id, start_from=start_from-count, count=count)}}"> <span class="glyphicon glyphicon-backward"></span> @@ -45,6 +47,7 @@ {%if start_from + count < total_markers%} <a href="{{url_for('species.populations.genotypes.list_markers', species_id=species.SpeciesId, + population_id=population.Id, start_from=start_from+count, count=count)}}"> Next diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html index 7c81943..acdd063 100644 --- a/uploader/templates/genotypes/select-population.html +++ b/uploader/templates/genotypes/select-population.html @@ -12,20 +12,14 @@ {{flash_all_messages()}} <div class="row"> - <p> - You have indicated that you intend to upload the genotypes for species - '{{species.FullName}}'. We now just require the population for your - experiment/study, and you should be good to go. - </p> -</div> - -<div class="row"> - {{select_population_form(url_for("species.populations.genotypes.select_population", - species_id=species.SpeciesId), - populations)}} + {{select_population_form(url_for("species.populations.genotypes.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html index d6f57eb..aa1414e 100644 --- a/uploader/templates/index.html +++ b/uploader/templates/index.html @@ -10,90 +10,98 @@ <div class="row"> {{flash_all_messages()}} <div class="explainer"> - <p>Welcome to the <strong>GeneNetwork Data Quality Control and Upload System</strong>. This system is provided to help in uploading your data onto GeneNetwork where you can do analysis on it.</p> + <p>Welcome to the <strong>GeneNetwork Data Upload and Quality Control + System</strong>.</p> + <p>This tool helps you prepare and upload research data to GeneNetwork for + analysis.</p> - <p>The sections below provide an overview of what features the menu items on - the left provide to you. Please peruse the information to get a good - big-picture understanding of what the system provides you and how to get - the most out of it.</p> + <h2 class="heading">Getting Started</h2> + <p>The sections below explain the features of the system. Review this guide + to learn how to use the system.</p> {%block extrapageinfo%}{%endblock%} - <h2>Species</h2> - - <p>The GeneNetwork service provides datasets and tools for doing genetic - studies — from - <a href="{{gn2server_intro}}" - target="_blank" - title="GeneNetwork introduction — opens in a new tab."> - its introduction</a>: - - <blockquote class="blockquote"> - <p>GeneNetwork is a group of linked data sets and tools used to study - complex networks of genes, molecules, and higher order gene function - and phenotypes. …</p> - </blockquote> - </p> - - <p>With this in mind, it follows that the data in the system is centered - aroud a variety of species. The <strong>species section</strong> will - list the currently available species in the system, and give you the - ability to add new ones, if the one you want to work on does not currently - exist on GeneNetwork</p> - - <h2>Populations</h2> - - <p>Your studies will probably focus on a particular subset of the entire - species you are interested in – your population.</p> - <p>Populations are a way to organise the species data so as to link data to - specific know populations for a particular species, e.g. The BXD - population of mice (Mus musculus)</p> - <p>In older GeneNetwork documentation, you might run into the term - <em>InbredSet</em>. Should you run into it, it is a term that we've - deprecated that essentially just means the population.</p> - - <h2>Samples</h2> - - <p>These are the samples or individuals (sometimes cases) that were involved - in the experiment, and from whom the data was derived.</p> - - <h2>Genotype Data</h2> - - <p>This section will allow you to view and upload the genetic markers for - your species, and the genotype encodings used for your particular - population.</p> - <p>While, technically, genetic markers relate to the species in general, and - not to a particular population, the data (allele information) itself - relates to the particular population it was generated from – - specifically, to the actual individuals used in the experiment.</p> - <p>This is the reason why the genotype data information comes under the - population, and will check for the prior existence of the related - samples/individuals before attempting an upload of your data.</p> - - <h2>Expression Data</h2> + <h3 class="subheading">Species</h3> - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>TODO</strong>: Document this …</p> + <p>GeneNetwork supports genetic studies across multiple species (e.g. mice + [Mus musculus], human [homo sapiens], rats [Rattus norvegicus], etc.) . + Here you can:</p> + <ul> + <li>View all species that are currently supported</li> + <li>Add new species not yet in the system</li> + </ul> + + <h3 class="subheading">Populations</h3> + + <p>A "population" refers to a specific subgroup within a species that you’re + studying (e.g., BXD mice). Here you can:</p> + <ul> + <li>View the populations that exist for a selected species</li> + <li>Add new populations of study for a selected species</li> + </ul> + + <h3 class="subheading">Samples</h3> + + <p>Manage individual specimens or cases used in your experiments. These + include:</p> + + <ul> + <li>Experimental subjects</li> + <li>Data sources (e.g., tissue samples, clinical cases)</li> + <li>Strain means (instead of entering multiple BXD1 individuals, for + example, the mean would be entered for a single BXD1 strain)</li> + </ul> + + + <h3 class="subheading">Genotype Data</h3> + + <p>Upload and review genetic markers and allele encodings for your + population. Key details:</p> + + <ul> + <li>Markers are species-level (e.g., mouse SNP databases).</li> + <li>Allele data is population-specific (tied to your experimental + samples).</li> + </ul> + + <p><strong>Requirement</strong>: Samples must already have been registered + in the system before uploading genotype data.</p> + + <h3 class="subheading">Phenotype Data</h3> + + <p>Phenotypes are the visible traits or features of a living thing. For + example, phenotypes include:</p> + + <ul> + <li>Weight</li> + <li>Height</li> + <li>Color (such as the color of fur or eyes)</li> + </ul> + + <p>This part of the system will allow you to upload and manage the values + for different phenotypes from various samples in your studies.</p> + + <!-- - <h2>Phenotype Data</h2> + <h3 class="subheading">Expression Data</h3> <p class="text-danger"> <span class="glyphicon glyphicon-exclamation-sign"></span> <strong>TODO</strong>: Document this …</p> - <h2>Individual Data</h2> + <h3 class="subheading">Individual Data</h3> <p class="text-danger"> <span class="glyphicon glyphicon-exclamation-sign"></span> <strong>TODO</strong>: Document this …</p> - <h2>RNA-Seq Data</h2> + <h3 class="subheading">RNA-Seq Data</h3> <p class="text-danger"> <span class="glyphicon glyphicon-exclamation-sign"></span> <strong>TODO</strong>: Document this …</p> </div> + --> </div> {%endblock%} diff --git a/uploader/templates/jobs/job-error.html b/uploader/templates/jobs/job-error.html new file mode 100644 index 0000000..b3015fc --- /dev/null +++ b/uploader/templates/jobs/job-error.html @@ -0,0 +1,17 @@ +{%extends "base.html"%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Background Jobs: Error{%endblock%} + +{%block pagetitle%}Background Jobs: Error{%endblock%} + +{%block contents%} + +<h1>Background Jobs: Error</h1> +<p>Job <strong>{{job["job_id"]}}</strong> failed!</p> +<p>The error details are in the "STDERR" section below.</p> + +<h2>STDERR</h2> +<pre>{{job["stderr"]}}</pre> +{%endblock%} diff --git a/uploader/templates/jobs/job-not-found.html b/uploader/templates/jobs/job-not-found.html new file mode 100644 index 0000000..a71e66f --- /dev/null +++ b/uploader/templates/jobs/job-not-found.html @@ -0,0 +1,11 @@ +{%extends "base.html"%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} +<p>Could not find job with ID: {{job_id}}</p> +{%endblock%} diff --git a/uploader/templates/jobs/job-status.html b/uploader/templates/jobs/job-status.html new file mode 100644 index 0000000..83c02fd --- /dev/null +++ b/uploader/templates/jobs/job-status.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5" /> +{%endblock%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} + +<p>Status: {{job["metadata"]["status"]}}</p> +<p>Job Type: {{job["metadata"]["job-type"]}}</p> + +<h2>STDOUT</h2> +<pre>{{job["stdout"]}}</pre> + +<h2>STDERR</h2> +<pre>{{job["stderr"]}}</pre> + +{%endblock%} diff --git a/uploader/templates/login.html b/uploader/templates/login.html index 1f71416..e76c644 100644 --- a/uploader/templates/login.html +++ b/uploader/templates/login.html @@ -5,7 +5,8 @@ {%block pagetitle%}log in{%endblock%} {%block extrapageinfo%} -<p class="text-dark text-primary"> - You <strong>do need to be logged in</strong> to upload data onto this system. - Please do that by clicking the "Log In" button at the top of the page.</p> +<p class="text-dark"> + You <strong>need to + <a href="{{authserver_authorise_uri()}}" + title="Sign in to the system">sign in</a></strong> to use this system.</p> {%endblock%} diff --git a/uploader/templates/macro-step-indicator.html b/uploader/templates/macro-step-indicator.html new file mode 100644 index 0000000..ac0be77 --- /dev/null +++ b/uploader/templates/macro-step-indicator.html @@ -0,0 +1,15 @@ +{%macro step_indicator(step, width=100)%} +<svg width="{{width}}" height="{{width}}" xmlns="http://www.w3.org/2000/svg"> + <circle cx="{{0.5*width}}" + cy="{{0.5*width}}" + r="{{0.5*width}}" + fill="#E5E5FF" /> + <text x="{{0.5*width}}" + y="{{0.6*width}}" + font-size="{{0.2*width}}" + text-anchor="middle" + fill="#555555"> + Step {{step}} + </text> +</svg> +{%endmacro%} diff --git a/uploader/templates/macro-table-pagination.html b/uploader/templates/macro-table-pagination.html new file mode 100644 index 0000000..292c531 --- /dev/null +++ b/uploader/templates/macro-table-pagination.html @@ -0,0 +1,26 @@ +{%macro table_pagination(start_at, page_count, total_count, base_uri, name)%} +{%set ns = namespace(forward_uri=base_uri, back_uri=base_uri)%} +{%set ns.forward_uri="brr"%} + <div class="row"> + <div class="col-md-2" style="text-align: start;"> + {%if start_at > 0%} + <a href="{{base_uri + + '?start_at='+((start_at-page_count)|string) + + '&count='+(page_count|string)}}"> + <span class="glyphicon glyphicon-backward"></span> + Previous + </a> + {%endif%} + </div> + <div class="col-md-8" style="text-align: center;"> + Displaying {{name}} {{start_at+1}} to {{start_at+page_count if start_at+page_count < total_count else total_count}} of {{total_count}}</div> + <div class="col-md-2" style="text-align: end;"> + {%if start_at + page_count < total_count%} + <a href="{{base_uri + + '?start_at='+((start_at+page_count)|string) + + '&count='+(page_count|string)}}"> + Next<span class="glyphicon glyphicon-forward"></span></a> + {%endif%} + </div> + </div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html new file mode 100644 index 0000000..9909c20 --- /dev/null +++ b/uploader/templates/phenotypes/add-phenotypes-base.html @@ -0,0 +1,166 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <form id="frm-add-phenotypes" + method="POST" + enctype="multipart/form-data" + action="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + use_bundle=use_bundle)}}" + data-resumable-target="{{url_for('files.resumable_upload_post')}}"> + <legend>Add New Phenotypes</legend> + + <div class="form-text help-block"> + {%block frm_add_phenotypes_documentation%}{%endblock%} + <p><strong class="text-warning">This will not update any existing phenotypes!</strong></p> + </div> + + {%block frm_add_phenotypes_elements%}{%endblock%} + + <fieldset id="fldset-publication-info"> + <legend>Publication Information</legend> + <input type="hidden" name="publication-id" id="txt-publication-id" /> + <span class="form-text text-muted"> + Select a publication for your data. <br /> + Can't find a publication you can use? Go ahead and + <a href="{{url_for( + 'publications.create_publication', + return_to='species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">create a new publication</a>.</span> + <table id="tbl-select-publication" class="table compact stripe"> + <thead> + <tr> + <th>#</th> + <th>PubMed ID</th> + <th>Title</th> + <th>Authors</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </fieldset> + + <div class="form-group"> + <input type="submit" + value="upload phenotypes" + class="btn btn-primary" /> + </div> + </form> +</div> + +<div class="row"> + {%block page_documentation%}{%endblock%} +</div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var publicationsDataTable = buildDataTable( + "#tbl-select-publication", + [], + [ + {data: "index"}, + { + searchable: true, + data: (pub) => { + if(pub.PubMed_ID) { + return `<a href="https://pubmed.ncbi.nlm.nih.gov/` + + `${pub.PubMed_ID}/" target="_blank" ` + + `title="Link to publication on NCBI.">` + + `${pub.PubMed_ID}</a>`; + } + return ""; + } + }, + { + searchable: true, + data: (pub) => { + var title = "⸻"; + if(pub.Title) { + title = pub.Title + } + return `<a href="/publications/view/${pub.Id}" ` + + `target="_blank" ` + + `title="Link to view publication details">` + + `${title}</a>`; + } + }, + { + searchable: true, + data: (pub) => { + authors = pub.Authors.split(",").map( + (item) => {return item.trim();}); + if(authors.length > 1) { + return authors[0] + ", et. al."; + } + return authors[0]; + } + } + ], + { + serverSide: true, + ajax: { + url: "/publications/list", + dataSrc: "publications" + }, + select: "single", + paging: true, + scrollY: 700, + deferRender: true, + scroller: true, + scrollCollapse: true, + layout: { + topStart: "info", + topEnd: "search" + } + }); + publicationsDataTable.on("select", (event, datatable, type, indexes) => { + indexes.forEach((element, index, thearray) => { + let row = datatable.row(element).node(); + console.debug(datatable.row(element).data()); + $("#frm-add-phenotypes #txt-publication-id").val( + datatable.row(element).data().Id); + }); + }); + publicationsDataTable.on("deselect", (event, datatable, type, indexes) => { + indexes.forEach((element, index, thearray) => { + let row = datatable.row(element).node(); + $("#frm-add-phenotypes #txt-publication-id").val(null); + }); + }); + }); +</script> + +{%block more_javascript%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/phenotypes/add-phenotypes-raw-files.html b/uploader/templates/phenotypes/add-phenotypes-raw-files.html new file mode 100644 index 0000000..67b56e3 --- /dev/null +++ b/uploader/templates/phenotypes/add-phenotypes-raw-files.html @@ -0,0 +1,847 @@ +{%extends "phenotypes/add-phenotypes-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} +{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%} +{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block frm_add_phenotypes_documentation%} +<p>This page will allow you to upload all the separate files that make up your + phenotypes. Here, you will have to upload each separate file individually. If + you want instead to upload all your files as a single ZIP file, + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + use_bundle=true)}}" + title="">click here</a>.</p> +{%endblock%} + +{%block frm_add_phenotypes_elements%} +<fieldset id="fldset-file-metadata"> + <legend>File(s) Metadata</legend> + <div class="form-group"> + <label for="txt-file-separator" class="form-label">File Separator</label> + <div class="input-group"> + <input id="txt-file-separator" + name="file-separator" + type="text" + value="	" + class="form-control" + maxlength="1" /> + <span class="input-group-btn"> + <button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button> + </span> + </div> + <span class="form-text text-muted"> + Provide the character that separates the fields in your file(s). It should + be the same character for all files (if more than one is provided).<br /> + A tab character will be assumed if you leave this field blank. See + <a href="#docs-file-separator" + title="Documentation for file-separator characters"> + documentation for more information</a>. + </span> + </div> + + <div class="form-group"> + <label for="txt-file-comment-character" class="form-label">File Comment-Character</label> + <div class="input-group"> + <input id="txt-file-comment-character" + name="file-comment-character" + type="text" + value="#" + class="form-control" + maxlength="1" /> + <span class="input-group-btn"> + <button id="btn-reset-file-comment-character" class="btn btn-info"> + Reset Default</button> + </span> + </div> + <span class="form-text text-muted"> + This specifies that lines that begin with the character provided will be + considered comment lines and ignored in their entirety. See + <a href="#docs-file-comment-character" + title="Documentation for comment characters"> + documentation for more information</a>. + </span> + </div> + + <div class="form-group"> + <label for="txt-file-na" class="form-label">File "No-Value" Indicators</label> + <div class="input-group"> + <input id="txt-file-na" + name="file-na" + type="text" + value="- NA N/A" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-reset-file-na" class="btn btn-info">Reset Default</button> + </span> + </div> + <span class="form-text text-muted"> + This specifies strings in your file indicate that there is no value for a + particular cell (a cell is where a column and row intersect). Provide a + space-separated list of strings if you have more than one way of + indicating no values. See + <a href="#docs-file-na" title="Documentation for no-value fields"> + documentation for more information</a>.</span> + </div> +</fieldset> + +<fieldset id="fldset-files"> + <legend>Data File(s)</legend> + + <fieldset id="fldset-descriptions-file"> + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-descriptions-transposed" + name="phenotype-descriptions-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-descriptions-transposed" + class="form-check-label"> + Description file transposed?</label> + </div> + + <div class="non-resumable-elements"> + <label for="finput-phenotype-descriptions" class="form-label"> + Phenotype Descriptions</label> + <input id="finput-phenotype-descriptions" + name="phenotype-descriptions" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-desc" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the phenotype descriptions, + <a href="#docs-file-phenotype-description" + title="Documentation of the phenotype data file format."> + the documentation for the expected format of the file</a>.</span> + </div> + {{display_resumable_elements( + "resumable-phenotype-descriptions", + "phenotype descriptions", + '<p>Drag and drop the CSV file that contains the descriptions of your + phenotypes here.</p> + + <p>The CSV file should be a matrix of + <strong>phenotypes × descriptions</strong> i.e. The first column + contains the phenotype names/identifiers whereas the first row is a list + of metadata fields like, "description", "units", etc.</p> + + <p>If the format is transposed (i.e. + <strong>descriptions × phenotypes</strong>) select the checkbox above. + </p> + + <p>Please see the + <a href="#docs-file-phenotype-description" + title="Documentation of the phenotype data file format."> + "Phenotypes Descriptions" documentation</a> section below for more + information on the expected format of the file provided here.</p>')}} + {{display_preview_table( + "tbl-preview-pheno-desc", "phenotype descriptions")}} + </div> + </fieldset> + + + <fieldset id="fldset-data-file"> + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-data-transposed" + name="phenotype-data-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-data-transposed" class="form-check-label"> + Data file transposed?</label> + </div> + + <div class="non-resumable-elements"> + <label for="finput-phenotype-data" class="form-label">Phenotype Data</label> + <input id="finput-phenotype-data" + name="phenotype-data" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-data" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the phenotype data. See + <a href="#docs-file-phenotype-data" + title="Documentation of the phenotype data file format."> + the documentation for the expected format of the file</a>.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-data", + "phenotype data", + '<p>Drag and drop a CSV file that contains the phenotypes numerical data + here. You can click the "Browse" button (below and to the right) to + select the file from your computer.</p> + + <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.</p> + + <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>) + select the checkbox above.</p> + <p>Please see the + <a href="#docs-file-phenotype-data" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below for more information + on the expected format for the file provided here.</p>')}} + {{display_preview_table("tbl-preview-pheno-data", "phenotype data")}} + </div> + </fieldset> + + + {%if population.Family in families_with_se_and_n%} + <fieldset id="fldset-se-file"> + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-se-transposed" + name="phenotype-se-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-se-transposed" class="form-check-label"> + Standard-Errors file transposed?</label> + </div> + <div class="group non-resumable-elements"> + <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label> + <input id="finput-phenotype-se" + name="phenotype-se" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-se" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the standard errors for the phenotypes, + computed from the data above.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-se", + "standard errors", + '<p>Drag and drop a CSV file that contains the phenotypes standard-errors + data here. You can click the "Browse" button (below and to the right) to + select the file from your computer.</p> + + <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.</p> + + <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>) + select the checkbox above.</p> + + <p>Please see the + <a href="#docs-file-phenotype-se" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below for more information + on the expected format of the file provided here.</p>')}} + + {{display_preview_table("tbl-preview-pheno-se", "standard errors")}} + </div> + </fieldset> + + + <fieldset id="fldset-n-file"> + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-n-transposed" + name="phenotype-n-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-n-transposed" class="form-check-label"> + Counts file transposed?</label> + </div> + <div class="non-resumable-elements"> + <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label> + <input id="finput-phenotype-n" + name="phenotype-n" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-n" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the number of samples/individuals used in + the computation of the standard errors above.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-n", + "number of samples/individuals", + '<p>Drag and drop a CSV file that contains the samples\' phenotypes counts + data here. You can click the "Browse" button (below and to the right) to + select the file from your computer.</p> + + <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.</p> + + <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>) + select the checkbox above.</p> + + <p>Please see the + <a href="#docs-file-phenotype-se" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below for more information + on the expected format of the file provided here.</p>')}} + + {{display_preview_table("tbl-preview-pheno-n", "number of samples/individuals")}} + </div> + </fieldset> +</fieldset> +{%endif%} +{%endblock%} + + +{%block page_documentation%} +<div class="row"> + <h2 class="heading" id="docs-help">Help</h2> + <h3 class="subheading">Common Features</h3> + <p>The following are the common expectations for <strong>ALL</strong> the + files provided in the form above: + <ul> + <li>The file <strong>MUST</strong> be character-separated values (CSV) + text file</li> + <li>The first row in the file <strong>MUST</strong> be a heading row, and + will be composed of the list identifiers for all of + samples/individuals/cases involved in your study.</li> + <li>The first column of data in the file <strong>MUST</strong> be the + identifiers for all of the phenotypes you wish to upload.</li> + </ul> + </p> + + <p>If you do not specify the separator character, then we will assume a + <strong>TAB</strong> character was used as your separator.</p> + + <p>We also assume you might include comments lines in your files. In that + case, if you do not specify what character denotes that a line in your files + is a comment line, we will assume the <strong>#</strong> character.<br /> + A comment <strong>MUST ALWAYS</strong> begin at the start of the line marked + with the comment character specified.</p> + + <h3 class="subheading" id="docs-file-metadata">File Metadata</h3> + <p>We request some details about your files to help us parse and process the + files correctly. The details we collect are:</p> + <dl> + <dt id="docs-file-separator">File separator</dt> + <dd>The files you provide should be character-separated value (CSV) files. + We need to know what character you used to separate the values in your + file. Some common ones are the Tab character, the comma, etc.<br /> + Providing that information makes it possible for the system to parse and + process your files correctly.<br> + <strong>NOTE:</strong> All the files you upload MUST use the same + separator.</dd> + + <dt id="docs-file-comment-character">Comment character</dt> + <dd>We support use of comment lines in your files. We only support one type + of comment style, the <em>line comment</em>.<br /> + This mean the comment begins at the start of the line, and the end of that + line indicates the end of that comment. If you have a really long comment, + then you need to break it across multiple lines, marking each line a + comment line.<br /> + The "comment character" is the character at the start of the line that + indicates that the line is a line comment.</dd> + + <dt id="docs-file-na">No-Value indicator(s)</dt> + <dd>Data in the real world is messy, and in some cases, entirely absent. You + need to indicate, in your files, that a particular field did not have a + value, and once you do that, you then need to let the system know how you + mark such fields. Common ways of indicating "empty values" are, leaving + the field blank, using a character such as '-', or using strings like + "NA", "N/A", "NULL", etc.<br /> + Providing this information will help with parsing and processing such + no-value fields the correct way.</dd> + </dl> + + <h3 class="subheading" id="docs-file-phenotype-description"> + file: Phenotypes Descriptions</h3> + <p>The data in this file is a matrix of <em>phenotypes × metadata-fields</em>. + Please note we use the term "metadata-fields" above loosely, due to lack of + a good word for this.</p> + <p>The file <strong>MUST</strong> have columns in this order: + <dl> + <dt>Phenotype Identifiers</dt> + <dd>These are the names/identifiers for your phenotypes. These + names/identifiers are the same ones you will have in all the other files you are + uploading.</dd> + + <dt>Descriptions</dt> + <dd>Each phenotype will need a description. Good description are necessary + to inform other people of what the data is about. Good description are + hard to construct, so we provide + <a href="https://info.genenetwork.org/faq.php#q-22" + title="How to write phenotype descriptions"> + advice on describing your phenotypes.</a></dd> + + <dt>Units</dt> + <dd>Each phenotype will need units for the measurements taken. If there are + none, then indicate the field is a no-value field.</dd> + </dl></p> + <p>You can add more columns after those three if you want to, but these 3 + <strong>MUST</strong> be present.</p> + <p>The file would, for example, look like the following:</p> + <code>id,description,units,…<br /> + pheno10001|Central nervous system, behavior, cognition; …|mg|…<br /> + pheno10002|Aging, metabolism, central nervous system: …|mg|…<br /> + ⋮<br /></code> + + <p><strong>Note 01</strong>: The first usable row is the heading row.</p> + <p><strong>Note 02: </strong>This example demonstrates a subtle issue that + could make your CSV file invalid — the choice of your field separator + character.<br > + In the example above, we use the pipe character (<code>|</code>) as our + field separator. This is because, if we follow the advice on how to write + good descriptions, then we cannot use the comma as our separator – if + we did, then our CSV file would be invalid because the system would have no + way to tell the difference between the comma as a field separator, and the + comma as a way to separate the "general category and ontology terms".</p> + + <h3 class="subheading">file: Phenotype Data, Standard Errors and/or Sample Counts</h3> + <span id="docs-file-phenotype-data"></span> + <span id="docs-file-phenotype-se"></span> + <span id="docs-file-phenotype-n"></span> + <p>The data is a matrix of <em>samples(or individuals) × phenotypes</em>, e.g.</p> + <code> + # num-cases: 2549 + # num-phenos: 13 + id,pheno10001,pheno10002,pheno10003,pheno10004,53.099998,…<br /> + IND001,61.400002,49,62.5,55.099998,…<br /> + IND002,54.099998,50.099998,53.299999,55.099998,…<br /> + IND003,483,403,501,403,…<br /> + IND004,49.799999,45.5,62.900002,NA,…<br /> + ⋮<br /></code> + + <p>where <code>IND001,IND002,IND003,IND004,…</code> are the + samples/individuals/cases in your study, and + <code>pheno10001,pheno10002,pheno10004,pheno10004,…</code> are the + identifiers for your phenotypes.</p> + <p>The lines beginning with the "<em>#</em>" symbol (i.e. + <code># num-cases: 2549</code> and <code># num-phenos: 13</code> are comment + lines and will be ignored</p> + <p>In this example, the comma (,) is used as the file separator.</p> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} + + +{%block more_javascript%} +<script src="{{url_for('base.node_modules', + filename='resumablejs/resumable.js')}}"></script> +<script type="text/javascript" src="/static/js/files.js"></script> + +<script type="text/javascript"> + $("#btn-reset-file-separator").on("click", (event) => { + event.preventDefault(); + $("#txt-file-separator").val("\t"); + $("#txt-file-separator").trigger("change"); + }); + $("#btn-reset-file-comment-character").on("click", (event) => { + event.preventDefault(); + $("#txt-file-comment-character").val("#"); + $("#txt-file-comment-character").trigger("change"); + }); + $("#btn-reset-file-na").on("click", (event) => { + event.preventDefault(); + $("#txt-file-na").val("- NA N/A"); + $("#txt-file-na").trigger("change"); + }); + + var update_preview = (table, filedata, formdata, numrows) => { + table.find("thead tr").remove() + table.find(".data-row").remove(); + var linenum = 0; + var tableheader = table.find("thead"); + var tablebody = table.find("tbody"); + var numheadings = 0; + var navalues = formdata + .na_strings + .split(" ") + .map((v) => {return v.trim();}) + .filter((v) => {return Boolean(v);}); + filedata.forEach((line) => { + if(line.startsWith(formdata.comment_char) || linenum >= numrows) { + return false; + } + var row = $("<tr></tr>"); + line.split(formdata.separator) + .map((field) => { + var value = field.trim(); + if(navalues.includes(value)) { + return "⋘NUL⋙"; + } + return value; + }) + .filter((field) => { + return (field !== "" && field != undefined && field != null); + }) + .forEach((field) => { + if(linenum == 0) { + numheadings += 1; + var tablefield = $("<th></th>"); + tablefield.text(field); + row.append(tablefield); + } else { + add_class(row, "data-row"); + var tablefield = $("<td></td>"); + tablefield.text(field); + row.append(tablefield); + } + }); + + if(linenum == 0) { + tableheader.append(row); + } else { + tablebody.append(row); + } + linenum += 1; + }); + + if(table.find("tbody tr.data-row").length > 0) { + add_class(table.find(".data-row-template"), "visually-hidden"); + } else { + remove_class(table.find(".data-row-template"), "visually-hidden"); + } + }; + + var makePreviewUpdater = (preview_table) => { + return (data) => { + update_preview( + preview_table, + data, + filesMetadata(), + PREVIEW_ROWS); + }; + }; + + var preview_tables_to_elements_map = { + "#tbl-preview-pheno-desc": "#finput-phenotype-descriptions", + "#tbl-preview-pheno-data": "#finput-phenotype-data", + "#tbl-preview-pheno-se": "#finput-phenotype-se", + "#tbl-preview-pheno-n": "#finput-phenotype-n" + }; + + var filesMetadata = () => { + return { + "separator": $("#txt-file-separator").val(), + "comment_char": $( + "#txt-file-comment-character").val(), + "na_strings": $("#txt-file-na").val() + } + }; + + var PREVIEW_ROWS = 5; + + var handler_update_previews = (event) => { + Object.entries(preview_tables_to_elements_map).forEach((mapentry) => { + var preview_table = $(mapentry[0]); + var file_input = $(mapentry[1]); + if(file_input[0].files.length > 0) { + readFirstNLines( + file_input[0].files[0], + 10, + [makePreviewUpdater(preview_table)]); + } + }); + + if(typeof(resumables) !== "undefined") { + resumables.forEach((resumable) => { + if(resumable.files.length > 0) { + readFirstNLines( + resumable.files[0].file, + 10, + [makePreviewUpdater(resumable.preview_table)]); + } + }); + } + }; + + [ + "#txt-file-separator", + "#txt-file-comment-character", + "#txt-file-na" + ].forEach((elementid) => { + $(elementid).on("change", handler_update_previews); + }); + + [ + "#finput-phenotype-descriptions", + "#finput-phenotype-data", + "#finput-phenotype-se", + "#finput-phenotype-n" + ].forEach((elementid) => { + $(elementid).on("change", (event) => { + readFirstNLines( + event.target.files[0], + 10, + [makePreviewUpdater( + $("#" + event.target.getAttribute("data-preview-table")))]); + }); + }); + + + var resumableDisplayFiles = (display_area, files) => { + files.forEach((file) => { + display_area.find(".file-display").remove(); + var display_element = display_area + .find(".file-display-template") + .clone(); + remove_class(display_element, "visually-hidden"); + remove_class(display_element, "file-display-template"); + add_class(display_element, "file-display"); + display_element.find(".filename").text(file.name + || file.fileName + || file.relativePath + || file.webkitRelativePath); + display_element.find(".filesize").text( + (file.size / (1024*1024)).toFixed(2) + "MB"); + display_element.find(".fileuniqueid").text(file.uniqueIdentifier); + display_element.find(".filemimetype").text(file.file.type); + display_area.append(display_element); + }); + }; + + + var indicateProgress = (resumable, progress_bar) => { + return () => {/*Has no event!*/ + var progress = (resumable.progress() * 100).toFixed(2); + var pbar = progress_bar.find(".progress-bar"); + remove_class(progress_bar, "visually-hidden"); + pbar.css("width", progress+"%"); + pbar.attr("aria-valuenow", progress); + pbar.text("Uploading: " + progress + "%"); + }; + }; + + var retryUpload = (retry_button, cancel_button) => { + retry_button.on("click", (event) => { + resumable.files.forEach((file) => {file.retry();}); + add_class(retry_button, "visually-hidden"); + remove_class(cancel_button, "visually-hidden"); + add_class(browse_button, "visually-hidden"); + }); + }; + + var cancelUpload = (cancel_button, retry_button) => { + cancel_button.on("click", (event) => { + resumable.files.forEach((file) => { + if(file.isUploading()) { + file.abort(); + } + }); + add_class(cancel_button, "visually-hidden"); + remove_class(retry_button, "visually-hidden"); + remove_class(browse_button, "visually-hidden"); + }); + }; + + + var startUpload = (browse_button, retry_button, cancel_button) => { + return (event) => { + remove_class(cancel_button, "visually-hidden"); + add_class(retry_button, "visually-hidden"); + add_class(browse_button, "visually-hidden"); + }; + }; + + var processForm = (form) => { + var formdata = new FormData(form); + uploaded_files.forEach((msg) => { + formdata.delete(msg["file-input-name"]); + formdata.append(msg["file-input-name"], JSON.stringify({ + "uploaded-file": msg["uploaded-file"], + "original-name": msg["original-name"] + })); + }); + formdata.append("resumable-upload", "true"); + formdata.append("publication-id", $("#txt-publication-id").val()); + return formdata; + } + + var uploaded_files = new Set(); + var submitForm = (new_file) => { + uploaded_files.add(new_file); + if(uploaded_files.size === resumables.length) { + var form = $("#frm-add-phenotypes"); + if(form.length !== 1) { + // TODO: Handle error somehow? + alert("Could not find form!!!"); + return false; + } + + $.ajax({ + "url": form.attr("action"), + "type": "POST", + "data": processForm(form[0]), + "processData": false, + "contentType": false, + "success": (data, textstatus, jqxhr) => { + // TODO: Redirect to endpoint that should come as part of the + // success/error message. + console.log("SUCCESS DATA: ", data); + console.log("SUCCESS STATUS: ", textstatus); + console.log("SUCCESS jqXHR: ", jqxhr); + window.location.assign(window.location.origin + data["redirect-to"]); + }, + }); + return false; + } + return false; + }; + + var uploadSuccess = (file_input_name) => { + return (file, message) => { + submitForm({...JSON.parse(message), "file-input-name": file_input_name}); + }; + }; + + + var uploadError = () => { + return (message, file) => { + $("#frm-add-phenotypes input[type=submit]").removeAttr("disabled"); + console.log("THE FILE:", file); + console.log("THE ERROR MESSAGE:", message); + }; + }; + + + + var makeResumableObject = (form_id, file_input_id, resumable_element_id, preview_table_id) => { + var the_form = $("#" + form_id); + var file_input = $("#" + file_input_id); + var submit_button = the_form.find("input[type=submit]"); + if(file_input.length != 1) { + return false; + } + var r = errorHandler( + fileSuccessHandler( + uploadStartHandler( + filesAddedHandler( + markResumableDragAndDropElement( + makeResumableElement( + the_form.attr("data-resumable-target"), + file_input.parent(), + $("#" + resumable_element_id), + submit_button, + ["csv", "tsv", "txt"]), + file_input.parent(), + $("#" + resumable_element_id), + $("#" + resumable_element_id + "-browse-button")), + (files) => { + // TODO: Also trigger preview! + resumableDisplayFiles( + $("#" + resumable_element_id + "-selected-files"), files); + files.forEach((file) => { + readFirstNLines( + file.file, + 10, + [makePreviewUpdater( + $("#" + preview_table_id))]) + }); + }), + startUpload($("#" + resumable_element_id + "-browse-button"), + $("#" + resumable_element_id + "-retry-button"), + $("#" + resumable_element_id + "-cancel-button"))), + uploadSuccess(file_input.attr("name"))), + uploadError()); + + /** Setup progress indicator **/ + progressHandler( + r, + indicateProgress(r, $("#" + resumable_element_id + "-progress-bar"))); + + return r; + }; + + var resumables = [ + ["frm-add-phenotypes", "finput-phenotype-descriptions", "resumable-phenotype-descriptions", "tbl-preview-pheno-desc"], + ["frm-add-phenotypes", "finput-phenotype-data", "resumable-phenotype-data", "tbl-preview-pheno-data"], + ["frm-add-phenotypes", "finput-phenotype-se", "resumable-phenotype-se", "tbl-preview-pheno-se"], + ["frm-add-phenotypes", "finput-phenotype-n", "resumable-phenotype-n", "tbl-preview-pheno-n"], + ].map((row) => { + r = makeResumableObject(row[0], row[1], row[2], row[3]); + r.preview_table = $("#" + row[3]); + return r; + }).filter((val) => { + return Boolean(val); + }); + + $("#frm-add-phenotypes input[type=submit]").on("click", (event) => { + event.preventDefault(); + console.debug(); + if ($("#txt-publication-id").val() == "") { + alert("You MUST provide a publication for the phenotypes."); + return false; + } + // TODO: Check all the relevant files exist + // TODO: Verify that files are not duplicated + var filenames = []; + var nondupfiles = []; + resumables.forEach((r) => { + var fname = r.files[0].file.name; + filenames.push(fname); + if(!nondupfiles.includes(fname)) { + nondupfiles.push(fname); + } + }); + + // Check that all files were provided + if(resumables.length !== filenames.length) { + window.alert("You MUST provide all the files requested."); + event.target.removeAttribute("disabled"); + return false; + } + + // Check that there are no duplicate files + var duplicates = Object.entries(filenames.reduce( + (acc, curr, idx, arr) => { + acc[curr] = (acc[curr] || 0) + 1; + return acc; + }, + {})).filter((entry) => {return entry[1] !== 1;}); + if(duplicates.length > 0) { + var msg = "The file(s):\r\n"; + msg = msg + duplicates.reduce( + (msgstr, afile) => { + return msgstr + " • " + afile[0] + "\r\n"; + }, + ""); + msg = msg + "is(are) duplicated. Please fix and try again."; + window.alert(msg); + event.target.removeAttribute("disabled"); + return false; + } + // TODO: Check all fields + // Start the uploads. + event.target.setAttribute("disabled", "disabled"); + resumables.forEach((r) => {r.upload();}); + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html new file mode 100644 index 0000000..898fc0c --- /dev/null +++ b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html @@ -0,0 +1,207 @@ +{%extends "phenotypes/add-phenotypes-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block frm_add_phenotypes_documentation%} +<p>Select the zip file bundle containing information on the phenotypes you + wish to upload, then click the "Upload Phenotypes" button below to + upload the data.</p> +<p>If you wish to upload the files individually instead, + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="">click here</a>.</p> +<p>See the <a href="#section-file-formats">File Formats</a> section below + to get an understanding of what is expected of the bundle files you + upload.</p> +{%endblock%} + +{%block frm_add_phenotypes_elements%} +<div class="form-group"> + <label for="finput-phenotypes-bundle" class="form-label"> + Phenotypes Bundle</label> + <input type="file" + id="finput-phenotypes-bundle" + name="phenotypes-bundle" + accept="application/zip, .zip" + required="required" + class="form-control" /> +</div> +{%endblock%} + +{%block page_documentation%} +<div class="row"> + <h2 class="heading" id="section-file-formats">File Formats</h2> + <p>We accept an extended form of the + <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html#format-of-the-data-files" + title="R/qtl2 software input file format documentation"> + input files' format used with the R/qtl2 software</a> as a single ZIP + file</p> + <p>The files that are used for this feature are: + <ul> + <li>the <em>control</em> file</li> + <li><em>pheno</em> file(s)</li> + <li><em>phenocovar</em> file(s)</li> + <li><em>phenose</em> files(s)</li> + </ul> + </p> + <p>Other files within the bundle will be ignored, for this feature.</p> + <p>The following section will detail the expectations for each of the + different file types within the uploaded ZIP file bundle for phenotypes:</p> + + <h3 class="subheading">Control File</h3> + <p>There <strong>MUST be <em>one, and only one</em></strong> file that acts + as the control file. This file can be: + <ul> + <li>a <em>JSON</em> file, or</li> + <li>a <em>YAML</em> file.</li> + </ul> + </p> + + <p>The control file is useful for defining things about the bundle such as:</p> + <ul> + <li>The field separator value (default: <code>sep: ','</code>). There can + only ever be one field separator and it <strong>MUST</strong> be the same + one for <strong>ALL</strong> files in the bundle.</li> + <li>The comment character (default: <code>comment.char: '#'</code>). Any + line that starts with this character will be considered a comment line and + be ignored in its entirety.</li> + <li>Code for missing values (default: <code>na.strings: 'NA'</code>). You + can specify more than one code to indicate missing values, e.g. + <code>{…, "na.strings": ["NA", "N/A", "-"], …}</code></li> + </ul> + + <h3 class="subheading"><em>pheno</em> File(s)</h3> + <p>These files are the main data files. You must have at least one of these + files in your bundle for it to be valid for this step.</p> + <p>The data is a matrix of <em>individuals × phenotypes</em> by default, as + below:<br /> + <code> + id,10001,10002,10003,10004,…<br /> + BXD1,61.400002,54.099998,483,49.799999,…<br /> + BXD2,49,50.099998,403,45.5,…<br /> + BXD5,62.5,53.299999,501,62.900002,…<br /> + BXD6,53.099998,55.099998,403,NA,…<br /> + ⋮<br /></code> + </p> + <p>If the <code>pheno_transposed</code> value is set to <code>True</code>, + then the data will be a <em>phenotypes × individuals</em> matrix as in the + example below:<br /> + <code> + id,BXD1,BXD2,BXD5,BXD6,…<br /> + 10001,61.400002,49,62.5,53.099998,…<br /> + 10002,54.099998,50.099998,53.299999,55.099998,…<br /> + 10003,483,403,501,403,…<br /> + 10004,49.799999,45.5,62.900002,NA,…<br /> + ⋮ + </code> + </p> + + + <h3 class="subheading"><em>phenocovar</em> File(s)</h3> + <p>At least one phenotypes metadata file with the metadata values such as + descriptions, PubMed Identifier, publication titles (if present), etc.</p> + <p>The data in this/these file(s) is a matrix of + <em>phenotypes × phenotypes-covariates</em>. The first column is always the + phenotype names/identifiers — same as in the R/qtl2 format.</p> + <p><em>phenocovar</em> files <strong>should never be transposed</strong>!</p> + <p>This file <strong>MUST</strong> be present in the bundle, and have data for + the bundle to be considered valid by our system for this step.<br /> + In addition to that, the following are the fields that <strong>must be + present</strong>, and + have values, in the file before the file is considered valid: + <ul> + <li><em>description</em>: A description for each phenotype. Useful + for users to know what the phenotype is about.</li> + <li><em>units</em>: The units of measurement for the phenotype, + e.g. milligrams for brain weight, centimetres/millimetres for + tail-length, etc.</li> + </ul></p> + + <p>The following <em>optional</em> fields can also be provided: + <ul> + <li><em>pubmedid</em>: A PubMed Identifier for the publication where + the phenotype is published. If this field is not provided, the system will + assume your phenotype is not published.</li> + </ul> + </p> + <p>These files will be marked up in the control file with the + <code>phenocovar</code> key, as in the examples below: + <ol> + <li>JSON: single file<br /> + <code>{<br /> + ⋮,<br /> + "phenocovar": "your_covariates_file.csv",<br /> + ⋮<br /> + } + </code> + </li> + <li>JSON: multiple files<br /> + <code>{<br /> + ⋮,<br /> + "phenocovar": [<br /> + "covariates_file_01.csv",<br /> + "covariates_file_01.csv",<br /> + ⋮<br /> + ],<br /> + ⋮<br /> + } + </code> + </li> + <li>YAML: single file or<br /> + <code> + ⋮<br /> + phenocovar: your_covariates_file.csv<br /> + ⋮ + </code> + </li> + <li>YAML: multiple files<br /> + <code> + ⋮<br /> + phenocovar:<br /> + - covariates_file_01.csv<br /> + - covariates_file_02.csv<br /> + - covariates_file_03.csv<br /> + …<br /> + ⋮ + </code> + </li> + </ol> + </p> + + <h3 class="subheading"><em>phenose</em> and <em>phenonum</em> File(s)</h3> + <p>These are extensions to the R/qtl2 standard, i.e. these types ofs file are + not supported by the original R/qtl2 file format</p> + <p>We use these files to upload the standard errors (<em>phenose</em>) when + the data file (<em>pheno</em>) is average data. In that case, the + <em>phenonum</em> file(s) contains the number of individuals that were + involved when computing the averages.</p> + <p>Both types of files are matrices of <em>individuals × phenotypes</em> by + default. Like the related <em>pheno</em> files, if + <code>pheno_transposed: True</code>, then the file will be a matrix of + <em>phenotypes × individuals</em>.</p> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html index 3bc5dea..adbc012 100644 --- a/uploader/templates/phenotypes/base.html +++ b/uploader/templates/phenotypes/base.html @@ -6,7 +6,14 @@ {%else%} class="breadcrumb-item" {%endif%}> + {%if dataset is mapping%} + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">{{dataset.Name}}</a> + {%else%} <a href="{{url_for('species.populations.phenotypes.index')}}">Phenotypes</a> + {%endif%} </li> {%block lvl4_breadcrumbs%}{%endblock%} {%endblock%} diff --git a/uploader/templates/phenotypes/bulk-edit-upload.html b/uploader/templates/phenotypes/bulk-edit-upload.html new file mode 100644 index 0000000..d0f38f5 --- /dev/null +++ b/uploader/templates/phenotypes/bulk-edit-upload.html @@ -0,0 +1,62 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +<div class="row"> + <p>Upload the edited file you downloaded and edited.</p> +</div> + +<div class="row"> + <form id="frm-bulk-edit-upload" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_upload_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + enctype="multipart/form-data"> + + <div class="form-group row"> + <label for="file-upload-bulk-edit-upload" + class="form-label col-form-label col-sm-2"> + Edited File</label> + <div class="col-sm-10"> + <input id="file-upload-bulk-edit-upload" + name="file-upload-bulk-edit-upload" + class="form-control" + type="file" + accept="text/tab-separated-values" + required="required" /> + </div> + </div> + + <input type="submit" class="btn btn-primary" + value="upload to edit" /> + + </form> +</div> +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html new file mode 100644 index 0000000..19a2b34 --- /dev/null +++ b/uploader/templates/phenotypes/create-dataset.html @@ -0,0 +1,108 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="create-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}">Create Datasets</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <p>Create a new phenotype dataset.</p> +</div> + +<div class="row"> + <form id="frm-create-pheno-dataset" + action="{{url_for('species.populations.phenotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}" + method="POST"> + + <div class="form-group"> + <label class="form-label" for="txt-dataset-name">Name</label> + {%if errors["dataset-name"] is defined%} + <small class="form-text text-muted danger"> + <p>{{errors["dataset-name"]}}</p></small> + {%endif%} + <input type="text" + name="dataset-name" + id="txt-dataset-name" + value="{{original_formdata.get('dataset-name') or (population.Name + 'Publish')}}" + {%if errors["dataset-name"] is defined%} + class="form-control danger" + {%else%} + class="form-control" + {%endif%} + required="required" /> + <small class="form-text text-muted"> + <p>A short representative name for the dataset.</p> + <p>Recommended: Use the population name and append "Publish" at the end. + <br />This field will only accept names composed of + letters ('A-Za-z'), numbers (0-9), hyphens and underscores.</p> + </small> + </div> + + <div class="form-group"> + <label class="form-label" for="txt-dataset-fullname">Full Name</label> + {%if errors["dataset-fullname"] is defined%} + <small class="form-text text-muted danger"> + <p>{{errors["dataset-fullname"]}}</p></small> + {%endif%} + <input id="txt-dataset-fullname" + name="dataset-fullname" + type="text" + value="{{original_formdata.get('dataset-fullname', '')}}" + {%if errors["dataset-fullname"] is defined%} + class="form-control danger" + {%else%} + class="form-control" + {%endif%} + required="required" /> + <small class="form-text text-muted"> + <p>A longer, descriptive name for the dataset. The name is meant for use + by humans, and therefore, it should be clear what the dataset contains + from the name.</p> + </small> + </div> + + <div class="form-group"> + <label class="form-label" for="txt-dataset-shortname">Short Name</label> + <input id="txt-dataset-shortname" + name="dataset-shortname" + type="text" + class="form-control" + value="{{original_formdata.get('dataset-shortname') or (population.Name + 'Publish')}}" /> + <small class="form-text text-muted"> + <p>An optional, short name for the dataset. <br /> + If this is not provided, it will default to the value provided for the + <strong>Name</strong> field above.</p></small> + </div> + + <div class="form-group"> + <input type="submit" + class="btn btn-primary" + value="create phenotype dataset" /> + </div> + + </form> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html new file mode 100644 index 0000000..115d6af --- /dev/null +++ b/uploader/templates/phenotypes/edit-phenotype.html @@ -0,0 +1,208 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="edit-phenotype"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">edit phenotype data</h2> + <p>The forms provided in this page help you update the data for the + phenotypes, and the publication information for the phenotype, + respectively.</p> +</div> + +<div class="row"> + <h3 class="subheading">Basic metadata</h3> + <form name="frm-phenotype-basic-metadata" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}"> + <input type="hidden" name="phenotype-id" value="{{phenotype.Id}}" /> + <div class="form-group"> + <label for="txt-pre-publication-description" + class="control-label col-sm-2">Pre-Publication Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-pre-publication-description" + name="pre-publication-description" + class="form-control" + value="{{phenotype['Pre_publication_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-pre-publication-abbreviation" + class="control-label col-sm-2">Pre-Publication Abbreviation</label> + <div class="col-sm-10"> + <input type="text" + id="txt-pre-publication-abbreviation" + name="pre-publication-abbreviation" + class="form-control" + value="{{phenotype['Pre_publication_abbreviation'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-post-publication-description" + class="control-label col-sm-2">Post-Publication Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-post-publication-description" + name="post-publication-description" + class="form-control" + value="{{phenotype['Post_publication_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-post-publication-abbreviation" + class="control-label col-sm-2">Post-Publication Abbreviation</label> + <div class="col-sm-10"> + <input type="text" + id="txt-post-publication-abbreviation" + name="post-publication-abbreviation" + class="form-control" + value="{{phenotype['Post_publication_abbreviation'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-original-description" + class="control-label col-sm-2">Original Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-original-description" + name="original-description" + class="form-control" + value="{{phenotype['Original_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-units" + class="control-label col-sm-2">units</label> + <div class="col-sm-10"> + <input type="text" + id="txt-units" + name="units" + class="form-control" + required="required" + value="{{phenotype['Units']}}" /> + </div> + </div> + + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary" + value="update basic metadata"> + </div> + </div> + </form> +</div> + + +<div class="row"> + <h3 class="subheading">phenotype data</h3> + <form id="frm-edit-phenotype-data" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}"> + <div style="max-height: 23.37em;overflow-y: scroll;"> + <table class="table table-striped table-responsive table-form-table"> + <thead style="position: sticky; top: 0;"> + <tr> + <th>#</th> + <th>Sample</th> + <th>Value</th> + {%if population.Family in families_with_se_and_n%} + <th>Standard-Error</th> + <th>Number of Samples</th> + {%endif%} + </tr> + </thead> + + <tbody> + {%for item in phenotype.data%} + <tr> + <td>{{loop.index}}</td> + <td>{{item.StrainName}}</td> + <td> + <input type="text" + name="value-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.value}}" + class="form-control" /> + <input type="hidden" + name="value-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.value}}" /></td> + {%if population.Family in families_with_se_and_n%} + <td> + <input type="text" + name="se-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.error or ''}}" + data-original-value="{{item.error or ''}}" + class="form-control" /> + <input type="hidden" + name="se-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.error or ''}}" /></td> + <td> + <input type="text" + name="n-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.count or ''}}" + data-original-value="{{item.count or "-"}}" + class="form-control" /> + <input type="hidden" + name="n-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.count or ''}}" /></td> + {%endif%} + </tr> + {%endfor%} + </tbody> + </table> + </div> + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary" + value="update data" /> + </div> + </div> + </form> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/index.html b/uploader/templates/phenotypes/index.html index 0c691e6..689c28e 100644 --- a/uploader/templates/phenotypes/index.html +++ b/uploader/templates/phenotypes/index.html @@ -11,16 +11,11 @@ {{flash_all_messages()}} <div class="row"> - <p>This section deals with phenotypes that - <span class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - … what are the characteristics of these phenotypes? …</span></p> - <p>Select the species to begin the process of viewing/uploading data about - your phenotypes</p> + {{select_species_form(url_for("species.populations.phenotypes.index"), species)}} </div> +{%endblock%} -<div class="row"> - {{select_species_form(url_for("species.populations.phenotypes.index"), - species)}} -</div> + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> {%endblock%} diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html new file mode 100644 index 0000000..257f726 --- /dev/null +++ b/uploader/templates/phenotypes/job-status.html @@ -0,0 +1,155 @@ +{%extends "phenotypes/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block extrameta%} +{%if job and job.status not in ("success", "completed:success", "error", "completed:error")%} +<meta http-equiv="refresh" content="5" /> +{%endif%} +{%endblock%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} + +{%if job%} +<h4 class="subheading">Progress</h4> +<div class="row" style="overflow:scroll;"> + <p><strong>Process Status:</strong> {{job.status}}</p> + {%if metadata%} + <table class="table table-responsive"> + <thead> + <tr> + <th>File</th> + <th>Status</th> + <th>Lines Processed</th> + <th>Total Errors</th> + </tr> + </thead> + + <tbody> + {%for file,meta in metadata.items()%} + <tr> + <td>{{file}}</td> + <td>{{meta.status}}</td> + <td>{{meta.linecount}}</td> + <td>{{meta["total-errors"]}}</td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} +</div> + +<div class="row"> + {%if job.status in ("completed:success", "success")%} + <p> + {%if errors | length == 0%} + <a href="{{url_for('species.populations.phenotypes.review_job_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" + class="btn btn-primary" + title="Continue to process data">Continue</a> + {%else%} + <span class="text-muted" + disabled="disabled" + style="border: solid 2px;border-radius: 5px;padding: 0.3em;"> + Cannot continue due to errors. Please fix the errors first. + </span> + {%endif%} + </p> + {%endif%} +</div> + +<h4 class="subheading">Errors</h4> +<div class="row" style="max-height: 20em; overflow: scroll;"> + {%if errors | length == 0 %} + <p class="text-info"> + <span class="glyphicon glyphicon-info-sign"></span> + No errors found so far + </p> + {%else%} + <table class="table table-responsive"> + <thead style="position: sticky; top: 0; background: white;"> + <tr> + <th>File</th> + <th>Row</th> + <th>Column</th> + <th>Value</th> + <th>Message</th> + </tr> + </thead> + + <tbody style="font-size: 0.9em;"> + {%for error in errors%} + <tr> + <td>{{error.filename}}</td> + <td>{{error.rowtitle}}</td> + <td>{{error.coltitle}}</td> + <td>{%if error.cellvalue is not none and error.cellvalue | length > 25%} + {{error.cellvalue[0:24]}}… + {%else%} + {{error.cellvalue}} + {%endif%} + </td> + <td> + {%if error.message | length > 250 %} + {{error.message[0:249]}}… + {%else%} + {{error.message}} + {%endif%} + </td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} +</div> + +<div class="row"> + {{cli_output(job, "stdout")}} +</div> + +<div class="row"> + {{cli_output(job, "stderr")}} +</div> + +{%else%} +<div class="row"> + <h3 class="text-danger">No Such Job</h3> + <p>Could not find a job with the ID: {{job_id}}</p> + <p> + Please go back to + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="'{{dataset.Name}}' dataset page"> + the '{{dataset.Name}}' dataset page</a> + to upload new phenotypes or edit existing ones.</p> +</div> +{%endif%} +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html index 360fd2c..2cf2c7f 100644 --- a/uploader/templates/phenotypes/list-datasets.html +++ b/uploader/templates/phenotypes/list-datasets.html @@ -48,11 +48,16 @@ </tbody> </table> {%else%} - <p class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no dataset for this population!</p> - <p><a href="#" - class="not-implemented btn btn-primary" + <p>Phenotypes need to go into a dataset. We do not currently have a dataset + for species <strong>'{{species["FullName"]}} ({{species["Name"]}})'</strong> + phenotypes.</p> + + <p>Do, please, create a new dataset by clicking on the "Create Dataset" button + below and following the prompts/instructions.</p> + <p><a href="{{url_for('species.populations.phenotypes.create_dataset', + species_id=species.SpeciesId, + population_id=population.Id)}}" + class="btn btn-primary" title="Create a new phenotype dataset.">create dataset</a></p> {%endif%} </div> diff --git a/uploader/templates/phenotypes/load-phenotypes-success.html b/uploader/templates/phenotypes/load-phenotypes-success.html new file mode 100644 index 0000000..645be16 --- /dev/null +++ b/uploader/templates/phenotypes/load-phenotypes-success.html @@ -0,0 +1,42 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="load-phenotypes-success"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">Add Phenotypes</a> +</li> +{%endblock%} + +{%block contents%} +<div class="row"> + <p>You have successfully loaded + <!-- maybe indicate the number of phenotypes here? -->your + new phenotypes into the database.</p> + <!-- TODO: Maybe notify user that they have sole access. --> + <!-- TODO: Maybe provide a link to go to GeneNetwork to view the data. --> + <p>View your data + <a href="{{search_page_uri}}" + target="_blank">on GeneNetwork2</a>. + You might need to login to GeneNetwork2 to view specific traits.</p> +</div> +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} + + +{%block more_javascript%}{%endblock%} diff --git a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html new file mode 100644 index 0000000..11b108b --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html @@ -0,0 +1,31 @@ +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%macro display_pheno_dataset_card(species, population, dataset)%} +{{display_population_card(species, population)}} + +<div class="card"> + <div class="card-body"> + <h5 class="card-title">Phenotypes' Dataset</h5> + <div class="card-text"> + <table class="table"> + <tbody> + <tr> + <td>Name</td> + <td>{{dataset.Name}}</td> + </tr> + + <tr> + <td>Full Name</td> + <td>{{dataset.FullName}}</td> + </tr> + + <tr> + <td>Short Name</td> + <td>{{dataset.ShortName}}</td> + </tr> + </tbody> + </table> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/macro-display-preview-table.html b/uploader/templates/phenotypes/macro-display-preview-table.html new file mode 100644 index 0000000..5a4c422 --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-preview-table.html @@ -0,0 +1,19 @@ +{%macro display_preview_table(tableid, filetype)%} +<div class="card"> + <div class="card-body"> + <h5 class="card-title">{{filetype | title}}: File Preview</h5> + <div class="card-text" style="overflow: scroll;"> + <table id="{{tableid}}" class="table table-condensed table-responsive"> + <thead> + <tr> + </tr> + <tbody> + <tr> + <td class="data-row-template text-info"></td> + </tr> + </tbody> + </table> + </div> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/macro-display-resumable-elements.html b/uploader/templates/phenotypes/macro-display-resumable-elements.html new file mode 100644 index 0000000..ed14ea5 --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-resumable-elements.html @@ -0,0 +1,60 @@ +{%macro display_resumable_elements(id, title, help)%} +<div id="{{id}}" + class="resumable-elements visually-hidden" + style="background:#D4D4EE;border-radius: 5px;;padding: 1em;border-left: solid #B2B2CC 1px;border-bottom: solid #B2B2CC 2px;margin-top:0.3em;"> + <strong style="line-height: 1.2em;">{{title | title}}</strong> + + <span class="form-text text-muted">{{help | safe}}</span> + + <div id="{{id}}-selected-files" + class="resumable-selected-files" + style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"> + <div class="panel panel-info file-display-template visually-hidden"> + <div class="panel-heading filename">The Filename Goes Here!</div> + <div class="panel-body"> + <ul> + <li> + <strong>Name</strong>: + <span class="filename">the file's name</span></li> + + <li><strong>Size</strong>: <span class="filesize">0 MB</span></li> + + <li> + <strong>Unique Identifier</strong>: + <span class="fileuniqueid">brrr</span></li> + + <li> + <strong>Mime</strong>: + <span class="filemimetype">text/csv</span></li> + </ul> + </div> + </div> + </div> + + <a id="{{id}}-browse-button" + class="resumable-browse-button btn btn-info" + href="#{{id}}" + style="margin-left: 80%;">Browse</a> + + <div id="{{id}}-progress-bar" class="progress visually-hidden"> + <div class="progress-bar" + role="progress-bar" + aria-valuenow="60" + aria-valuemin="0" + aria-valuemax="100" + style="width: 0%;"> + Uploading: 60% + </div> + </div> + + <div id="{{id}}-cancel-resume-buttons"> + <a id="{{id}}-resume-button" + class="resumable-resume-button btn btn-info visually-hidden" + href="#">resume upload</a> + + <a id="{{id}}-cancel-button" + class="resumable-cancel-button btn btn-danger visually-hidden" + href="#">cancel upload</a> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html new file mode 100644 index 0000000..859df74 --- /dev/null +++ b/uploader/templates/phenotypes/review-job-data.html @@ -0,0 +1,125 @@ +{%extends "phenotypes/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block extrameta%} +{%if not job%} +<meta http-equiv="refresh" + content="20; url={{url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" /> +{%endif%} +{%endblock%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} + +{%if job%} +<div class="row"> + <h3 class="heading">Data Review</h3> + <p class="text-info"><strong> + The data has <em>NOT</em> been added/saved yet. Review the details below + and click "Continue" to save the data.</strong></p> + <p>The “<strong>{{dataset.FullName}}</strong>” dataset from the + “<strong>{{population.FullName}}</strong>” population of the + species “<strong>{{species.SpeciesName}} ({{species.FullName}})</strong>” + will be updated as follows:</p> + + <ul> + {%if publication%} + <li>All {{summary.get("pheno", {}).get("total-data-rows", "0")}} phenotypes + are linked to the following publication: + <ul> + <li><strong>Publication Title:</strong> + {{publication.Title or "—"}}</li> + <li><strong>Author(s):</strong> + {{publication.Authors or "—"}}</li> + </ul> + </li> + {%endif%} + {%for ftype in ("phenocovar", "pheno", "phenose", "phenonum")%} + {%if summary.get(ftype, False)%} + <li>A total of {{summary[ftype]["number-of-files"]}} files will be processed + adding {%if ftype == "phenocovar"%}(possibly){%endif%} + {{summary[ftype]["total-data-rows"]}} new + {%if ftype == "phenocovar"%} + phenotypes + {%else%} + {{summary[ftype]["description"]}} rows + {%endif%} + to the database. + </li> + {%endif%} + {%endfor%} + </ul> + + <form id="frm-review-phenotype-data" + method="POST" + action="{{url_for('species.populations.phenotypes.load_data_to_database', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}"> + <input type="hidden" name="data-qc-job-id" value="{{job.jobid}}" /> + <input type="submit" + value="continue" + class="btn btn-primary" /> + </form> +</div> +{%else%} +<div class="row"> + <h4 class="subheading">Invalid Job</h3> + <p class="text-danger"> + Could not find a job with the ID: <strong>{{job_id}}.</p> + <p>You will be redirected in + <span id="countdown-element" class="text-info">20</span> second(s)</p> + <p class="text-muted"> + <small> + If you are not redirected, please + <a href="{{url_for( + 'species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">click here</a> to continue + </small> + </p> +</div> +{%endif%} +{%endblock%} + +{%block sidebarcontents%} +{{display_pheno_dataset_card(species, population, dataset)}} +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(document).ready(function() { + var countdown = 20; + var countdown_element = $("#countdown-element"); + if(countdown_element.length === 1) { + intv = window.setInterval(function() { + countdown = countdown - 1; + countdown_element.html(countdown); + }, 1000); + } + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/select-population.html b/uploader/templates/phenotypes/select-population.html index eafd4a7..48c19b1 100644 --- a/uploader/templates/phenotypes/select-population.html +++ b/uploader/templates/phenotypes/select-population.html @@ -11,18 +11,16 @@ {%block contents%} {{flash_all_messages()}} -<div class="row"> - <p>Select the population for your phenotypes to view and manage the phenotype - datasets that relate to it.</p> -</div> <div class="row"> - {{select_population_form(url_for("species.populations.phenotypes.select_population", - species_id=species.SpeciesId), - populations)}} + {{select_population_form(url_for("species.populations.phenotypes.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index 959b02a..306dcce 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -1,5 +1,6 @@ {%extends "phenotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} {%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Phenotypes{%endblock%} @@ -15,7 +16,7 @@ <a href="{{url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, population_id=population.Id, - dataset_id=dataset.Id)}}">View Datasets</a> + dataset_id=dataset.Id)}}">View</a> </li> {%endblock%} @@ -36,10 +37,7 @@ <tbody> <tr> - <td><a href="{{url_for('species.populations.phenotypes.view_dataset', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}">{{dataset.Name}}</a></td> + <td>{{dataset.Name}}</td> <td>{{dataset.FullName}}</td> <td>{{dataset.ShortName}}</td> </tr> @@ -48,40 +46,37 @@ </div> <div class="row"> + <p><a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="Add a bunch of phenotypes" + class="btn btn-primary">Add phenotypes</a></p> +</div> + +<div class="row"> <h2>Phenotype Data</h2> - <p>This dataset has a total of {{phenotype_count}} phenotypes.</p> - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - Limit access here, according to the authorisation privileges the user has on - this dataset!</p> + <p>Click on any of the phenotypes in the table below to view and edit that + phenotype's data.</p> + <p>Use the search to filter through all the phenotypes and find specific + phenotypes of interest.</p> +</div> + - <table class="table"> +<div class="row"> + + <table id="tbl-phenotypes-list" class="table compact stripe cell-border"> <thead> <tr> - <th>#</th> + <th></th> + <th>Index</th> <th>Record</th> <th>Description</th> </tr> </thead> - <tbody> - {%for pheno in phenotypes%} - <tr> - <td>{{pheno.sequence_number}}</td> - <td><a href="{{url_for('species.populations.phenotypes.view_phenotype', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id, - xref_id=pheno['pxr.Id'])}}" - title="View phenotype details"> - {{pheno.InbredSetCode}}_{{pheno["pxr.Id"]}}</a></td> - <td>{{pheno.Post_publication_description or pheno.Pre_publication_abbreviation or pheno.Original_description}}</td> - </tr> - {%else%} - <tr><td colspan="5"></td></tr> - {%endfor%} - </tbody> + <tbody></tbody> </table> </div> {%endblock%} @@ -89,3 +84,67 @@ {%block sidebarcontents%} {{display_population_card(species, population)}} {%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var species_id = {{species.SpeciesId}}; + var population_id = {{population.Id}}; + var dataset_id = {{dataset.Id}}; + var dataset_name = "{{dataset.Name}}"; + var data = {{phenotypes | tojson}}; + + var dtPhenotypesList = buildDataTable( + "#tbl-phenotypes-list", + data, + [ + { + data: function(pheno) { + return `<input type="checkbox" name="selected-phenotypes" ` + + `id="chk-selected-phenotypes-${pheno.InbredSetCode}_${pheno.xref_id}" ` + + `value="${pheno.InbredSetCode}_${pheno.xref_id}" ` + + `class="chk-row-select" />` + } + }, + {data: "sequence_number"}, + { + data: function(pheno, type, set, meta) { + var spcs_id = {{species.SpeciesId}}; + var pop_id = {{population.Id}}; + var dtst_id = {{dataset.Id}}; + return `<a href="/species/${spcs_id}` + + `/populations/${pop_id}` + + `/phenotypes/datasets/${dtst_id}` + + `/phenotype/${pheno.xref_id}` + + `" target="_blank">` + + `${pheno.InbredSetCode}_${pheno.xref_id}` + + `</a>`; + } + }, + { + data: function(pheno) { + return (pheno.Post_publication_description || + pheno.Original_description || + pheno.Pre_publication_description); + } + } + ], + { + select: "multi+shift", + layout: { + top1Start: { + pageLength: { + text: "Show _MENU_ of _TOTAL_" + } + }, + topStart: "info", + top1End: null + }, + rowId: function(pheno) { + return `${pheno.InbredSetCode}_${pheno.xref_id}`; + } + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html index 5aef6c1..21ac501 100644 --- a/uploader/templates/phenotypes/view-phenotype.html +++ b/uploader/templates/phenotypes/view-phenotype.html @@ -16,7 +16,7 @@ species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id, - xref_id=xref_id)}}">View Datasets</a> + xref_id=xref_id)}}">View Phenotype</a> </li> {%endblock%} @@ -24,18 +24,110 @@ {{flash_all_messages()}} <div class="row"> - {{resource}} + <div class="panel panel-default"> + <div class="panel-heading"><strong>Basic Phenotype Details</strong></div> + + <table class="table"> + <tbody> + <tr> + <td><strong>Phenotype</strong></td> + <td>{{phenotype.Post_publication_description or phenotype.Pre_publication_abbreviation or phenotype.Original_description}} + </tr> + <tr> + <td><strong>Database</strong></td> + <td>{{dataset.FullName}}</td> + </tr> + <tr> + <td><strong>Units</strong></td> + <td>{{phenotype.Units}}</td> + </tr> + {%for key,value in publish_data.items()%} + <tr> + <td><strong>{{key}}</strong></td> + <td>{{value}}</td> + </tr> + {%else%} + <tr> + <td colspan="2" class="text-muted"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + No publication data found. + </td> + </tr> + {%endfor%} + </tbody> + </table> + </div> </div> +{%if "group:resource:edit-resource" in privileges +or "group:resource:delete-resource" in privileges%} <div class="row"> - <h2>Dataset</h2> - {{dataset}} + <div class="btn-group btn-group-justified"> + <div class="btn-group"> + {%if "group:resource:edit-resource" in privileges%} + <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}" + title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values." + class="btn btn-primary">Edit</a> + {%endif%} + </div> + <div class="btn-group"></div> + <div class="btn-group"> + {%if "group:resource:delete-resource" in privileges%} + <a href="#" + title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype." + class="btn btn-danger not-implemented" + disabled="disabled">delete</a> + {%endif%} + </div> + </div> </div> +{%endif%} <div class="row"> - <h2>Phenotype</h2> - {{phenotype}} + <div class="panel panel-default"> + <div class="panel-heading"><strong>Phenotype Data</strong></div> + {%if "group:resource:view-resource" in privileges%} + <table class="table"> + <thead> + <tr> + <th>#</th> + <th>Sample</th> + <th>Value</th> + {%if has_se%} + <th>SE</th> + <th>N</th> + {%endif%} + </tr> + </thead> + + <tbody> + {%for item in phenotype.data%} + <tr> + <td>{{loop.index}}</td> + <td>{{item.StrainName}}</td> + <td>{{item.value}}</td> + {%if has_se%} + <td>{{item.error or "-"}}</td> + <td>{{item.count or "-"}}</td> + {%endif%} + </tr> + {%endfor%} + </tbody> + </table> + {%else%} + <p class="text-danger"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + You do not currently have privileges to view this phenotype in greater + detail. + </p> + {%endif%} + </div> </div> + {%endblock%} {%block sidebarcontents%} diff --git a/uploader/templates/platforms/index.html b/uploader/templates/platforms/index.html index 35b6464..555b444 100644 --- a/uploader/templates/platforms/index.html +++ b/uploader/templates/platforms/index.html @@ -19,3 +19,7 @@ {{select_species_form(url_for("species.platforms.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/platforms/list-platforms.html b/uploader/templates/platforms/list-platforms.html index 718dd1d..a6bcfdc 100644 --- a/uploader/templates/platforms/list-platforms.html +++ b/uploader/templates/platforms/list-platforms.html @@ -58,7 +58,7 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th></th> <th>Platform Name</th> <th><a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}" title="Gene Expression Omnibus: Platforms section" diff --git a/uploader/templates/populations/base.html b/uploader/templates/populations/base.html index d763fc1..9db8083 100644 --- a/uploader/templates/populations/base.html +++ b/uploader/templates/populations/base.html @@ -6,7 +6,13 @@ {%else%} class="breadcrumb-item" {%endif%}> + {%if population is mapping%} + <a href="{{url_for('species.populations.view_population', + species_id=species.SpeciesId, + population_id=population.Id)}}">{{population.Name}}</a> + {%else%} <a href="{{url_for('species.populations.index')}}">Populations</a> + {%endif%} </li> {%block lvl3_breadcrumbs%}{%endblock%} {%endblock%} diff --git a/uploader/templates/populations/create-population.html b/uploader/templates/populations/create-population.html index b05ce37..007b6bf 100644 --- a/uploader/templates/populations/create-population.html +++ b/uploader/templates/populations/create-population.html @@ -37,12 +37,15 @@ <div class="row"> <form method="POST" action="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}"> + species_id=species.SpeciesId, + return_to=return_to)}}"> <legend>Create Population</legend> {{flash_all_messages()}} + <input type="hidden" name="return_to" value="{{return_to}}"> + <div {%if errors.population_fullname%} class="form-group has-error" {%else%} @@ -107,9 +110,12 @@ value="{{error_values.population_code or ''}}" class="form-control" /> <small class="form-text text-muted"> - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - What is this field is for? Confirm with Arthur and the rest. + <p class="form-text text-muted"> + This is a 3-character code for your population, that is prepended to + the phenotype identifiers. e.g. For the "BXD Family" population, the + code is "BXD" and therefore, the phenotype identifiers for the + population look like the following examples: <em>BXD_10148</em>, + <em>BXD_10180</em>, <em>BXD_10197</em>, etc. </p> </small> </div> @@ -148,24 +154,35 @@ {%else%} class="form-group" {%endif%}> - <label for="select-population-family" class="form-label">Family</label> - <select id="select-population-family" - name="population_family" - class="form-control" - required="required"> - <option value="">Please select a family</option> + <label for="txt-population-family" class="form-label">Family</label> + <input type="text" + id="txt-population-family" + name="population_family" + class="form-control" + list="families-list" /> + <datalist id="families-list"> {%for family in families%} - <option value="{{family}}" - {%if error_values.population_family == family%} - selected="selected" - {%endif%}>{{family}}</option> + <option value="{{family}}">{{family}}</option> {%endfor%} - </select> + </datalist> <small class="form-text text-muted"> <p> - This is a rough grouping of the populations in GeneNetwork into lists - of common types of populations. - </p> + This is <strong>optional</strong> metadata. It is used to group + populations into "families" for presentation in the menus. + {%if families | length > 0%} + Examples of currently existing families are: + <ul> + {%for family in families[0:7]%} + <li>{{family}}</li> + {%endfor%} + <li>etc.</li> + </ul> + {%endif%} + + You can + {%if families|length>0%} select from existing families, or {%endif%} + create a new family by typing in the input box above. You can also + leave the family blank.</p> </small> </div> diff --git a/uploader/templates/populations/index.html b/uploader/templates/populations/index.html index 4354e02..d2bee77 100644 --- a/uploader/templates/populations/index.html +++ b/uploader/templates/populations/index.html @@ -22,3 +22,7 @@ {{select_species_form(url_for("species.populations.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/populations/list-populations.html b/uploader/templates/populations/list-populations.html index 7c7145f..f780e94 100644 --- a/uploader/templates/populations/list-populations.html +++ b/uploader/templates/populations/list-populations.html @@ -51,7 +51,7 @@ <caption>Populations for {{species.FullName}}</caption> <thead> <tr> - <th>#</th> + <th></th> <th>Name</th> <th>Full Name</th> <th>Description</th> diff --git a/uploader/templates/populations/macro-display-population-card.html b/uploader/templates/populations/macro-display-population-card.html index e68f8e3..16b477f 100644 --- a/uploader/templates/populations/macro-display-population-card.html +++ b/uploader/templates/populations/macro-display-population-card.html @@ -7,25 +7,34 @@ <div class="card-body"> <h5 class="card-title">Population</h5> <div class="card-text"> - <dl> - <dt>Name</dt> - <dd>{{population.Name}}</dd> + <table class="table"> + <tbody> + <tr> + <td>Name</td> + <td>{{population.Name}}</td> + </tr> - <dt>Full Name</dt> - <dd>{{population.FullName}}</dd> + <tr> + <td>Full Name</td> + <td>{{population.FullName}}</td> + </tr> - <dt>Code</dt> - <dd>{{population.InbredSetCode}}</dd> + <tr> + <td>Code</td> + <td>{{population.InbredSetCode}}</td> + </tr> - <dt>Genetic Type</dt> - <dd>{{population.GeneticType}}</dd> + <tr> + <td>Genetic Type</td> + <td>{{population.GeneticType}}</td> + </tr> - <dt>Family</dt> - <dd>{{population.Family}}</dd> - - <dt>Description</dt> - <dd>{{population.Description or "-"}}</dd> - </dl> + <tr> + <td>Family</td> + <td>{{population.Family}}</td> + </tr> + </tbody> + </table> </div> </div> </div> diff --git a/uploader/templates/populations/macro-select-population.html b/uploader/templates/populations/macro-select-population.html index af4fd3a..14b0510 100644 --- a/uploader/templates/populations/macro-select-population.html +++ b/uploader/templates/populations/macro-select-population.html @@ -1,30 +1,52 @@ -{%macro select_population_form(form_action, populations)%} -<form method="GET" action="{{form_action}}"> - <legend>Select Population</legend> - - <div class="form-group"> - <label for="select-population" class="form-label">Select Population</label> - <select id="select-population" - name="population_id" - class="form-control" - required="required"> - <option value="">Select Population</option> - {%for family in populations%} - <optgroup {%if family[0][1] is not none%} - label="{{family[0][1]}}" - {%else%} - label="Undefined" - {%endif%}> - {%for population in family[1]%} - <option value="{{population.Id}}">{{population.FullName}}</option> - {%endfor%} - </optgroup> - {%endfor%} - </select> +{%from "macro-step-indicator.html" import step_indicator%} + +{%macro select_population_form(form_action, species, populations)%} +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("2")}} What population do you want to work with?</h2> + + {%if populations | length != 0%} + + <p class="form-text">Search for, and select the population from the table + below and click "Continue"</p> + + <div class="radio"> + <label class="control-label" for="rdo-cant-find-population"> + <input type="radio" id="rdo-cant-find-population" + name="population_id" value="CREATE-POPULATION" /> + I cannot find the population I want — create it! + </label> + </div> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> + </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-population" class="table compact stripe" + data-populations-list='{{populations | tojson}}'> + <thead> + <tr> + <th></th> + <th>Population</th> + </tr> + </thead> + + <tbody></tbody> + </table> </div> - <div class="form-group"> - <input type="submit" value="Select" class="btn btn-primary" /> + {%else%} + <p class="form-text"> + There are no populations currently defined for {{species['FullName']}} + ({{species['SpeciesName']}}).<br /> + Click "Continue" to create the first!</p> + <input type="hidden" name="population_id" value="CREATE-POPULATION" /> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> </div> + {%endif%} + </form> {%endmacro%} diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html index 1e2964e..b23caeb 100644 --- a/uploader/templates/populations/view-population.html +++ b/uploader/templates/populations/view-population.html @@ -15,7 +15,7 @@ {%endif%}> <a href="{{url_for('species.populations.view_population', species_id=species.SpeciesId, - population_id=population.InbredSetId)}}">view population</a> + population_id=population.InbredSetId)}}">view</a> </li> {%endblock%} @@ -62,29 +62,35 @@ <nav class="nav"> <ul> <li> - <a href="{{url_for('species.populations.genotypes.list_genotypes', + <a href="{{url_for('species.populations.samples.list_samples', species_id=species.SpeciesId, population_id=population.Id)}}" - title="Upload genotypes for {{species.FullName}}">Upload Genotypes</a> + title="Manage samples: Add new or delete existing."> + manage samples</a> </li> <li> - <a href="{{url_for('species.populations.samples.list_samples', + <a href="{{url_for('species.populations.genotypes.list_genotypes', species_id=species.SpeciesId, population_id=population.Id)}}" - title="Manage samples: Add new or delete existing."> - manage samples</a> + title="Manage genotypes for {{species.FullName}}">Manage Genotypes</a> </li> <li> - <a href="#" title="Upload expression data">upload expression data</a> + <a href="{{url_for('species.populations.phenotypes.list_datasets', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Manage phenotype data.">manage phenotype data</a> </li> <li> - <a href="#" title="Upload phenotype data">upload phenotype data</a> + <a href="#" title="Manage expression data" + class="not-implemented">manage expression data</a> </li> <li> - <a href="#" title="Upload individual data">upload individual data</a> + <a href="#" title="Manage individual data" + class="not-implemented">manage individual data</a> </li> <li> - <a href="#" title="Upload RNA-Seq data">upload RNA-Seq data</a> + <a href="#" title="Manage RNA-Seq data" + class="not-implemented">manage RNA-Seq data</a> </li> </ul> </nav> diff --git a/uploader/templates/publications/base.html b/uploader/templates/publications/base.html new file mode 100644 index 0000000..db80bfa --- /dev/null +++ b/uploader/templates/publications/base.html @@ -0,0 +1,12 @@ +{%extends "base.html"%} + +{%block lvl1_breadcrumbs%} +<li {%if activelink=="publications"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('publications.index')}}">Publications</a> +</li> +{%block lvl2_breadcrumbs%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html new file mode 100644 index 0000000..3f828a9 --- /dev/null +++ b/uploader/templates/publications/create-publication.html @@ -0,0 +1,191 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}View Publication{%endblock%} + +{%block pagetitle%}View Publication{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <form id="frm-create-publication" + method="POST" + action="{{url_for('publications.create_publication', **request.args)}}" + class="form-horizontal"> + + <div class="row mb-3"> + <label for="txt-pubmed-id" class="col-sm-2 col-form-label"> + PubMed ID</label> + <div class="col-sm-10"> + <div class="input-group"> + <input type="text" + id="txt-pubmed-id" + name="pubmed-id" + class="form-control"/> + <div class="input-group-text"> + <button class="btn btn-outline-primary" + id="btn-search-pubmed-id">search</button> + </div> + </div> + <span id="search-pubmed-id-error" + class="form-text text-muted text-danger visually-hidden"> + </span> + <span class="form-text text-muted">This is the publication's ID on + <a href="https://pubmed.ncbi.nlm.nih.gov/" + title="Link to NCBI's PubMed service">NCBI's Pubmed Service</a> + </span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-title" class="col-sm-2 col-form-label"> + Title</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-title" + name="publication-title" + class="form-control" /> + <span class="form-text text-muted">Provide the publication's title here.</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-authors" class="col-sm-2 col-form-label"> + Authors</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-authors" + name="publication-authors" + required="required" + class="form-control" /> + <span class="form-text text-muted"> + A publication <strong>MUST</strong> have an author. You <em>must</em> + provide a value for the authors field. + </span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-journal" class="col-sm-2 col-form-label"> + Journal</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-journal" + name="publication-journal" + class="form-control" /> + <span class="form-text text-muted">Provide the name journal where the + publication was done, here.</span> + </div> + </div> + + <div class="row mb-3"> + <label for="select-publication-month" + class="col-sm-2 col-form-label"> + Month</label> + <div class="col-sm-4"> + <select class="form-control" + id="select-publication-month" + name="publication-month"> + <option value="">Select a month</option> + <option value="january">January</option> + <option value="february">February</option> + <option value="march">March</option> + <option value="april">April</option> + <option value="may">May</option> + <option value="june">June</option> + <option value="july">July</option> + <option value="august">August</option> + <option value="september">September</option> + <option value="october">October</option> + <option value="november">November</option> + <option value="december">December</option> + </select> + <span class="form-text text-muted">Month of publication</span> + </div> + + <label for="txt-publication-year" + class="col-sm-2 col-form-label"> + Year</label> + <div class="col-sm-4"> + <input type="number" + id="txt-publication-year" + name="publication-year" + class="form-control" + min="1960" /> + <span class="form-text text-muted">Year of publication</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-volume" + class="col-sm-2 col-form-label"> + Volume</label> + <div class="col-sm-4"> + <input type="text" + id="txt-publication-volume" + name="publication-volume" + class="form-control"> + <span class="form-text text-muted">Journal volume</span> + </div> + + <label for="txt-publication-pages" + class="col-sm-2 col-form-label"> + Pages</label> + <div class="col-sm-4"> + <input type="text" + id="txt-publication-pages" + name="publication-pages" + class="form-control" /> + <span class="form-text text-muted">Journal pages for the publication</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-abstract" class="col-sm-2 col-form-label">Abstract</label> + <div class="col-sm-10"> + <textarea id="txt-publication-abstract" + name="publication-abstract" + class="form-control" + rows="7"></textarea> + </div> + </div> + + <div class="row mb-3"> + <div class="col-sm-2"></div> + <div class="col-sm-8"> + <input type="submit" class="btn btn-primary" value="Add" /> + <input type="reset" class="btn btn-danger" /> + </div> + </div> + +</form> +</div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript" src="/static/js/pubmed.js"></script> +<script type="text/javascript"> + $(function() { + $("#btn-search-pubmed-id").on("click", (event) => { + event.preventDefault(); + var search_button = event.target; + var pubmed_id = $("#txt-pubmed-id").val().trim(); + remove_class($("#txt-pubmed-id").parent(), "has-error"); + if(pubmed_id == "") { + add_class($("#txt-pubmed-id").parent(), "has-error"); + return false; + } + + search_button.disabled = true; + // Fetch publication details + fetch_publication_details(pubmed_id, + [() => {search_button.disabled = false;}]); + return false; + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/publications/index.html b/uploader/templates/publications/index.html new file mode 100644 index 0000000..369812b --- /dev/null +++ b/uploader/templates/publications/index.html @@ -0,0 +1,100 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Publications{%endblock%} + +{%block pagetitle%}Publications{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row" style="padding-bottom: 1em;"> + <a href="{{url_for('publications.create_publication')}}" + class="btn btn-primary"> + add new publication</a> +</div> + +<div class="row"> + <table id="tbl-list-publications" class="table compact stripe"> + <thead> + <tr> + <th>#</th> + <th>PubMed ID</th> + <th>Title</th> + <th>Authors</th> + </tr> + </thead> + + <tbody></tbody> + </table> +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var publicationsDataTable = buildDataTable( + "#tbl-list-publications", + [], + [ + {data: "index"}, + { + searchable: true, + data: (pub) => { + if(pub.PubMed_ID) { + return `<a href="https://pubmed.ncbi.nlm.nih.gov/` + + `${pub.PubMed_ID}/" target="_blank" ` + + `title="Link to publication on NCBI.">` + + `${pub.PubMed_ID}</a>`; + } + return ""; + } + }, + { + searchable: true, + data: (pub) => { + var title = "⸻"; + if(pub.Title) { + title = pub.Title + } + return `<a href="/publications/view/${pub.Id}" ` + + `target="_blank" ` + + `title="Link to view publication details">` + + `${title}</a>`; + } + }, + { + searchable: true, + data: (pub) => { + authors = pub.Authors.split(",").map( + (item) => {return item.trim();}); + if(authors.length > 1) { + return authors[0] + ", et. al."; + } + return authors[0]; + } + } + ], + { + serverSide: true, + ajax: { + url: "/publications/list", + dataSrc: "publications" + }, + scrollY: 700, + scroller: true, + scrollCollapse: true, + paging: true, + deferRender: true, + layout: { + topStart: "info", + topEnd: "search", + bottomStart: "pageLength", + bottomEnd: false + } + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/publications/view-publication.html b/uploader/templates/publications/view-publication.html new file mode 100644 index 0000000..388547a --- /dev/null +++ b/uploader/templates/publications/view-publication.html @@ -0,0 +1,78 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}View Publication{%endblock%} + +{%block pagetitle%}View Publication{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <table class="table"> + <tr> + <th>PubMed</th> + <td> + {%if publication.PubMed_ID%} + <a href="https://pubmed.ncbi.nlm.nih.gov/{{publication.PubMed_ID}}/" + target="_blank">{{publication.PubMed_ID}}</a> + {%else%} + — + {%endif%} + </td> + </tr> + <tr> + <th>Title</th> + <td>{{publication.Title or "—"}}</td> + </tr> + <tr> + <th>Authors</th> + <td>{{publication.Authors or "—"}}</td> + </tr> + <tr> + <th>Journal</th> + <td>{{publication.Journal or "—"}}</td> + </tr> + <tr> + <th>Published</th> + <td>{{publication.Month or ""}} {{publication.Year or "—"}}</td> + </tr> + <tr> + <th>Volume</th> + <td>{{publication.Volume or "—"}}</td> + </tr> + <tr> + <th>Pages</th> + <td>{{publication.Pages or "—"}}</td> + </tr> + <tr> + <th>Abstract</th> + <td> + {%for line in (publication.Abstract or "—").replace("\r\n", "<br />").replace("\n", "<br />").split("<br />")%} + <p>{{line}}</p> + {%endfor%} + </td> + </tr> + </table> +</div> + +<div class="row"> + <form id="frm-edit-delete-publication" method="POST" action="#"> + <input type="hidden" name="publication_id" value="{{publication.Id}}" /> + <div class="form-group"> + <input type="submit" value="edit" class="btn btn-primary not-implemented" /> + {%if linked_phenotypes | length == 0%} + <input type="submit" value="delete" class="btn btn-danger not-implemented" /> + {%endif%} + </div> + </form> +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() {}); +</script> +{%endblock%} diff --git a/uploader/templates/samples/index.html b/uploader/templates/samples/index.html index ee4a63e..ee98734 100644 --- a/uploader/templates/samples/index.html +++ b/uploader/templates/samples/index.html @@ -17,3 +17,7 @@ {{select_species_form(url_for("species.populations.samples.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/list-samples.html b/uploader/templates/samples/list-samples.html index 13e5cec..aed27c3 100644 --- a/uploader/templates/samples/list-samples.html +++ b/uploader/templates/samples/list-samples.html @@ -29,6 +29,19 @@ </p> </div> +<div class="row"> + <p> + <a href="{{url_for('species.populations.samples.upload_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Add samples for population '{{population.FullName}}' from species + '{{species.FullName}}'." + class="btn btn-primary"> + add samples + </a> + </p> +</div> + {%if samples | length > 0%} <div class="row"> <p> @@ -73,7 +86,7 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th></th> <th>Name</th> <th>Auxilliary Name</th> <th>Symbol</th> @@ -96,32 +109,17 @@ <p> <a href="#" - title="Add samples for population '{{population.FullName}}' from species + title="Delete samples from population '{{population.FullName}}' from species '{{species.FullName}}'." - class="btn btn-danger"> + class="btn btn-danger not-implemented"> delete all samples </a> </p> </div> - {%else%} - <div class="row"> - <p> - There are no samples entered for this population. Do please go ahead and add - the samples for this population by clicking on the button below. - </p> - - <p> - <a href="{{url_for('species.populations.samples.upload_samples', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="Add samples for population '{{population.FullName}}' from species - '{{species.FullName}}'." - class="btn btn-primary"> - add samples - </a> - </p> + <p>There are no samples entered for this population. Click the "Add Samples" + button above, to add some new samples.</p> </div> {%endif%} diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html index f437780..1cc7573 100644 --- a/uploader/templates/samples/select-population.html +++ b/uploader/templates/samples/select-population.html @@ -12,28 +12,15 @@ {{flash_all_messages()}} <div class="row"> - <p>You have selected "{{species.FullName}}" as the species that your data relates to.</p> - <p>Next, we need information regarding the population your data relates to. Do please select the population from the existing ones below</p> -</div> - -<div class="row"> {{select_population_form( - url_for("species.populations.samples.select_population", species_id=species.SpeciesId), - populations)}} -</div> - -<div class="row"> - <p> - If you cannot find the population your data relates to in the drop-down - above, you might want to - <a href="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}" - title="Create a new population for species '{{species.FullName}},"> - add a new population to GeneNetwork</a> - instead. + url_for("species.populations.samples.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html index 458ab55..2cf8053 100644 --- a/uploader/templates/samples/upload-failure.html +++ b/uploader/templates/samples/upload-failure.html @@ -15,7 +15,7 @@ <h3>Debugging Information</h3> <ul> - <li><strong>job id</strong>: {{job.job_id}}</li> + <li><strong>job id</strong>: {{job.jobid}}</li> <li><strong>status</strong>: {{job.status}}</li> <li><strong>job type</strong>: {{job["job-type"]}}</li> </ul> diff --git a/uploader/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html index 25d3290..6422094 100644 --- a/uploader/templates/samples/upload-samples.html +++ b/uploader/templates/samples/upload-samples.html @@ -66,7 +66,7 @@ <div class="form-group"> <label for="file-samples" class="form-label">select file</label> <input type="file" name="samples_file" id="file:samples" - accept="text/csv, text/tab-separated-values" + accept="text/csv, text/tab-separated-values, text/plain" class="form-control" /> </div> diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html index 04391db..f64f72b 100644 --- a/uploader/templates/species/base.html +++ b/uploader/templates/species/base.html @@ -6,7 +6,12 @@ {%else%} class="breadcrumb-item" {%endif%}> + {%if species is mapping%} + <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}"> + {{species.Name}}</a> + {%else%} <a href="{{url_for('species.list_species')}}">Species</a> + {%endif%} </li> {%block lvl2_breadcrumbs%}{%endblock%} {%endblock%} diff --git a/uploader/templates/species/create-species.html b/uploader/templates/species/create-species.html index 0d0bedf..138dbaa 100644 --- a/uploader/templates/species/create-species.html +++ b/uploader/templates/species/create-species.html @@ -19,72 +19,88 @@ <div class="row"> <form id="frm-create-species" method="POST" - action="{{url_for('species.create_species')}}"> + action="{{url_for('species.create_species', return_to=return_to)}}" + class="form-horizontal"> <legend>Create Species</legend> {{flash_all_messages()}} + <input type="hidden" name="return_to" value="{{return_to}}"> + <div class="form-group"> - <label for="txt-taxonomy-id" class="form-label"> + <label for="txt-taxonomy-id" class="control-label col-sm-2"> Taxonomy ID</label> - <div class="input-group"> - <input id="txt-taxonomy-id" - name="species_taxonomy_id" - type="text" - class="form-control" /> - <span class="input-group-btn"> - <button id="btn-search-taxonid" class="btn btn-info">Search</button> - </span> + <div class="col-sm-10"> + <div class="input-group"> + <input id="txt-taxonomy-id" + name="species_taxonomy_id" + type="text" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-search-taxonid" class="btn btn-info">Search</button> + </span> + </div> + <small class="form-text text-small text-muted"> + Use + <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/" + title="NCBI's Taxonomy Browser homepage" + target="_blank"> + NCBI's Taxonomy Browser homepage</a> to search for the species you + want. If the species exists on NCBI, they will have a Taxonomy ID. Copy + that Taxonomy ID to this field, and click "Search" to auto-fill the + details.<br /> + This field is optional.</small> </div> - <small class="form-text text-small text-muted">Provide the taxonomy ID for - your species that can be used to link to external sites like NCBI. Enter - the taxonomy ID and click "Search" to auto-fill the form with data. - <br /> - While it is recommended to provide a value for this field, doing so is - optional. - </small> </div> <div class="form-group"> - <label for="txt-species-name" class="form-label">Common Name</label> - <input id="txt-species-name" - name="common_name" - type="text" - class="form-control" - required="required" /> - <small class="form-text text-muted">Provide the common, possibly - non-scientific name for the species here, e.g. Human, Mouse, etc.</small> + <label for="txt-species-name" class="control-label col-sm-2">Common Name</label> + <div class="col-sm-10"> + <input id="txt-species-name" + name="common_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the day-to-day term used by + laymen, e.g. Mouse (instead of Mus musculus), round worm (instead of + Ascaris lumbricoides), etc.<br /> + For species without this, just enter the scientific name. + </small> + </div> </div> <div class="form-group"> - <label for="txt-species-scientific" class="form-label"> + <label for="txt-species-scientific" class="control-label col-sm-2"> Scientific Name</label> - <input id="txt-species-scientific" - name="scientific_name" - type="text" - class="form-control" - required="required" /> - <small class="form-text text-muted">Provide the scientific name for the - species you are creating, e.g. Homo sapiens, Mus musculus, etc.</small> + <div class="col-sm-10"> + <input id="txt-species-scientific" + name="scientific_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the scientific name for the + species e.g. Homo sapiens, Mus musculus, etc.</small> + </div> </div> <div class="form-group"> - <label for="select-species-family" class="form-label">Family</label> - <select id="select-species-family" - name="species_family" - required="required" - class="form-control"> - <option value="">Please select a grouping</option> - {%for family in families%} - <option value="{{family}}">{{family}}</option> - {%endfor%} - </select> - <small class="form-text text-muted"> - This is a generic grouping for the species that determines under which - grouping the species appears in the GeneNetwork menus</small> + <label for="select-species-family" class="control-label col-sm-2">Family</label> + <div class="col-sm-10"> + <select id="select-species-family" + name="species_family" + required="required" + class="form-control"> + <option value="ungrouped">I do not know what to pick</option> + {%for family in families%} + <option value="{{family}}">{{family}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + This is a rough grouping of the species.</small> + </div> </div> - <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> <input type="submit" value="create new species" class="btn btn-primary" /> @@ -113,7 +129,7 @@ } msg = ( "Request to '${uri}' failed with message '${textStatus}'. " - + "Please try again later, or fill the details manually."); + + "Please try again later, or fill the details manually."); alert(msg); console.error(msg, data, textStatus); return false; diff --git a/uploader/templates/species/list-species.html b/uploader/templates/species/list-species.html index 85c9d40..64084b0 100644 --- a/uploader/templates/species/list-species.html +++ b/uploader/templates/species/list-species.html @@ -29,7 +29,7 @@ <caption>Available Species</caption> <thead> <tr> - <th>#</td> + <th></td> <th title="A common, layman's name for the species.">Common Name</th> <th title="The scientific name for the species">Organism Name</th> <th title="An identifier for the species in the NCBI taxonomy database"> diff --git a/uploader/templates/species/macro-display-species-card.html b/uploader/templates/species/macro-display-species-card.html index 857c0f0..166c7b9 100644 --- a/uploader/templates/species/macro-display-species-card.html +++ b/uploader/templates/species/macro-display-species-card.html @@ -3,13 +3,19 @@ <div class="card-body"> <h5 class="card-title">Species</h5> <div class="card-text"> - <dl> - <dt>Common Name</dt> - <dd>{{species.SpeciesName}}</dd> + <table class="table"> + <tbody> + <tr> + <td>Common Name</td> + <td>{{species.SpeciesName}}</td> + </tr> - <dt>Scientific Name</dt> - <dd>{{species.FullName}}</dd> - </dl> + <tr> + <td>Scientific Name</td> + <td>{{species.FullName}}</td> + </tr> + </tbody> + </table> </div> </div> </div> diff --git a/uploader/templates/species/macro-select-species.html b/uploader/templates/species/macro-select-species.html index dd086c0..3714ae4 100644 --- a/uploader/templates/species/macro-select-species.html +++ b/uploader/templates/species/macro-select-species.html @@ -1,36 +1,59 @@ +{%from "macro-step-indicator.html" import step_indicator%} + {%macro select_species_form(form_action, species)%} -{%if species | length > 0%} -<form method="GET" action="{{form_action}}"> - <div class="form-group"> - <label for="select-species" class="form-label">Species</label> - <select id="select-species" - name="species_id" - class="form-control" - required="required"> - <option value="">Select Species</option> - {%for group in species%} - {{group}} - <optgroup {%if group[0][1] is not none%} - label="{{group[0][1].capitalize()}}" - {%else%} - label="Undefined" - {%endif%}> - {%for aspecies in group[1]%} - <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option> - {%endfor%} - </optgroup> - {%endfor%} - </select> +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("1")}} What species do you want to work with?</h2> + + {%if species | length != 0%} + + <p class="form-text">Search for, and select the species from the table below + and click "Continue"</p> + + <div class="radio"> + <label for="rdo-cant-find-species" + style="font-weight: 1;"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + I could not find the species I want (create it). + </label> </div> - <div class="form-group"> - <input type="submit" value="Select" class="btn btn-primary" /> + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary" + value="continue" /> </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-species" class="table compact stripe" + data-species-list='{{species | tojson}}'> + <div class=""> + <thead> + <tr> + <th></th> + <th>Species Name</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </div> + + {%else%} + + <label class="control-label" for="rdo-cant-find-species"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + There are no species to select from. Create the first one.</label> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary col-sm-offset-1" + value="continue" /> + </div> + + {%endif%} + </form> -{%else%} -<p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-mark"></span> - We could not find species to select from! -</p> -{%endif%} {%endmacro%} diff --git a/uploader/templates/species/view-species.html b/uploader/templates/species/view-species.html index b01864d..2d02f7e 100644 --- a/uploader/templates/species/view-species.html +++ b/uploader/templates/species/view-species.html @@ -45,6 +45,12 @@ title="Create/Edit populations for {{species.FullName}}"> Manage populations</a> </li> + <li> + <a href="{{url_for('species.platforms.list_platforms', + species_id=species.SpeciesId)}}" + title="Create/Edit sequencing platforms for {{species.FullName}}"> + Manage sequencing platforms</a> + </li> </ol> |