diff options
182 files changed, 11080 insertions, 2956 deletions
diff --git a/.dev/run-checks.sh b/.dev/run-checks.sh new file mode 100644 index 0000000..66ac681 --- /dev/null +++ b/.dev/run-checks.sh @@ -0,0 +1,2 @@ +pylint setup.py tests quality_control uploader r_qtl scripts && \ + mypy --show-error-codes . @@ -5,4 +5,5 @@ /build /dist /bdist -/*.egg-info
\ No newline at end of file +/*.egg-info +/flask_session/**
\ No newline at end of file diff --git a/.guix-channel b/.guix-channel index e8ea99d..f1a8fa6 100644 --- a/.guix-channel +++ b/.guix-channel @@ -4,18 +4,57 @@ (keyring-reference "keyring") (dependencies (channel + (name gn-machines) + (url "https://git.genenetwork.org/gn-machines") + (branch "main")) + ;; Until https://issues.guix.gnu.org/68797 is resolved, we need to + ;; explicitly list guix-bioinformatics, guix-forge, guix-past and + ;; guix-rust-past-crates—the dependencies of the gn-machines channel—here. + (channel + (name guix) + (url "https://codeberg.org/guix/guix") + (branch "master") + (commit "0a4740705090acc4c8a10d4f53afc58c9f62e980") + (introduction + (channel-introduction + (version 0) + (commit "9edb3f66fd807b096b48283debdcddccfea34bad") + (signer + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) + (channel + (name guix-forge) + (url "https://git.systemreboot.net/guix-forge/") + (branch "main") + (commit "e43fd9a4d73654d3876e2c698af7da89f3408f89") + (introduction + (channel-introduction + (version 0) + (commit "0432e37b20dd678a02efee21adf0b9525a670310") + (signer + "7F73 0343 F2F0 9F3C 77BF 79D3 2E25 EE8B 6180 2BB3")))) + (channel (name guix-bioinformatics) (url "https://git.genenetwork.org/guix-bioinformatics") - (branch "master")) - ;; FIXME: guix-bioinformatics depends on guix-past. So, there - ;; should be no reason to explicitly depend on guix-past. But, the - ;; channel does not build otherwise. This is probably a guix bug. + (commit "9b0955f14ec725990abb1f6af3b9f171e4943f77")) (channel (name guix-past) - (url "https://gitlab.inria.fr/guix-hpc/guix-past") + (url "https://codeberg.org/guix-science/guix-past") + (branch "master") + (commit "473c942b509ab3ead35159d27dfbf2031a36cd4d") + (introduction + (channel-introduction + (version 0) + (commit "c3bc94ee752ec545e39c1b8a29f739405767b51c") + (signer + "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5")))) + (channel + (name guix-rust-past-crates) + (url "https://codeberg.org/guix/guix-rust-past-crates.git") + (branch "trunk") + (commit "b8b7ffbd1cec9f56f93fae4da3a74163bbc9c570") (introduction (channel-introduction (version 0) - (commit "0c119db2ea86a389769f4d2b9c6f5c41c027e336") + (commit "1db24ca92c28255b28076792b93d533eabb3dc6a") (signer - "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5")))))) + "F4C2 D1DF 3FDE EA63 D1D3 0776 ACC6 6D09 CA52 8292")))))) diff --git a/.guix/modules/gn-uploader.scm b/.guix/modules/gn-uploader.scm index 4e198fa..ea6c982 100644 --- a/.guix/modules/gn-uploader.scm +++ b/.guix/modules/gn-uploader.scm @@ -7,7 +7,7 @@ #:use-module ((gnu packages check) #:select (python-pylint)) #:use-module ((gnu packages python-check) #:select (python-mypy)) - #:use-module ((gn packages genenetwork) #:select (gn-uploader) #:prefix gn:)) + #:use-module ((gn-machines genenetwork) #:select (gn-uploader) #:prefix gn:)) (define %source-dir (dirname (dirname (current-source-directory)))) @@ -2,6 +2,10 @@ #+TITLE: GeneNetwork Quality Control Application #+OPTIONS: ^:{} +[[https://ci.genenetwork.org/jobs/gn-uploader][https://ci.genenetwork.org/badge/gn-uploader.svg]] +[[https://ci.genenetwork.org/jobs/gn-uploader-all-tests][https://ci.genenetwork.org/badge/gn-uploader-all-tests.svg]] + + ** Project Goals The project seeks to handle the checking of data files for correct syntax and @@ -219,7 +223,7 @@ To check for correct type usage in the application, run: Run unit tests with: #+BEGIN_SRC shell $ export UPLOADER_CONF=</path/to/configuration/file.py> - $ pytest -m unit_test + $ pytest -m unit_test -n auto #+END_SRC To run ALL tests (not just unit tests): @@ -1,5 +1,8 @@ [mypy] +[mypy-lxml.*] +ignore_missing_imports = True + [mypy-flask.*] ignore_missing_imports = True diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..cdd4f06 --- /dev/null +++ b/pylintrc @@ -0,0 +1,647 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list=lxml + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked and +# will not be imported (useful for modules/projects where namespaces are +# manipulated during runtime and thus existing member attributes cannot be +# deduced by static analysis). It supports qualified module names, as well as +# Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Resolve imports to .pyi stubs if available. May reduce no-member messages and +# increase not-an-iterable messages. +prefer-stubs=no + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.10 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=5 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of positional arguments for function / method. +max-positional-arguments=5 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )?<?https?://\S+>?$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + +# Let 'consider-using-join' be raised when the separator to join on would be +# non-empty (resulting in expected fixes of the type: ``"- " + " - +# ".join(items)``) +suggest-join-with-non-empty-separator=yes + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are: text, parseable, colorized, +# json2 (improved json format), json (old json format) and msvs (visual +# studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/qc_app/default_settings.py b/qc_app/default_settings.py index 7a9da0f..7bb0bf8 100644 --- a/qc_app/default_settings.py +++ b/qc_app/default_settings.py @@ -7,7 +7,7 @@ import os LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") SECRET_KEY = b"<Please! Please! Please! Change This!>" -UPLOAD_FOLDER = "/tmp/qc_app_files" +UPLOADS_DIRECTORY = "/tmp/qc_app_files" REDIS_URL = "redis://" JOBS_TTL_SECONDS = 1209600 # 14 days GNQC_REDIS_PREFIX="GNQC" diff --git a/quality_control/checks.py b/quality_control/checks.py index bdfd12b..bb05e31 100644 --- a/quality_control/checks.py +++ b/quality_control/checks.py @@ -52,12 +52,15 @@ def decimal_places_pattern(mini: int, maxi: Optional[int] = None) -> re.Pattern: + r")$" ) -def decimal_points_error(filename: str,# pylint: disable=[too-many-arguments] - lineno: int, - field: str, - value: str, - mini: int, - maxi: Optional[int] = None) -> Optional[InvalidValue]: +def decimal_points_error( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] + filename: str, + lineno: int, + field: str, + value: str, + mini: int, + maxi: Optional[int] = None +) -> Optional[InvalidValue]: """ Check that 'value' in a decimal number with the appropriate decimal places. """ diff --git a/quality_control/parsing.py b/quality_control/parsing.py index f1d21fc..7a8185d 100644 --- a/quality_control/parsing.py +++ b/quality_control/parsing.py @@ -104,23 +104,22 @@ def collect_errors( if line_number == 1: consistent_columns_checker = make_column_consistency_checker( filename, line) - for error in __process_errors__( - filename, line_number, line, - partial(header_errors, strains=strains), - errors): - yield error + yield from __process_errors__( + filename, line_number, line, + partial(header_errors, strains=strains), + errors) if line_number != 1: - col_consistency_error = consistent_columns_checker(line_number, line) + col_consistency_error = consistent_columns_checker(# pylint: disable=[possibly-used-before-assignment] + line_number, line) if col_consistency_error: yield col_consistency_error - for error in __process_errors__( + yield from __process_errors__( filename, line_number, line, ( average_errors if filetype == FileType.AVERAGE else se_errors), - errors): - yield error + errors) if update_progress: update_progress(line_number, line) diff --git a/r_qtl/r_qtl2.py b/r_qtl/r_qtl2/__init__.py index dfa84ba..ce1dbf8 100644 --- a/r_qtl/r_qtl2.py +++ b/r_qtl/r_qtl2/__init__.py @@ -16,7 +16,7 @@ from r_qtl.exceptions import InvalidFormat, MissingFileException FILE_TYPES = ( "geno", "founder_geno", "pheno", "covar", "phenocovar", "gmap", "pmap", - "phenose") + "phenose", "phenonum") __CONTROL_FILE_ERROR_MESSAGE__ = ( "The zipped bundle that was provided does not contain a valid control file " @@ -411,22 +411,19 @@ def file_data(zfile: ZipFile, try: if isinstance(cdata[member_key], list): - for row in (line for lines in + yield from (line for lines in (file_data( zfile, member_key, {**cdata, member_key: innerfile}, process_value, process_transposed_value) for innerfile in cdata[member_key]) - for line in lines): - yield row + for line in lines) return if not cdata.get(f"{member_key}_transposed", False): - for row in with_non_transposed(zfile, member_key, cdata, process_value): - yield row + yield from with_non_transposed(zfile, member_key, cdata, process_value) return - for row in with_transposed( - zfile, member_key, cdata, process_transposed_value): - yield row + yield from with_transposed( + zfile, member_key, cdata, process_transposed_value) except KeyError as exc: raise MissingFileException(*exc.args) from exc @@ -477,8 +474,7 @@ def raw_file_data(zipfilepath: Union[str, Path], with (ZipFile(str(zipfilepath), "r") as zfile, zfile.open(memberfilename) as innerfile): wrappedfile = io.TextIOWrapper(innerfile) - for line in wrappedfile: - yield line + yield from wrappedfile def strip_comments(rawdata: Iterator[str], commentchar) -> Iterator[str]: """Remove comments from raw text.""" @@ -568,15 +564,36 @@ def load_samples(zipfilepath: Union[str, Path], def read_text_file(filepath: Union[str, Path]) -> Iterator[str]: """Read the raw text from a text file.""" with open(filepath, "r", encoding="utf8") as _file: - for line in _file: - yield line + yield from _file def read_csv_file(filepath: Union[str, Path], separator: str = ",", comment_char: str = "#") -> Iterator[tuple[str, ...]]: - """Read a file as a csv file.""" + """Read a file as a csv file. This does not process the N/A values.""" for line in read_text_file(filepath): if line.startswith(comment_char): continue yield tuple(field.strip() for field in line.split(separator)) + + +def read_csv_file_headers( + filepath: Union[str, Path], + transposed: bool, + separator: str = ",", + comment_char: str = "#" +) -> tuple[str, ...]: + """Read the 'true' headers of a CSV file.""" + headers: tuple[str, ...] = tuple() + for line in read_text_file(filepath): + if line.startswith(comment_char): + continue + + row = tuple(field.strip() for field in line.split(separator)) + if not transposed: + return row + + headers = headers + (row[0],) + continue + + return headers diff --git a/r_qtl/r_qtl2/types.py b/r_qtl/r_qtl2/types.py new file mode 100644 index 0000000..13b8db4 --- /dev/null +++ b/r_qtl/r_qtl2/types.py @@ -0,0 +1,164 @@ +"""The types used by our R/qtl2 system.""" +from enum import Enum +from pathlib import Path +from dataclasses import field, dataclass +from typing import (Union, + Literal, + Optional, + Sequence, + TypeAlias, + Annotated) + + +_ALLELE_COUNTS_BY_CROSSTYPE = { + "bc": 2, "f2": 2, "riself": 2, "risib": 2, "dh": 2, "haploid": 2, "ail": 2, + "ail3": 3, + "riself4": 4, "risib4": 4, + "dh6": 6, + "hs": 8, "do": 8, "riself8": 8, "risib8": 8, + "dof1": 9, # 8 DO founders + 1 base inbred line + "riself16": 16, + "magic19": 19 +} + + + +PathLike = Union[Path, str] + +AlleleLabel: TypeAlias = str + +GenoCodeAlleleValue = Union[ + Literal[-1, 0, 1], + Literal[1, 2, 3],# T0D0: reconcile this to genocodes of 0, 1, 2 instead + Annotated[float, "Value must be between 0.0 and 2.0"] +] + +GenoCode = dict[AlleleLabel, GenoCodeAlleleValue] + +# Sex information +SexKey = Union[str, int]# codes used in the covariate file +@dataclass(frozen=True) +class ControlDataSex: + """Sex information in the control file.""" + covar: Optional[str] = None + file: Optional[PathLike] = None + female: Optional[SexKey] = None + male: Optional[SexKey] = None + + +# Cross information +CrossType = Literal[ + # T0D0: Look at https://kbroman.org/qtl2/assets/vignettes/input_files.html#Backcross to implement appropriate QC dependent on the crosstype. + "bc",# Backcross + "f2",# F2 intercross + "riself",# RIL by self -- RIL: Recombinant Inbred Lines + "risib",# RIL by sib + "dh",# Double haploid + "haploid",# Haploid + "ail",# AIL -- Advanced Inbred Lines + "hs",# Heterogeneous stock + "do",# Diversity outbreds + "riself4",# Multi-parent recombinant inbred lines + "riself8",# Multi-parent recombinant inbred lines + "riself16",# Multi-parent recombinant inbred lines + "risib4",# Multi-parent recombinant inbred lines + "risib8",# Multi-parent recombinant inbred lines + "magic19",# 19-way MAGIC lines + "dh6",# 6-way doubled haploids + "dof1",# DOF1 + "ail3",# 3-way advanced intercross lines + "genail",# General advanced intercross lines + "genril",# General recombinant inbred lines +] + +class CrossDirection(Enum): + FORWARD = 0 + REVERSE = 1 + + +@dataclass(frozen=True) +class ShortCrossInfo: + """Cross information in the control file.""" + # "covar": indicates the name of the column in the covariate data + covar: Optional[str] = None + file: Optional[PathLike] = None + forward_crosses: Sequence[str] = field(default_factory=tuple) + reverse_crosses: Sequence[str] = field(default_factory=tuple) + + def __post_init__(self): + """Validate the cross info.""" + # A valid short config MUST have either a `covar` or a `file` source + # but not both. + if not (self.covar and self.file): + raise ValueError("ShortCrossInfo requires either a 'covar' or a " + "'file' parameter.") + if self.covar and self.file: + raise ValueError("ShortCrossInfo cannot define both 'covar' and " + "'file' simultaneously.") + + +LongCrossInfo: TypeAlias = PathLike +CrossInfo = Union[LongCrossInfo, ShortCrossInfo] + + +def expected_allele_count_for_crosstype( + alleles: Sequence[str], + crosstype: CrossType, + expected_counts: dict[CrossType, int] +) -> bool: + """Check that the number of alleles matches what crosstype expects.""" + def __validate__(expected, alleles): + if expected is not None and len(alleles) != expected: + raise ValueError( + f"Cross type '{crosstype}' expects exactly {expected} allele " + f"labels. Received {len(alleles)}: {alleles}") + + if crosstype.startswith(("genail", "genril")): + # genail and genril are dynamic (e.g., genail8) + # Extract the trailing digits to determine expected founder count + num_part = "".join(filter(str.isdigit, crosstype)) + if num_part: + return __validate__(int(num_part), alleles) + + return __validate__(expected_counts.get(crosstype), alleles) + + + +@dataclass(frozen=True) +class ControlData: + """Class for the R/qtl2 control data.""" + # File names: Force listings, rather than singular strings + geno: Sequence[PathLike] = field(default_factory=tuple) + founder_geno: Sequence[PathLike] = field(default_factory=tuple) + pheno: Sequence[PathLike] = field(default_factory=tuple) + covar: Sequence[PathLike] = field(default_factory=tuple) + phenocovar: Sequence[PathLike] = field(default_factory=tuple) + gmap: Sequence[PathLike] = field(default_factory=tuple) + pmap: Sequence[PathLike] = field(default_factory=tuple) + + # X Chromosome + x_chr: Optional[str] = None + + # Allele labels + alleles: Sequence[str] = field(default_factory=tuple) + + # Genotype codes + genotypes: GenoCode = field(default_factory=dict) + + # sex + sex: Optional[ControlDataSex] = None + + # Cross info + crosstype: Optional[CrossType] = None + cross_info: Optional[CrossInfo] = None + + + # CSV fields + na_strings: Sequence[str] = ("-", "NA", "N/A") + sep: str = "," + comment_char: Optional[str] = "#" + + def __post_init__(self): + if self.alleles and self.crosstype: + expected_allele_count_for_crosstype( + self.alleles, self.crosstype, _ALLELE_COUNTS_BY_CROSSTYPE) diff --git a/scripts/cli/__init__.py b/scripts/cli/__init__.py new file mode 100644 index 0000000..45bbda9 --- /dev/null +++ b/scripts/cli/__init__.py @@ -0,0 +1,3 @@ +"""Package to hold CLI-specific utilities.""" + +from . import options diff --git a/scripts/cli/logging.py b/scripts/cli/logging.py new file mode 100644 index 0000000..30ecf17 --- /dev/null +++ b/scripts/cli/logging.py @@ -0,0 +1,18 @@ +"""Logging for scripts.""" +import logging + +def setup_logging( + script_logger: logging.Logger, + loglevel: str, + modules: tuple[str, ...] = tuple() +): + """Setup module-level loggers to the same log-level as the application.""" + logging.basicConfig( + encoding="utf-8", + format=("%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s: " + "%(message)s"), + level=logging.INFO) + script_logger.setLevel(getattr(logging, loglevel.upper())) + effective_loglevel = logging.getLevelName(script_logger.getEffectiveLevel()) + for module in modules: + logging.getLogger(module).setLevel(effective_loglevel) diff --git a/scripts/cli/options.py b/scripts/cli/options.py new file mode 100644 index 0000000..58d3df4 --- /dev/null +++ b/scripts/cli/options.py @@ -0,0 +1,56 @@ +"""General options to be added to CLI scripts.""" +from argparse import ArgumentParser + + +def add_logging(parser: ArgumentParser) -> ArgumentParser: + """Add optional log-level option""" + loglevels = ("debug", "info", "warning", "error", "critical") + parser.add_argument( + "--log_level", + "--log-level", + "--loglevel", + metavar="LOG-LEVEL", + type=str, + default="INFO", + choices=loglevels, + help=("Controls the severity of events to log. Valid values are: " + + ", ".join(f"'{level}'" for level in loglevels))) + return parser + + +def add_mariadb_uri(parser: ArgumentParser) -> ArgumentParser: + """Add the MySQL/MariaDB URI argument.""" + parser.add_argument("db_uri", + metavar="DB-URI", + type=str, + help="MariaDB/MySQL connection URL") + return parser + + +def add_species_id(parser: ArgumentParser) -> ArgumentParser: + """Add species-id as a mandatory argument.""" + parser.add_argument("species_id", + metavar="SPECIES-ID", + type=int, + help="The species to operate on.") + return parser + + +def add_population_id(parser: ArgumentParser) -> ArgumentParser: + """Add population-id as a mandatory argument.""" + parser = add_species_id(parser) + parser.add_argument("population_id", + metavar="POPULATION-ID", + type=int, + help="The ID for the population to operate on.") + return parser + + +def add_dataset_id(parser: ArgumentParser) -> ArgumentParser: + """Add dataset-id as a mandatory argument.""" + parser = add_population_id(parser) + parser.add_argument("dataset_id", + metavar="DATASET-ID", + type=int, + help="The ID for the dataset to operate on.") + return parser diff --git a/scripts/cli/validators.py b/scripts/cli/validators.py new file mode 100644 index 0000000..6d16e4c --- /dev/null +++ b/scripts/cli/validators.py @@ -0,0 +1,10 @@ +"""CLI options validators.""" +from pathlib import Path + + +def directory_exists(val: str) -> Path: + """Check that directory path specified actually exists.""" + _dir = Path(val).absolute() + if _dir.is_dir() and _dir.exists(): + return _dir + raise FileNotFoundError(f"The path '{_dir}' MUST exist and be a directory.") diff --git a/scripts/cli_parser.py b/scripts/cli_parser.py index d42ae66..bf39731 100644 --- a/scripts/cli_parser.py +++ b/scripts/cli_parser.py @@ -3,6 +3,20 @@ from uuid import UUID from typing import Optional from argparse import ArgumentParser + +def add_logging_option(parser: ArgumentParser) -> ArgumentParser: + """Add optional log-level option""" + parser.add_argument( + "--log-level", + "--loglevel", + type=str, + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL", + "debug", "info", "warning", "error", "critical"], + help="The severity of events to track with the logger.") + return parser + + def init_cli_parser(program: str, description: Optional[str] = None) -> ArgumentParser: """Initialise the CLI arguments parser.""" parser = ArgumentParser(prog=program, description=description) @@ -19,13 +33,8 @@ def init_cli_parser(program: str, description: Optional[str] = None) -> Argument type=int, default=86400, help="How long to keep any redis keys around.") - parser.add_argument( - "--loglevel", - type=str, - default="INFO", - choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], - help="The severity of events to track with the logger.") - return parser + return add_logging_option(parser) + def add_global_data_arguments(parser: ArgumentParser) -> ArgumentParser: """Add the global (present in nearly ALL scripts) CLI arguments.""" diff --git a/scripts/compute_phenotype_means.py b/scripts/compute_phenotype_means.py new file mode 100644 index 0000000..6d39ace --- /dev/null +++ b/scripts/compute_phenotype_means.py @@ -0,0 +1,101 @@ +"""Compute phenotype means.""" +import sys +import logging +from pathlib import Path +from typing import TypeVar +from argparse import Namespace, ArgumentParser + +import MySQLdb + +from gn_libs import mysqldb +from uploader import setup_modules_logging + +from .cli_parser import add_logging_option +from .load_phenotypes_to_db import update_means + +logger = logging.getLogger(__name__) +logging.basicConfig( + encoding="utf-8", + format="%(asctime)s - %(name)s - %(levelname)s — %(message)s", + level=logging.INFO) + + +def fetch_xref_id(conn: mysqldb.Connection, population_id: int) -> tuple[int, ...]: + """Fetch a population's cross-reference IDs.""" + logger.debug("Fetching the xref IDs.") + with conn.cursor(cursorclass=MySQLdb.cursors.DictCursor) as cursor: + query = "SELECT Id FROM PublishXRef WHERE InbredSetId=%(population_id)s" + cursor.execute(query, {"population_id": population_id}) + return tuple(int(row["Id"]) for row in cursor.fetchall()) + + +def run(args) -> int: + """Run the script.""" + logger.debug("Running the script!") + with mysqldb.database_connection(args.db_uri) as mariadb_conn: + xref_ids = args.cross_ref_ids or fetch_xref_id(mariadb_conn, args.population_id) + if len(xref_ids): + update_means(mariadb_conn, + args.population_id, + xref_ids) + logger.debug("Successfully computed means for %02d phenotypes.", + len(xref_ids)) + return 0 + _reasons = ( + f"no population exists with the ID {args.population_id}", + "the population exists but it has no phenotypes linked to it yet") + logger.error( + "No cross-reference IDs to run against. Likely causes are: %s", + " OR ".join(_reasons) + ".") + return 1 + + +T = TypeVar("T") +def comma_separated_list(val: str, itemstype: type = str) -> tuple[T, ...]: + """Convert val into a list of items of type 'itemstype'.""" + return tuple(itemstype(item.strip()) for item in val.split(",")) + + +def comma_separated_list_of_integers(val: str) -> tuple[int, ...]: + """Convert 'val' into list of items of type 'int'.""" + return comma_separated_list(val, int) + + +if __name__ == "__main__": + def parse_args() -> Namespace: + """Define and parse the CLI parsers accepted by this script.""" + parser = ArgumentParser( + "compute-phenotype-means", + description="Compute/Recompute the phenotype means.") + parser.add_argument("db_uri", + metavar="db-uri", + type=str, + help="MariaDB/MySQL connection URL") + parser.add_argument("jobs_db_path", + metavar="jobs-db-path", + type=Path, + help="Path to jobs' SQLite database.") + parser.add_argument("population_id", + metavar="population-id", + type=int, + help=("Identifier for the InbredSet group/" + "population to run means against.")) + ## Optional arguments + parser = add_logging_option(parser) + parser.add_argument( + "--cross-ref-ids", + type=comma_separated_list_of_integers, + help=("Provide cross-reference IDs to narrow the number of " + "phenotypes that the means are computed against."), + default=[]) + + return parser.parse_args() + + def main() -> int: + """compute-phenotype-means: Entry-point function.""" + args = parse_args() + logger.setLevel(getattr(logging, args.log_level.upper())) + setup_modules_logging(logger, ("scripts.load_phenotypes_to_db",)) + return run(args) + + sys.exit(main()) diff --git a/scripts/insert_data.py b/scripts/insert_data.py index 67038f8..aec0251 100644 --- a/scripts/insert_data.py +++ b/scripts/insert_data.py @@ -197,7 +197,7 @@ def probeset_ids(dbconn: mdb.Connection, break yield row -def insert_means(# pylint: disable=[too-many-locals, too-many-arguments] +def insert_means(# pylint: disable=[too-many-locals, too-many-arguments, too-many-positional-arguments] filepath: str, speciesid: int, platform_id: int, datasetid: int, dbconn: mdb.Connection, rconn: Redis) -> int: # pylint: disable=[unused-argument] "Insert the means/averages data into the database" @@ -232,7 +232,7 @@ def insert_means(# pylint: disable=[too-many-locals, too-many-arguments] item for sublist in read_datavalues(filepath, headings, strains).values() for item in sublist), - start=(last_data_id(dbconn)+1))) + start=last_data_id(dbconn)+1)) with dbconn.cursor(cursorclass=DictCursor) as cursor: while True: means = tuple(take(the_means, 10000)) @@ -245,7 +245,7 @@ def insert_means(# pylint: disable=[too-many-locals, too-many-arguments] cursor.executemany(xref_query, means) return 0 -def insert_se(# pylint: disable = [too-many-arguments,too-many-locals] +def insert_se(# pylint: disable = [too-many-arguments,too-many-locals, too-many-positional-arguments] filepath: str, speciesid: int, platformid: int, datasetid: int, dbconn: mdb.Connection, rconn: Redis) -> int: # pylint: disable=[unused-argument] "Insert the standard-error data into the database" diff --git a/scripts/insert_samples.py b/scripts/insert_samples.py index 1b0a052..96ae8e2 100644 --- a/scripts/insert_samples.py +++ b/scripts/insert_samples.py @@ -3,12 +3,13 @@ import sys import logging import pathlib import argparse +import traceback import MySQLdb as mdb -from redis import Redis + from gn_libs.mysqldb import database_connection -from uploader.check_connections import check_db, check_redis +from uploader.check_connections import check_db from uploader.species.models import species_by_id from uploader.population.models import population_by_id from uploader.samples.models import ( @@ -33,8 +34,7 @@ class SeparatorAction(argparse.Action): """Process the value passed in.""" setattr(namespace, self.dest, (chr(9) if values == "\\t" else values)) -def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments] - rconn: Redis,# pylint: disable=[unused-argument] +def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments, too-many-positional-arguments] speciesid: int, populationid: int, samplesfile: pathlib.Path, @@ -73,6 +73,7 @@ def insert_samples(conn: mdb.Connection,# pylint: disable=[too-many-arguments] print("Samples upload successfully completed.") return 0 + if __name__ == "__main__": def cli_args(): @@ -117,33 +118,30 @@ if __name__ == "__main__": help=("The character used to delimit (surround?) the value in " "each column.")) - # == Script-specific extras == - parser.add_argument("--redisuri", - help="URL to initialise connection to redis", - default="redis:///") - args = parser.parse_args() return args def main(): """Run script to insert samples into the database.""" - + status_code = 1 # Exit with an Exception args = cli_args() check_db(args.databaseuri) - check_redis(args.redisuri) if not args.samplesfile.exists(): logging.error("File not found: '%s'.", args.samplesfile) return 2 - with (Redis.from_url(args.redisuri, decode_responses=True) as rconn, - database_connection(args.databaseuri) as dbconn): - return insert_samples(dbconn, - rconn, - args.speciesid, - args.populationid, - args.samplesfile, - args.separator, - args.firstlineheading, - args.quotechar) + with database_connection(args.databaseuri) as dbconn: + try: + status_code = insert_samples(dbconn, + args.speciesid, + args.populationid, + args.samplesfile, + args.separator, + args.firstlineheading, + args.quotechar) + except Exception as _exc:# pylint: disable=[broad-exception-caught] + print(traceback.format_exc(), file=sys.stderr) + + return status_code sys.exit(main()) diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py new file mode 100644 index 0000000..31eb715 --- /dev/null +++ b/scripts/load_phenotypes_to_db.py @@ -0,0 +1,551 @@ +"""Load phenotypes and their data provided in files into the database.""" +import sys +import uuid +import json +import time +import logging +import argparse +from pathlib import Path +from zipfile import ZipFile +from datetime import datetime +from typing import Any, Iterable +from urllib.parse import urljoin +from functools import reduce, partial + +from MySQLdb.cursors import DictCursor + +from gn_libs import jobs, mysqldb, sqlite3, monadic_requests as mrequests + +from r_qtl import r_qtl2 as rqtl2 +from uploader.species.models import species_by_id +from uploader.population.models import population_by_species_and_id +from uploader.samples.models import samples_by_species_and_population +from uploader.phenotypes.models import ( + dataset_by_id, + save_phenotypes_data, + create_new_phenotypes, + quick_save_phenotypes_data) +from uploader.publications.models import fetch_publication_by_id + +from scripts.rqtl2.bundleutils import build_line_joiner, build_line_splitter + +from functional_tools import take + +logging.basicConfig( + format="%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s: %(message)s") +logger = logging.getLogger(__name__) + + + +def __replace_na_strings__(line, na_strings): + return ((None if value in na_strings else value) for value in line) + + +def save_phenotypes( + conn: mysqldb.Connection, + control_data: dict[str, Any], + population_id, + publication_id, + filesdir: Path +) -> tuple[dict, ...]: + """Read `phenofiles` and save the phenotypes therein.""" + phenofiles = tuple(filesdir.joinpath(_file) for _file in control_data["phenocovar"]) + if len(phenofiles) <= 0: + return tuple() + + if control_data["phenocovar_transposed"]: + logger.info("Undoing transposition of the files rows and columns.") + phenofiles = tuple( + rqtl2.transpose_csv_with_rename( + _file, + build_line_splitter(control_data), + build_line_joiner(control_data)) + for _file in phenofiles) + + _headers = rqtl2.read_csv_file_headers(phenofiles[0], + control_data["phenocovar_transposed"], + control_data["sep"], + control_data["comment.char"]) + return create_new_phenotypes( + conn, + population_id, + publication_id, + (dict(zip(_headers, + __replace_na_strings__(line, control_data["na.strings"]))) + for filecontent + in (rqtl2.read_csv_file(path, + separator=control_data["sep"], + comment_char=control_data["comment.char"]) + for path in phenofiles) + for idx, line in enumerate(filecontent) + if idx != 0)) + + +def __row_to_dataitems__( + sample_row: dict, + dataidmap: dict, + pheno_name2id: dict[str, int], + samples: dict +) -> Iterable[dict]: + samplename = sample_row["id"] + + return ({ + "phenotype_id": dataidmap[pheno_name2id[phenoname]]["phenotype_id"], + "data_id": dataidmap[pheno_name2id[phenoname]]["data_id"], + "sample_name": samplename, + "sample_id": samples[samplename]["Id"], + "value": phenovalue + } for phenoname, phenovalue in sample_row.items() if phenoname != "id") + + +def __build_dataitems__( + phenofiles, + control_data, + samples, + dataidmap, + pheno_name2id +): + _headers = rqtl2.read_csv_file_headers( + phenofiles[0], + False, # Any transposed files have been un-transposed by this point + control_data["sep"], + control_data["comment.char"]) + _filescontents = ( + rqtl2.read_csv_file(path, + separator=control_data["sep"], + comment_char=control_data["comment.char"]) + for path in phenofiles) + _linescontents = ( + __row_to_dataitems__( + dict(zip(("id",) + _headers[1:], + __replace_na_strings__(line, control_data["na.strings"]))), + dataidmap, + pheno_name2id, + samples) + for linenum, line in (enumline for filecontent in _filescontents + for enumline in enumerate(filecontent)) + if linenum > 0) + return (item for items in _linescontents + for item in items + if item["value"] is not None) + + +def save_numeric_data(# pylint: disable=[too-many-positional-arguments,too-many-arguments] + conn: mysqldb.Connection, + dataidmap: dict, + pheno_name2id: dict[str, int], + samples: dict, + control_data: dict, + filesdir: Path, + filetype: str, + table: str +): + """Read data from files and save to the database.""" + phenofiles = tuple( + filesdir.joinpath(_file) for _file in control_data[filetype]) + if len(phenofiles) <= 0: + return tuple() + + if control_data[f"{filetype}_transposed"]: + logger.info("Undoing transposition of the files rows and columns.") + phenofiles = tuple( + rqtl2.transpose_csv_with_rename( + _file, + build_line_splitter(control_data), + build_line_joiner(control_data)) + for _file in phenofiles) + + try: + logger.debug("Attempt quick save with `LOAD … INFILE`.") + return quick_save_phenotypes_data( + conn, + table, + __build_dataitems__( + phenofiles, + control_data, + samples, + dataidmap, + pheno_name2id), + filesdir) + except Exception as _exc:# pylint: disable=[broad-exception-caught] + logger.debug("Could not use `LOAD … INFILE`, using raw query", + exc_info=True) + time.sleep(60) + return save_phenotypes_data( + conn, + table, + __build_dataitems__( + phenofiles, + control_data, + samples, + dataidmap, + pheno_name2id)) + + +save_pheno_data = partial(save_numeric_data, + filetype="pheno", + table="PublishData") + + +save_phenotypes_se = partial(save_numeric_data, + filetype="phenose", + table="PublishSE") + + +save_phenotypes_n = partial(save_numeric_data, + filetype="phenonum", + table="NStrain") + + +def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments,too-many-arguments] + auth_details, + resource_details, + species, + population, + dataset, + xrefdata): + """Grant the user access to their data.""" + logger.info("Updating authorisation for the data.") + logger.debug("Resource details for the authorisation: %s", resource_details) + authserver, token = auth_details + _tries = 0 + _delay = 1 + headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + } + def authserveruri(endpoint): + return urljoin(authserver, endpoint) + + def __fetch_user_details__(): + logger.info("… Fetching user details") + return mrequests.get( + authserveruri("/auth/user/"), + headers=headers + ) + + def __link_data__(user): + logger.info("… linking uploaded data to user's group") + return mrequests.post( + authserveruri("/auth/data/link/phenotype"), + headers=headers, + json={ + "species_name": species["Name"], + "group_id": user["group"]["group_id"], + "selected": [ + { + "SpeciesId": species["SpeciesId"], + "InbredSetId": population["Id"], + "PublishFreezeId": dataset["Id"], + "dataset_name": dataset["Name"], + "dataset_fullname": dataset["FullName"], + "dataset_shortname": dataset["ShortName"], + "PublishXRefId": item["xref_id"] + } + for item in xrefdata + ], + "using-raw-ids": "on" + }).then(lambda ld_results: (user, ld_results)) + + def __fetch_phenotype_category_details__(user, linkeddata): + logger.info("… fetching phenotype category details") + return mrequests.get( + authserveruri("/auth/resource/categories"), + headers=headers + ).then( + lambda categories: ( + user, + linkeddata, + next(category for category in categories + if category["resource_category_key"] == "phenotype")) + ) + + def __create_resource__(user, linkeddata, category): + logger.info("… creating authorisation resource object") + return mrequests.post( + authserveruri("/auth/resource/create"), + headers=headers, + json={ + **resource_details, + "resource_category": category["resource_category_id"], + "public": "off" + }).then(lambda cr_results: (user, linkeddata, cr_results)) + + def __attach_data_to_resource__(user, linkeddata, resource): + logger.info("… attaching data to authorisation resource object") + return mrequests.post( + authserveruri("/auth/resource/data/link"), + headers=headers, + json={ + "dataset_type": "phenotype", + "resource_id": resource["resource_id"], + "data_link_ids": [ + item["data_link_id"] for item in linkeddata["traits"]] + }).then(lambda attc: (user, linkeddata, resource, attc)) + + def __handle_error__(resp): + error = resp.json() + if error.get("error") == "IntegrityError": + # This is hacky. If the auth already exists, something went wrong + # somewhere. + # This needs investigation to recover correctly. + logger.error( + "Error: The authorisation for the data was already set up.") + return 0 + logger.error("ERROR: Updating the authorisation for the data failed.") + logger.debug( + "ERROR: The response from the authorisation server was:\n\t%s", + error) + return 1 + + def __handle_success__(_val): + logger.info( + "The authorisation for the data has been updated successfully.") + return 0 + + return __fetch_user_details__().then(__link_data__).then( + lambda result: __fetch_phenotype_category_details__(*result) + ).then( + lambda result: __create_resource__(*result) + ).then( + lambda result: __attach_data_to_resource__(*result) + ).either(__handle_error__, __handle_success__) + + +def load_data(# pylint: disable=[too-many-locals] + conn: mysqldb.Connection, job: dict +) -> tuple[dict, dict, dict, tuple[int, ...]]: + """Load the data attached in the given job.""" + _job_metadata = job["metadata"] + # Steps + # 0. Read data from the files: can be multiple files per type + # + _species = species_by_id(conn, int(_job_metadata["species_id"])) + _population = population_by_species_and_id( + conn, + _species["SpeciesId"], + int(_job_metadata["population_id"])) + _dataset = dataset_by_id( + conn, + _species["SpeciesId"], + _population["Id"], + int(_job_metadata["dataset_id"])) + # 1. Just retrive the publication: Don't create publications for now. + _publication = fetch_publication_by_id( + conn, int(_job_metadata.get("publication_id", "0"))) or {"Id": 0} + # 2. Save all new phenotypes: + # -> return phenotype IDs + bundle = Path(_job_metadata["bundle_file"]) + _control_data = rqtl2.control_data(bundle) + logger.info("Extracting the zipped bundle of files.") + _outdir = Path(bundle.parent, f"bundle_{bundle.stem}") + with ZipFile(str(bundle), "r") as zfile: + _files = rqtl2.extract(zfile, _outdir) + logger.info("Saving new phenotypes.") + _phenos = save_phenotypes(conn, + _control_data, + _population["Id"], + _publication["Id"], + _outdir) + + def __build_phenos_maps__(accumulator, row): + return ({ + **accumulator[0], + row["phenotype_id"]: { + "population_id": _population["Id"], + "phenotype_id": row["phenotype_id"], + "data_id": row["data_id"], + "publication_id": row["publication_id"], + } + }, { + **accumulator[1], + row["pre_publication_abbreviation"]: row["phenotype_id"] + }, ( + accumulator[2] + ({ + "xref_id": row["xref_id"], + "population_id": row["population_id"], + "phenotype_id": row["phenotype_id"], + "publication_id": row["publication_id"], + "data_id": row["data_id"] + },))) + dataidmap, pheno_name2id, _xrefs = reduce(# type: ignore[var-annotated] + __build_phenos_maps__, _phenos, ({},{}, tuple())) + # 3. a. Fetch the strain names and IDS: create name->ID map + samples = { + row["Name"]: row + for row in samples_by_species_and_population( + conn, _species["SpeciesId"], _population["Id"])} + # b. Save all the data items (DataIds are vibes), return new IDs + logger.info("Saving new phenotypes data.") + _num_data_rows = save_pheno_data(conn=conn, + dataidmap=dataidmap, + pheno_name2id=pheno_name2id, + samples=samples, + control_data=_control_data, + filesdir=_outdir) + logger.info("Saved %s new phenotype data rows.", _num_data_rows) + + # 4. If standard errors and N exist, save them too + # (use IDs returned in `3. b.` above). + if _control_data.get("phenose"): + logger.info("Saving new phenotypes standard errors.") + _num_se_rows = save_phenotypes_se(conn=conn, + dataidmap=dataidmap, + pheno_name2id=pheno_name2id, + samples=samples, + control_data=_control_data, + filesdir=_outdir) + logger.info("Saved %s new phenotype standard error rows.", _num_se_rows) + + if _control_data.get("phenonum"): + logger.info("Saving new phenotypes sample counts.") + _num_n_rows = save_phenotypes_n(conn=conn, + dataidmap=dataidmap, + pheno_name2id=pheno_name2id, + samples=samples, + control_data=_control_data, + filesdir=_outdir) + logger.info("Saved %s new phenotype sample counts rows.", _num_n_rows) + + return (_species, _population, _dataset, _xrefs) + + +def update_means( + conn: mysqldb.Connection, + population_id: int, + xref_ids: tuple[int, ...] +): + """Compute the means from the data and update them in the database.""" + logger.info("Computing means for %02d phenotypes.", len(xref_ids)) + query = ( + "UPDATE PublishXRef SET mean = " + "(SELECT AVG(value) FROM PublishData" + " WHERE PublishData.Id=PublishXRef.DataId) " + "WHERE PublishXRef.Id=%(xref_id)s " + "AND PublishXRef.InbredSetId=%(population_id)s") + _xref_iterator = (_xref_id for _xref_id in xref_ids) + with conn.cursor(cursorclass=DictCursor) as cursor: + while True: + batch = take(_xref_iterator, 10000) + if len(batch) == 0: + break + logger.info("\tComputing means for batch of %02d phenotypes.", len(batch)) + cursor.executemany( + query, + tuple({ + "population_id": population_id, + "xref_id": _xref_id + } for _xref_id in batch)) + + +if __name__ == "__main__": + def parse_args(): + """Setup command-line arguments.""" + parser = argparse.ArgumentParser( + prog="load_phenotypes_to_db", + description="Process the phenotypes' data and load it into the database.") + parser.add_argument("db_uri", type=str, help="MariaDB/MySQL connection URL") + parser.add_argument( + "jobs_db_path", type=Path, help="Path to jobs' SQLite database.") + parser.add_argument("job_id", type=uuid.UUID, help="ID of the running job") + parser.add_argument( + "--log-level", + type=str, + help="Determines what is logged out.", + choices=("debug", "info", "warning", "error", "critical"), + default="info") + return parser.parse_args() + + def setup_logging(log_level: str): + """Setup logging for the script.""" + logger.setLevel(log_level) + logging.getLogger("uploader.phenotypes.models").setLevel(log_level) + + + def __parse_resource_details__(meta) -> dict: + """Parse out details regarding the wrapper resource from the metadata.""" + _key_mappings_ = { + # allow both 'data_*' and 'data*' for the metadata. + "data_description": "description", + "datadescription": "description" + } + return { + "resource_name": meta.get( + "dataname", + meta.get("data_name", + "Unnamed phenotypes - " + datetime.now().isoformat())), + "resource_metadata": { + rkey: meta[mkey] + for mkey, rkey in _key_mappings_.items() if mkey in meta + } + } + + + def main(): + """Entry-point for this script.""" + args = parse_args() + setup_logging(args.log_level.upper()) + + with (mysqldb.database_connection(args.db_uri) as conn, + conn.cursor(cursorclass=DictCursor) as cursor, + sqlite3.connection(args.jobs_db_path) as jobs_conn): + job = jobs.job(jobs_conn, args.job_id) + + # Lock the PublishXRef/PublishData/PublishSE/NStrain here: Why? + # The `DataId` values are sequential, but not auto-increment + # Can't convert `PublishXRef`.`DataId` to AUTO_INCREMENT. + # `SELECT MAX(DataId) FROM PublishXRef;` + # How do you check for a table lock? + # https://oracle-base.com/articles/mysql/mysql-identify-locked-tables + # `SHOW OPEN TABLES LIKE 'Publish%';` + _db_tables_ = ( + "Species", + "InbredSet", + "Strain", + "StrainXRef", + "Publication", + "Phenotype", + "PublishXRef", + "PublishFreeze", + "PublishData", + "PublishSE", + "NStrain") + + logger.debug( + ("Locking database tables for the connection:" + + "".join("\n\t- %s" for _ in _db_tables_) + "\n"), + *_db_tables_) + cursor.execute(# Lock the tables to avoid race conditions + "LOCK TABLES " + ", ".join( + f"{_table} WRITE" for _table in _db_tables_)) + + db_results = load_data(conn, job) + _xref_ids = tuple(xref["xref_id"] for xref in db_results[3]) + jobs.update_metadata( + jobs_conn, + args.job_id, + "xref_ids", + json.dumps(_xref_ids)) + + logger.info("Unlocking all database tables.") + cursor.execute("UNLOCK TABLES") + + logger.info("Updating means.") + update_means(conn, db_results[1]["Id"], _xref_ids) + + # Update authorisations (break this down) — maybe loop until it works? + logger.info("Updating authorisation.") + _job_metadata = job["metadata"] + + return update_auth((_job_metadata["authserver"], + _job_metadata["token"]), + __parse_resource_details__(_job_metadata), + *db_results) + + + try: + sys.exit(main()) + except Exception as _exc:# pylint: disable=[broad-exception-caught] + logger.debug("Data loading failed… Halting!", + exc_info=True) + sys.exit(1) diff --git a/scripts/phenotypes/__init__.py b/scripts/phenotypes/__init__.py new file mode 100644 index 0000000..73ad839 --- /dev/null +++ b/scripts/phenotypes/__init__.py @@ -0,0 +1 @@ +"Scripts for dealing with phenotypes." diff --git a/scripts/phenotypes/delete_phenotypes.py b/scripts/phenotypes/delete_phenotypes.py new file mode 100644 index 0000000..461f3ec --- /dev/null +++ b/scripts/phenotypes/delete_phenotypes.py @@ -0,0 +1,173 @@ +"""Delete phenotypes.""" +import sys +import logging +from pathlib import Path +from typing import Optional +from urllib.parse import urljoin +from argparse import Namespace, ArgumentParser + +import requests +from MySQLdb.cursors import DictCursor, BaseCursor + +from gn_libs.mysqldb import database_connection + +from uploader.phenotypes.models import delete_phenotypes +from scripts.cli.logging import setup_logging +from scripts.cli.options import (add_logging, + add_mariadb_uri, + add_population_id) + +logger = logging.getLogger(__name__) + +def read_xref_ids_file(filepath: Optional[Path]) -> tuple[int, ...]: + """Read the phenotypes' cross-reference IDS from file.""" + if filepath is None: + return tuple() + + logger.debug("Using file '%s' to retrieve XREF IDs for deletion.", + filepath.name) + _ids: tuple[int, ...] = tuple() + with filepath.open(mode="r") as infile: + for line in infile.readlines(): + try: + _ids += (int(line.strip()),) + except TypeError: + pass + + return _ids + + +def fetch_all_xref_ids( + cursor: BaseCursor, population_id: int) -> tuple[int, ...]: + """Fetch all cross-reference IDs.""" + cursor.execute("SELECT Id FROM PublishXRef WHERE InbredSetId=%s", + (population_id,)) + return tuple(int(row["Id"]) for row in cursor.fetchall()) + + +def update_auth( + auth_details: tuple[str, str], + species_id: int, + population_id: int, + dataset_id: int, + xref_ids: tuple[int, ...] = tuple() +): + """Update the authorisation server: remove items to delete.""" + authserver, token = auth_details + resp = requests.post( + urljoin(authserver, + (f"/auth/data/phenotypes/{species_id}/{population_id}" + f"/{dataset_id}/delete")), + timeout=(9.13, 20), + headers={ + "Authorization": f"Bearer {token}", + "Content-Type": "application/json" + }, + json={"xref_ids": xref_ids}) + resp.raise_for_status() + + +def delete_the_phenotypes( + cursor: BaseCursor, + population_id: int, + xref_ids: tuple[int, ...] = tuple()) -> int: + """Process and delete the phenotypes.""" + delete_phenotypes(cursor, population_id, xref_ids) + + return 0 + +if __name__ == "__main__": + def parse_args() -> Namespace: + """Parse CLI arguments.""" + parser = add_logging( + add_population_id( + add_mariadb_uri( + ArgumentParser( + prog="delete-phenotypes", + description=( + "Script to delete phenotypes from the database."))))) + parser.add_argument( + "dataset_id", + metavar="DATASET-ID", + type=int, + help="The dataset identifier for phenotypes to delete.") + parser.add_argument( + "auth_server_uri", + metavar="AUTH-SERVER-URI", + type=str, + help="URI to the authorisation server.") + parser.add_argument( + "auth_token", + metavar="AUTH-TOKEN", + type=str, + help=("Token to use to update the authorisation system with the " + "deletions done.")) + parser.add_argument( + "--xref_ids_file", + metavar="XREF-IDS-FILE", + type=Path, + help=("Path to a file with phenotypes cross-reference IDs to " + "delete.")) + parser.add_argument( + "--delete-all", + action="store_true", + help=("If no 'XREF-IDS-FILE' is provided, this flag determines " + "whether or not all the phenotypes for the given population " + "will be deleted.")) + return parser.parse_args() + + + def main(): + """The `delete-phenotypes` script's entry point.""" + args = parse_args() + setup_logging(logger, args.log_level.upper(), tuple()) + with (database_connection(args.db_uri) as conn, + conn.cursor(cursorclass=DictCursor) as cursor): + xref_ids = read_xref_ids_file(args.xref_ids_file) + try: + assert not (len(xref_ids) > 0 and args.delete_all) + xref_ids = (fetch_all_xref_ids(cursor, args.population_id) + if args.delete_all else xref_ids) + logger.debug("Will delete %s phenotypes and related data", + len(xref_ids)) + if len(xref_ids) == 0: + print("No cross-reference IDs were provided. Aborting.") + return 0 + + print("Updating authorisations: ", end="") + update_auth((args.auth_server_uri, args.auth_token), + args.species_id, + args.population_id, + args.dataset_id, + xref_ids) + print("OK.") + print("Deleting the data: ", end="") + delete_phenotypes(cursor, args.population_id, xref_ids=xref_ids) + print("OK.") + if args.xref_ids_file is not None: + print("Deleting temporary file: ", end="") + args.xref_ids_file.unlink() + print("OK.") + + return 0 + except AssertionError: + logger.error( + "'DELETE-ALL' and 'XREF-IDS' are mutually exclusive. " + "If you specify the list of XREF-IDS (in a file) to delete " + "and also specify to 'DELETE-ALL' phenotypes in the " + "population, we have no way of knowing what it is you want.") + return 1 + except requests.exceptions.HTTPError as _exc: + resp = _exc.response + resp_data = resp.json() + logger.debug("%s: %s", + resp_data["error"], + resp_data["error_description"], + exc_info=True) + return 1 + except Exception as _exc:# pylint: disable=[broad-exception-caught] + logger.debug("Failed while attempting to delete phenotypes.", + exc_info=True) + return 1 + + sys.exit(main()) diff --git a/scripts/process_rqtl2_bundle.py b/scripts/process_rqtl2_bundle.py index 8b7a0fb..e2ce420 100644 --- a/scripts/process_rqtl2_bundle.py +++ b/scripts/process_rqtl2_bundle.py @@ -104,7 +104,7 @@ def process_bundle(dbconn: mdb.Connection, rqtl2bundle=Path(meta["rqtl2-bundle-file"])), logger) if genoexit != 0: - raise Exception("Processing 'geno' file failed.") + raise Exception("Processing 'geno' file failed.")# pylint: disable=[broad-exception-raised] logger.debug( "geno file processing completed successfully. (ExitCode: %s)", genoexit) @@ -122,7 +122,7 @@ def process_bundle(dbconn: mdb.Connection, rqtl2bundle=Path(meta["rqtl2-bundle-file"])), logger) if phenoexit != 0: - raise Exception("Processing 'pheno' file failed.") + raise Exception("Processing 'pheno' file failed.")# pylint: disable=[broad-exception-raised] logger.debug( "pheno file processing completed successfully. (ExitCode: %s)", phenoexit) diff --git a/scripts/qc_on_rqtl2_bundle.py b/scripts/qc_on_rqtl2_bundle.py index 9f9248c..4e6ef00 100644 --- a/scripts/qc_on_rqtl2_bundle.py +++ b/scripts/qc_on_rqtl2_bundle.py @@ -40,7 +40,7 @@ def add_to_errors(rconn: Redis, """Add `errors` to a given list of errors""" errs = tuple(dict(item) for item in set( [dict2tuple(old) for old in - json.loads(rconn.hget(fqjobid, key) or "[]")] + + json.loads(rconn.hget(fqjobid, key) or "[]")] +# type: ignore[arg-type] [dict2tuple({"type": type(error).__name__, **error._asdict()}) for error in errors])) rconn.hset(fqjobid, key, json.dumps(errs)) @@ -83,7 +83,8 @@ def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals count = 0 checked = 0 cdata = rqtl2.control_data(zfile) - rconn.hset(fqjobid, f"{filetype}-filesize", compute_filesize(zfile, filetype)) + rconn.hset( + fqjobid, f"{filetype}-filesize", str(compute_filesize(zfile, filetype))) def __update_processed__(value): nonlocal checked checked = checked + len(value) @@ -104,7 +105,7 @@ def retrieve_errors_with_progress(rconn: Redis,#pylint: disable=[too-many-locals yield error __update_processed__(value) - rconn.hset(fqjobid, f"{filetype}-linecount", count) + rconn.hset(fqjobid, f"{filetype}-linecount", count)# type: ignore[arg-type] except rqe.MissingFileException: fname = cdata.get(filetype) yield rqfe.MissingFile(filetype, fname, ( @@ -191,7 +192,7 @@ def check_pheno_samples( return allerrors -def qc_pheno_errors(# pylint: disable=[too-many-arguments] +def qc_pheno_errors(# pylint: disable=[too-many-arguments, too-many-positional-arguments] rconn, fqjobid, dburi, speciesid, zfile, logger) -> bool: """Check for errors in `pheno` file(s).""" cdata = rqtl2.control_data(zfile) @@ -260,7 +261,7 @@ def run_qc(rconn: Redis, if qc_missing_files(rconn, fqjobid, zfile, logger): return 1 - def with_zipfile(# pylint: disable=[too-many-arguments] + def with_zipfile(# pylint: disable=[too-many-arguments,too-many-positional-arguments] rconn, fqjobid, dbconn, speciesid, filename, logger, func ): with ZipFile(filename, "r") as zfile: @@ -295,7 +296,7 @@ def run_qc(rconn: Redis, return 1 def __fetch_errors__(rkey: str) -> tuple: - return tuple(json.loads(rconn.hget(fqjobid, rkey) or "[]")) + return tuple(json.loads(rconn.hget(fqjobid, rkey) or "[]")) # type: ignore[arg-type] return (1 if any(( bool(__fetch_errors__(key)) diff --git a/scripts/qc_on_rqtl2_bundle2.py b/scripts/qc_on_rqtl2_bundle2.py deleted file mode 100644 index 7e5d253..0000000 --- a/scripts/qc_on_rqtl2_bundle2.py +++ /dev/null @@ -1,346 +0,0 @@ -"""Run Quality Control checks on R/qtl2 bundle.""" -import os -import sys -import json -from time import sleep -from pathlib import Path -from zipfile import ZipFile -from argparse import Namespace -from datetime import timedelta -import multiprocessing as mproc -from functools import reduce, partial -from logging import Logger, getLogger, StreamHandler -from typing import Union, Sequence, Callable, Iterator - -import MySQLdb as mdb -from redis import Redis - -from quality_control.errors import InvalidValue -from quality_control.checks import decimal_points_error - -from uploader import jobs -from uploader.db_utils import database_connection -from uploader.check_connections import check_db, check_redis - -from r_qtl import r_qtl2 as rqtl2 -from r_qtl import r_qtl2_qc as rqc -from r_qtl import exceptions as rqe -from r_qtl import fileerrors as rqfe - -from scripts.process_rqtl2_bundle import parse_job -from scripts.redis_logger import setup_redis_logger -from scripts.cli_parser import init_cli_parser, add_global_data_arguments -from scripts.rqtl2.bundleutils import build_line_joiner, build_line_splitter - - -def check_for_missing_files( - rconn: Redis, fqjobid: str, extractpath: Path, logger: Logger) -> bool: - """Check that all files listed in the control file do actually exist.""" - logger.info("Checking for missing files.") - missing = rqc.missing_files(extractpath) - # add_to_errors(rconn, fqjobid, "errors-generic", tuple( - # rqfe.MissingFile( - # mfile[0], mfile[1], ( - # f"File '{mfile[1]}' is listed in the control file under " - # f"the '{mfile[0]}' key, but it does not actually exist in " - # "the bundle.")) - # for mfile in missing)) - if len(missing) > 0: - logger.error(f"Missing files in the bundle!") - return True - return False - - -def open_file(file_: Path) -> Iterator: - """Open file and return one line at a time.""" - with open(file_, "r", encoding="utf8") as infile: - for line in infile: - yield line - - -def check_markers( - filename: str, - row: tuple[str, ...], - save_error: lambda val: val -) -> tuple[rqfe.InvalidValue]: - """Check that the markers are okay""" - errors = tuple() - counts = {} - for marker in row: - counts = {**counts, marker: counts.get(marker, 0) + 1} - if marker is None or marker == "": - errors = errors + (save_error(rqfe.InvalidValue( - filename, - "markers" - "-", - marker, - "A marker MUST be a valid value.")),) - - return errors + tuple( - save_error(rqfe.InvalidValue( - filename, - "markers", - key, - f"Marker '{key}' was repeated {value} times")) - for key,value in counts.items() if value > 1) - - -def check_geno_line( - filename: str, - headers: tuple[str, ...], - row: tuple[Union[str, None]], - cdata: dict, - save_error: lambda val: val -) -> tuple[rqfe.InvalidValue]: - """Check that the geno line is correct.""" - errors = tuple() - # Verify that line has same number of columns as headers - if len(headers) != len(row): - errors = errors + (save_error(rqfe.InvalidValue( - filename, - headers[0], - row[0], - row[0], - "Every line MUST have the same number of columns.")),) - - # First column is the individuals/cases/samples - if not bool(row[0]): - errors = errors + (save_error(rqfe.InvalidValue( - filename, - headers[0], - row[0], - row[0], - "The sample/case MUST be a valid value.")),) - - def __process_value__(val): - if val in cdata["na.strings"]: - return None - if val in cdata["alleles"]: - return cdata["genotypes"][val] - - genocode = cdata.get("genotypes", {}) - for coltitle, cellvalue in zip(headers[1:],row[1:]): - if ( - bool(genocode) and - cellvalue is not None and - cellvalue not in genocode.keys() - ): - errors = errors + (save_error(rqfe.InvalidValue( - filename, row[0], coltitle, cellvalue, - f"Value '{cellvalue}' is invalid. Expected one of " - f"'{', '.join(genocode.keys())}'.")),) - - return errors - - -def push_file_error_to_redis(rconn: Redis, key: str, error: InvalidValue) -> InvalidValue: - """Push the file error to redis a json string - - Parameters - ---------- - rconn: Connection to redis - key: The name of the list where we push the errors - error: The file error to save - - Returns - ------- - Returns the file error it saved - """ - if bool(error): - rconn.rpush(key, json.dumps(error._asdict())) - return error - - -def file_errors_and_details( - redisargs: dict[str, str], - file_: Path, - filetype: str, - cdata: dict, - linesplitterfn: Callable, - linejoinerfn: Callable, - headercheckers: tuple[Callable, ...], - bodycheckers: tuple[Callable, ...] -) -> dict: - """Compute errors, and other file metadata.""" - errors = tuple() - if cdata[f"{filetype}_transposed"]: - rqtl2.transpose_csv_with_rename(file_, linesplitterfn, linejoinerfn) - - with Redis.from_url(redisargs["redisuri"], decode_responses=True) as rconn: - save_error_fn = partial(push_file_error_to_redis, - rconn, - error_list_name(filetype, file_.name)) - for lineno, line in enumerate(open_file(file_), start=1): - row = linesplitterfn(line) - if lineno == 1: - headers = tuple(row) - errors = errors + reduce( - lambda errs, fnct: errs + fnct( - file_.name, row[1:], save_error_fn), - headercheckers, - tuple()) - continue - - errors = errors + reduce( - lambda errs, fnct: errs + fnct( - file_.name, headers, row, cdata, save_error_fn), - bodycheckers, - tuple()) - - filedetails = { - "filename": file_.name, - "filesize": os.stat(file_).st_size, - "linecount": lineno - } - rconn.hset(redisargs["fqjobid"], - f"file-details:{filetype}:{file_.name}", - json.dumps(filedetails)) - return {**filedetails, "errors": errors} - - -def error_list_name(filetype: str, filename: str): - """Compute the name of the list where the errors will be pushed. - - Parameters - ---------- - filetype: The type of file. One of `r_qtl.r_qtl2.FILE_TYPES` - filename: The name of the file. - """ - return f"errors:{filetype}:{filename}" - - -def check_for_geno_errors( - redisargs: dict[str, str], - extractdir: Path, - cdata: dict, - linesplitterfn: Callable[[str], tuple[Union[str, None]]], - linejoinerfn: Callable[[tuple[Union[str, None], ...]], str], - logger: Logger -) -> bool: - """Check for errors in genotype files.""" - if "geno" in cdata or "founder_geno" in cdata: - genofiles = tuple( - extractdir.joinpath(fname) for fname in cdata.get("geno", [])) - fgenofiles = tuple( - extractdir.joinpath(fname) for fname in cdata.get("founder_geno", [])) - allgenofiles = genofiles + fgenofiles - with Redis.from_url(redisargs["redisuri"], decode_responses=True) as rconn: - error_list_names = [ - error_list_name("geno", file_.name) for file_ in allgenofiles] - for list_name in error_list_names: - rconn.delete(list_name) - rconn.hset( - redisargs["fqjobid"], - "geno-errors-lists", - json.dumps(error_list_names)) - processes = [ - mproc.Process(target=file_errors_and_details, - args=( - redisargs, - file_, - ftype, - cdata, - linesplitterfn, - linejoinerfn, - (check_markers,), - (check_geno_line,)) - ) - for ftype, file_ in ( - tuple(("geno", file_) for file_ in genofiles) + - tuple(("founder_geno", file_) for file_ in fgenofiles)) - ] - for process in processes: - process.start() - # Set expiry for any created error lists - for key in error_list_names: - rconn.expire(name=key, - time=timedelta(seconds=redisargs["redisexpiry"])) - - # TOD0: Add the errors to redis - if any(rconn.llen(errlst) > 0 for errlst in error_list_names): - logger.error("At least one of the 'geno' files has (an) error(s).") - return True - logger.info("No error(s) found in any of the 'geno' files.") - - else: - logger.info("No 'geno' files to check.") - - return False - - -# def check_for_pheno_errors(...): -# """Check for errors in phenotype files.""" -# pass - - -# def check_for_phenose_errors(...): -# """Check for errors in phenotype, standard-error files.""" -# pass - - -# def check_for_phenocovar_errors(...): -# """Check for errors in phenotype-covariates files.""" -# pass - - -def run_qc(rconn: Redis, args: Namespace, fqjobid: str, logger: Logger) -> int: - """Run quality control checks on R/qtl2 bundles.""" - thejob = parse_job(rconn, args.redisprefix, args.jobid) - print(f"THE JOB =================> {thejob}") - jobmeta = thejob["job-metadata"] - inpath = Path(jobmeta["rqtl2-bundle-file"]) - extractdir = inpath.parent.joinpath(f"{inpath.name}__extraction_dir") - with ZipFile(inpath, "r") as zfile: - rqtl2.extract(zfile, extractdir) - - ### BEGIN: The quality control checks ### - cdata = rqtl2.control_data(extractdir) - splitter = build_line_splitter(cdata) - joiner = build_line_joiner(cdata) - - redisargs = { - "fqjobid": fqjobid, - "redisuri": args.redisuri, - "redisexpiry": args.redisexpiry - } - check_for_missing_files(rconn, fqjobid, extractdir, logger) - # check_for_pheno_errors(...) - check_for_geno_errors(redisargs, extractdir, cdata, splitter, joiner, logger) - # check_for_phenose_errors(...) - # check_for_phenocovar_errors(...) - ### END: The quality control checks ### - - def __fetch_errors__(rkey: str) -> tuple: - return tuple(json.loads(rconn.hget(fqjobid, rkey) or "[]")) - - return (1 if any(( - bool(__fetch_errors__(key)) - for key in - ("errors-geno", "errors-pheno", "errors-phenos", "errors-phenocovar"))) - else 0) - - -if __name__ == "__main__": - def main(): - """Enter R/qtl2 bundle QC runner.""" - args = add_global_data_arguments(init_cli_parser( - "qc-on-rqtl2-bundle", "Run QC on R/qtl2 bundle.")).parse_args() - check_redis(args.redisuri) - check_db(args.databaseuri) - - logger = getLogger("qc-on-rqtl2-bundle") - logger.addHandler(StreamHandler(stream=sys.stderr)) - logger.setLevel("DEBUG") - - fqjobid = jobs.job_key(args.redisprefix, args.jobid) - with Redis.from_url(args.redisuri, decode_responses=True) as rconn: - logger.addHandler(setup_redis_logger( - rconn, fqjobid, f"{fqjobid}:log-messages", - args.redisexpiry)) - - exitcode = run_qc(rconn, args, fqjobid, logger) - rconn.hset( - jobs.job_key(args.redisprefix, args.jobid), "exitcode", exitcode) - return exitcode - - sys.exit(main()) diff --git a/scripts/redis_logger.py b/scripts/redis_logger.py index d3fde5f..a74e5e4 100644 --- a/scripts/redis_logger.py +++ b/scripts/redis_logger.py @@ -6,7 +6,7 @@ from redis import Redis class RedisLogger(logging.Handler): """Log out to redis for our worker scripts""" - def __init__(self,#pylint: disable=[too-many-arguments] + def __init__(self,#pylint: disable=[too-many-arguments, too-many-positional-arguments] rconn: Redis, fullyqualifiedjobid: str, messageslistname: str, diff --git a/scripts/rqtl2/entry.py b/scripts/rqtl2/entry.py index 327ed2c..7423a4b 100644 --- a/scripts/rqtl2/entry.py +++ b/scripts/rqtl2/entry.py @@ -20,27 +20,23 @@ def build_main( [Redis, Connection, str, Namespace, logging.Logger], int ], - loggername: str + logger: logging.Logger ) -> Callable[[],int]: """Build a function to be used as an entry-point for scripts.""" def main(): - try: - logging.basicConfig( - format=( - "%(asctime)s - %(levelname)s %(name)s: " - "(%(pathname)s: %(lineno)d) %(message)s"), - level=args.loglevel) - logger = logging.getLogger(loggername) - with (Redis.from_url(args.redisuri, decode_responses=True) as rconn, - database_connection(args.databaseuri) as dbconn): - fqjobid = jobs.job_key(args.redisprefix, args.jobid) + with (Redis.from_url(args.redisuri, decode_responses=True) as rconn, + database_connection(args.databaseuri) as dbconn): + logger.setLevel(args.log_level.upper()) + fqjobid = jobs.job_key(args.redisprefix, args.jobid) + + try: rconn.hset(fqjobid, "status", "started") logger.addHandler(setup_redis_logger( rconn, fqjobid, f"{fqjobid}:log-messages", args.redisexpiry)) - logger.addHandler(StreamHandler(stream=sys.stdout)) + logger.addHandler(StreamHandler(stream=sys.stderr)) check_db(args.databaseuri) check_redis(args.redisuri) @@ -48,15 +44,15 @@ def build_main( logger.error("File not found: '%s'.", args.rqtl2bundle) return 2 - returncode = run_fn(rconn, dbconn, fqjobid, args, logger) + returncode = run_fn(rconn, dbconn, fqjobid, args) if returncode == 0: rconn.hset(fqjobid, "status", "completed:success") return returncode rconn.hset(fqjobid, "status", "completed:error") return returncode - except Exception as _exc:# pylint: disable=[broad-except] - logger.error("The process failed!", exc_info=True) - rconn.hset(fqjobid, "status", "completed:error") - return 4 + except Exception as _exc:# pylint: disable=[broad-except] + logger.error("The process failed!", exc_info=True) + rconn.hset(fqjobid, "status", "completed:error") + return 4 return main diff --git a/scripts/rqtl2/install_genotypes.py b/scripts/rqtl2/install_genotypes.py index 8762655..5e6abb0 100644 --- a/scripts/rqtl2/install_genotypes.py +++ b/scripts/rqtl2/install_genotypes.py @@ -20,7 +20,7 @@ from scripts.rqtl2.entry import build_main from scripts.rqtl2.cli_parser import add_common_arguments from scripts.cli_parser import init_cli_parser, add_global_data_arguments -__MODULE__ = "scripts.rqtl2.install_genotypes" +logger = getLogger(__name__) def insert_markers( dbconn: mdb.Connection, @@ -191,7 +191,7 @@ def install_genotypes(#pylint: disable=[too-many-locals] dbconn: mdb.Connection, fullyqualifiedjobid: str,#pylint: disable=[unused-argument] args: argparse.Namespace, - logger: Logger = getLogger(__name__) + logger: Logger = logger # pylint: disable=[redefined-outer-name] ) -> int: """Load any existing genotypes into the database.""" (speciesid, populationid, datasetid, rqtl2bundle) = ( @@ -257,5 +257,5 @@ if __name__ == "__main__": return parser.parse_args() - main = build_main(cli_args(), install_genotypes, __MODULE__) + main = build_main(cli_args(), install_genotypes, logger) sys.exit(main()) diff --git a/scripts/rqtl2/install_phenos.py b/scripts/rqtl2/install_phenos.py index 9059cd6..11ac8a4 100644 --- a/scripts/rqtl2/install_phenos.py +++ b/scripts/rqtl2/install_phenos.py @@ -19,7 +19,7 @@ from r_qtl import r_qtl2_qc as rqc from functional_tools import take -__MODULE__ = "scripts.rqtl2.install_phenos" +logger = getLogger(__name__) def insert_probesets(dbconn: mdb.Connection, platformid: int, @@ -101,7 +101,8 @@ def install_pheno_files(#pylint: disable=[too-many-locals] dbconn: mdb.Connection, fullyqualifiedjobid: str,#pylint: disable=[unused-argument] args: argparse.Namespace, - logger: Logger = getLogger()) -> int: + logger: Logger = logger # pylint: disable=[redefined-outer-name] +) -> int: """Load data in `pheno` files and other related files into the database.""" (speciesid, platformid, datasetid, rqtl2bundle) = ( args.speciesid, args.platformid, args.datasetid, args.rqtl2bundle) @@ -159,5 +160,5 @@ if __name__ == "__main__": return parser.parse_args() - main = build_main(cli_args(), install_pheno_files, __MODULE__) + main = build_main(cli_args(), install_pheno_files, logger) sys.exit(main()) diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index ba28ed0..084c876 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -36,8 +36,15 @@ from scripts.cli_parser import init_cli_parser, add_global_data_arguments from scripts.rqtl2.bundleutils import build_line_joiner, build_line_splitter __MODULE__ = "scripts.rqtl2.phenotypes_qc" +logging.basicConfig( + format=("%(asctime)s - %(levelname)s %(name)s: " + "(%(pathname)s: %(lineno)d) %(message)s")) +logger = logging.getLogger(__MODULE__) -def validate(phenobundle: Path, logger: Logger) -> dict: +def validate( + phenobundle: Path, + logger: Logger# pylint: disable=[redefined-outer-name] +) -> dict: """Check that the bundle is generally valid""" try: rqc.validate_bundle(phenobundle) @@ -59,7 +66,7 @@ def validate(phenobundle: Path, logger: Logger) -> dict: def check_for_mandatory_pheno_keys( phenobundle: Path, - logger: Logger, + logger: Logger,# pylint: disable=[redefined-outer-name] **kwargs ) -> dict: """Check that the mandatory keys exist for phenotypes.""" @@ -86,7 +93,7 @@ def check_for_mandatory_pheno_keys( def check_for_averages_files( phenobundle: Path, - logger: Logger, + logger: Logger,# pylint: disable=[redefined-outer-name] **kwargs ) -> dict: """Check that averages files appear together""" @@ -140,15 +147,15 @@ def redis_logger( ) -> Iterator[logging.Logger]: """Build a Redis message-list logger.""" rconn = Redis.from_url(redisuri, decode_responses=True) - logger = logging.getLogger(loggername) - logger.propagate = False + _logger = logging.getLogger(loggername) + _logger.propagate = False handler = RedisMessageListHandler( rconn, fullyqualifiedkey(fqkey, filename))#type: ignore[arg-type] handler.setFormatter(logging.getLogger().handlers[0].formatter) - logger.addHandler(handler) + _logger.addHandler(handler) try: - yield logger + yield _logger finally: rconn.close() @@ -175,9 +182,9 @@ def qc_phenocovar_file( redisuri, f"{__MODULE__}.qc_phenocovar_file", filepath.name, - f"{fqkey}:logs") as logger, + f"{fqkey}:logs") as _logger, Redis.from_url(redisuri, decode_responses=True) as rconn): - logger.info("Running QC on file: %s", filepath.name) + print("Running QC on file: ", filepath.name) _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) _headings = tuple(heading.lower() for heading in next(_csvfile)) _errors: tuple[InvalidValue, ...] = tuple() @@ -191,11 +198,11 @@ def qc_phenocovar_file( "-", "-", (f"File {filepath.name} is missing the {heading} heading " - "in the header line."))),) + "in the header row/line."))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount - logger.info("Testing record '%s'", line[0]) + _logger.info("Testing record '%s'", line[0]) if len(line) != len(_headings): _errs = _errs + (save_error(InvalidValue( filepath.name, @@ -205,12 +212,12 @@ def qc_phenocovar_file( (f"Record {_lc} in file {filepath.name} has a different " "number of columns than the number of headings"))),) _line = dict(zip(_headings, line)) - if not bool(_line["description"]): + if not bool(_line.get("description")): _errs = _errs + ( save_error(InvalidValue(filepath.name, _line[_headings[0]], "description", - _line["description"], + _line.get("description"), "The description is not provided!")),) rconn.hset(file_fqkey(fqkey, "metadata", filepath), @@ -236,7 +243,7 @@ def merge_dicts(*dicts): return reduce(lambda merged, dct: {**merged, **dct}, dicts, {}) -def decimal_points_error(# pylint: disable=[too-many-arguments] +def decimal_points_error(# pylint: disable=[too-many-arguments,too-many-positional-arguments] filename: str, rowtitle: str, coltitle: str, @@ -267,7 +274,7 @@ def integer_error( return InvalidValue(filename, rowtitle, coltitle, cellvalue, message) -def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments] +def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-many-positional-arguments] filepath: Path, redisuri: str, fqkey: str, @@ -283,29 +290,35 @@ def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments] redisuri, f"{__MODULE__}.qc_pheno_file", filepath.name, - f"{fqkey}:logs") as logger, + f"{fqkey}:logs") as _logger, Redis.from_url(redisuri, decode_responses=True) as rconn): - logger.info("Running QC on file: %s", filepath.name) + print("Running QC on file: ", filepath.name) save_error = partial( push_error, rconn, file_fqkey(fqkey, "errors", filepath)) _csvfile = rqtl2.read_csv_file(filepath, separator, comment_char) _headings: tuple[str, ...] = tuple( + # select lowercase for comparison purposes heading.lower() for heading in next(_csvfile)) _errors: tuple[InvalidValue, ...] = tuple() - _absent = tuple(pheno for pheno in _headings[1:] if pheno not in phenonames) + _absent = tuple(pheno for pheno in _headings[1:] if pheno + not in tuple( + # lower to have consistent case with headings for + # comparison + phe.lower() for phe in phenonames)) if len(_absent) > 0: _errors = _errors + (save_error(InvalidValue( filepath.name, "header row", "-", ", ".join(_absent), - ("The following phenotype names do not exist in any of the " - f"provided phenocovar files: ({', '.join(_absent)})"))),) + ("The following trait names/identifiers do not exist in any of " + "the provided descriptions/covariates files: " + f"({', '.join(_absent)})"))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount - logger.debug("Checking row %s", line[0]) + _logger.debug("Checking row %s", line[0]) if line[0] not in samples: _errs = _errs + (save_error(InvalidValue( filepath.name, @@ -365,10 +378,10 @@ def run_qc(# pylint: disable=[too-many-locals] dbconn: mdb.Connection, fullyqualifiedjobid: str, args: Namespace, - logger: Logger + logger: Logger = logger # pylint: disable=[redefined-outer-name] ) -> int: """Run quality control checks on the bundle.""" - logger.debug("Beginning the quality assurance checks.") + print("Beginning the quality assurance checks.") results = check_for_averages_files( **check_for_mandatory_pheno_keys( **validate(args.rqtl2bundle, logger))) @@ -393,7 +406,7 @@ def run_qc(# pylint: disable=[too-many-locals] for ftype in ("pheno", "phenocovar", "phenose", "phenonum"))) # - Fetch samples/individuals from database. - logger.debug("Fetching samples/individuals from the database.") + print("Fetching samples/individuals from the database.") samples = tuple(#type: ignore[var-annotated] item for item in set(reduce( lambda acc, item: acc + ( @@ -410,7 +423,7 @@ def run_qc(# pylint: disable=[too-many-locals] json.dumps(tuple(f"{fullyqualifiedjobid}:phenocovar:{_file}" for _file in cdata.get("phenocovar", [])))) with mproc.Pool(mproc.cpu_count() - 1) as pool: - logger.debug("Check for errors in 'phenocovar' file(s).") + print("Check for errors in 'phenocovar' file(s).") _phenocovar_qc_res = merge_dicts(*pool.starmap(qc_phenocovar_file, tuple( (extractiondir.joinpath(_file), args.redisuri, @@ -432,7 +445,7 @@ def run_qc(# pylint: disable=[too-many-locals] "Expected a non-negative number with at least one decimal " "place.")) - logger.debug("Check for errors in 'pheno' file(s).") + print("Check for errors in 'pheno' file(s).") _pheno_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( extractiondir.joinpath(_file), args.redisuri, @@ -451,7 +464,7 @@ def run_qc(# pylint: disable=[too-many-locals] # - Check the 3 checks above for phenose and phenonum values too # qc_phenose_files(…) # qc_phenonum_files(…) - logger.debug("Check for errors in 'phenose' file(s).") + print("Check for errors in 'phenose' file(s).") _phenose_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( extractiondir.joinpath(_file), args.redisuri, @@ -467,7 +480,7 @@ def run_qc(# pylint: disable=[too-many-locals] dec_err_fn ) for _file in cdata.get("phenose", [])))) - logger.debug("Check for errors in 'phenonum' file(s).") + print("Check for errors in 'phenonum' file(s).") _phenonum_qc_res = merge_dicts(*pool.starmap(qc_pheno_file, tuple(( extractiondir.joinpath(_file), args.redisuri, @@ -504,5 +517,5 @@ if __name__ == "__main__": type=Path) return parser.parse_args() - main = build_main(cli_args(), run_qc, __MODULE__) + main = build_main(cli_args(), run_qc, logger) sys.exit(main()) diff --git a/scripts/run_qtlreaper.py b/scripts/run_qtlreaper.py new file mode 100644 index 0000000..a461d9a --- /dev/null +++ b/scripts/run_qtlreaper.py @@ -0,0 +1,238 @@ +"""Script to run rust-qtlreaper and update database with results.""" +import os +import sys +import csv +import time +import secrets +import logging +import subprocess +import multiprocessing +from pathlib import Path +from functools import reduce +from typing import Union, Iterator +from argparse import Namespace, ArgumentParser + +from gn_libs import mysqldb + +from uploader.phenotypes.models import phenotypes_vector_data +from uploader.population.models import population_by_species_and_id +from uploader.samples.models import samples_by_species_and_population + +from scripts.cli.logging import setup_logging +from scripts.cli.validators import directory_exists +from scripts.cli.options import add_logging, add_mariadb_uri, add_population_id + +logger = logging.getLogger(__name__) + + +def retrieve_genotype_file(genotypes_dir: Path, population_code: str) -> Path: + """Retrieves the genotype file""" + _genofile = genotypes_dir.joinpath(f"{population_code}.geno") + if _genofile.exists(): + return _genofile + raise FileNotFoundError(f"Could not find the genotype file '{population_code}.geno'") + + +def samples_from_genofile(genofile: Path) -> tuple[str, ...]: + """Read samples from the genotype file.""" + with genofile.open(mode="r", encoding="utf-8") as inptr: + while True: + line = inptr.readline() + if (line.startswith("#") # comment line + or line.startswith("@") # allele? heterozygosity? + or line.strip() == "" # empty line + ): + continue + return tuple(line.strip().split("\t")[4:]) + + +def reconcile_samples( + genosamples: tuple[str, ...], + dbsamples: tuple[str, ...] +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """merge samples in genosamples and dbsamples and retain order in genosamples.""" + in_db_not_geno = set(dbsamples).difference(genosamples) + return genosamples, tuple(in_db_not_geno) + + +def generate_qtlreaper_traits_file( + outdir: Path, + samples: tuple[str, ...], + traits_data: tuple[dict[str, Union[int, float]], ...], + filename_prefix: str = "" +) -> Path: + """Generate a file for use with qtlreaper that contains the traits' data.""" + _dialect = csv.unix_dialect() + _dialect.delimiter="\t" + _dialect.quoting=0 + + _traitsfile = outdir.joinpath( + f"{filename_prefix}_{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] + with _traitsfile.open(mode="w", encoding="utf-8") as outptr: + writer = csv.DictWriter( + outptr, fieldnames=("Trait",) + samples, dialect=_dialect) + writer.writeheader() + for row in traits_data: + writer.writerow({ + "Trait": row["xref_id"], + **{sample: row.get(sample, "") for sample in samples} + }) + + return _traitsfile + + +def parse_tsv_file(results_file: Path) -> Iterator[dict]: + """Parse the rust-qtlreaper output into usable python objects.""" + with results_file.open("r", encoding="utf-8") as readptr: + _dialect = csv.unix_dialect() + _dialect.delimiter = "\t" + reader = csv.DictReader(readptr, dialect=_dialect) + yield from reader + + +def __qtls_by_trait__(qtls, current): + """Organise QTL results by trait""" + return { + **qtls, + current["ID"]: qtls.get(current["ID"], tuple()) + (current,) + } + + +def save_qtl_values_to_db(conn, qtls: tuple[dict, ...]): + """Save computed QTLs to the database.""" + with conn.cursor() as cursor: + cursor.executemany( + "UPDATE PublishXRef SET " + "Locus=%(Locus)s, LRS=%(LRS)s, additive=%(Additive)s " + "WHERE Id=%(ID)s", + qtls) + + +def dispatch(args: Namespace) -> int:# pylint: disable=[too-many-locals] + """Dispatch the actual logic.""" + exitcode = 1 + with mysqldb.database_connection(args.db_uri) as conn: + try: + population = population_by_species_and_id(conn, args.species_id, args.population_id) + assert population, (f"No population with ID '{args.population_id} for " + f"species with ID '{args.species_id}'.") + _genofile = retrieve_genotype_file(args.genotypes_dir, population["Name"]) + logger.debug("Genotype file: %s", _genofile) + samples, _samples_not_in_genofile = reconcile_samples( + samples_from_genofile(_genofile), + tuple( + sample["Name"] for sample in + samples_by_species_and_population( + conn, args.species_id, args.population_id))) + if len(_samples_not_in_genofile) > 0: + logger.warning( + "Ignoring %d samples that are in the database but not in " + "the provided genotype file.", + len(_samples_not_in_genofile)) + logger.debug("Ignored the following samples: %s", + ", ".join(_samples_not_in_genofile)) + + # Fetch traits data: provided list, or all traits in db + _traitsdata = tuple(phenotypes_vector_data( + conn, + args.species_id, + args.population_id, + xref_ids=tuple(args.xref_ids)).values()) + logger.debug("Successfully got traits data. Generating the QTLReaper's traits file…") + _traitsfile = generate_qtlreaper_traits_file( + args.working_dir, + samples, + _traitsdata, + filename_prefix="qtlreaper_input_traits_file") + logger.debug("QTLReaper's Traits file: %s", _traitsfile) + + _qtlreaper_main_output = args.working_dir.joinpath( + f"main-output-{secrets.token_urlsafe(15)}.tsv")#type: ignore[attr-defined] + _qtlreaper_permu_output = args.working_dir.joinpath( + f"permu-output-{secrets.token_urlsafe(15)}.tsv") + logger.debug("Main output filename: %s", _qtlreaper_main_output) + with subprocess.Popen( + ("qtlreaper", + "--n_permutations", "1000", + "--geno", _genofile, + "--traits", _traitsfile, + "--main_output", _qtlreaper_main_output, + "--permu_output", _qtlreaper_permu_output, + "--threads", str(int(1+(multiprocessing.cpu_count()/2)))), + env=({**os.environ, "RUST_BACKTRACE": "full"} + if logger.getEffectiveLevel() == logging.DEBUG + else dict(os.environ))) as _qtlreaper: + while _qtlreaper.poll() is None: + logger.debug("QTLReaper process running…") + time.sleep(1) + results = ( + tuple(#type: ignore[var-annotated] + max(qtls, key=lambda qtl: qtl["LRS"]) + for qtls in + reduce(__qtls_by_trait__, + parse_tsv_file(_qtlreaper_main_output), + {}).values()) + if _qtlreaper_main_output.exists() + else tuple()) + logger.debug("Cleaning up temporary files.") + + # short-circuits to delete file if exists + if _traitsfile.exists(): + _traitsfile.unlink() + logger.info("Deleted generated traits' file for QTLReaper.") + + if _qtlreaper_main_output.exists(): + _qtlreaper_main_output.unlink() + logger.info("Deleted QTLReaper's main output file.") + + if _qtlreaper_permu_output.exists(): + _qtlreaper_permu_output.unlink() + logger.info("Deleted QTLReaper's permutations file.") + + if _qtlreaper.returncode != 0: + return _qtlreaper.returncode + + save_qtl_values_to_db(conn, results) + logger.info("Successfully computed p values for %s traits.", len(_traitsdata)) + return 0 + except FileNotFoundError as fnf: + logger.error(", ".join(str(arg) for arg in fnf.args), exc_info=False) + except AssertionError as aserr: + logger.error(", ".join(aserr.args), exc_info=False) + except Exception as _exc:# pylint: disable=[broad-exception-caught] + logger.debug("Type of exception: %s", type(_exc)) + logger.error("General exception!", exc_info=True) + + return exitcode + + +if __name__ == "__main__": + def main(): + """run_qtlreaper.py: entry point.""" + parser = add_logging(add_population_id(add_mariadb_uri( + ArgumentParser("run_qtlreaper")))) + parser.add_argument( + "genotypes_dir", + metavar="GENOTYPES-DIRECTORY", + type=directory_exists, + help="Path to directory with the genotypes.") + parser.add_argument( + "working_dir", + metavar="WORKING-DIRECTORY", + type=directory_exists, + help="Directory where the script will write temporary files.") + parser.add_argument( + "xref_ids", + metavar="CROSS-REFERENCE-IDS", + type=int, + nargs="*", + help=("Optional list of specific cross-reference IDs to narrow down" + " to. If provided, QTLReaper will only run against them. " + "If NOT provided, QTLReaper will run against all the traits " + "in the population.")) + args = parser.parse_args() + setup_logging(logger, args.log_level) + + return dispatch(args) + + sys.exit(main()) diff --git a/scripts/worker.py b/scripts/worker.py index 91b0332..3165fe7 100644 --- a/scripts/worker.py +++ b/scripts/worker.py @@ -79,7 +79,7 @@ def main(): fqjobid = jobs.job_key(args.redisprefix, args.jobid) rconn.hset(fqjobid, "stderr", f"No such job. '{args.job_id}'.") rconn.expire(name=jobs.job_key(args.redisprefix, args.job_id), - time=timedelta(seconds=(2 * 60 * 60))) + time=timedelta(seconds=2 * 60 * 60)) print(f"No such job. '{args.job_id}'.", file=sys.stderr) return 2 return 3 diff --git a/tests/conftest.py b/tests/conftest.py index 9012221..2009aab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,8 @@ import io import os import uuid +import shutil +from pathlib import Path from hashlib import sha256 import redis @@ -46,17 +48,20 @@ def cleanup_redis(redisuri: str, prefix: str): @pytest.fixture(scope="module") def client(): "Fixture for test client" - app = create_app() test_prefix = sha256(f"test:{uuid.uuid4()}".encode("utf8")).hexdigest() - app.config.update({ + tests_work_dir = Path("/tmp/{test_prefix}") + tests_work_dir.mkdir(exist_ok=True) + app = create_app({ "TESTING": True, "GNQC_REDIS_PREFIX": f"{test_prefix}:GNQC", - "JOBS_TTL_SECONDS": 2 * 60 * 60# 2 hours + "JOBS_TTL_SECONDS": 2 * 60 * 60,# 2 hours + "ASYNCHRONOUS_JOBS_SQLITE_DB": f"{tests_work_dir}/jobs.db" }) with app.app_context(): yield app.test_client() cleanup_redis(app.config["REDIS_URL"], test_prefix) + shutil.rmtree(tests_work_dir, ignore_errors=True) @pytest.fixture(scope="module") def db_url(client):#pylint: disable=[redefined-outer-name] @@ -178,7 +183,7 @@ def redis_conn_with_completed_job_some_errors(redis_url, redis_ttl, jobs_prefix, def uploads_dir(client): # pylint: disable=[redefined-outer-name] """Returns the configured, uploads directory, creating it if it does not exist.""" - the_dir = client.application.config["UPLOAD_FOLDER"] + the_dir = client.application.config["UPLOADS_DIRECTORY"] if not os.path.exists(the_dir): os.mkdir(the_dir) diff --git a/tests/r_qtl/test_r_qtl2_control_file.py b/tests/r_qtl/test_r_qtl2_control_file.py index 316307d..5b9fef6 100644 --- a/tests/r_qtl/test_r_qtl2_control_file.py +++ b/tests/r_qtl/test_r_qtl2_control_file.py @@ -16,6 +16,7 @@ __DEFAULTS__ = { "pheno_transposed": False, "covar_transposed": False, "phenocovar_transposed": False, + "phenonum_transposed": False, "gmap_transposed": False, "pmap_transposed": False, "phenose_transposed": False diff --git a/tests/test_instance_dir/config.py b/tests/test_instance_dir/config.py index 2ee569b..f04b3df 100644 --- a/tests/test_instance_dir/config.py +++ b/tests/test_instance_dir/config.py @@ -6,6 +6,6 @@ import os LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING") SECRET_KEY = b"<Please! Please! Please! Change This!>" -UPLOAD_FOLDER = "/tmp/qc_app_files" +UPLOADS_DIRECTORY = "/tmp/qc_app_files" REDIS_URL = "redis://" JOBS_TTL_SECONDS = 600 # 10 minutes diff --git a/tests/uploader/phenotypes/__init__.py b/tests/uploader/phenotypes/__init__.py new file mode 100644 index 0000000..1e0a932 --- /dev/null +++ b/tests/uploader/phenotypes/__init__.py @@ -0,0 +1 @@ +"""phenotypes tests""" diff --git a/tests/uploader/phenotypes/test_misc.py b/tests/uploader/phenotypes/test_misc.py new file mode 100644 index 0000000..cf475ad --- /dev/null +++ b/tests/uploader/phenotypes/test_misc.py @@ -0,0 +1,387 @@ +"""Test miscellaneous phenotypes functions.""" + +import pytest + +from uploader.phenotypes.misc import phenotypes_data_differences + +__sample_db_phenotypes_data__ = ( + { + "PhenotypeId": 4, + "xref_id": 10001, + "DataId": 8967043, + "data": { + "B6D2F1": {"StrainId": 1, "value": None}, + "C57BL/6J": {"StrainId": 2, "value": None}, + "DBA/2J": {"StrainId": 3, "value": None}, + "BXD1": {"StrainId": 4, "value": 61.4}, + "BXD2": {"StrainId": 5, "value": 49}, + "BXD5": {"StrainId": 6, "value": 62.5}, + "BXD6": {"StrainId": 7, "value": 53.1} + } + }, + { + "PhenotypeId": 10, + "xref_id": 10002, + "DataId": 8967044, + "data": { + "B6D2F1": {"StrainId": 1, "value": None}, + "C57BL/6J": {"StrainId": 2, "value": None}, + "DBA/2J": {"StrainId": 3, "value": None}, + "BXD1": {"StrainId": 4, "value": 54.1}, + "BXD2": {"StrainId": 5, "value": 50.1}, + "BXD5": {"StrainId": 6, "value": 53.3}, + "BXD6": {"StrainId": 7, "value": 55.1} + } + }, + { + "PhenotypeId": 15, + "xref_id": 10003, + "DataId": 8967045, + "data": { + "B6D2F1": {"StrainId": 1, "value": None}, + "C57BL/6J": {"StrainId": 2, "value": None}, + "DBA/2J": {"StrainId": 3, "value": None}, + "BXD1": {"StrainId": 4, "value": 483}, + "BXD2": {"StrainId": 5, "value": 403}, + "BXD5": {"StrainId": 6, "value": 501}, + "BXD6": {"StrainId": 7, "value": 403} + } + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "data": { + "B6D2F1": {"StrainId": 1, "value": None}, + "C57BL/6J": {"StrainId": 2, "value": None}, + "DBA/2J": {"StrainId": 3, "value": None}, + "BXD1": {"StrainId": 4, "value": 49.8}, + "BXD2": {"StrainId": 5, "value": 45.5}, + "BXD5": {"StrainId": 6, "value": 62.9}, + "BXD6": {"StrainId": 7, "value": None} + } + }, + { + "PhenotypeId": 25, + "xref_id": 10005, + "DataId": 8967047, + "data": { + "B6D2F1": {"StrainId": 1, "value": None}, + "C57BL/6J": {"StrainId": 2, "value": None}, + "DBA/2J": {"StrainId": 3, "value": None}, + "BXD1": {"StrainId": 4, "value": 46}, + "BXD2": {"StrainId": 5, "value": 44.9}, + "BXD5": {"StrainId": 6, "value": 52.5}, + "BXD6": {"StrainId": 7, "value": None} + } + }) + + +@pytest.mark.unit_test +@pytest.mark.parametrize( + "filedata,dbdata,expected", + ((tuple(), tuple(), tuple()), # No data + + # No data difference + (({ + "phenotype_id": 4, + "xref_id": 10001, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 61.4, + "BXD2": 49, + "BXD5":62.5, + "BXD6": 53.1 + } + }, + { + "phenotype_id": 10, + "xref_id": 10002, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 54.1, + "BXD2": 50.1, + "BXD5": 53.3, + "BXD6": 55.1 + } + }, + { + "phenotype_id": 15, + "xref_id": 10003, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 483, + "BXD2": 403, + "BXD5": 501, + "BXD6": 403 + } + }, + { + "phenotype_id": 20, + "xref_id": 10004, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 49.8, + "BXD2": 45.5, + "BXD5": 62.9, + "BXD6": None + } + }, + { + "phenotype_id": 25, + "xref_id": 10005, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 46, + "BXD2": 44.9, + "BXD5": 52.5, + "BXD6": None + } + }), + __sample_db_phenotypes_data__, + tuple()), + + # Change values: No deletions + (({ + "phenotype_id": 4, + "xref_id": 10001, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 77.2, + "BXD2": 49, + "BXD5":62.5, + "BXD6": 53.1 + } + }, + { + "phenotype_id": 10, + "xref_id": 10002, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 54.1, + "BXD2": 50.1, + "BXD5": 53.3, + "BXD6": 55.1 + } + }, + { + "phenotype_id": 15, + "xref_id": 10003, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 483, + "BXD2": 403, + "BXD5": 503, + "BXD6": 903 + } + }, + { + "phenotype_id": 20, + "xref_id": 10004, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": 1, + "BXD1": 8, + "BXD2": 9, + "BXD5": 62.9, + "BXD6": None + } + }, + { + "phenotype_id": 25, + "xref_id": 10005, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 46, + "BXD2": 44.9, + "BXD5": 52.5, + "BXD6": None + } + }), + __sample_db_phenotypes_data__, + ({ + "PhenotypeId": 4, + "xref_id": 10001, + "DataId": 8967043, + "StrainId": 4, + "StrainName": "BXD1", + "value": 77.2 + }, + { + "PhenotypeId": 15, + "xref_id": 10003, + "DataId": 8967045, + "StrainId": 6, + "StrainName": "BXD5", + "value": 503 + }, + { + "PhenotypeId": 15, + "xref_id": 10003, + "DataId": 8967045, + "StrainId": 7, + "StrainName": "BXD6", + "value": 903 + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "StrainId": 3, + "StrainName": "DBA/2J", + "value": 1 + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "StrainId": 4, + "StrainName": "BXD1", + "value": 8 + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "StrainId": 5, + "StrainName": "BXD2", + "value": 9 + })), + + # Changes — with deletions + (({ + "phenotype_id": 4, + "xref_id": 10001, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": None, + "BXD2": 49, + "BXD5":62.5, + "BXD6": 53.1 + } + }, + { + "phenotype_id": 10, + "xref_id": 10002, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 54.1, + "BXD2": 50.1, + "BXD5": 53.3, + "BXD6": 55.1 + } + }, + { + "phenotype_id": 15, + "xref_id": 10003, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 483, + "BXD2": 403, + "BXD5": None, + "BXD6": None + } + }, + { + "phenotype_id": 20, + "xref_id": 10004, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": 15, + "BXD1": None, + "BXD2": 24, + "BXD5": 62.9, + "BXD6": None + } + }, + { + "phenotype_id": 25, + "xref_id": 10005, + "data": { + "B6D2F1": None, + "C57BL/6J": None, + "DBA/2J": None, + "BXD1": 46, + "BXD2": 44.9, + "BXD5": 52.5, + "BXD6": None + } + }), + __sample_db_phenotypes_data__, + ({ + "PhenotypeId": 4, + "xref_id": 10001, + "DataId": 8967043, + "StrainId": 4, + "StrainName": "BXD1", + "value": None + }, + { + "PhenotypeId": 15, + "xref_id": 10003, + "DataId": 8967045, + "StrainId": 6, + "StrainName": "BXD5", + "value": None + }, + { + "PhenotypeId": 15, + "xref_id": 10003, + "DataId": 8967045, + "StrainId": 7, + "StrainName": "BXD6", + "value": None + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "StrainId": 3, + "StrainName": "DBA/2J", + "value": 15 + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "StrainId": 4, + "StrainName": "BXD1", + "value": None + }, + { + "PhenotypeId": 20, + "xref_id": 10004, + "DataId": 8967046, + "StrainId": 5, + "StrainName": "BXD2", + "value": 24 + })))) +def test_phenotypes_data_differences(filedata, dbdata, expected): + """Test differences are computed correctly.""" + assert phenotypes_data_differences(filedata, dbdata) == expected diff --git a/tests/uploader/publications/__init__.py b/tests/uploader/publications/__init__.py new file mode 100644 index 0000000..de15e08 --- /dev/null +++ b/tests/uploader/publications/__init__.py @@ -0,0 +1 @@ +"""publications tests""" diff --git a/tests/uploader/publications/test_misc.py b/tests/uploader/publications/test_misc.py new file mode 100644 index 0000000..8c7e567 --- /dev/null +++ b/tests/uploader/publications/test_misc.py @@ -0,0 +1,68 @@ +"""Tests for functions used for bulk editing.""" +import pytest + +from uploader.publications.misc import publications_differences + + +@pytest.mark.unit_test +@pytest.mark.parametrize( + "filedata,dbdata,pubmed2pubidmap,expected", + (((), (), {}, tuple()), # no data + + # Same Data + (({"phenotype_id": 1, "xref_id": 10001, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10002, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10003, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10005, "PubMed_ID": 9999999999999}), + ({"PhenotypeId": 1, "xref_id": 10001, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10002, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10003, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10004, "PublicationId": 15, + "PubMed_ID": 9999999999999}), + {9999999999999: 15}, + tuple()), + + # Differences: no new pubmeds (all pubmeds in db) + (({"phenotype_id": 1, "xref_id": 10001, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10002, "PubMed_ID": 9999999999998}, + {"phenotype_id": 1, "xref_id": 10003, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10004, "PubMed_ID": 9999999999997}), + ({"PhenotypeId": 1, "xref_id": 10001, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10002, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10003, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10004, "PublicationId": 15, + "PubMed_ID": None}), + {9999999999999: 15, 9999999999998: 18, 9999999999997: 12}, + ({"PhenotypeId": 1, "xref_id": 10002, "PublicationId": 18, + "PubMed_ID": 9999999999998}, + {"PhenotypeId": 1, "xref_id": 10004, "PublicationId": 12, + "PubMed_ID": 9999999999997})), + + # Differences: Deletions of pubmeds + (({"phenotype_id": 1, "xref_id": 10001, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10002, "PubMed_ID": None}, + {"phenotype_id": 1, "xref_id": 10003, "PubMed_ID": 9999999999999}, + {"phenotype_id": 1, "xref_id": 10004, "PubMed_ID": None}), + ({"PhenotypeId": 1, "xref_id": 10001, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10002, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10003, "PublicationId": 15, + "PubMed_ID": 9999999999999}, + {"PhenotypeId": 1, "xref_id": 10004, "PublicationId": 15, + "PubMed_ID": 9999999999999}), + {9999999999999: 15, 9999999999998: 18, 9999999999997: 12}, + ({"PhenotypeId": 1, "xref_id": 10002, "PublicationId": None, + "PubMed_ID": None}, + {"PhenotypeId": 1, "xref_id": 10004, "PublicationId": None, + "PubMed_ID": None})))) +def test_publications_differences(filedata, dbdata, pubmed2pubidmap, expected): + """Test publication differences — flesh out description…""" + assert publications_differences( + filedata, dbdata, pubmed2pubidmap) == expected diff --git a/tests/uploader/test_parse.py b/tests/uploader/test_parse.py index 076c47c..56e1b41 100644 --- a/tests/uploader/test_parse.py +++ b/tests/uploader/test_parse.py @@ -8,7 +8,8 @@ from uploader.jobs import job, jobsnamespace from tests.conftest import uploadable_file_object -def test_parse_with_existing_uploaded_file(#pylint: disable=[too-many-arguments] +def test_parse_with_existing_uploaded_file( + #pylint: disable=[too-many-arguments,too-many-positional-arguments] client, db_url, redis_url, @@ -49,7 +50,7 @@ def test_parse_with_existing_uploaded_file(#pylint: disable=[too-many-arguments] assert the_job["command"] == " ".join([ sys.executable, "-m", "scripts.validate_file", db_url, redis_url, jobs_prefix, job_id, "--redisexpiry", str(redis_ttl), str(speciesid), - filetype, f"{client.application.config['UPLOAD_FOLDER']}/{filename}"]) + filetype, f"{client.application.config['UPLOADS_DIRECTORY']}/{filename}"]) @pytest.mark.parametrize( "filename,uri,error_msgs", diff --git a/uploader/__init__.py b/uploader/__init__.py index cae531b..afaa78d 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -3,9 +3,16 @@ import os import sys import logging from pathlib import Path +from typing import Optional from flask import Flask, request -from flask_session import Session + +from cachelib import FileSystemCache + +from gn_libs import jobs as gnlibs_jobs + +from flask_session import Session# type: ignore[attr-defined] + from uploader.oauth2.client import user_logged_in, authserver_authorise_uri @@ -13,9 +20,17 @@ from . import session from .base_routes import base from .files.views import files from .species import speciesbp +from .publications import pubbp from .oauth2.views import oauth2 +from .flask_extensions import url_for from .expression_data import exprdatabp from .errors import register_error_handlers +from .background_jobs import background_jobs_bp + +logging.basicConfig( + format=("%(asctime)s — %(filename)s:%(lineno)s — %(levelname)s " + "(%(thread)d:%(threadName)s): %(message)s") +) def override_settings_with_envvars( app: Flask, ignore: tuple[str, ...]=tuple()) -> None: @@ -50,10 +65,48 @@ def setup_logging(app: Flask) -> Flask: "SERVER_SOFTWARE", "").split('/') return __log_gunicorn__(app) if bool(software) else __log_dev__(app) +def setup_modules_logging(app_logger, modules): + """Setup module-level loggers to the same log-level as the application.""" + loglevel = logging.getLevelName(app_logger.getEffectiveLevel()) + for module in modules: + _logger = logging.getLogger(module) + _logger.setLevel(loglevel) + + +def __setup_scratch_directory__(app: Flask) -> Flask: + app.config["SCRATCH_DIRECTORY"] = Path( + app.config["SCRATCH_DIRECTORY"]).absolute() + return app + +def __setup_upload_directory__(app: Flask) -> Flask: + if app.config.get("UPLOADS_DIRECTORY", "").strip() == "": + app.config["UPLOADS_DIRECTORY"] = app.config[ + "SCRATCH_DIRECTORY"].joinpath("uploads") + else: + app.config["UPLOADS_DIRECTORY"] = Path( + app.config["UPLOADS_DIRECTORY"].strip()).absolute() + + return app + + +def update_unspecified_defaults(app: Flask) -> Flask: + """Setup the defaults for necessary configurations that do not have values + specified for them.""" + return __setup_upload_directory__(__setup_scratch_directory__(app)) + + +def create_app(config: Optional[dict] = None): + """The application factory. + + config: dict + Useful to override settings in the settings files and environment + especially in environments such as testing.""" + if config is None: + config = {} -def create_app(): - """The application factory""" app = Flask(__name__) + + ### BEGIN: Application configuration app.config.from_pyfile( Path(__file__).parent.joinpath("default_settings.py")) if "UPLOADER_CONF" in os.environ: @@ -68,16 +121,30 @@ def create_app(): if secretsfile.exists(): # Silently ignore secrets if the file does not exist. app.config.from_pyfile(secretsfile) + app.config.update(config) # Override everything with passed in config + update_unspecified_defaults(app) + ### END: Application configuration + + app.config["SESSION_CACHELIB"] = FileSystemCache( + cache_dir=str(Path(app.config["SESSION_FILESYSTEM_CACHE_PATH"]).absolute()), + threshold=int(app.config["SESSION_FILESYSTEM_CACHE_THRESHOLD"]), + default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) + setup_modules_logging( + app.logger, tuple(app.config.get("LOGGABLE_MODULES", []))) # setup jinja2 symbols - app.add_template_global(lambda : request.url, name="request_url") + app.add_template_global(user_logged_in) + app.add_template_global(url_for, name="url_for") app.add_template_global(authserver_authorise_uri) + app.add_template_global(lambda : request.url, name="request_url") app.add_template_global(lambda: app.config["GN2_SERVER_URL"], name="gn2server_uri") - app.add_template_global(user_logged_in) - app.add_template_global(lambda : session.user_details()["email"], name="user_email") + app.add_template_global(lambda : session.user_details()["email"], + name="user_email") + app.add_template_global(lambda: app.config["FEATURE_FLAGS_HTTP"], + name="http_feature_flags") Session(app) @@ -86,6 +153,9 @@ def create_app(): app.register_blueprint(files, url_prefix="/files") app.register_blueprint(oauth2, url_prefix="/oauth2") app.register_blueprint(speciesbp, url_prefix="/species") + app.register_blueprint(pubbp, url_prefix="/publications") + app.register_blueprint(background_jobs_bp, url_prefix="/background-jobs/") register_error_handlers(app) + gnlibs_jobs.init_app(app) return app diff --git a/uploader/authorisation.py b/uploader/authorisation.py index ee8fe97..3cf3585 100644 --- a/uploader/authorisation.py +++ b/uploader/authorisation.py @@ -16,13 +16,12 @@ def require_login(function): @wraps(function) def __is_session_valid__(*args, **kwargs): """Check that the user is logged in and their token is valid.""" - def __clear_session__(_no_token): - session.clear_session_info() - flash("You need to be logged in.", "alert-danger") + def __alert_needs_sign_in__(_no_token): + flash("You need to be signed in.", "alert alert-danger big-alert") return redirect("/") return session.user_token().either( - __clear_session__, + __alert_needs_sign_in__, lambda token: function(*args, **kwargs)) return __is_session_valid__ @@ -49,7 +48,7 @@ def require_token(func: Callable) -> Callable: """ def __invalid_token__(_whatever): logging.debug("==========> Failure log: %s", _whatever) - raise Exception( + raise Exception(# pylint: disable=[broad-exception-raised] "You attempted to access a feature of the system that requires " "authorisation. Unfortunately, we could not verify you have the " "appropriate authorisation to perform the action you requested. " diff --git a/uploader/background_jobs.py b/uploader/background_jobs.py new file mode 100644 index 0000000..a71dd44 --- /dev/null +++ b/uploader/background_jobs.py @@ -0,0 +1,225 @@ +"""Generic views and utilities to handle background jobs.""" +import uuid +import datetime +import importlib +from typing import Callable +from functools import partial + +from werkzeug.wrappers.response import Response +from flask import ( + flash, + request, + redirect, + Blueprint, + current_app as app) + +from gn_libs import jobs +from gn_libs import sqlite3 +from gn_libs.jobs.jobs import JobNotFound + +from uploader import session +from uploader.authorisation import require_login +from uploader.flask_extensions import url_for, render_template + +background_jobs_bp = Blueprint("background-jobs", __name__) +HandlerType = Callable[[dict], Response] + + +def make_datetime_formatter(dtformat: str = "%A, %d %B %Y at %H:%M %Z") -> Callable[[str], str]: + """Make a datetime formatter with the provided `dtformat`""" + def __formatter__(val: str) -> str: + dt = datetime.datetime.fromisoformat(val) + return dt.strftime(dtformat.strip()) + + return __formatter__ + +__default_datetime_formatter__ = make_datetime_formatter() + + +def __default_handler__(_job): + return render_template("background-jobs/job-summary.html", + job=_job, + display_datetime=__default_datetime_formatter__) + +def register_handlers( + job_type: str, + success_handler: HandlerType, + # pylint: disable=[redefined-outer-name] + error_handler: HandlerType = __default_handler__ + # pylint: disable=[redefined-outer-name] +) -> str: + """Register success and error handlers for each job type.""" + if not bool(app.config.get("background-jobs")): + app.config["background-jobs"] = {} + + if not bool(app.config["background-jobs"].get(job_type)): + app.config["background-jobs"][job_type] = { + "success": success_handler, + "error": error_handler + } + + return job_type + + +def register_job_handlers(job: dict): + """Related to register handlers above.""" + def __load_handler__(absolute_function_path): + _parts = absolute_function_path.split(".") + app.logger.debug("THE PARTS ARE: %s", _parts) + assert len(_parts) > 1, f"Invalid path: {absolute_function_path}" + module = importlib.import_module(f".{_parts[-2]}", + package=".".join(_parts[0:-2])) + return getattr(module, _parts[-1]) + + metadata = job["metadata"] + if metadata.get("success_handler"): + _success_handler = __load_handler__(metadata["success_handler"]) + try: + _error_handler = __load_handler__(metadata["error_handler"]) + except Exception as _exc:# pylint: disable=[broad-exception-caught] + _error_handler = __default_handler__ + register_handlers( + metadata["job-type"], _success_handler, _error_handler) + + +def handler(job: dict, handler_type: str) -> HandlerType: + """Fetch a handler for the job.""" + _job_type = job["metadata"]["job-type"] + _handler = app.config.get( + "background-jobs", {} + ).get( + _job_type, {} + ).get(handler_type) + if bool(_handler): + return _handler(job) + + return __default_handler__(job) + + +error_handler = partial(handler, handler_type="error") +success_handler = partial(handler, handler_type="success") + + +@background_jobs_bp.route("/status/<uuid:job_id>") +@require_login +def job_status(job_id: uuid.UUID): + """View the job status.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + + register_job_handlers(job) + if status in ("error", "stopped"): + return error_handler(job) + + if status == "completed": + return success_handler(job) + + return render_template("background-jobs/job-status.html", + job=job, + display_datetime=__default_datetime_formatter__) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/error/<uuid:job_id>") +@require_login +def job_error(job_id: uuid.UUID): + """Handle job errors in a generic manner.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + return render_template("jobs/job-error.html", job=job) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/list") +@require_login +def list_jobs(): + """List background jobs.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + return render_template( + "background-jobs/list-jobs.html", + jobs=jobs.jobs_by_external_id( + conn, session.user_details()["user_id"]), + display_datetime=__default_datetime_formatter__) + + +@background_jobs_bp.route("/summary/<uuid:job_id>") +@require_login +def job_summary(job_id: uuid.UUID): + """Provide a summary for completed jobs.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + + if status in ("completed", "error", "stopped"): + return render_template("background-jobs/job-summary.html", + job=job, + display_datetime=__default_datetime_formatter__) + return redirect(url_for( + "background-jobs.job_status", job_id=job["job_id"])) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/delete/<uuid:job_id>", methods=["GET", "POST"]) +@require_login +def delete_single(job_id: uuid.UUID): + """Delete a single job.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + if status not in ("completed", "error", "stopped"): + flash("We cannot delete a running job.", "alert alert-danger") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + + if request.method == "GET": + return render_template("background-jobs/delete-job.html", + job=job, + display_datetime=__default_datetime_formatter__) + + if request.form["btn-confirm-delete"] == "delete": + jobs.delete_job(conn, job_id) + flash("Job was deleted successfully.", "alert alert-success") + return redirect(url_for("background-jobs.list_jobs")) + flash("Delete cancelled.", "alert alert-info") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@background_jobs_bp.route("/stop/<uuid:job_id>", methods=["GET", "POST"]) +@require_login +def stop_job(job_id: uuid.UUID): + """Stop a running job.""" + with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn: + try: + job = jobs.job(conn, job_id, fulldetails=True) + status = job["metadata"]["status"] + if status != "running": + flash("Cannot stop a job that is not running.", "alert alert-danger") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + + if request.method == "GET": + return render_template("background-jobs/stop-job.html", + job=job, + display_datetime=__default_datetime_formatter__) + + if request.form["btn-confirm-stop"] == "stop": + jobs.kill_job(conn, job_id) + flash("Job was stopped successfully.", "alert alert-success") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + flash("Stop cancelled.", "alert alert-info") + return redirect(url_for( + "background-jobs.job_summary", job_id=job_id)) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) diff --git a/uploader/base_routes.py b/uploader/base_routes.py index 742a254..72a8402 100644 --- a/uploader/base_routes.py +++ b/uploader/base_routes.py @@ -1,15 +1,22 @@ """Basic routes required for all pages""" import os +import logging from urllib.parse import urljoin -from flask import (Blueprint, +from gn_libs.mysqldb import database_connection +from flask import (flash, + request, + redirect, + Blueprint, current_app as app, send_from_directory) +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer -from uploader.oauth2.client import user_logged_in +from uploader.species.models import all_species, species_by_id base = Blueprint("base", __name__) +logger = logging.getLogger(__name__) render_template = make_template_renderer("home") @@ -24,9 +31,25 @@ def favicon(): @base.route("/", methods=["GET"]) def index(): """Load the landing page""" - return render_template("index.html" if user_logged_in() else "login.html", - gn2server_intro=urljoin(app.config["GN2_SERVER_URL"], - "/intro")) + streamlined_ui = request.args.get("streamlined_ui") + with database_connection(app.config["SQL_URI"]) as conn: + print("We found a species ID. Processing...") + if not bool(request.args.get("species_id")): + return render_template( + "index.html", + gn2server_intro=urljoin(app.config["GN2_SERVER_URL"], "/intro"), + species=all_species(conn), + view_under_construction=request.args.get( + "view_under_construction", False)) + + species = species_by_id(conn, request.args.get("species_id")) + if not bool(species): + flash("Selected species was not found!", "alert alert-danger") + return redirect(url_for("base.index", streamlined_ui=streamlined_ui)) + + return redirect(url_for("species.view_species", + species_id=species["SpeciesId"])) + def appenv(): """Get app's guix environment path.""" @@ -35,8 +58,8 @@ def appenv(): @base.route("/bootstrap/<path:filename>") def bootstrap(filename): """Fetch bootstrap files.""" - return send_from_directory( - appenv(), f"share/genenetwork2/javascript/bootstrap/{filename}") + return send_from_directory(appenv(), f"share/web/bootstrap/{filename}") + @base.route("/jquery/<path:filename>") @@ -46,6 +69,19 @@ def jquery(filename): appenv(), f"share/genenetwork2/javascript/jquery/{filename}") +@base.route("/datatables/<path:filename>") +def datatables(filename): + """Fetch DataTables files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/DataTables/{filename}") + +@base.route("/datatables-extensions/<path:filename>") +def datatables_extensions(filename): + """Fetch DataTables files.""" + return send_from_directory( + appenv(), f"share/genenetwork2/javascript/DataTablesExtensions/{filename}") + + @base.route("/node-modules/<path:filename>") def node_modules(filename): """Fetch node-js modules.""" diff --git a/uploader/configutils.py b/uploader/configutils.py new file mode 100644 index 0000000..c5db50b --- /dev/null +++ b/uploader/configutils.py @@ -0,0 +1,13 @@ +"""Functions to fetch settings.""" +from pathlib import Path + +def fetch_setting(app, setting): + """Fetch a specified configuration `setting` from the `app` object.""" + return app.config[setting] + +def uploads_dir(app) -> Path: + """Fetch the uploads directory""" + _dir = Path(fetch_setting(app, "UPLOADS_DIRECTORY")).absolute() + assert _dir.exists() and _dir.is_dir(), ( + f"'{_dir}' needs to be an existing directory.") + return _dir diff --git a/uploader/datautils.py b/uploader/datautils.py index 46a55c4..d132c42 100644 --- a/uploader/datautils.py +++ b/uploader/datautils.py @@ -1,5 +1,7 @@ """Generic data utilities: Rename module.""" import math +import json +import base64 from functools import reduce from typing import Union, Sequence @@ -36,3 +38,13 @@ def safe_int(val: Union[str, int, float]) -> int: return int(val) except ValueError: return 0 + + +def base64_encode_dict(dct: dict, **kwargs) -> bytes: + """Base64 encode a dictionary. Takes the same keywords as `json.dumps` function.""" + return base64.urlsafe_b64encode(json.dumps(dct, **kwargs).encode("utf-8")) + + +def base64_decode_to_dict(value: str, **kwargs) -> dict: + """Base64 encode a dictionary. Takes the same keywords as `json.loads` function.""" + return json.loads(base64.urlsafe_b64decode(value), **kwargs) diff --git a/uploader/db/datasets.py b/uploader/db/datasets.py index 767ec41..4b263f5 100644 --- a/uploader/db/datasets.py +++ b/uploader/db/datasets.py @@ -53,7 +53,7 @@ def probeset_study_by_id(conn: mdb.Connection, studyid) -> Optional[dict]: _study = cursor.fetchone() return dict(_study) if bool(_study) else None -def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments] +def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-arguments, too-many-positional-arguments] populationid: int, platformid: int, tissueid: int, @@ -87,7 +87,7 @@ def probeset_create_study(conn: mdb.Connection,#pylint: disable=[too-many-argume (studyid, studyid)) return {**studydata, "studyid": studyid} -def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments] +def probeset_create_dataset(conn: mdb.Connection,#pylint: disable=[too-many-arguments, too-many-positional-arguments] studyid: int, averageid: int, datasetname: str, diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 1acb247..04e1c0a 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -2,9 +2,17 @@ The default configuration file. The values here should be overridden in the actual configuration file used for the production and staging systems. """ + LOG_LEVEL = "WARNING" SECRET_KEY = b"<Please! Please! Please! Change This!>" -UPLOAD_FOLDER = "/tmp/qc_app_files" + +# Scratch directory and uploads: +# *** The scratch directory *** +# We avoid `/tmp` entirely for the scratch directory to avoid shared global +# mutable state with other users/applications/processes. +SCRATCH_DIRECTORY = "~/tmp/gn-uploader-scratchdir" +UPLOADS_DIRECTORY = ""# If not set, will be under scratch directory. + REDIS_URL = "redis://" JOBS_TTL_SECONDS = 1209600 # 14 days GNQC_REDIS_PREFIX="gn-uploader" @@ -12,9 +20,25 @@ SQL_URI = "" GN2_SERVER_URL = "https://genenetwork.org/" -SESSION_TYPE = "redis" SESSION_PERMANENT = True SESSION_USE_SIGNER = True +SESSION_TYPE = "cachelib" +## --- Settings for CacheLib session type --- ## +## --- These are on flask-session config variables --- ## +## --- https://cachelib.readthedocs.io/en/stable/file/ --- ## +SESSION_FILESYSTEM_CACHE_PATH = "./flask_session" +SESSION_FILESYSTEM_CACHE_THRESHOLD = 500 +SESSION_FILESYSTEM_CACHE_TIMEOUT = 300 +SESSION_FILESYSTEM_CACHE_MODE = 0o600 +SESSION_FILESYSTEM_CACHE_HASH_METHOD = None # default: hashlib.md5 +## --- END: Settings for CacheLib session type --- ## JWKS_ROTATION_AGE_DAYS = 7 # Days (from creation) to keep a JWK in use. JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before deleting it. + + +## --- Feature flags --- +FEATURE_FLAGS_HTTP: list[str] = [] + +## --- Modules for which to log output --- +LOGGABLE_MODULES: list[str] = [] diff --git a/uploader/errors.py b/uploader/errors.py index 3e7c893..2ac48b8 100644 --- a/uploader/errors.py +++ b/uploader/errors.py @@ -3,7 +3,8 @@ import traceback from werkzeug.exceptions import HTTPException import MySQLdb as mdb -from flask import Flask, request, render_template, current_app as app +from flask import Flask, request, current_app as app +from uploader.flask_extensions import render_template def handle_general_exception(exc: Exception): """Handle generic exceptions.""" diff --git a/uploader/expression_data/dbinsert.py b/uploader/expression_data/dbinsert.py index 6d8ce80..7040698 100644 --- a/uploader/expression_data/dbinsert.py +++ b/uploader/expression_data/dbinsert.py @@ -94,7 +94,7 @@ def select_platform(): job = jobs.job(rconn, jobs.jobsnamespace(), job_id) if job: filename = job["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + filepath = f"{app.config['UPLOADS_DIRECTORY']}/{filename}" if os.path.exists(filepath): default_species = 1 gchips = genechips() @@ -367,7 +367,7 @@ def insert_data(): assert form.get("datasetid"), "dataset" filename = form["filename"] - filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}" + filepath = f"{app.config['UPLOADS_DIRECTORY']}/{filename}" redisurl = app.config["REDIS_URL"] if os.path.exists(filepath): with Redis.from_url(redisurl, decode_responses=True) as rconn: @@ -377,7 +377,7 @@ def insert_data(): form["species"], form["genechipid"], form["datasetid"], app.config["SQL_URI"], redisurl, app.config["JOBS_TTL_SECONDS"]), - redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") + redisurl, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("dbinsert.insert_status", job_id=job["jobid"])) return render_error(f"File '{filename}' no longer exists.") diff --git a/uploader/expression_data/views.py b/uploader/expression_data/views.py index 7629f3e..0e9b072 100644 --- a/uploader/expression_data/views.py +++ b/uploader/expression_data/views.py @@ -11,7 +11,6 @@ from werkzeug.utils import secure_filename from gn_libs.mysqldb import database_connection from flask import (flash, request, - url_for, redirect, Blueprint, current_app as app) @@ -19,6 +18,7 @@ from flask import (flash, from quality_control.errors import InvalidValue, DuplicateHeading from uploader import jobs +from uploader.flask_extensions import url_for from uploader.datautils import order_by_family from uploader.ui import make_template_renderer from uploader.authorisation import require_login @@ -162,7 +162,7 @@ def upload_file(species_id: int, population_id: int): species=species, population=population) - upload_dir = app.config["UPLOAD_FOLDER"] + upload_dir = app.config["UPLOADS_DIRECTORY"] request_errors = errors(request) if request_errors: for error in request_errors: @@ -225,7 +225,7 @@ def parse_file(species_id: int, population_id: int): _errors = True if filename: - filepath = os.path.join(app.config["UPLOAD_FOLDER"], filename) + filepath = os.path.join(app.config["UPLOADS_DIRECTORY"], filename) if not os.path.exists(filepath): flash("Selected file does not exist (any longer)", "alert-danger") _errors = True @@ -241,7 +241,7 @@ def parse_file(species_id: int, population_id: int): species_id, filepath, filetype,# type: ignore[arg-type] app.config["JOBS_TTL_SECONDS"]), redisurl, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("species.populations.expression-data.parse_status", species_id=species_id, @@ -263,7 +263,7 @@ def parse_status(species_id: int, population_id: int, job_id: str): return render_template("no_such_job.html", job_id=job_id), 400 error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + job_id, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: @@ -345,7 +345,7 @@ def fail(species_id: int, population_id: int, job_id: str): if job: error_filename = jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors") + job_id, f"{app.config['UPLOADS_DIRECTORY']}/job_errors") if os.path.exists(error_filename): stat = os.stat(error_filename) if stat.st_size > 0: diff --git a/uploader/files/__init__.py b/uploader/files/__init__.py index 60d2f3b..53c3176 100644 --- a/uploader/files/__init__.py +++ b/uploader/files/__init__.py @@ -1,3 +1,4 @@ +"""General files and chunks utilities.""" from .chunks import chunked_binary_read from .functions import (fullpath, save_file, diff --git a/uploader/files/chunks.py b/uploader/files/chunks.py index c4360b5..f63f32f 100644 --- a/uploader/files/chunks.py +++ b/uploader/files/chunks.py @@ -5,6 +5,8 @@ from typing import Iterator from flask import current_app as app from werkzeug.utils import secure_filename +from uploader.configutils import uploads_dir + def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator: """Read a file in binary mode in chunks.""" @@ -29,4 +31,4 @@ def chunks_directory(uniqueidentifier: str) -> Path: """Compute the directory where chunks are temporarily stored.""" if uniqueidentifier == "": raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") + return Path(uploads_dir(app), f"tempdir_{uniqueidentifier}") diff --git a/uploader/files/functions.py b/uploader/files/functions.py index 7b9f06b..68f4e16 100644 --- a/uploader/files/functions.py +++ b/uploader/files/functions.py @@ -8,6 +8,8 @@ from flask import current_app from werkzeug.utils import secure_filename from werkzeug.datastructures import FileStorage +from uploader.configutils import uploads_dir + from .chunks import chunked_binary_read def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Path: @@ -30,7 +32,7 @@ def save_file(fileobj: FileStorage, upload_dir: Path, hashed: bool = True) -> Pa def fullpath(filename: str): """Get a file's full path. This makes use of `flask.current_app`.""" - return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute() + return Path(uploads_dir(current_app), filename).absolute() def sha256_digest_over_file(filepath: Path) -> str: diff --git a/uploader/files/views.py b/uploader/files/views.py index 8d81654..ea0e827 100644 --- a/uploader/files/views.py +++ b/uploader/files/views.py @@ -1,16 +1,20 @@ """Module for generic files endpoints.""" +import time +import random import traceback from pathlib import Path from flask import request, jsonify, Blueprint, current_app as app +from uploader.configutils import uploads_dir + from .chunks import chunk_name, chunks_directory files = Blueprint("files", __name__) def target_file(fileid: str) -> Path: """Compute the full path for the target file.""" - return Path(app.config["UPLOAD_FOLDER"], fileid) + return Path(uploads_dir(app), fileid) @files.route("/upload/resumable", methods=["GET"]) @@ -56,10 +60,13 @@ def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: """Merge the chunks into a single file.""" with open(targetfile, "ab") as _target: for chunkfile in chunkpaths: + app.logger.error("Merging chunk: %s", chunkfile) with open(chunkfile, "rb") as _chunkdata: _target.write(_chunkdata.read()) - chunkfile.unlink() + chunkfile.unlink() # Don't use `missing_ok=True` — chunk MUST exist + # If chunk does't exist, it might indicate a race condition. Handle + # that instead. return targetfile @@ -92,15 +99,51 @@ def resumable_upload_post(): Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) for _achunk in range(1, _totalchunks+1)) if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() + ### HACK: Break possible race condition ### + # Looks like sometimes, there are multiple threads/requests trying + # to merge one file, leading to race conditions and in some rare + # instances, actual data corruption. This hack is meant to break + # that race condition. + _delays = ( + 101, 103, 107, 109, 113, 127, 131, 137, 139, 149, 151, 157, 163, + 167, 173, 179, 181, 191, 193, 197, 199, 211, 223, 227, 229, 233, + 239, 241, 251, 257, 263, 269, 271, 277, 281, 283, 293) + _lockfile = Path(chunks_directory(_fileid), "merge.lock") + while True: + time.sleep(random.choice(_delays) / 1000) + if (chunks_directory(_fileid).exists() + and not (_lockfile.exists() and _targetfile.exists())): + # merge_files and clean up chunks + _lockfile.touch() + __merge_chunks__(_targetfile, chunkpaths) + _lockfile.unlink() + chunks_directory(_fileid).rmdir() + continue + + if (_targetfile.exists() + and not ( + chunks_directory(_fileid).exists() + and _lockfile.exists())): + # merge complete + break + + # There is still a thread that's merging this file + continue + ### END: HACK: Break possible race condition ### + + if _targetfile.exists(): + return jsonify({ + "uploaded-file": _targetfile.name, + "original-name": _uploadfilename, + "message": "File was uploaded successfully!", + "statuscode": 200 + }), 200 return jsonify({ "uploaded-file": _targetfile.name, "original-name": _uploadfilename, - "message": "File was uploaded successfully!", - "statuscode": 200 - }), 200 + "message": "Uploaded file is missing!", + "statuscode": 404 + }), 404 return jsonify({ "message": f"Chunk {int(_chunk)} uploaded successfully.", "statuscode": 201 diff --git a/uploader/flask_extensions.py b/uploader/flask_extensions.py new file mode 100644 index 0000000..0fc774a --- /dev/null +++ b/uploader/flask_extensions.py @@ -0,0 +1,52 @@ +"""Custom extensions to the default flask functions/classes.""" +import logging +from typing import Any, Optional + +from flask import ( + request, + current_app as app, + url_for as flask_url_for, + render_template as flask_render_template) + +logger = logging.getLogger(__name__) + + +def fetch_flags(): + """Fetch get arguments that are defined as feature flags.""" + flags = {} + for flag in app.config["FEATURE_FLAGS_HTTP"]: + flag_value = (request.args.get(flag) or request.form.get(flag) or "").strip() + if bool(flag_value): + flags[flag] = flag_value + continue + continue + logger.debug("HTTP FEATURE FLAGS: %s", flags) + return flags + + +def url_for( + endpoint: str, + _anchor: Optional[str] = None, + _method: Optional[str] = None, + _scheme: Optional[str] = None, + _external: Optional[bool] = None, + **values: Any) -> str: + """Extension to flask's `url_for` function.""" + logger.debug("other variables: %s", values) + return flask_url_for(endpoint=endpoint, + _anchor=_anchor, + _method=_method, + _scheme=_scheme, + _external=_external, + **values, + **fetch_flags()) + + +def render_template(template_name_or_list, **context: Any) -> str: + """Extend flask's `render_template` function""" + return flask_render_template( + template_name_or_list, + **{ + **context, + **fetch_flags() # override any flag values + }) diff --git a/uploader/genotypes/models.py b/uploader/genotypes/models.py index 4c3e634..41270da 100644 --- a/uploader/genotypes/models.py +++ b/uploader/genotypes/models.py @@ -1,13 +1,17 @@ """Functions for handling genotypes.""" +import logging from typing import Optional +from functools import reduce from datetime import datetime import MySQLdb as mdb from MySQLdb.cursors import Cursor, DictCursor -from flask import current_app as app from gn_libs.mysqldb import debug_query +logger = logging.getLogger(__name__) + + def genocode_by_population( conn: mdb.Connection, population_id: int) -> tuple[dict, ...]: """Get the allele/genotype codes.""" @@ -29,18 +33,102 @@ def genotype_markers_count(conn: mdb.Connection, species_id: int) -> int: def genotype_markers( conn: mdb.Connection, species_id: int, + population_id: int, + offset: int = 0, + limit: int = -1# no limit if negative, zero returns empty list. +) -> tuple[tuple[dict, ...], int]: + """Retrieve markers from the database. + + Return: A tuple of: + - Listing of the markers, + - The total number of markers found in the system. + """ + _query_template = ( + "SELECT %%COLS%% " + "FROM Species AS spc " + "INNER JOIN InbredSet AS iset " + "ON spc.Id = iset.SpeciesId " + "INNER JOIN GenoFreeze AS gfr " + "ON iset.Id = gfr.InbredSetId " + "INNER JOIN GenoXRef AS gxr " + "ON gfr.Id = gxr.GenoFreezeId " + "INNER JOIN Geno AS gno " + "ON gxr.GenoId = gno.Id " + "WHERE spc.Id=%s " + "AND iset.Id=%s " + "%%LIMIT%%") + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + _query_template.replace("%%LIMIT%%", "").replace( + "%%COLS%%", "COUNT(gno.Id) AS total_records"), + (species_id, population_id)) + _total_records = cursor.fetchone()["total_records"] + cursor.execute( + _query_template.replace("%%COLS%%", "gno.*, gxr.cM").replace( + "%%LIMIT%%", + (f"LIMIT {int(limit)} OFFSET {int(offset)}" + if bool(limit) and limit >= 0 + else "")), + (species_id, population_id)) + debug_query(cursor, logger) + _records = tuple(dict(row) for row in cursor.fetchall()) + return _records, _total_records + + +def genotype_records( + conn: mdb.Connection, + species_id: int, + population_id: int, offset: int = 0, - limit: Optional[int] = None -) -> tuple[dict, ...]: - """Retrieve markers from the database.""" - _query = "SELECT * FROM Geno WHERE SpeciesId=%s" - if bool(limit) and limit > 0:# type: ignore[operator] - _query = _query + f" LIMIT {limit} OFFSET {offset}" + limit: int = -1# no limit if negative, zero returns empty list. +) -> tuple[tuple[dict, ...], int]: + """Retrieve the actual genotype records from the database. + + Returns: A tuple of: + - the listing of the genotype data, + - the total number of genotype records for this population. + """ + def __organise_geno_records__(acc, row): + _current_row = acc.get(row["GenoId"], { + "GenoId": row["GenoId"], + "data": {} + }) + _current_row["data"][row["StrainName"]] = row["value"] + return { + **acc, + _current_row["GenoId"]: _current_row + } + + _query_template = ( + "SELECT gxr.GenoId, gxr.DataId, gdt.value, strn.Name AS StrainName " + "FROM GenoXRef AS gxr " + "INNER JOIN GenoData AS gdt ON gxr.DataId = gdt.Id " + "INNER JOIN Strain AS strn ON gdt.StrainId = strn.Id " + "WHERE gxr.GenoId IN (%%PARAMS_STR%%)") with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (species_id,)) - debug_query(cursor, app.logger) - return tuple(dict(row) for row in cursor.fetchall()) + _markers, _num_records = genotype_markers( + conn, species_id, population_id, offset, limit) + if len(_markers) == 0: + return (tuple(), 0) + + _genoids = tuple(_marker["Id"] for _marker in _markers) + cursor.execute( + _query_template.replace( + "%%PARAMS_STR%%", ",".join(["%s"] * len(_genoids))), + _genoids) + debug_query(cursor, logger) + _records: dict[str, dict] = reduce( + __organise_geno_records__, cursor.fetchall(), {}) + return ( + tuple({ + **_marker, + "data": _records.get( + _marker["Id"], {} + ).get("data", {}) + } for _marker in _markers), + _num_records) def genotype_dataset( @@ -65,7 +153,7 @@ def genotype_dataset( with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute(_query, _params) - debug_query(cursor, app.logger) + debug_query(cursor, logger) result = cursor.fetchone() if bool(result): return dict(result) diff --git a/uploader/genotypes/views.py b/uploader/genotypes/views.py index 0433420..454fee7 100644 --- a/uploader/genotypes/views.py +++ b/uploader/genotypes/views.py @@ -1,130 +1,124 @@ """Views for the genotypes.""" +import logging +from uuid import uuid4 + from MySQLdb.cursors import DictCursor +from pymonad.either import Left, Right, Either +from gn_libs.requests import request_json from gn_libs.mysqldb import database_connection +from werkzeug.exceptions import UnsupportedMediaType from flask import (flash, request, - url_for, + jsonify, redirect, Blueprint, + make_response, render_template, current_app as app) +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post from uploader.authorisation import require_login -from uploader.species.models import all_species, species_by_id +from uploader.species.models import species_by_id from uploader.monadic_requests import make_either_error_handler -from uploader.request_checks import with_species, with_population -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import (populations_by_species, - population_by_species_and_id) +from uploader.population.models import population_by_species_and_id +from uploader.request_checks import with_dataset, with_population + from .models import (genotype_markers, + genotype_records, genotype_dataset, save_new_dataset, - genotype_markers_count, genocode_by_population) +logger = logging.getLogger(__name__) genotypesbp = Blueprint("genotypes", __name__) render_template = make_template_renderer("genotypes") -@genotypesbp.route("populations/genotypes", methods=["GET"]) -@require_login -def index(): - """Direct entry-point for genotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("species_id")): - return render_template("genotypes/index.html", - species=order_by_family(all_species(conn)), - activelink="genotypes") - species = species_by_id(conn, request.args.get("species_id")) - if not bool(species): - flash(f"Could not find species with ID '{request.args.get('species_id')}'!", - "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) - return redirect(url_for("species.populations.genotypes.select_population", - species_id=species["SpeciesId"])) - -@genotypesbp.route("/<int:species_id>/populations/genotypes/select-population", - methods=["GET"]) +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes", + methods=["GET", "POST"]) @require_login -@with_species(redirect_uri="species.populations.genotypes.index") -def select_population(species: dict, species_id: int): - """Select the population under which the genotypes go.""" +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") +def index(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] + """Entry-point to the genotypes management section.""" with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("population_id")): - return render_template("genotypes/select-population.html", - species=species, - populations=order_by_family( - populations_by_species(conn, species_id), - order_key="FamilyOrder"), - activelink="genotypes") + form = request_json() + offset = int(form.get("start", "0")) + number_of_records = int(form.get("length", "10")) + _markers, _total_markers, = genotype_markers( + conn, species["SpeciesId"], population["Id"]) + _genotype_records, _count = genotype_records( + conn, + species["SpeciesId"], + population["Id"], + offset, + number_of_records) + _genotype_records = tuple( + {**_record, "index": _idx} + for _idx, _record + in enumerate(_genotype_records, start=offset+1)) - population = population_by_species_and_id( - conn, species_id, request.args.get("population_id")) - if not bool(population): - flash("Invalid population selected!", "alert-danger") - return redirect(url_for( - "species.populations.genotypes.select_population", - species_id=species_id)) + ## Order these correctly + _samples = (tuple() if len(_genotype_records) == 0 + else tuple(_genotype_records[0]["data"].keys())) - return redirect(url_for("species.populations.genotypes.list_genotypes", - species_id=species_id, - population_id=population["Id"])) + if "application/json" in request.headers["Accept"]: + return make_response( + jsonify({ + "genotype_records": _genotype_records, + "total_genotype_records": _count, + "fetched_genotype_records": len(_genotype_records), + "samples_order": _samples, + "draw": int(request.args.get("draw", "0")) + }), 200) + if "text/html" in request.headers["Accept"]: + return render_template( + "genotypes/index.html", + species=species, + population=population, + genocode=genocode_by_population(conn, population["Id"]), + dataset=genotype_dataset( + conn, species["SpeciesId"], population["Id"]), + genotype_records=_genotype_records, + samples=_samples, + activelink="list-genotypes") -@genotypesbp.route( - "/<int:species_id>/populations/<int:population_id>/genotypes", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") -def list_genotypes(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] - """List genotype details for species and population.""" - with database_connection(app.config["SQL_URI"]) as conn: - return render_template("genotypes/list-genotypes.html", - species=species, - population=population, - genocode=genocode_by_population( - conn, population["Id"]), - total_markers=genotype_markers_count( - conn, species["SpeciesId"]), - dataset=genotype_dataset(conn, - species["SpeciesId"], - population["Id"]), - activelink="list-genotypes") + raise UnsupportedMediaType("This endpoint can only server HTML or JSON") @genotypesbp.route( - "/<int:species_id>/populations/<int:population_id>/genotypes/list-markers", + "/<int:species_id>/populations/<int:population_id>/genotypes/<int:dataset_id>/list-markers", methods=["GET"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") -def list_markers( - species: dict, - population: dict, - **kwargs -):# pylint: disable=[unused-argument] - """List a species' genetic markers.""" +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.index") +def list_markers(species: dict, population: dict, **_kwargs): + """List the markers that exist for this species.""" + args = request.args + offset = int(args.get("start") or 0) with database_connection(app.config["SQL_URI"]) as conn: - start_from = max(safe_int(request.args.get("start_from") or 0), 0) - count = safe_int(request.args.get("count") or 20) - return render_template("genotypes/list-markers.html", - species=species, - population=population, - total_markers=genotype_markers_count( - conn, species["SpeciesId"]), - start_from=start_from, - count=count, - markers=enumerate_sequence( - genotype_markers(conn, - species["SpeciesId"], - offset=start_from, - limit=count), - start=start_from+1), - activelink="list-markers") + markers, total_records = genotype_markers( + conn, + species["SpeciesId"], + population["Id"], + offset=offset, + limit=int(args.get("length") or 0)) + return jsonify({ + **({"draw": int(args.get("draw", "0"))} + if bool(args.get("draw")) + else {}), + "recordsTotal": total_records, + "recordsFiltered": len(markers), + "markers": tuple({**marker, "index": idx} + for idx, marker in + enumerate(markers, start=offset+1)) + }) + @genotypesbp.route( "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" @@ -137,14 +131,14 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): species = species_by_id(conn, species_id) if not bool(species): flash("Invalid species provided!", "alert-danger") - return redirect(url_for("species.populations.genotypes.index")) + return redirect(url_for("species.list_species")) population = population_by_species_and_id( conn, species_id, population_id) if not bool(population): flash("Invalid population selected!", "alert-danger") return redirect(url_for( - "species.populations.genotypes.select_population", + "species.populations.list_species_populations", species_id=species_id)) dataset = genotype_dataset(conn, species_id, population_id, dataset_id) @@ -167,25 +161,32 @@ def view_dataset(species_id: int, population_id: int, dataset_id: int): "create", methods=["GET", "POST"]) @require_login -@with_population(species_redirect_uri="species.populations.genotypes.index", - redirect_uri="species.populations.genotypes.select_population") +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """Create a genotype dataset.""" + if request.method == "GET": + return render_template("genotypes/create-dataset.html", + species=species, + population=population, + activelink="create-dataset") + with (database_connection(app.config["SQL_URI"]) as conn, conn.cursor(cursorclass=DictCursor) as cursor): - if request.method == "GET": - return render_template("genotypes/create-dataset.html", - species=species, - population=population, - activelink="create-dataset") - - form = request.form - new_dataset = save_new_dataset( - cursor, - population["Id"], - form["geno-dataset-name"], - form["geno-dataset-fullname"], - form["geno-dataset-shortname"]) + + def __save_dataset__() -> Either: + form = request.form + try: + return Right(save_new_dataset( + cursor, + population["Id"], + form["geno-dataset-name"], + form["geno-dataset-fullname"], + form["geno-dataset-shortname"])) + except Exception:# pylint: disable=[broad-exception-caught] + msg = "Error adding new Genotype dataset to database." + logger.error(msg, exc_info=True) + return Left(Exception(msg)) def __success__(_success): flash("Successfully created genotype dataset.", "alert-success") @@ -194,19 +195,72 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= species_id=species["SpeciesId"], population_id=population["Id"])) - return oauth2_post( - "auth/resource/genotypes/create", - json={ - **dict(request.form), - "species_id": species["SpeciesId"], - "population_id": population["Id"], - "dataset_id": new_dataset["Id"], - "dataset_name": form["geno-dataset-name"], - "dataset_fullname": form["geno-dataset-fullname"], - "dataset_shortname": form["geno-dataset-shortname"], - "public": "on" - } + return __save_dataset__().then( + lambda new_dataset: oauth2_post( + "auth/resource/genotypes/create", + json={ + **dict(request.form), + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": new_dataset["Id"], + "dataset_name": new_dataset["Name"], + "dataset_fullname": new_dataset["FullName"], + "dataset_shortname": new_dataset["ShortName"], + "public": "on" + } + ) ).either( make_either_error_handler( "There was an error creating the genotype dataset."), __success__) + + +def genotype_csv_to_r_qtl2(uploadsdir: Path, csvfile, csv_meta: dict) -> Path: + """Convert given CSV genotype file into the R/qtl2 format.""" + bundlepath = Path(uploadsdir, f"{uuid.uuid4()}.zip".replace("-", "")) + raise NotImplementedError("This is not implemented yet.") + + +@genotypesbp.route( + "/<int:species_id>/populations/<int:population_id>/genotypes/datasets/" + "<int:dataset_id>/add-records", + methods=["GET", "POST"]) +@require_login +@with_population(species_redirect_uri="species.list_species", + redirect_uri="species.populations.list_species_populations") +@with_dataset(species_redirect_uri="species.list_species", + population_redirect_uri="species.populations.list_species_populations", + redirect_uri="species.populations.genotypes.index", + dataset_by_id=genotype_dataset) +def add_genotype_records(species: dict, population: dict, dataset: dict, **kwargs): + """Add new Genotype records to the dataset.""" + if request.method == "GET": + return render_template("genotypes/add-genotypes-records-csv.html", + species=species, + population=population, + dataset=dataset, + activelink="add-genotypes-records") + + # request.method is POST from here + # T0D0: Handle direct uploads (i.e. Not via javascript) + form = dict(request.form) # Request comes in as multipart/formdata + + bundlepath = genotype_csv_to_r_qtl2( + # T0D0: Actually, rather than generating the R/qtl2 bundle here, first + # off, do basic check through the data to collect the alleles and other + # necessary information. Also do basic QC. + Path(uploads_dir(app)), + form["uploaded-file"], + csv_meta: { + "sep": form.get("file-separator", ","), + "comment.char": form.get("file-comment-char", "#"), + "na.strings": form.get("file-na", "- NA N/A").split(" ") + }) + + if "application/json" in request.headers.get("Accept"): + return make_response( + jsonify({ + "message": ("Upload successful. Follow the 'redirect-to' URI " + "to continue") + }), + 200) diff --git a/uploader/jobs.py b/uploader/jobs.py index e86ee05..b2de54b 100644 --- a/uploader/jobs.py +++ b/uploader/jobs.py @@ -41,7 +41,8 @@ def error_filename(jobid, error_dir): "Compute the path of the file where errors will be dumped." return f"{error_dir}/job_{jobid}.error" -def initialise_job(# pylint: disable=[too-many-arguments] +def initialise_job( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str, ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict: "Initialise a job 'object' and put in on redis" @@ -54,7 +55,8 @@ def initialise_job(# pylint: disable=[too-many-arguments] name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds)) return the_job -def build_file_verification_job(#pylint: disable=[too-many-arguments] +def build_file_verification_job( + #pylint: disable=[too-many-arguments, too-many-positional-arguments] redis_conn: Redis, dburi: str, redisuri: str, @@ -77,7 +79,8 @@ def build_file_verification_job(#pylint: disable=[too-many-arguments] "filename": os.path.basename(filepath), "percent": 0 }) -def data_insertion_job(# pylint: disable=[too-many-arguments] +def data_insertion_job( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] redis_conn: Redis, filepath: str, filetype: str, totallines: int, speciesid: int, platformid: int, datasetid: int, databaseuri: str, redisuri: str, ttl_seconds: int) -> dict: @@ -144,8 +147,8 @@ def job_errors( return take( ( json.loads(error) - for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*") - for error in rconn.lrange(key, 0, -1)), + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*")# type: ignore[union-attr] + for error in rconn.lrange(key, 0, -1)),# type: ignore[union-attr] count) @@ -157,8 +160,8 @@ def job_files_metadata( """Get the metadata for specific job file.""" return { key.split(":")[-1]: { - **rconn.hgetall(key), + **rconn.hgetall(key),# type: ignore[dict-item] "filetype": key.split(":")[-3] } - for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*") + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*")# type: ignore[union-attr] } diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py index c492df5..eda42d0 100644 --- a/uploader/monadic_requests.py +++ b/uploader/monadic_requests.py @@ -5,12 +5,12 @@ from typing import Union, Optional, Callable import requests from requests.models import Response from pymonad.either import Left, Right, Either +from markupsafe import escape as markupsafe_escape from flask import (flash, request, redirect, render_template, - current_app as app, - escape as flask_escape) + current_app as app) # HTML Status codes indicating a successful request. SUCCESS_CODES = (200, 201, 202, 203, 204, 205, 206, 207, 208, 226) @@ -39,9 +39,9 @@ def make_error_handler( trace=traceback.format_exception(resp_or_exc)) if isinstance(resp_or_exc, Response): flash("The authorisation server responded with " - f"({flask_escape(resp_or_exc.status_code)}, " - f"{flask_escape(resp_or_exc.reason)}) for the request to " - f"'{flask_escape(resp_or_exc.request.url)}'", + f"({markupsafe_escape(resp_or_exc.status_code)}, " + f"{markupsafe_escape(resp_or_exc.reason)}) for the request to " + f"'{markupsafe_escape(resp_or_exc.request.url)}'", "alert-danger") return redirect_to @@ -59,6 +59,11 @@ def get(url, params=None, **kwargs) -> Either: :rtype: pymonad.either.Either """ + timeout = kwargs.get("timeout") + kwargs = {key: val for key,val in kwargs.items() if key != "timeout"} + if timeout is None: + timeout = (9.13, 20) + try: resp = requests.get(url, params=params, **kwargs) if resp.status_code in SUCCESS_CODES: @@ -76,6 +81,11 @@ def post(url, data=None, json=None, **kwargs) -> Either: :rtype: pymonad.either.Either """ + timeout = kwargs.get("timeout") + kwargs = {key: val for key,val in kwargs.items() if key != "timeout"} + if timeout is None: + timeout = (9.13, 20) + try: resp = requests.post(url, data=data, json=json, **kwargs) if resp.status_code in SUCCESS_CODES: @@ -95,10 +105,10 @@ def make_either_error_handler(msg): try: _data = error.json() except Exception as _exc: - raise Exception(error.content) from _exc - raise Exception(_data) + raise Exception(error.content) from _exc# pylint: disable=[broad-exception-raised] + raise Exception(_data)# pylint: disable=[broad-exception-raised] app.logger.debug("\n\n%s\n\n", msg) - raise Exception(error) + raise Exception(error)# pylint: disable=[broad-exception-raised] return __fail__ diff --git a/uploader/oauth2/client.py b/uploader/oauth2/client.py index 1efa299..e37816d 100644 --- a/uploader/oauth2/client.py +++ b/uploader/oauth2/client.py @@ -1,9 +1,10 @@ """OAuth2 client utilities.""" import json import time +import uuid import random from datetime import datetime, timedelta -from urllib.parse import urljoin, urlparse +from urllib.parse import urljoin, urlparse, urlencode import requests from flask import request, current_app as app @@ -17,6 +18,7 @@ from authlib.integrations.requests_client import OAuth2Session from uploader import session import uploader.monadic_requests as mrequests +from uploader.flask_extensions import fetch_flags SCOPE = ("profile group role resource register-client user masquerade " "introspect migrate-data") @@ -42,7 +44,8 @@ def __fetch_auth_server_jwks__() -> KeySet: return KeySet([ JsonWebKey.import_key(key) for key in requests.get( - urljoin(authserver_uri(), "auth/public-jwks") + urljoin(authserver_uri(), "auth/public-jwks"), + timeout=(9.13, 20) ).json()["jwks"]]) @@ -146,9 +149,27 @@ def oauth2_client(): __client__) +def fetch_user_details() -> Either: + """Retrieve user details from the auth server""" + suser = session.session_info()["user"] + if suser["email"] == "anon@ymous.user": + udets = oauth2_get("auth/user/").then( + lambda usrdets: session.set_user_details({ + "user_id": uuid.UUID(usrdets["user_id"]), + "name": usrdets["name"], + "email": usrdets["email"], + "token": session.user_token(), + "logged_in": session.user_token().either( + lambda _e: False, lambda _t: True) + })) + return udets + return Right(suser) + + def user_logged_in(): """Check whether the user has logged in.""" suser = session.session_info()["user"] + fetch_user_details() return suser["logged_in"] and suser["token"].is_right() @@ -156,11 +177,13 @@ def authserver_authorise_uri(): """Build up the authorisation URI.""" req_baseurl = urlparse(request.base_url, scheme=request.scheme) host_uri = f"{req_baseurl.scheme}://{req_baseurl.netloc}/" - return urljoin( - authserver_uri(), - "auth/authorise?response_type=code" - f"&client_id={oauth2_clientid()}" - f"&redirect_uri={urljoin(host_uri, 'oauth2/code')}") + args = { + "response_type": "code", + "client_id": oauth2_clientid(), + "redirect_uri": ( + f"{urljoin(host_uri, 'oauth2/code')}?{urlencode(fetch_flags())}") + } + return f"{urljoin(authserver_uri(), 'auth/authorise')}?{urlencode(args)}" def __no_token__(_err) -> Left: diff --git a/uploader/oauth2/tokens.py b/uploader/oauth2/tokens.py new file mode 100644 index 0000000..eb650f6 --- /dev/null +++ b/uploader/oauth2/tokens.py @@ -0,0 +1,47 @@ +"""Utilities for dealing with tokens.""" +import uuid +from typing import Union +from urllib.parse import urljoin +from datetime import datetime, timedelta + +from authlib.jose import jwt +from flask import current_app as app + +from uploader import monadic_requests as mrequests + +from . import jwks +from .client import (SCOPE, authserver_uri, oauth2_clientid) + + +def request_token(token_uri: str, user_id: Union[uuid.UUID, str], **kwargs): + """Request token from the auth server.""" + issued = datetime.now() + jwtkey = jwks.newest_jwk_with_rotation( + jwks.jwks_directory(app, "UPLOADER_SECRETS"), + int(app.config["JWKS_ROTATION_AGE_DAYS"])) + _mins2expiry = kwargs.get("minutes_to_expiry", 5) + return mrequests.post( + token_uri, + json={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "scope": kwargs.get("scope", SCOPE), + "assertion": jwt.encode( + header={ + "alg": "RS256", + "typ": "JWT", + "kid": jwtkey.as_dict()["kid"] + }, + payload={ + "iss": str(oauth2_clientid()), + "sub": str(user_id), + "aud": urljoin(authserver_uri(), "auth/token"), + "exp": (issued + timedelta(minutes=_mins2expiry)).timestamp(), + "nbf": int(issued.timestamp()), + "iat": int(issued.timestamp()), + "jti": str(uuid.uuid4()) + }, + key=jwtkey).decode("utf8"), + "client_id": oauth2_clientid(), + **kwargs.get("extra_params", {}) + } + ) diff --git a/uploader/oauth2/views.py b/uploader/oauth2/views.py index 61037f3..b1b740f 100644 --- a/uploader/oauth2/views.py +++ b/uploader/oauth2/views.py @@ -1,99 +1,69 @@ """Views for OAuth2 related functionality.""" -import uuid -from datetime import datetime, timedelta from urllib.parse import urljoin, urlparse, urlunparse -from authlib.jose import jwt from flask import ( flash, jsonify, - url_for, request, redirect, Blueprint, current_app as app) from uploader import session +from uploader.flask_extensions import url_for from uploader import monadic_requests as mrequests from uploader.monadic_requests import make_error_handler from . import jwks +from .tokens import request_token from .client import ( - SCOPE, - oauth2_get, user_logged_in, authserver_uri, oauth2_clientid, + fetch_user_details, oauth2_clientsecret) oauth2 = Blueprint("oauth2", __name__) + @oauth2.route("/code") def authorisation_code(): """Receive authorisation code from auth server and use it to get token.""" - def __process_error__(resp_or_exception): - app.logger.debug("ERROR: (%s)", resp_or_exception) + def __process_error__(error_response): + app.logger.debug("ERROR: (%s)", error_response.content) flash("There was an error retrieving the authorisation token.", - "alert-danger") - return redirect("/") + "alert alert-danger") + return redirect(url_for("base.index")) def __fail_set_user_details__(_failure): app.logger.debug("Fetching user details fails: %s", _failure) - flash("Could not retrieve the user details", "alert-danger") - return redirect("/") + flash("Could not retrieve the user details", "alert alert-danger") + return redirect(url_for("base.index")) def __success_set_user_details__(_success): app.logger.debug("Session info: %s", _success) - return redirect("/") + return redirect(url_for("base.index")) def __success__(token): session.set_user_token(token) - return oauth2_get("auth/user/").then( - lambda usrdets: session.set_user_details({ - "user_id": uuid.UUID(usrdets["user_id"]), - "name": usrdets["name"], - "email": usrdets["email"], - "token": session.user_token(), - "logged_in": True})).either( + return fetch_user_details().either( __fail_set_user_details__, __success_set_user_details__) code = request.args.get("code", "").strip() if not bool(code): - flash("AuthorisationError: No code was provided.", "alert-danger") - return redirect("/") + flash("AuthorisationError: No code was provided.", "alert alert-danger") + return redirect(url_for("base.index")) baseurl = urlparse(request.base_url, scheme=request.scheme) - issued = datetime.now() - jwtkey = jwks.newest_jwk_with_rotation( - jwks.jwks_directory(app, "UPLOADER_SECRETS"), - int(app.config["JWKS_ROTATION_AGE_DAYS"])) - return mrequests.post( - urljoin(authserver_uri(), "auth/token"), - json={ - "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + return request_token( + token_uri=urljoin(authserver_uri(), "auth/token"), + user_id=request.args["user_id"], + extra_params={ "code": code, - "scope": SCOPE, "redirect_uri": urljoin( urlunparse(baseurl), url_for("oauth2.authorisation_code")), - "assertion": jwt.encode( - header={ - "alg": "RS256", - "typ": "JWT", - "kid": jwtkey.as_dict()["kid"] - }, - payload={ - "iss": str(oauth2_clientid()), - "sub": request.args["user_id"], - "aud": urljoin(authserver_uri(),"auth/token"), - "exp": (issued + timedelta(minutes=5)).timestamp(), - "nbf": int(issued.timestamp()), - "iat": int(issued.timestamp()), - "jti": str(uuid.uuid4()) - }, - key=jwtkey).decode("utf8"), - "client_id": oauth2_clientid() }).either(__process_error__, __success__) @oauth2.route("/public-jwks") @@ -116,8 +86,8 @@ def logout(): _user = session_info["user"] _user_str = f"{_user['name']} ({_user['email']})" session.clear_session_info() - flash("Successfully logged out.", "alert-success") - return redirect("/") + flash("Successfully signed out.", "alert alert-success") + return redirect(url_for("base.index")) if user_logged_in(): return session.user_token().then( @@ -130,9 +100,9 @@ def logout(): "client_secret": oauth2_clientsecret() })).either( make_error_handler( - redirect_to=redirect("/"), + redirect_to=redirect(url_for("base.index")), cleanup_thunk=lambda: __unset_session__( session.session_info())), lambda res: __unset_session__(session.session_info())) - flash("There is no user that is currently logged in.", "alert-info") - return redirect("/") + flash("There is no user that is currently logged in.", "alert alert-info") + return redirect(url_for("base.index")) diff --git a/uploader/phenotypes/misc.py b/uploader/phenotypes/misc.py new file mode 100644 index 0000000..1924c07 --- /dev/null +++ b/uploader/phenotypes/misc.py @@ -0,0 +1,26 @@ +"""Miscellaneous functions handling phenotypes and phenotypes data.""" +import logging + +logger = logging.getLogger(__name__) + + +def phenotypes_data_differences( + filedata: tuple[dict, ...], dbdata: tuple[dict, ...] +) -> tuple[dict, ...]: + """Compute differences between file data and db data""" + diff: tuple[dict, ...] = tuple() + for filerow, dbrow in zip( + sorted(filedata, key=lambda item: (item["phenotype_id"], item["xref_id"])), + sorted(dbdata, key=lambda item: (item["PhenotypeId"], item["xref_id"]))): + for samplename, value in filerow["data"].items(): + if value != dbrow["data"].get(samplename, {}).get("value"): + diff = diff + ({ + "PhenotypeId": filerow["phenotype_id"], + "xref_id": filerow["xref_id"], + "DataId": dbrow["DataId"], + "StrainId": dbrow["data"].get(samplename, {}).get("StrainId"), + "StrainName": samplename, + "value": value + },) + + return diff diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index 73b1cce..3d656d2 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -1,16 +1,35 @@ """Database and utility functions for phenotypes.""" -from typing import Optional +import time +import random +import logging +import tempfile +from pathlib import Path from functools import reduce from datetime import datetime +from typing import Union, Optional, Iterable -import MySQLdb as mdb -from MySQLdb.cursors import Cursor, DictCursor -from flask import current_app as app +from MySQLdb.connections import Connection +from MySQLdb.cursors import Cursor, DictCursor, BaseCursor from gn_libs.mysqldb import debug_query +from functional_tools import take + +logger = logging.getLogger(__name__) + + +__PHENO_DATA_TABLES__ = { + "PublishData": { + "table": "PublishData", "valueCol": "value", "DataIdCol": "Id"}, + "PublishSE": { + "table": "PublishSE", "valueCol": "error", "DataIdCol": "DataId"}, + "NStrain": { + "table": "NStrain", "valueCol": "count", "DataIdCol": "DataId"} +} + + def datasets_by_population( - conn: mdb.Connection, + conn: Connection, species_id: int, population_id: int ) -> tuple[dict, ...]: @@ -25,22 +44,22 @@ def datasets_by_population( return tuple(dict(row) for row in cursor.fetchall()) -def dataset_by_id(conn: mdb.Connection, +def dataset_by_id(conn: Connection, species_id: int, population_id: int, dataset_id: int) -> dict: """Fetch dataset details by identifier""" with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( - "SELECT s.SpeciesId, pf.* FROM Species AS s " - "INNER JOIN InbredSet AS iset ON s.Id=iset.SpeciesId " - "INNER JOIN PublishFreeze AS pf ON iset.Id=pf.InbredSetId " - "WHERE s.Id=%s AND iset.Id=%s AND pf.Id=%s", + "SELECT Species.SpeciesId, PublishFreeze.* FROM Species " + "INNER JOIN InbredSet ON Species.Id=InbredSet.SpeciesId " + "INNER JOIN PublishFreeze ON InbredSet.Id=PublishFreeze.InbredSetId " + "WHERE Species.Id=%s AND InbredSet.Id=%s AND PublishFreeze.Id=%s", (species_id, population_id, dataset_id)) return dict(cursor.fetchone()) -def phenotypes_count(conn: mdb.Connection, +def phenotypes_count(conn: Connection, population_id: int, dataset_id: int) -> int: """Count the number of phenotypes in the dataset.""" @@ -54,50 +73,94 @@ def phenotypes_count(conn: mdb.Connection, return int(cursor.fetchone()["total_phenos"]) -def dataset_phenotypes(conn: mdb.Connection, - population_id: int, - dataset_id: int, - offset: int = 0, - limit: Optional[int] = None) -> tuple[dict, ...]: +def phenotype_publication_data(conn, phenotype_id) -> Optional[dict]: + """Retrieve the publication data for a phenotype if it exists.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT DISTINCT pxr.PhenotypeId, pub.* FROM PublishXRef AS pxr " + "INNER JOIN Publication as pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.PhenotypeId=%s", + (phenotype_id,)) + res = cursor.fetchone() + if res is None: + return res + return dict(res) + + +def dataset_phenotypes(# pylint: disable=[too-many-arguments, too-many-positional-arguments] + conn: Connection, + population_id: int, + dataset_id: int, + offset: int = 0, + limit: Optional[int] = None, + xref_ids: tuple[int, ...] = tuple() +) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" - _query = ( - "SELECT pheno.*, pxr.Id, ist.InbredSetCode FROM Phenotype AS pheno " + _narrow_by_ids = ( + f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" + if len(xref_ids) > 0 else "") + _narrow_by_limit = ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + _pub_query = ( + "SELECT pub.* " + "FROM PublishXRef AS pxr " + "INNER JOIN Publication AS pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.InbredSetId=%s") + _narrow_by_ids + _pheno_query = (( + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, pxr.PublicationId, " + "ist.InbredSetCode " + "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " - "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( - f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + + _narrow_by_ids + + _narrow_by_limit) with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (population_id, dataset_id)) - debug_query(cursor, app.logger) - return tuple(dict(row) for row in cursor.fetchall()) + cursor.execute(_pub_query, (population_id,) + xref_ids) + debug_query(cursor, logger) + _pubs = {row["Id"]: dict(row) for row in cursor.fetchall()} + cursor.execute(_pheno_query, (population_id, dataset_id) + xref_ids) + debug_query(cursor, logger) + return tuple({**dict(row), "publication": _pubs[row["PublicationId"]]} + for row in cursor.fetchall()) -def __phenotype_se__(cursor: Cursor, - species_id: int, - population_id: int, - dataset_id: int, - xref_id: str) -> dict: +def __phenotype_se__(cursor: BaseCursor, xref_id, dataids_and_strainids): """Fetch standard-error values (if they exist) for a phenotype.""" - _sequery = ( - "SELECT pxr.Id AS xref_id, pxr.DataId, str.Id AS StrainId, pse.error, nst.count " - "FROM Phenotype AS pheno " - "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " - "INNER JOIN PublishSE AS pse ON pxr.DataId=pse.DataId " - "INNER JOIN NStrain AS nst ON pse.DataId=nst.DataId " - "INNER JOIN Strain AS str ON nst.StrainId=str.Id " - "INNER JOIN StrainXRef AS sxr ON str.Id=sxr.StrainId " - "INNER JOIN PublishFreeze AS pf ON sxr.InbredSetId=pf.InbredSetId " - "INNER JOIN InbredSet AS iset ON pf.InbredSetId=iset.InbredSetId " - "WHERE (str.SpeciesId, pxr.InbredSetId, pf.Id, pxr.Id)=(%s, %s, %s, %s)") - cursor.execute(_sequery, - (species_id, population_id, dataset_id, xref_id)) - return {(row["DataId"], row["StrainId"]): { - "xref_id": row["xref_id"], - "DataId": row["DataId"], - "error": row["error"], - "count": row["count"] - } for row in cursor.fetchall()} + paramstr = ", ".join(["(%s, %s)"] * len(dataids_and_strainids)) + flat = tuple(item for sublist in dataids_and_strainids for item in sublist) + cursor.execute("SELECT * FROM PublishSE WHERE (DataId, StrainId) IN " + f"({paramstr})", + flat) + debug_query(cursor, logger) + _se = { + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "error": row["error"] + } + for row in cursor.fetchall() + } + + cursor.execute("SELECT * FROM NStrain WHERE (DataId, StrainId) IN " + f"({paramstr})", + flat) + debug_query(cursor, logger) + _n = { + (row["DataId"], row["StrainId"]): { + "DataId": row["DataId"], + "StrainId": row["StrainId"], + "count": row["count"] + } + for row in cursor.fetchall() + } + + keys = set(tuple(_se.keys()) + tuple(_n.keys())) + return { + key: {"xref_id": xref_id, **_se.get(key,{}), **_n.get(key,{})} + for key in keys + } def __organise_by_phenotype__(pheno, row): """Organise disparate data rows into phenotype 'objects'.""" @@ -113,10 +176,12 @@ def __organise_by_phenotype__(pheno, row): "Pre_publication_abbreviation": row["Pre_publication_abbreviation"], "Post_publication_abbreviation": row["Post_publication_abbreviation"], "xref_id": row["pxr.Id"], + "DataId": row["DataId"], "data": { **(_pheno["data"] if bool(_pheno) else {}), (row["DataId"], row["StrainId"]): { "DataId": row["DataId"], + "StrainId": row["StrainId"], "mean": row["mean"], "Locus": row["Locus"], "LRS": row["LRS"], @@ -143,7 +208,7 @@ def __merge_pheno_data_and_se__(data, sedata) -> dict: def phenotype_by_id( - conn: mdb.Connection, + conn: Connection, species_id: int, population_id: int, dataset_id: int, @@ -170,20 +235,18 @@ def phenotype_by_id( **_pheno, "data": tuple(__merge_pheno_data_and_se__( _pheno["data"], - __phenotype_se__(cursor, - species_id, - population_id, - dataset_id, - xref_id)).values()) + __phenotype_se__( + cursor, xref_id, tuple(_pheno["data"].keys())) + ).values()) } if bool(_pheno) and len(_pheno.keys()) > 1: - raise Exception( + raise Exception(# pylint: disable=[broad-exception-raised] "We found more than one phenotype with the same identifier!") return None -def phenotypes_data(conn: mdb.Connection, +def phenotypes_data(conn: Connection, population_id: int, dataset_id: int, offset: int = 0, @@ -202,11 +265,64 @@ def phenotypes_data(conn: mdb.Connection, f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute(_query, (population_id, dataset_id)) - debug_query(cursor, app.logger) + debug_query(cursor, logger) return tuple(dict(row) for row in cursor.fetchall()) -def save_new_dataset(cursor: Cursor, +def phenotypes_vector_data(# pylint: disable=[too-many-arguments, too-many-positional-arguments] + conn: Connection, + species_id: int, + population_id: int, + xref_ids: tuple[int, ...] = tuple(), + offset: int = 0, + limit: Optional[int] = None +) -> dict[tuple[int, int, int], dict[str, Union[int,float]]]: + """Retrieve the vector data values for traits in the database.""" + _params: tuple[int, ...] = (species_id, population_id) + _query = ("SELECT " + "Species.Id AS SpeciesId, iset.Id AS InbredSetId, " + "pxr.Id AS xref_id, pdata.*, Strain.Id AS StrainId, " + "Strain.Name AS StrainName " + "FROM " + "Species INNER JOIN InbredSet AS iset " + "ON Species.Id=iset.SpeciesId " + "INNER JOIN PublishXRef AS pxr " + "ON iset.Id=pxr.InbredSetId " + "INNER JOIN PublishData AS pdata " + "ON pxr.DataId=pdata.Id " + "INNER JOIN Strain " + "ON pdata.StrainId=Strain.Id " + "WHERE Species.Id=%s AND iset.Id=%s") + if len(xref_ids) > 0: + _paramstr = ", ".join(["%s"] * len(xref_ids)) + _query = _query + f" AND pxr.Id IN ({_paramstr})" + _params = _params + xref_ids + + def __organise__(acc, row): + _rowid = (species_id, population_id, row["xref_id"]) + _phenodata = { + **acc.get( + _rowid, { + "species_id": species_id, + "population_id": population_id, + "xref_id": row["xref_id"] + }), + row["StrainName"]: row["value"] + } + return { + **acc, + _rowid: _phenodata + } + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + _query + (f" LIMIT {limit} OFFSET {offset}" if bool(limit) else ""), + _params) + debug_query(cursor, logger) + return reduce(__organise__, cursor.fetchall(), {}) + + +def save_new_dataset(cursor: BaseCursor, population_id: int, dataset_name: str, dataset_fullname: str, @@ -229,5 +345,344 @@ def save_new_dataset(cursor: Cursor, "%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, " "%(users)s)", params) - debug_query(cursor, app.logger) + debug_query(cursor, logger) return {**params, "Id": cursor.lastrowid} + + +def __pre_process_phenotype_data__(row): + _desc = row.get("description", "") + _pre_pub_desc = row.get("pre_publication_description", _desc) + _orig_desc = row.get("original_description", _desc) + _post_pub_desc = row.get("post_publication_description", _orig_desc) + _pre_pub_abbr = row.get("pre_publication_abbreviation", row["id"]) + _post_pub_abbr = row.get("post_publication_abbreviation", _pre_pub_abbr) + return { + "pre_publication_description": _pre_pub_desc, + "post_publication_description": _post_pub_desc, + "original_description": _orig_desc, + "units": row["units"], + "pre_publication_abbreviation": _pre_pub_abbr, + "post_publication_abbreviation": _post_pub_abbr + } + + +def create_new_phenotypes(# pylint: disable=[too-many-locals] + conn: Connection, + population_id: int, + publication_id: int, + phenotypes: Iterable[dict] +) -> tuple[dict, ...]: + """Add entirely new phenotypes to the database. WARNING: Not thread-safe.""" + _phenos: tuple[dict, ...] = tuple() + with conn.cursor(cursorclass=DictCursor) as cursor: + def make_next_id(idcol, table): + cursor.execute(f"SELECT MAX({idcol}) AS last_id FROM {table}") + _last_id = int(cursor.fetchone()["last_id"]) + def __next_id__(): + _next_id = _last_id + 1 + while True: + yield _next_id + _next_id = _next_id + 1 + + return __next_id__ + + ### Bottleneck: Everything below makes this function not ### + ### thread-safe because we have to retrieve the last IDs from ### + ### the database and increment those to compute the next IDs. ### + ### This is an unfortunate result from the current schema that ### + ### has a cross-reference table that requires that a phenotype ### + ### be linked to an existing publication, and have data IDs to ### + ### link to that phenotype's data. ### + ### The fact that the IDs are sequential also compounds the ### + ### bottleneck. ### + ### + ### For extra safety, ensure the following tables are locked ### + ### for `WRITE`: ### + ### - PublishXRef ### + ### - Phenotype ### + ### - PublishXRef ### + __next_xref_id = make_next_id("Id", "PublishXRef")() + __next_pheno_id__ = make_next_id("Id", "Phenotype")() + __next_data_id__ = make_next_id("DataId", "PublishXRef")() + + def __build_params_and_prepubabbrevs__(acc, row): + processed = __pre_process_phenotype_data__(row) + return ( + acc[0] + ({ + **processed, + "population_id": population_id, + "publication_id": publication_id, + "phenotype_id": next(__next_pheno_id__), + "xref_id": next(__next_xref_id), + "data_id": next(__next_data_id__) + },), + acc[1] + (processed["pre_publication_abbreviation"],)) + while True: + batch = take(phenotypes, 1000) + if len(batch) == 0: + break + + params, abbrevs = reduce(#type: ignore[var-annotated] + __build_params_and_prepubabbrevs__, + batch, + (tuple(), tuple())) + # Check for uniqueness for all "Pre_publication_description" values + abbrevs_paramsstr = ", ".join(["%s"] * len(abbrevs)) + _query = ("SELECT PublishXRef.PhenotypeId, Phenotype.* " + "FROM PublishXRef " + "INNER JOIN Phenotype " + "ON PublishXRef.PhenotypeId=Phenotype.Id " + "WHERE PublishXRef.InbredSetId=%s " + "AND Phenotype.Pre_publication_abbreviation IN " + f"({abbrevs_paramsstr})") + cursor.execute(_query, + ((population_id,) + abbrevs)) + existing = tuple(row["Pre_publication_abbreviation"] + for row in cursor.fetchall()) + if len(existing) > 0: + # Narrow this exception, perhaps? + raise Exception(# pylint: disable=[broad-exception-raised] + "Found already existing phenotypes with the following " + "'Pre-publication abbreviations':\n\t" + "\n\t".join(f"* {item}" for item in existing)) + + cursor.executemany( + ( + "INSERT INTO " + "Phenotype(" + "Id, " + "Pre_publication_description, " + "Post_publication_description, " + "Original_description, " + "Units, " + "Pre_publication_abbreviation, " + "Post_publication_abbreviation, " + "Authorized_Users" + ")" + "VALUES (" + "%(phenotype_id)s, " + "%(pre_publication_description)s, " + "%(post_publication_description)s, " + "%(original_description)s, " + "%(units)s, " + "%(pre_publication_abbreviation)s, " + "%(post_publication_abbreviation)s, " + "'robwilliams'" + ")"), + params) + _comments = f"Created at {datetime.now().isoformat()}" + cursor.executemany( + ("INSERT INTO PublishXRef(" + "Id, " + "InbredSetId, " + "PhenotypeId, " + "PublicationId, " + "DataId, " + "comments" + ")" + "VALUES(" + "%(xref_id)s, " + "%(population_id)s, " + "%(phenotype_id)s, " + "%(publication_id)s, " + "%(data_id)s, " + f"'{_comments}'" + ")"), + params) + _phenos = _phenos + params + + return _phenos + + +def save_phenotypes_data( + conn: Connection, + table: str, + data: Iterable[dict] +) -> int: + """Save new phenotypes data into the database.""" + _table_details = __PHENO_DATA_TABLES__[table] + with conn.cursor(cursorclass=DictCursor) as cursor: + _count = 0 + while True: + batch = take(data, 100000) + if len(batch) == 0: + logger.warning("Got an empty batch. This needs investigation.") + break + + logger.debug("Saving batch of %s items.", len(batch)) + cursor.executemany( + (f"INSERT INTO {_table_details['table']}" + f"({_table_details['DataIdCol']}, StrainId, {_table_details['valueCol']}) " + "VALUES " + f"(%(data_id)s, %(sample_id)s, %(value)s) "), + tuple(batch)) + debug_query(cursor, logger) + _count = _count + len(batch) + + + logger.debug("Saved a total of %s data rows", _count) + return _count + + +def quick_save_phenotypes_data( + conn: Connection, + table: str, + dataitems: Iterable[dict], + tmpdir: Path +) -> int: + """Save data items to the database, but using """ + _table_details = __PHENO_DATA_TABLES__[table] + with (tempfile.NamedTemporaryFile( + prefix=f"{table}_data", mode="wt", dir=tmpdir) as tmpfile, + conn.cursor(cursorclass=DictCursor) as cursor): + _count = 0 + logger.debug("Write data rows to text file.") + for row in dataitems: + tmpfile.write( + f'{row["data_id"]}\t{row["sample_id"]}\t{row["value"]}\n') + _count = _count + 1 + tmpfile.flush() + + logger.debug("Load text file into database (table: %s)", + _table_details["table"]) + cursor.execute( + f"LOAD DATA LOCAL INFILE '{tmpfile.name}' " + f"INTO TABLE {_table_details['table']} " + "(" + f"{_table_details['DataIdCol']}, " + "StrainId, " + f"{_table_details['valueCol']}" + ")") + debug_query(cursor, logger) + return _count + + +def __sleep_random__(): + """Sleep a random amount of time chosen from 0.05s to 1s in increments of 0.05""" + time.sleep(random.choice(tuple(i / 20.0 for i in range(1, 21)))) + + +def delete_phenotypes_data( + cursor: BaseCursor, + data_ids: tuple[int, ...] +) -> tuple[int, int, int]: + """Delete numeric data for phenotypes with the given data IDs.""" + if len(data_ids) == 0: + return (0, 0, 0) + + # Loop to handle big deletes i.e. ≥ 10000 rows + _dcount, _secount, _ncount = (0, 0, 0)# Count total rows deleted + while True: + _paramstr = ", ".join(["%s"] * len(data_ids)) + cursor.execute( + "DELETE FROM PublishData " + f"WHERE Id IN ({_paramstr}) " + "ORDER BY Id ASC, StrainId ASC "# Make deletions deterministic + "LIMIT 1000", + data_ids) + _dcount_curr = cursor.rowcount + _dcount += _dcount_curr + + cursor.execute( + "DELETE FROM PublishSE " + f"WHERE DataId IN ({_paramstr}) " + "ORDER BY DataId ASC, StrainId ASC "# Make deletions deterministic + "LIMIT 1000", + data_ids) + _secount_curr = cursor.rowcount + _secount += _secount_curr + + cursor.execute( + "DELETE FROM NStrain " + f"WHERE DataId IN ({_paramstr}) " + "ORDER BY DataId ASC, StrainId ASC "# Make deletions deterministic + "LIMIT 1000", + data_ids) + _ncount_curr = cursor.rowcount + _ncount += _ncount_curr + __sleep_random__() + + if all((_dcount_curr == 0, _secount_curr == 0, _ncount_curr == 0)): + # end loop if there are no more rows to delete. + break + + return (_dcount, _secount, _ncount) + + +def __linked_ids__( + cursor: BaseCursor, + population_id: int, + xref_ids: tuple[int, ...] +) -> tuple[tuple[int, int, int], ...]: + """Retrieve `DataId` values from `PublishXRef` table.""" + _paramstr = ", ".join(["%s"] * len(xref_ids)) + cursor.execute("SELECT PhenotypeId, PublicationId, DataId " + "FROM PublishXRef " + f"WHERE InbredSetId=%s AND Id IN ({_paramstr})", + (population_id,) + xref_ids) + return tuple( + (int(row["PhenotypeId"]), int(row["PublicationId"]), int(row["DataId"])) + for row in cursor.fetchall()) + + +def delete_phenotypes( + conn_or_cursor: Union[Connection, Cursor], + population_id: int, + xref_ids: tuple[int, ...] +) -> tuple[int, int, int, int]: + """Delete phenotypes and all their data.""" + def __delete_phenos__(cursor: BaseCursor, pheno_ids: tuple[int, ...]) -> int: + """Delete data from the `Phenotype` table.""" + _paramstr = ", ".join(["%s"] * len(pheno_ids)) + + _pcount = 0 + while True: + cursor.execute( + "DELETE FROM Phenotype " + f"WHERE Id IN ({_paramstr}) " + "ORDER BY Id " + "LIMIT 1000", + pheno_ids) + _pcount_curr = cursor.rowcount + _pcount += _pcount_curr + __sleep_random__() + if _pcount_curr == 0: + break + + return cursor.rowcount + + def __delete_xrefs__(cursor: BaseCursor) -> int: + _paramstr = ", ".join(["%s"] * len(xref_ids)) + + _xcount = 0 + while True: + cursor.execute( + "DELETE FROM PublishXRef " + f"WHERE InbredSetId=%s AND Id IN ({_paramstr}) " + "ORDER BY Id " + "LIMIT 10000", + (population_id,) + xref_ids) + _xcount_curr = cursor.rowcount + _xcount += _xcount_curr + __sleep_random__() + if _xcount_curr == 0: + break + + return _xcount + + def __with_cursor__(cursor): + _phenoids, _pubids, _dataids = reduce( + lambda acc, curr: (acc[0] + (curr[0],), + acc[1] + (curr[1],), + acc[2] + (curr[2],)), + __linked_ids__(cursor, population_id, xref_ids), + (tuple(), tuple(), tuple())) + __delete_phenos__(cursor, _phenoids) + return (__delete_xrefs__(cursor),) + delete_phenotypes_data( + cursor, _dataids) + + if isinstance(conn_or_cursor, BaseCursor): + return __with_cursor__(conn_or_cursor) + + with conn_or_cursor.cursor(cursorclass=DictCursor) as cursor: + return __with_cursor__(cursor) diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index ec4c840..85d6357 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -1,40 +1,57 @@ -"""Views handling ('classical') phenotypes.""" +"""Views handling ('classical') phenotypes."""# pylint: disable=[too-many-lines] +import io +import csv import sys import uuid import json -import datetime +import logging +from typing import Any from pathlib import Path from zipfile import ZipFile -from functools import wraps -from logging import INFO, ERROR, DEBUG, FATAL, CRITICAL, WARNING +from functools import reduce +from urllib.parse import urljoin, urlparse, ParseResult, urlunparse, urlencode + +import datetime from redis import Redis +from pymonad.either import Left from requests.models import Response from MySQLdb.cursors import DictCursor + +from gn_libs import sqlite3 +from gn_libs import jobs as gnlibs_jobs +from gn_libs.jobs.jobs import JobNotFound from gn_libs.mysqldb import database_connection + +from werkzeug.datastructures import Headers from flask import (flash, request, - url_for, jsonify, redirect, Blueprint, - current_app as app) + current_app as app, + Response as FlaskResponse) -# from r_qtl import r_qtl2 as rqtl2 from r_qtl import r_qtl2_qc as rqc from r_qtl import exceptions as rqe from uploader import jobs -from uploader.files import save_file#, fullpath +from uploader import session +from uploader.files import save_file +from uploader.configutils import uploads_dir +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_post +from uploader.oauth2.tokens import request_token from uploader.authorisation import require_login +from uploader.oauth2 import client as oauth2client +from uploader.route_utils import build_next_argument +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler -from uploader.request_checks import with_species, with_population -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import (populations_by_species, - population_by_species_and_id) +from uploader.publications.models import fetch_publication_by_id +from uploader.request_checks import with_species, with_dataset, with_population from uploader.input_validation import (encode_errors, decode_errors, is_valid_representative_name) @@ -44,11 +61,16 @@ from .models import (dataset_by_id, phenotypes_count, save_new_dataset, dataset_phenotypes, - datasets_by_population) + datasets_by_population, + phenotype_publication_data) +logger = logging.getLogger(__name__) phenotypesbp = Blueprint("phenotypes", __name__) render_template = make_template_renderer("phenotypes") +_FAMILIES_WITH_SE_AND_N_ = ( + "Reference Populations (replicate average, SE, N)",) + @phenotypesbp.route("/phenotypes", methods=["GET"]) @require_login def index(): @@ -56,10 +78,16 @@ def index(): with database_connection(app.config["SQL_URI"]) as conn: if not bool(request.args.get("species_id")): return render_template("phenotypes/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="phenotypes") - species = species_by_id(conn, request.args.get("species_id")) + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.phenotypes.select_population")) + + species = species_by_id(conn, species_id) if not bool(species): flash("No such species!", "alert-danger") return redirect(url_for("species.populations.phenotypes.index")) @@ -73,27 +101,14 @@ def index(): @with_species(redirect_uri="species.populations.phenotypes.index") def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population for your phenotypes.""" - with database_connection(app.config["SQL_URI"]) as conn: - if not bool(request.args.get("population_id")): - return render_template("phenotypes/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, species["SpeciesId"]), - order_key="FamilyOrder"), - activelink="phenotypes") - - population = population_by_species_and_id( - conn, species["SpeciesId"], int(request.args["population_id"])) - if not bool(population): - flash("No such population found!", "alert-danger") - return redirect(url_for( - "species.populations.phenotypes.select_population", - species_id=species["SpeciesId"])) - - return redirect(url_for("species.populations.phenotypes.list_datasets", - species_id=species["SpeciesId"], - population_id=population["Id"])) + return generic_select_population( + species, + "phenotypes/select-population.html", + request.args.get("population_id") or "", + "species.populations.phenotypes.select_population", + "species.populations.phenotypes.list_datasets", + "phenotypes", + "No such population found!") @@ -121,45 +136,6 @@ def list_datasets(species: dict, population: dict, **kwargs):# pylint: disable=[ activelink="list-datasets") -def with_dataset( - species_redirect_uri: str, - population_redirect_uri: str, - redirect_uri: str -): - """Ensure the dataset actually exists.""" - def __decorator__(func): - @wraps(func) - @with_population(species_redirect_uri, population_redirect_uri) - def __with_dataset__(**kwargs): - try: - _spcid = int(kwargs["species_id"]) - _popid = int(kwargs["population_id"]) - _dsetid = int(kwargs.get("dataset_id")) - select_dataset_uri = redirect(url_for( - redirect_uri, species_id=_spcid, population_id=_popid)) - if not bool(_dsetid): - flash("You need to select a valid 'dataset_id' value.", - "alert-danger") - return select_dataset_uri - with database_connection(app.config["SQL_URI"]) as conn: - dataset = dataset_by_id(conn, _spcid, _popid, _dsetid) - if not bool(dataset): - flash("You must select a valid dataset.", - "alert-danger") - return select_dataset_uri - except ValueError as _verr: - app.logger.debug( - "Exception converting 'dataset_id' to integer: %s", - kwargs.get("dataset_id"), - exc_info=True) - flash("Expected 'dataset_id' value to be an integer." - "alert-danger") - return select_dataset_uri - return func(dataset=dataset, **kwargs) - return __with_dataset__ - return __decorator__ - - @phenotypesbp.route( "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" "/<int:dataset_id>/view", @@ -168,7 +144,8 @@ def with_dataset( @with_dataset( species_redirect_uri="species.populations.phenotypes.index", population_redirect_uri="species.populations.phenotypes.select_population", - redirect_uri="species.populations.phenotypes.list_datasets") + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) def view_dataset(# pylint: disable=[unused-argument] species: dict, population: dict, dataset: dict, **kwargs): """View a specific dataset""" @@ -191,12 +168,10 @@ def view_dataset(# pylint: disable=[unused-argument] phenotype_count=phenotypes_count( conn, population["Id"], dataset["Id"]), phenotypes=enumerate_sequence( - dataset_phenotypes(conn, - population["Id"], - dataset["Id"], - offset=start_at, - limit=count), - start=start_at+1), + dataset_phenotypes( + conn, + population["Id"], + dataset["Id"])), start_from=start_at, count=count, activelink="view-dataset") @@ -210,7 +185,8 @@ def view_dataset(# pylint: disable=[unused-argument] @with_dataset( species_redirect_uri="species.populations.phenotypes.index", population_redirect_uri="species.populations.phenotypes.select_population", - redirect_uri="species.populations.phenotypes.list_datasets") + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) def view_phenotype(# pylint: disable=[unused-argument] species: dict, population: dict, @@ -220,22 +196,27 @@ def view_phenotype(# pylint: disable=[unused-argument] ): """View an individual phenotype from the dataset.""" def __render__(privileges): + phenotype = phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id) return render_template( "phenotypes/view-phenotype.html", species=species, population=population, dataset=dataset, - phenotype=phenotype_by_id(conn, - species["SpeciesId"], - population["Id"], - dataset["Id"], - xref_id), - privileges=(privileges - ### For demo! Do not commit this part - + ("group:resource:edit-resource", - "group:resource:delete-resource",) - ### END: For demo! Do not commit this part - ), + xref_id=xref_id, + phenotype=phenotype, + has_se=any(bool(item.get("error")) for item in phenotype["data"]), + publication=(phenotype_publication_data(conn, phenotype["Id"]) or {}), + privileges=privileges, + next=build_next_argument( + uri="species.populations.phenotypes.view_phenotype", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id), activelink="view-phenotype") def __fail__(error): @@ -299,6 +280,11 @@ def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable= dataset_shortname = ( form["dataset-shortname"] or form["dataset-name"]).strip() _pheno_dataset = save_new_dataset( + # It's not necessary to update the authorisation server to register + # new phenotype resource here, since each phenotype trait can, in + # theory, have its own access control allowing/disallowing access to + # it. In practice, however, we tend to gather multiple traits into a + # single resource for access control. cursor, population["Id"], form["dataset-name"].strip(), @@ -316,7 +302,7 @@ def process_phenotypes_rqtl2_bundle(error_uri): try: ## Handle huge files here... phenobundle = save_file(request.files["phenotypes-bundle"], - Path(app.config["UPLOAD_FOLDER"])) + uploads_dir(app)) rqc.validate_bundle(phenobundle) return phenobundle except AssertionError as _aerr: @@ -339,22 +325,29 @@ def process_phenotypes_individual_files(error_uri): "comment.char": form["file-comment-character"], "na.strings": form["file-na"].split(" "), } - bundlepath = Path(app.config["UPLOAD_FOLDER"], + bundlepath = Path(uploads_dir(app), f"{str(uuid.uuid4()).replace('-', '')}.zip") with ZipFile(bundlepath,mode="w") as zfile: - for rqtlkey, formkey in (("phenocovar", "phenotype-descriptions"), - ("pheno", "phenotype-data"), - ("phenose", "phenotype-se"), - ("phenonum", "phenotype-n")): + for rqtlkey, formkey, _type in ( + ("phenocovar", "phenotype-descriptions", "mandatory"), + ("pheno", "phenotype-data", "mandatory"), + ("phenose", "phenotype-se", "optional"), + ("phenonum", "phenotype-n", "optional")): + if _type == "optional" and not bool(form.get(formkey)): + continue # skip if an optional key does not exist. + + cdata[f"{rqtlkey}_transposed"] = ( + (form.get(f"{formkey}-transposed") or "off") == "on") + if form.get("resumable-upload", False): # Chunked upload of large files was used filedata = json.loads(form[formkey]) zfile.write( - Path(app.config["UPLOAD_FOLDER"], filedata["uploaded-file"]), + Path(uploads_dir(app), filedata["uploaded-file"]), arcname=filedata["original-name"]) cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filedata["original-name"]] else: - # TODO: Check this path: fix any bugs. + # T0DO: Check this path: fix any bugs. _sentfile = request.files[formkey] if not bool(_sentfile): flash(f"Expected file ('{formkey}') was not provided.", @@ -362,12 +355,13 @@ def process_phenotypes_individual_files(error_uri): return error_uri filepath = save_file( - _sentfile, Path(app.config["UPLOAD_FOLDER"]), hashed=False) + _sentfile, uploads_dir(app), hashed=False) zfile.write( - Path(app.config["UPLOAD_FOLDER"], filepath), + Path(uploads_dir(app), filepath), arcname=filepath.name) cdata[rqtlkey] = cdata.get(rqtlkey, []) + [filepath.name] + zfile.writestr("control_data.json", data=json.dumps(cdata, indent=2)) return bundlepath @@ -381,7 +375,8 @@ def process_phenotypes_individual_files(error_uri): @with_dataset( species_redirect_uri="species.populations.phenotypes.index", population_redirect_uri="species.populations.phenotypes.select_population", - redirect_uri="species.populations.phenotypes.list_datasets") + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals] """Add one or more phenotypes to the dataset.""" use_bundle = request.args.get("use_bundle", "").lower() == "true" @@ -392,15 +387,13 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p dataset_id=dataset["Id"])) _redisuri = app.config["REDIS_URL"] _sqluri = app.config["SQL_URI"] - with (Redis.from_url(_redisuri, decode_responses=True) as rconn, - # database_connection(_sqluri) as conn, - # conn.cursor(cursorclass=DictCursor) as cursor - ): + with Redis.from_url(_redisuri, decode_responses=True) as rconn: if request.method == "GET": today = datetime.date.today() return render_template( ("phenotypes/add-phenotypes-with-rqtl2-bundle.html" - if use_bundle else "phenotypes/add-phenotypes-raw-files.html"), + if use_bundle + else "phenotypes/add-phenotypes-raw-files.html"), species=species, population=population, dataset=dataset, @@ -411,8 +404,7 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p "December"), current_month=today.strftime("%B"), current_year=int(today.strftime("%Y")), - families_with_se_and_n=( - "Reference Populations (replicate average, SE, N)",), + families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_, use_bundle=use_bundle, activelink="add-phenotypes") @@ -431,26 +423,22 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p [sys.executable, "-m", "scripts.rqtl2.phenotypes_qc", _sqluri, _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]), str(population["Id"]), - # str(dataset["Id"]), str(phenobundle), "--loglevel", - { - INFO: "INFO", - ERROR: "ERROR", - DEBUG: "DEBUG", - FATAL: "FATAL", - CRITICAL: "CRITICAL", - WARNING: "WARNING" - }[app.logger.getEffectiveLevel()], + logging.getLevelName( + app.logger.getEffectiveLevel() + ).lower(), "--redisexpiry", str(_ttl_seconds)], "phenotype_qc", _ttl_seconds, {"job-metadata": json.dumps({ "speciesid": species["SpeciesId"], "populationid": population["Id"], "datasetid": dataset["Id"], - "bundle": str(phenobundle.absolute())})}), + "bundle": str(phenobundle.absolute()), + **({"publicationid": request.form["publication-id"]} + if request.form.get("publication-id") else {})})}), _redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{uploads_dir(app)}/job_errors") app.logger.debug("JOB DETAILS: %s", _job) jobstatusuri = url_for("species.populations.phenotypes.job_status", @@ -477,7 +465,8 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p @with_dataset( species_redirect_uri="species.populations.phenotypes.index", population_redirect_uri="species.populations.phenotypes.select_population", - redirect_uri="species.populations.phenotypes.list_datasets") + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) def job_status( species: dict, population: dict, @@ -491,6 +480,7 @@ def job_status( job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) except jobs.JobNotFound as _jnf: job = None + return render_template("phenotypes/job-status.html", species=species, population=population, @@ -502,3 +492,833 @@ def job_status( metadata=jobs.job_files_metadata( rconn, jobs.jobsnamespace(), job['jobid']), activelink="add-phenotypes") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>/download-errors", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def download_errors( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs):# pylint: disable=[unused-argument] + """Download the list of errors as a CSV file.""" + with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + _prefix_ = jobs.jobsnamespace() + _jobid_ = job['jobid'] + def __generate_chunks__(): + _errors_ = ( + json.loads(error) + for key in rconn.keys( + f"{_prefix_}:{str(_jobid_)}:*:errors:*") + for error in rconn.lrange(key, 0, -1)) + _chunk_no_ = 0 + _all_errors_printed_ = False + while not _all_errors_printed_: + _chunk_ = [] + try: + for _ in range(0, 1000): + _chunk_.append(next(_errors_)) + except StopIteration: + _all_errors_printed_ = True + if len(_chunk_) <= 0: + raise + + _out_ = io.StringIO() + _writer_ = csv.DictWriter(_out_, fieldnames=tuple(_chunk_[0].keys())) + if _chunk_no_ == 0: + _writer_.writeheader() + _writer_.writerows(_chunk_) + _chunk_no_ += 1 + yield _out_.getvalue() + if _all_errors_printed_: + return + + headers = Headers() + headers.set("Content-Disposition", + "attachment", + filename=f"{job['job-type']}_{job['jobid']}.csv") + return FlaskResponse( + __generate_chunks__(), mimetype="text/csv", headers=headers) + except jobs.JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/job/<uuid:job_id>/review", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def review_job_data( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Review data one more time before entering it into the database.""" + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + database_connection(app.config["SQL_URI"]) as conn): + try: + job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) + except jobs.JobNotFound as _jnf: + job = None + + def __metadata_by_type__(by_type, item): + filetype = item[1]["filetype"] + return { + **by_type, + filetype: (by_type.get(filetype, tuple()) + + ({"filename": item[0], **item[1]},)) + } + metadata: dict[str, Any] = reduce( + __metadata_by_type__, + (jobs.job_files_metadata( + rconn, jobs.jobsnamespace(), job['jobid']) + if job else {}).items(), + {}) + + def __desc__(filetype): + match filetype: + case "phenocovar": + desc = "phenotypes" + case "pheno": + desc = "phenotypes data" + case "phenose": + desc = "phenotypes standard-errors" + case "phenonum": + desc = "phenotypes samples" + case _: + desc = f"unknown file type '{filetype}'." + + return desc + + def __summarise__(filetype, files): + return { + "filetype": filetype, + "number-of-files": len(files), + "total-data-rows": sum( + int(afile["linecount"]) - 1 for afile in files), + "description": __desc__(filetype) + } + + summary = { + filetype: __summarise__(filetype, meta) + for filetype,meta in metadata.items() + } + _job_metadata = json.loads(job["job-metadata"]) + return render_template("phenotypes/review-job-data.html", + species=species, + population=population, + dataset=dataset, + job_id=job_id, + job=job, + summary=summary, + publication=( + fetch_publication_by_id( + conn, int(_job_metadata["publicationid"])) + if _job_metadata.get("publicationid") + else None), + user=session.user_details(), + timestamp=datetime.datetime.now().isoformat(), + activelink="add-phenotypes") + + +def load_phenotypes_success_handler(job): + """Handle loading new phenotypes into the database successfully.""" + return redirect(url_for( + "species.populations.phenotypes.load_data_success", + species_id=job["metadata"]["species_id"], + population_id=job["metadata"]["population_id"], + dataset_id=job["metadata"]["dataset_id"], + job_id=job["job_id"])) + + +def proceed_to_job_status(job): + """A generic 'job success' handler for asynchronous phenotype jobs.""" + app.logger.debug("The new job: %s", job) + return redirect(url_for("background-jobs.job_status", job_id=job["job_id"])) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/load-data-to-database", + methods=["POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def load_data_to_database( + species: dict, + population: dict, + dataset: dict, + **kwargs +):# pylint: disable=[unused-argument] + """Load the data from the given QC job into the database.""" + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with (Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn, + sqlite3.connection(_jobs_db) as conn): + # T0DO: Maybe break the connection between the jobs here, pass: + # - the bundle name (rebuild the full path here.) + # - publication details, where separate + # - details about the files: e.g. total lines, etc + qc_job = jobs.job(rconn, jobs.jobsnamespace(), request.form["data-qc-job-id"]) + _meta = json.loads(qc_job["job-metadata"]) + _load_job_id = uuid.uuid4() + _loglevel = logging.getLevelName(app.logger.getEffectiveLevel()).lower() + command = [ + sys.executable, + "-u", + "-m", + "scripts.load_phenotypes_to_db", + app.config["SQL_URI"], + _jobs_db, + str(_load_job_id), + "--log-level", + _loglevel + ] + + def __handle_error__(resp): + return render_template("http-error.html", *resp.json()) + + + return request_token( + token_uri=urljoin(oauth2client.authserver_uri(), "auth/token"), + user_id=session.user_details()["user_id"] + ).then( + lambda token: gnlibs_jobs.initialise_job( + conn, + _load_job_id, + command, + "load-new-phenotypes-data", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "bundle_file": _meta["bundle"], + "publication_id": _meta["publicationid"], + "authserver": oauth2client.authserver_uri(), + "token": token["access_token"], + "dataname": request.form["data_name"].strip(), + "success_handler": ( + "uploader.phenotypes.views" + ".load_phenotypes_success_handler"), + **{ + key: request.form[key] + for key in ("data_description",) + if key in request.form.keys() + } + }, + external_id=session.logged_in_user_id()) + ).then( + lambda job: gnlibs_jobs.launch_job( + job, + _jobs_db, + Path(f"{uploads_dir(app)}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + ).either(__handle_error__, proceed_to_job_status) + + +def update_phenotype_metadata(conn, metadata: dict): + """Update a phenotype's basic metadata values.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Phenotype WHERE Id=%(phenotype-id)s", + metadata) + res = { + **{ + _key: _val for _key,_val in { + key.lower().replace("_", "-"): value + for key, value in (cursor.fetchone() or {}).items() + }.items() + if _key in metadata.keys() + }, + "phenotype-id": metadata.get("phenotype-id") + } + if res == metadata: + return False + + cursor.execute( + "UPDATE Phenotype SET " + "Pre_publication_description=%(pre-publication-description)s, " + "Post_publication_description=%(post-publication-description)s, " + "Original_description=%(original-description)s, " + "Units=%(units)s, " + "Pre_publication_abbreviation=%(pre-publication-abbreviation)s, " + "Post_publication_abbreviation=%(post-publication-abbreviation)s " + "WHERE Id=%(phenotype-id)s", + metadata) + return cursor.rowcount + + +def update_phenotype_values(conn, values): + """Update a phenotype's data values.""" + with conn.cursor() as cursor: + cursor.executemany( + "UPDATE PublishData SET value=%(new)s " + "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in values if item["new"] is not None)) + cursor.executemany( + "DELETE FROM PublishData " + "WHERE Id=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in values if item["new"] is None)) + return len(values) + return 0 + + +def update_phenotype_se(conn, serrs): + """Update a phenotype's standard-error values.""" + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO PublishSE(DataId, StrainId, error) " + "VALUES(%(data_id)s, %(strain_id)s, %(new)s) " + "ON DUPLICATE KEY UPDATE error=VALUES(error)", + tuple(item for item in serrs if item["new"] is not None)) + cursor.executemany( + "DELETE FROM PublishSE " + "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in serrs if item["new"] is None)) + return len(serrs) + return 0 + + +def update_phenotype_n(conn, counts): + """Update a phenotype's strain counts.""" + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO NStrain(DataId, StrainId, count) " + "VALUES(%(data_id)s, %(strain_id)s, %(new)s) " + "ON DUPLICATE KEY UPDATE count=VALUES(count)", + tuple(item for item in counts if item["new"] is not None)) + cursor.executemany( + "DELETE FROM NStrain " + "WHERE DataId=%(data_id)s AND StrainId=%(strain_id)s", + tuple(item for item in counts if item["new"] is None)) + return len(counts) + + return 0 + + +def update_phenotype_data(conn, data: dict): + """Update the numeric data for a phenotype.""" + def __organise_by_dataid_and_strainid__(acc, current): + _key, dataid, strainid = current[0].split("::") + _keysrc, _keytype = _key.split("-") + newkey = f"{dataid}::{strainid}" + newitem = acc.get(newkey, {}) + newitem[_keysrc] = newitem.get(_keysrc, {}) + newitem[_keysrc][_keytype] = current[1] + return {**acc, newkey: newitem} + + def __separate_items__(acc, row): + key, val = row + return ({ + **acc[0], + key: { + **val["value"], + "changed?": (not val["value"]["new"] == val["value"]["original"]) + } + }, { + **acc[1], + key: { + **val["se"], + "changed?": (not val["se"]["new"] == val["se"]["original"]) + } + },{ + **acc[2], + key: { + **val["n"], + "changed?": (not val["n"]["new"] == val["n"]["original"]) + } + }) + + values, serrs, counts = tuple(# type: ignore[var-annotated] + tuple({ + "data_id": row[0].split("::")[0], + "strain_id": row[0].split("::")[1], + "new": row[1]["new"] + } for row in item) + for item in ( + filter(lambda val: val[1]["changed?"], item.items())# type: ignore[arg-type] + for item in reduce(# type: ignore[var-annotated] + __separate_items__, + reduce(__organise_by_dataid_and_strainid__, + data.items(), + {}).items(), + ({}, {}, {})))) + + return (update_phenotype_values(conn, values), + update_phenotype_se(conn, serrs), + update_phenotype_n(conn, counts)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/phenotype/<int:xref_id>/edit", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def edit_phenotype_data(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + xref_id: int, + **kwargs +): + """Edit the data for a particular phenotype.""" + def __render__(**kwargs): + processed_kwargs = { + **kwargs, + "privileges": kwargs.get("privileges", tuple()) + } + return render_template( + "phenotypes/edit-phenotype.html", + species=species, + population=population, + dataset=dataset, + xref_id=xref_id, + families_with_se_and_n=_FAMILIES_WITH_SE_AND_N_, + **processed_kwargs, + activelink="edit-phenotype") + + with database_connection(app.config["SQL_URI"]) as conn: + if request.method == "GET": + def __fetch_phenotype__(privileges): + phenotype = phenotype_by_id(conn, + species["SpeciesId"], + population["Id"], + dataset["Id"], + xref_id) + if phenotype is None: + msg = ("Could not find the phenotype with cross-reference ID" + f" '{xref_id}' from dataset '{dataset['FullName']}' " + f" from the '{population['FullName']}' population of " + f" species '{species['FullName']}'.") + return Left({"privileges": privileges, "phenotype-error": msg}) + return {"privileges": privileges, "phenotype": phenotype} + + def __fetch_publication_data__(**kwargs): + pheno = kwargs["phenotype"] + return { + **kwargs, + "publication_data": phenotype_publication_data( + conn, pheno["Id"]) + } + + def __fail__(failure_object): + # process the object + return __render__(failure_object=failure_object) + + return oauth2_post( + "/auth/resource/phenotypes/individual/linked-resource", + json={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "xref_id": xref_id + } + ).then( + lambda resource: tuple( + privilege["privilege_id"] for role in resource["roles"] + for privilege in role["privileges"]) + ).then( + __fetch_phenotype__ + ).then( + lambda args: __fetch_publication_data__(**args) + ).either(__fail__, lambda args: __render__(**args)) + + ## POST + _change = False + match request.form.get("submit", "invalid-action"): + case "update basic metadata": + _change = update_phenotype_metadata(conn, { + key: value.strip() if bool(value.strip()) else None + for key, value in request.form.items() + if key not in ("submit",) + }) + msg = "Basic metadata was updated successfully." + case "update data": + _update = update_phenotype_data(conn, { + key: value.strip() if bool(value.strip()) else None + for key, value in request.form.items() + if key not in ("submit",) + }) + msg = (f"{_update[0]} value rows, {_update[1]} standard-error " + f"rows and {_update[2]} 'N' rows were updated.") + _change = any(item != 0 for item in _update) + case "update publication": + flash("NOT IMPLEMENTED: Would update publication data.", "alert-success") + case _: + flash("Invalid phenotype editing action.", "alert-danger") + + if _change: + flash(msg, "alert-success") + return redirect(url_for( + "species.populations.phenotypes.view_phenotype", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id)) + + flash("No change was made by the user.", "alert-info") + return redirect(url_for( + "species.populations.phenotypes.edit_phenotype_data", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"], + xref_id=xref_id)) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/load-data-success/<uuid:job_id>", + methods=["GET"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def load_data_success( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] + """Display success page if loading data to database was successful.""" + with (database_connection(app.config["SQL_URI"]) as conn, + sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) + as jobsconn): + try: + gn2_uri = urlparse(app.config["GN2_SERVER_URL"]) + job = gnlibs_jobs.job(jobsconn, job_id, fulldetails=True) + app.logger.debug("THE JOB: %s", job) + _xref_ids = tuple( + str(item) for item + in json.loads(job["metadata"].get("xref_ids", "[]"))) + _publication = fetch_publication_by_id( + conn, int(job["metadata"].get("publication_id", "0"))) + _search_terms = (item for item in + (str(_publication["PubMed_ID"] or ""), + _publication["Authors"], + (_publication["Title"] or "")) + if item != "") + return render_template( + "phenotypes/load-phenotypes-success.html", + species=species, + population=population, + dataset=dataset, + job=job, + search_page_uri=urlunparse(ParseResult( + scheme=gn2_uri.scheme, + netloc=gn2_uri.netloc, + path="/search", + params="", + query=urlencode({ + "species": species["Name"], + "group": population["Name"], + "type": "Phenotypes", + "dataset": dataset["Name"], + "search_terms_or": ( + # Very long URLs will cause + # errors. + " ".join(_xref_ids) + if len(_xref_ids) <= 100 + else ""), + "search_terms_and": " ".join( + _search_terms).strip(), + "accession_id": "None", + "FormID": "searchResult" + }), + fragment=""))) + except JobNotFound as _jnf: + return render_template("jobs/job-not-found.html", job_id=job_id) + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/recompute-means", + methods=["POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def recompute_means(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + **kwargs +): + """Compute/Recompute the means for phenotypes in a particular population.""" + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + _job_id = uuid.uuid4() + _xref_ids = tuple(int(item.split("_")[-1]) + for item in request.form.getlist("selected-phenotypes")) + + _loglevel = logging.getLevelName(app.logger.getEffectiveLevel()).lower() + command = [ + sys.executable, + "-u", + "-m", + "scripts.compute_phenotype_means", + app.config["SQL_URI"], + _jobs_db, + str(population["Id"]), + "--log-level", + _loglevel] + ( + ["--cross-ref-ids", ",".join(str(_id) for _id in _xref_ids)] + if len(_xref_ids) > 0 else + []) + logger.debug("%s.recompute_means: command (%s)", __name__, command) + + with sqlite3.connection(_jobs_db) as conn: + _job = gnlibs_jobs.launch_job( + gnlibs_jobs.initialise_job( + conn, + _job_id, + command, + "(re)compute-phenotype-means", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "success_handler": ( + "uploader.phenotypes.views." + "recompute_phenotype_means_success_handler") + }, + external_id=session.logged_in_user_id()), + _jobs_db, + Path(f"{uploads_dir(app)}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + return redirect(url_for("background-jobs.job_status", + job_id=_job["job_id"])) + + +def return_to_dataset_view_handler(job, msg: str): + """Handler for background jobs: Returns to `View Dataset` page.""" + flash(msg, "alert alert-success") + return redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=job["metadata"]["species_id"], + population_id=job["metadata"]["population_id"], + dataset_id=job["metadata"]["dataset_id"], + job_id=job["job_id"])) + +def recompute_phenotype_means_success_handler(job): + """Handle loading new phenotypes into the database successfully.""" + return return_to_dataset_view_handler(job, "Means computed successfully!") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/rerun-qtlreaper", + methods=["POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def rerun_qtlreaper(# pylint: disable=[unused-argument] + species: dict, + population: dict, + dataset: dict, + **kwargs +): + """(Re)run QTLReaper for phenotypes in a particular population.""" + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + _job_id = uuid.uuid4() + _loglevel = logging.getLevelName(app.logger.getEffectiveLevel()).lower() + + _workingdir = Path(app.config["SCRATCH_DIRECTORY"]).joinpath("qtlreaper") + _workingdir.mkdir(exist_ok=True) + command = [ + sys.executable, + "-u", + "-m", + "scripts.run_qtlreaper", + "--log-level", _loglevel, + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["Id"]), + str(Path(app.config["GENOTYPE_FILES_DIRECTORY"]).joinpath( + "genotype")), + str(_workingdir) + ] + [ + str(_xref_id) for _xref_id in ( + int(item.split("_")[-1]) + for item in request.form.getlist("selected-phenotypes")) + ] + logger.debug("(Re)run QTLReaper: %s", command) + with sqlite3.connection(_jobs_db) as conn: + _job_id = uuid.uuid4() + _job = gnlibs_jobs.launch_job( + gnlibs_jobs.initialise_job( + conn, + _job_id, + command, + "(re)run-qtlreaper", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "success_handler": ( + "uploader.phenotypes.views." + "rerun_qtlreaper_success_handler") + }, + external_id=session.logged_in_user_id()), + _jobs_db, + Path(f"{uploads_dir(app)}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + return redirect(url_for("background-jobs.job_status", + job_id=_job["job_id"])) + return redirect(url_for( + "background-jobs.job_status", job_id=_job["job_id"])) + + +def rerun_qtlreaper_success_handler(job): + """Handle success (re)running QTLReaper script.""" + return return_to_dataset_view_handler(job, "QTLReaper ran successfully!") + + +def delete_phenotypes_success_handler(job): + """Handle success running the 'delete-phenotypes' script.""" + return return_to_dataset_view_handler( + job, "Phenotypes deleted successfully.") + + +@phenotypesbp.route( + "<int:species_id>/populations/<int:population_id>/phenotypes/datasets" + "/<int:dataset_id>/delete", + methods=["GET", "POST"]) +@require_login +@with_dataset( + species_redirect_uri="species.populations.phenotypes.index", + population_redirect_uri="species.populations.phenotypes.select_population", + redirect_uri="species.populations.phenotypes.list_datasets", + dataset_by_id=dataset_by_id) +def delete_phenotypes(# pylint: disable=[unused-argument, too-many-locals] + species: dict, + population: dict, + dataset: dict, + **kwargs +): + """Delete the specified phenotype data.""" + _dataset_page = redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + + def __handle_error__(resp): + flash( + "Error retrieving authorisation token. Phenotype deletion " + "failed. Please try again later.", + "alert alert-danger") + return _dataset_page + + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with (database_connection(app.config["SQL_URI"]) as conn, + sqlite3.connection(_jobs_db) as jobsconn): + form = request.form + xref_ids = tuple(int(item) for item in set(form.getlist("xref_ids"))) + + match form.get("action"): + case "cancel": + return redirect(url_for( + "species.populations.phenotypes.view_dataset", + species_id=species["SpeciesId"], + population_id=population["Id"], + dataset_id=dataset["Id"])) + case "delete": + _loglevel = logging.getLevelName( + app.logger.getEffectiveLevel()).lower() + if form.get("confirm_delete_all_phenotypes", "") == "on": + _cmd = ["--delete-all"] + else: + # setup phenotypes xref_ids file + _xref_ids_file = Path( + app.config["SCRATCH_DIRECTORY"], + f"delete-phenotypes-{uuid.uuid4()}.txt") + with _xref_ids_file.open(mode="w", encoding="utf8") as ptr: + ptr.write("\n".join(str(_id) for _id in xref_ids)) + + _cmd = ["--xref_ids_file", str(_xref_ids_file)] + + _job_id = uuid.uuid4() + return request_token( + token_uri=urljoin( + oauth2client.authserver_uri(), "auth/token"), + user_id=session.user_details()["user_id"] + ).then( + lambda token: gnlibs_jobs.initialise_job( + jobsconn, + _job_id, + [ + sys.executable, + "-u", + "-m", + "scripts.phenotypes.delete_phenotypes", + "--log-level", _loglevel, + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["Id"]), + str(dataset["Id"]), + app.config["AUTH_SERVER_URL"], + token["access_token"]] + _cmd, + "delete-phenotypes", + extra_meta={ + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "dataset_id": dataset["Id"], + "success_handler": ( + "uploader.phenotypes.views." + "delete_phenotypes_success_handler") + }, + external_id=session.logged_in_user_id()) + ).then( + lambda _job: gnlibs_jobs.launch_job( + _job, + _jobs_db, + Path(f"{uploads_dir(app)}/job_errors"), + worker_manager="gn_libs.jobs.launcher", + loglevel=_loglevel) + ).either(__handle_error__, proceed_to_job_status) + case _: + _phenos: tuple[dict, ...] = tuple() + if len(xref_ids) > 0: + _phenos = dataset_phenotypes( + conn, population["Id"], dataset["Id"], xref_ids=xref_ids) + + return render_template( + "phenotypes/confirm-delete-phenotypes.html", + species=species, + population=population, + dataset=dataset, + phenotypes=_phenos) diff --git a/uploader/platforms/models.py b/uploader/platforms/models.py index a859371..0dd9368 100644 --- a/uploader/platforms/models.py +++ b/uploader/platforms/models.py @@ -56,7 +56,8 @@ def platform_by_species_and_id( return None -def save_new_platform(# pylint: disable=[too-many-arguments] +def save_new_platform( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] cursor: Cursor, species_id: int, geo_platform: str, diff --git a/uploader/platforms/views.py b/uploader/platforms/views.py index c20ab44..ba0f0ef 100644 --- a/uploader/platforms/views.py +++ b/uploader/platforms/views.py @@ -4,15 +4,15 @@ from gn_libs.mysqldb import database_connection from flask import ( flash, request, - url_for, redirect, Blueprint, current_app as app) +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.authorisation import require_login from uploader.species.models import all_species, species_by_id -from uploader.datautils import safe_int, order_by_family, enumerate_sequence +from uploader.datautils import safe_int, enumerate_sequence from .models import (save_new_platform, platforms_by_species, @@ -29,9 +29,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "platforms/index.html", - species=order_by_family(all_species(conn)), + species=all_species(conn), activelink="platforms") + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.platforms.list_platforms")) + species = species_by_id(conn, request.args["species_id"]) if not bool(species): flash("No species selected.", "alert-danger") diff --git a/uploader/population/models.py b/uploader/population/models.py index 6dcd85e..4d95065 100644 --- a/uploader/population/models.py +++ b/uploader/population/models.py @@ -26,13 +26,23 @@ def populations_by_species(conn: mdb.Connection, speciesid) -> tuple: return tuple() +__GENERIC_POPULATION_FAMILIES__ = ( + "Reference Populations (replicate average, SE, N)", + "Crosses and Heterogeneous Stock (individuals)", + "Groups Without Genotypes") -def population_families(conn) -> tuple: +def population_families(conn, species_id: int) -> tuple[str]: """Fetch the families under which populations are grouped.""" with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(__GENERIC_POPULATION_FAMILIES__)) cursor.execute( - "SELECT DISTINCT(Family) FROM InbredSet WHERE Family IS NOT NULL") - return tuple(row["Family"] for row in cursor.fetchall()) + "SELECT DISTINCT(Family) FROM InbredSet " + "WHERE SpeciesId=%s " + "AND Family IS NOT NULL " + f"AND Family NOT IN ({paramstr})", + (species_id, *__GENERIC_POPULATION_FAMILIES__)) + return __GENERIC_POPULATION_FAMILIES__ + tuple( + row["Family"] for row in cursor.fetchall()) def population_genetic_types(conn) -> tuple: @@ -47,9 +57,11 @@ def population_genetic_types(conn) -> tuple: def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dict: """Save the population details to the db.""" cursor.execute("SELECT DISTINCT(Family), FamilyOrder FROM InbredSet " - "WHERE Family IS NOT NULL AND Family != '' " + "WHERE SpeciesId=%s " + "AND Family IS NOT NULL AND Family != '' " "AND FamilyOrder IS NOT NULL " - "ORDER BY FamilyOrder ASC") + "ORDER BY FamilyOrder ASC", + (population_details["SpeciesId"],)) _families = { row["Family"]: int(row["FamilyOrder"]) for row in cursor.fetchall() @@ -61,7 +73,7 @@ def save_population(cursor: mdb.cursors.Cursor, population_details: dict) -> dic **population_details, "FamilyOrder": _families.get( population_details["Family"], - max(_families.values())+1) + max((0,) + tuple(_families.values()))+1) } cursor.execute( "INSERT INTO InbredSet(" diff --git a/uploader/population/rqtl2.py b/uploader/population/rqtl2.py index 436eca0..bb5066e 100644 --- a/uploader/population/rqtl2.py +++ b/uploader/population/rqtl2.py @@ -11,13 +11,11 @@ from typing import Union, Callable, Optional import MySQLdb as mdb from redis import Redis from MySQLdb.cursors import DictCursor -from werkzeug.utils import secure_filename from gn_libs.mysqldb import database_connection +from markupsafe import escape from flask import ( flash, - escape, request, - jsonify, url_for, redirect, Response, @@ -136,7 +134,7 @@ def upload_rqtl2_bundle(species_id: int, population_id: int): try: app.logger.debug("Files in the form: %s", request.files) the_file = save_file(request.files["rqtl2_bundle_file"], - Path(app.config["UPLOAD_FOLDER"])) + Path(app.config["UPLOADS_DIRECTORY"])) except AssertionError: app.logger.debug(traceback.format_exc()) flash("Please provide a valid R/qtl2 zip bundle.", @@ -187,131 +185,10 @@ def trigger_rqtl2_bundle_qc( "rqtl2-bundle-file": str(rqtl2bundle.absolute()), "original-filename": originalfilename})}), redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return jobid -def chunk_name(uploadfilename: str, chunkno: int) -> str: - """Generate chunk name from original filename and chunk number""" - if uploadfilename == "": - raise ValueError("Name cannot be empty!") - if chunkno < 1: - raise ValueError("Chunk number must be greater than zero") - return f"{secure_filename(uploadfilename)}_part_{chunkno:05d}" - - -def chunks_directory(uniqueidentifier: str) -> Path: - """Compute the directory where chunks are temporarily stored.""" - if uniqueidentifier == "": - raise ValueError("Unique identifier cannot be empty!") - return Path(app.config["UPLOAD_FOLDER"], f"tempdir_{uniqueidentifier}") - - -@rqtl2.route(("<int:species_id>/populations/<int:population_id>/rqtl2/" - "/rqtl2-bundle-chunked"), - methods=["GET"]) -@require_login -def upload_rqtl2_bundle_chunked_get(# pylint: disable=["unused-argument"] - species_id: int, - population_id: int -): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that provides a way - for testing whether all the chunks have been uploaded and to assist with - resuming a failed expression-data. - """ - fileid = request.args.get("resumableIdentifier", type=str) or "" - filename = request.args.get("resumableFilename", type=str) or "" - chunk = request.args.get("resumableChunkNumber", type=int) or 0 - if not(fileid or filename or chunk): - return jsonify({ - "message": "At least one required query parameter is missing.", - "error": "BadRequest", - "statuscode": 400 - }), 400 - - if Path(chunks_directory(fileid), - chunk_name(filename, chunk)).exists(): - return "OK" - - return jsonify({ - "message": f"Chunk {chunk} was not found.", - "error": "NotFound", - "statuscode": 404 - }), 404 - - -def __merge_chunks__(targetfile: Path, chunkpaths: tuple[Path, ...]) -> Path: - """Merge the chunks into a single file.""" - with open(targetfile, "ab") as _target: - for chunkfile in chunkpaths: - with open(chunkfile, "rb") as _chunkdata: - _target.write(_chunkdata.read()) - - chunkfile.unlink() - return targetfile - - -@rqtl2.route(("<int:species_id>/population/<int:population_id>/rqtl2/upload/" - "/rqtl2-bundle-chunked"), - methods=["POST"]) -@require_login -def upload_rqtl2_bundle_chunked_post(species_id: int, population_id: int): - """ - Extension to the `upload_rqtl2_bundle` endpoint above that allows large - files to be uploaded in chunks. - - This should hopefully speed up uploads, and if done right, even enable - resumable uploads - """ - _totalchunks = request.form.get("resumableTotalChunks", type=int) or 0 - _chunk = request.form.get("resumableChunkNumber", default=1, type=int) - _uploadfilename = request.form.get( - "resumableFilename", default="", type=str) or "" - _fileid = request.form.get( - "resumableIdentifier", default="", type=str) or "" - _targetfile = Path(app.config["UPLOAD_FOLDER"], _fileid) - - if _targetfile.exists(): - return jsonify({ - "message": ( - "A file with a similar unique identifier has previously been " - "uploaded and possibly is/has being/been processed."), - "error": "BadRequest", - "statuscode": 400 - }), 400 - - try: - # save chunk data - chunks_directory(_fileid).mkdir(exist_ok=True, parents=True) - request.files["file"].save(Path(chunks_directory(_fileid), - chunk_name(_uploadfilename, _chunk))) - - # Check whether upload is complete - chunkpaths = tuple( - Path(chunks_directory(_fileid), chunk_name(_uploadfilename, _achunk)) - for _achunk in range(1, _totalchunks+1)) - if all(_file.exists() for _file in chunkpaths): - # merge_files and clean up chunks - __merge_chunks__(_targetfile, chunkpaths) - chunks_directory(_fileid).rmdir() - jobid = trigger_rqtl2_bundle_qc( - species_id, population_id, _targetfile, _uploadfilename) - return url_for( - "expression-data.rqtl2.rqtl2_bundle_qc_status", jobid=jobid) - except Exception as exc:# pylint: disable=[broad-except] - msg = "Error processing uploaded file chunks." - app.logger.error(msg, exc_info=True, stack_info=True) - return jsonify({ - "message": msg, - "error": type(exc).__name__, - "error-description": " ".join(str(arg) for arg in exc.args), - "error-trace": traceback.format_exception(exc) - }), 500 - - return "OK" - - @rqtl2.route("/upload/species/rqtl2-bundle/qc-status/<uuid:jobid>", methods=["GET", "POST"]) @require_login @@ -1018,7 +895,7 @@ def confirm_bundle_details(species_id: int, population_id: int): }) }), redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") + f"{app.config['UPLOADS_DIRECTORY']}/job_errors") return redirect(url_for("expression-data.rqtl2.rqtl2_processing_status", jobid=jobid)) diff --git a/uploader/population/views.py b/uploader/population/views.py index 4f985f5..795ce81 100644 --- a/uploader/population/views.py +++ b/uploader/population/views.py @@ -2,16 +2,17 @@ import json import base64 +from markupsafe import escape from MySQLdb.cursors import DictCursor from gn_libs.mysqldb import database_connection from flask import (flash, request, - url_for, redirect, Blueprint, current_app as app) from uploader.samples.views import samplesbp +from uploader.flask_extensions import url_for from uploader.oauth2.client import oauth2_post from uploader.ui import make_template_renderer from uploader.authorisation import require_login @@ -19,11 +20,11 @@ from uploader.genotypes.views import genotypesbp from uploader.datautils import enumerate_sequence from uploader.phenotypes.views import phenotypesbp from uploader.expression_data.views import exprdatabp +from uploader.species.models import all_species, species_by_id from uploader.monadic_requests import make_either_error_handler from uploader.input_validation import is_valid_representative_name -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) +from uploader.phenotypes.models import (dataset_phenotypes, + datasets_by_population) from .models import (save_population, population_families, @@ -48,7 +49,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "populations/index.html", - species=order_species_by_family(all_species(conn))) + species=all_species(conn), + activelink="populations") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.list_species_populations")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("Invalid species identifier provided!", "alert-danger") @@ -93,7 +102,7 @@ def create_population(species_id: int): return render_template( "populations/create-population.html", species=species, - families = population_families(conn), + families = population_families(conn, species["SpeciesId"]), genetic_types = population_genetic_types(conn), mapping_methods=( {"id": "0", "value": "No mapping support"}, @@ -101,6 +110,7 @@ def create_population(species_id: int): {"id": "2", "value": "GEMMA"}, {"id": "3", "value": "R/qtl"}, {"id": "4", "value": "GEMMA, PLINK"}), + return_to=(request.args.get("return_to") or ""), activelink="create-population", **error_values) @@ -145,13 +155,21 @@ def create_population(species_id: int): "FullName": population_fullname, "InbredSetCode": request.form.get("population_code") or None, "Description": request.form.get("population_description") or None, - "Family": request.form.get("population_family") or None, + "Family": request.form.get("population_family", "").strip() or None, "MappingMethodId": request.form.get("population_mapping_method_id"), "GeneticType": request.form.get("population_genetic_type") or None }) def __flash_success__(_success): - flash("Successfully created resource.", "alert-success") + flash("Successfully created population " + f"{escape(new_population['FullName'])}.", + "alert-success") + return_to = request.form.get("return_to") or "" + if return_to: + return redirect(url_for( + return_to, + species_id=species["SpeciesId"], + population_id=new_population["InbredSetId"])) return redirect(url_for( "species.populations.view_population", species_id=species["SpeciesId"], @@ -177,10 +195,15 @@ def create_population(species_id: int): @require_login def view_population(species_id: int, population_id: int): """View the details of a population.""" + streamlined_ui = request.args.get("streamlined_ui") with database_connection(app.config["SQL_URI"]) as conn: species = species_by_id(conn, species_id) population = population_by_species_and_id(conn, species_id, population_id) + datasets = datasets_by_population(conn, species_id, population_id) error = False + if len(datasets) > 1: + error = True + flash("Got more than one dataset for the population.", "alert alert-danger") if not bool(species): flash("You must select a species.", "alert-danger") @@ -191,9 +214,32 @@ def view_population(species_id: int, population_id: int): error = True if error: - return redirect(url_for("species.populations.index")) + return redirect(url_for(("species.view_species" + if bool(streamlined_ui) + else "species.populations.index"), + species_id=species["SpeciesId"], + streamlined_ui=streamlined_ui)) + + _datasets = datasets_by_population( + conn, species["SpeciesId"], population["Id"]) + assert len(datasets) == 0 or len(datasets) == 1, ( + "We expect only one phenotypes dataset per population.") + _kwargs = { + "species": species, + "population": population, + "activelink": "view-population", + "streamlined_ui": streamlined_ui, + "view_under_construction": request.args.get( + "view_under_construction", False) + } + + if len(_datasets) == 1: + _dataset = _datasets[0] + _kwargs = { + **_kwargs, + "dataset": _dataset, + "phenotypes": enumerate_sequence( + dataset_phenotypes(conn, population["Id"], _dataset["Id"])) + } - return render_template("populations/view-population.html", - species=species, - population=population, - activelink="view-population") + return render_template("populations/view-population.html", **_kwargs) diff --git a/uploader/publications/__init__.py b/uploader/publications/__init__.py new file mode 100644 index 0000000..7efcabb --- /dev/null +++ b/uploader/publications/__init__.py @@ -0,0 +1,2 @@ +"""Package for handling publications.""" +from .views import pubbp diff --git a/uploader/publications/datatables.py b/uploader/publications/datatables.py new file mode 100644 index 0000000..8b3d4a0 --- /dev/null +++ b/uploader/publications/datatables.py @@ -0,0 +1,52 @@ +"""Fetch data for datatables.""" +import logging +from typing import Optional + +from MySQLdb.cursors import DictCursor + +from gn_libs.mysqldb import Connection, debug_query + +logger = logging.getLogger(__name__) + +def fetch_publications( + conn: Connection, + search: Optional[str] = None, + offset: int = 0, + limit: int = -1 +) -> tuple[tuple[dict, ...], int, int, int]: + """Fetch publications from the database.""" + _query = "SELECT * FROM Publication" + _count_query = "SELECT COUNT(*) FROM Publication" + _params = None + _where_clause = "" + _limit_clause = "" + if search is not None and bool(search): + _where_clause = ("WHERE PubMed_ID LIKE %s " + "OR Authors LIKE %s " + "OR Title LIKE %s") + _params = (f"%{search}%",) * 3 + + if limit > 0: + _limit_clause = f"LIMIT {limit} OFFSET {offset}" + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT COUNT(*) FROM Publication") + _total_rows = int(cursor.fetchone()["COUNT(*)"]) + + cursor.execute(f"{_count_query} {_where_clause}", _params) + debug_query(cursor, logger) + _result = cursor.fetchone() + _total_filtered = int(_result["COUNT(*)"] if bool(_result) else 0) + + cursor.execute(f"{_query} {_where_clause} {_limit_clause}", _params) + debug_query(cursor, logger) + _current_filtered = tuple( + {**dict(row), "index": idx} + for idx, row + in enumerate(cursor.fetchall(), start=offset+1)) + + return ( + _current_filtered, + len(_current_filtered), + _total_filtered, + _total_rows) diff --git a/uploader/publications/misc.py b/uploader/publications/misc.py new file mode 100644 index 0000000..f0ff9c7 --- /dev/null +++ b/uploader/publications/misc.py @@ -0,0 +1,25 @@ +"""Miscellaneous functions dealing with publications.""" + + +def publications_differences( + filedata: tuple[dict, ...], + dbdata: tuple[dict, ...], + pubmedid2pubidmap: dict[int, int] +) -> tuple[dict, ...]: + """Compute the differences between file data and db data""" + diff: tuple[dict, ...] = tuple() + for filerow, dbrow in zip( + sorted(filedata, key=lambda item: ( + item["phenotype_id"], item["xref_id"])), + sorted(dbdata, key=lambda item: ( + item["PhenotypeId"], item["xref_id"]))): + if filerow["PubMed_ID"] == dbrow["PubMed_ID"]: + continue + + newpubmed = filerow["PubMed_ID"] + diff = diff + ({ + **dbrow, + "PubMed_ID": newpubmed, + "PublicationId": pubmedid2pubidmap.get(newpubmed)},) + + return diff diff --git a/uploader/publications/models.py b/uploader/publications/models.py new file mode 100644 index 0000000..d913144 --- /dev/null +++ b/uploader/publications/models.py @@ -0,0 +1,133 @@ +"""Module to handle persistence and retrieval of publication to/from MariaDB""" +import logging +from typing import Iterable + +from MySQLdb.cursors import DictCursor + +from gn_libs.mysqldb import Connection, debug_query + +logger = logging.getLogger(__name__) + + +def fetch_phenotype_publications( + conn: Connection, + ids: tuple[tuple[int, int], ...] +) -> tuple[dict, ...]: + """Fetch publication from database by ID.""" + paramstr = ",".join(["(%s, %s)"] * len(ids)) + query = ( + "SELECT " + "pxr.PhenotypeId, pxr.Id AS xref_id, pxr.PublicationId, pub.PubMed_ID " + "FROM PublishXRef AS pxr INNER JOIN Publication AS pub " + "ON pxr.PublicationId=pub.Id " + f"WHERE (pxr.PhenotypeId, pxr.Id) IN ({paramstr})") + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(query, tuple(item for row in ids for item in row)) + return tuple(dict(row) for row in cursor.fetchall()) + + +def create_new_publications( + conn: Connection, + publications: tuple[dict, ...] +) -> tuple[dict, ...]: + """Create new publications in the database.""" + if len(publications) > 0: + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.executemany( + ("INSERT INTO " + "Publication( " + "PubMed_ID, Abstract, Authors, Title, Journal, Volume, Pages, " + "Month, Year" + ") " + "VALUES(" + "%(pubmed_id)s, %(abstract)s, %(authors)s, %(title)s, " + "%(journal)s, %(volume)s, %(pages)s, %(month)s, %(year)s" + ") " + "RETURNING *"), + publications) + return tuple({ + **row, "publication_id": row["Id"] + } for row in cursor.fetchall()) + + return tuple() + + +def update_publications(conn: Connection , publications: tuple[dict, ...]) -> tuple[dict, ...]: + """Update details for multiple publications""" + if len(publications) > 0: + with conn.cursor(cursorclass=DictCursor) as cursor: + logger.debug("UPDATING PUBLICATIONS: %s", publications) + cursor.executemany( + ("UPDATE Publication SET " + "PubMed_ID=%(pubmed_id)s, Abstract=%(abstract)s, " + "Authors=%(authors)s, Title=%(title)s, Journal=%(journal)s, " + "Volume=%(volume)s, Pages=%(pages)s, Month=%(month)s, " + "Year=%(year)s " + "WHERE Id=%(publication_id)s"), + publications) + debug_query(cursor, logger) + return publications + return tuple() + return tuple() + + +def delete_publications(conn: Connection , publications: tuple[dict, ...]): + """Delete multiple publications""" + publications = tuple(pub for pub in publications if bool(pub)) + if len(publications) > 0: + _pub_ids = tuple(pub["Id"] for pub in publications) + _paramstr = ", ".join(["%s"] * len(_pub_ids)) + _phenos_query = ( + "SELECT PublicationId, COUNT(PhenotypeId) FROM PublishXRef " + f"WHERE PublicationId IN ({_paramstr}) GROUP BY PublicationId;") + + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute(_phenos_query, _pub_ids) + _linked_phenos = cursor.fetchall() + if len(_linked_phenos) > 0: + raise Exception(# pylint: disable=[broad-exception-raised] + "Cannot delete publications with linked phenotypes.") + + cursor.execute( + f"DELETE FROM Publication WHERE Id IN ({_paramstr})", _pub_ids) + + +def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict: + """Fetch a specific publication from the database.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute("SELECT * FROM Publication WHERE Id=%s", + (publication_id,)) + _res = cursor.fetchone() + return dict(_res) if _res else {} + + +def fetch_publications_by_ids( + conn: Connection, publications_ids: tuple[int, ...] +) -> tuple[dict, ...]: + """Fetch publications with the given IDs.""" + if len(publications_ids) == 0: + return tuple() + + with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(publications_ids)) + cursor.execute(f"SELECT * FROM Publication WHERE Id IN ({paramstr})", + tuple(publications_ids)) + return tuple(dict(row) for row in cursor.fetchall()) + + +def fetch_publication_phenotypes( + conn: Connection, publication_id: int) -> Iterable[dict]: + """Fetch all phenotypes linked to this publication.""" + with conn.cursor(cursorclass=DictCursor) as cursor: + cursor.execute( + "SELECT pxr.Id AS xref_id, pxr.PublicationId, phe.* " + "FROM PublishXRef AS pxr INNER JOIN Phenotype AS phe " + "ON pxr.PhenotypeId=phe.Id " + "WHERE pxr.PublicationId=%s", + (publication_id,)) + while True: + row = cursor.fetchone() + if row: + yield row + else: + break diff --git a/uploader/publications/pubmed.py b/uploader/publications/pubmed.py new file mode 100644 index 0000000..15bf701 --- /dev/null +++ b/uploader/publications/pubmed.py @@ -0,0 +1,103 @@ +"""Module to interact with NCBI's PubMed""" +import logging +from typing import Optional + +import requests +from lxml import etree + +logger = logging.getLogger(__name__) + + +def __pub_date__(pubdate: etree.Element): + pubyear = pubdate.find("Year") + pubmonth = pubdate.find("Month") + pubday = pubdate.find("Day") + return { + "year": pubyear.text if pubyear is not None else None, + "month": pubmonth.text if pubmonth is not None else None, + "day": pubday.text if pubday is not None else None + } + + +def __journal__(journal: etree.Element) -> dict: + volume = journal.find("JournalIssue/Volume") + issue = journal.find("JournalIssue/Issue") + return { + "volume": volume.text if volume is not None else None, + "issue": issue.text if issue is not None else None, + **__pub_date__(journal.find("JournalIssue/PubDate")), + "journal": journal.find("Title").text + } + +def __author__(author: etree.Element) -> str: + return f'{author.find("LastName").text} {author.find("Initials").text}' + + +def __pages__(pagination: etree.Element) -> str: + start = pagination.find("StartPage") + end = pagination.find("EndPage") + return (start.text + ( + f"-{end.text}" if end is not None else "" + )) if start is not None else "" + + +def __abstract__(article: etree.Element) -> Optional[str]: + abstract = article.find("Abstract/AbstractText") + return abstract.text if abstract is not None else None + + +def __article__(pubmed_article: etree.Element) -> dict: + article = pubmed_article.find("MedlineCitation/Article") + return { + "pubmed_id": int(pubmed_article.find("MedlineCitation/PMID").text), + "title": article.find("ArticleTitle").text, + **__journal__(article.find("Journal")), + "abstract": __abstract__(article), + "pages": __pages__(article.find("Pagination")), + "authors": ", ".join(__author__(author) + for author in article.findall("AuthorList/Author")) + } + + +def __process_pubmed_publication_data__(text) -> tuple[dict, ...]: + """Process the data from PubMed into usable data.""" + doc = etree.XML(text) + articles = doc.xpath("//PubmedArticle") + logger.debug("Retrieved %s publications from NCBI", len(articles)) + return tuple(__article__(article) for article in articles) + +def fetch_publications(pubmed_ids: tuple[int, ...]) -> tuple[dict, ...]: + """Retrieve data on new publications from NCBI.""" + # See whether we can retrieve multiple publications in one go + # Parse data and save to DB + # Return PublicationId(s) for new publication(s). + if len(pubmed_ids) == 0: + logger.debug("There are no new PubMed IDs to fetch") + return tuple() + + logger.info("Fetching publications data for the following PubMed IDs: %s", + ", ".join((str(pid) for pid in pubmed_ids))) + + # Should we, perhaps, pass this in from a config variable? + uri = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi" + try: + response = requests.get( + uri, + params={ + "db": "pubmed", + "retmode": "xml", + "id": ",".join(str(item) for item in pubmed_ids) + }, + timeout=(9.13, 20)) + + if response.status_code == 200: + return __process_pubmed_publication_data__(response.text) + + logger.error( + "Could not fetch the new publication from %s (status code: %s)", + uri, + response.status_code) + except requests.exceptions.ConnectionError: + logger.error("Could not find the domain %s", uri) + + return tuple() diff --git a/uploader/publications/views.py b/uploader/publications/views.py new file mode 100644 index 0000000..89e9f5d --- /dev/null +++ b/uploader/publications/views.py @@ -0,0 +1,198 @@ +"""Endpoints for publications""" +import json +import datetime + +from gn_libs.mysqldb import database_connection +from flask import ( + flash, + request, + redirect, + Blueprint, + render_template, + current_app as app) + +from uploader.flask_extensions import url_for +from uploader.authorisation import require_login +from uploader.route_utils import redirect_to_next + +from .models import ( + delete_publications, + update_publications, + fetch_publication_by_id, + create_new_publications, + fetch_publication_phenotypes) + +from .datatables import fetch_publications + +pubbp = Blueprint("publications", __name__) + + +@pubbp.route("/", methods=["GET"]) +@require_login +def index(): + """Index page for publications.""" + return render_template("publications/index.html") + + +@pubbp.route("/list", methods=["GET"]) +@require_login +def list_publications(): + """Fetch publications that fulfill a specific search, or all of them, if + there is no search term.""" + # request breakdown: + # https://datatables.net/manual/server-side + _page = int(request.args.get("draw")) + _length = int(request.args.get("length") or '-1') + _start = int(request.args.get("start") or '0') + _search = request.args["search[value]"] + with database_connection(app.config["SQL_URI"]) as conn: + _publications, _current_rows, _totalfiltered, _totalrows = fetch_publications( + conn, + _search, + offset=_start, + limit=_length) + + return json.dumps({ + "draw": _page, + "recordsTotal": _totalrows, + "recordsFiltered": _totalfiltered, + "publications": _publications, + "status": "success" + }) + + +@pubbp.route("/view/<int:publication_id>", methods=["GET"]) +@require_login +def view_publication(publication_id: int): + """View more details on a particular publication.""" + with database_connection(app.config["SQL_URI"]) as conn: + publication = fetch_publication_by_id(conn, publication_id) + + if not bool(publication): + flash("Requested publication was not found!", "alert-warning") + return redirect(url_for('publications.index')) + + return render_template( + "publications/view-publication.html", + publication=publication, + linked_phenotypes=tuple(fetch_publication_phenotypes( + conn, publication_id))) + + +@pubbp.route("/create", methods=["GET", "POST"]) +@require_login +def create_publication(): + """Create a new publication.""" + _get_args = { + key: request.args[key] + for key in ("species_id", "population_id", "dataset_id", "return_to") + if bool(request.args.get(key)) + } + + if request.method == "GET": + now = datetime.datetime.now() + return render_template( + "publications/create-publication.html", + get_args=_get_args, + current_year=now.year, + current_month=now.strftime("%B")) + form = request.form + authors = form.get("publication-authors").encode("utf8") + if authors is None or authors == "": + flash("The publication's author(s) MUST be provided!", "alert alert-danger") + return redirect(url_for("publications.create")) + + with database_connection(app.config["SQL_URI"]) as conn: + publications = create_new_publications(conn, ({ + "pubmed_id": form.get("pubmed-id") or None, + "abstract": form.get("publication-abstract").encode("utf8") or None, + "authors": authors, + "title": form.get("publication-title").encode("utf8") or None, + "journal": form.get("publication-journal").encode("utf8") or None, + "volume": form.get("publication-volume").encode("utf8") or None, + "pages": form.get("publication-pages").encode("utf8") or None, + "month": (form.get("publication-month") or "").encode("utf8").capitalize() or None, + "year": form.get("publication-year").encode("utf8") or None + },)) + flash("New publication created!", "alert alert-success") + return redirect(url_for( + request.args.get("return_to") or "publications.view_publication", + publication_id=publications[0]["publication_id"], + **_get_args)) + + flash("Publication creation failed!", "alert alert-danger") + app.logger.debug("Failed to create the new publication.", exc_info=True) + return redirect(url_for("publications.create_publication")) + + +@pubbp.route("/edit/<int:publication_id>", methods=["GET", "POST"]) +@require_login +def edit_publication(publication_id: int): + """Edit a publication's details.""" + with database_connection(app.config["SQL_URI"]) as conn: + if request.method == "GET": + return render_template( + "publications/edit-publication.html", + publication=fetch_publication_by_id(conn, publication_id), + linked_phenotypes=tuple(fetch_publication_phenotypes( + conn, publication_id)), + publication_id=publication_id) + + form = request.form + _pub = update_publications(conn, ({ + "publication_id": publication_id, + "pubmed_id": form.get("pubmed-id") or None, + "abstract": (form.get("publication-abstract") or "").encode("utf8") or None, + "authors": (form.get("publication-authors") or "").encode("utf8"), + "title": (form.get("publication-title") or "").encode("utf8") or None, + "journal": (form.get("publication-journal") or "").encode("utf8") or None, + "volume": (form.get("publication-volume") or "").encode("utf8") or None, + "pages": (form.get("publication-pages") or "").encode("utf8") or None, + "month": (form.get("publication-month") or "").encode("utf8").capitalize() or None, + "year": (form.get("publication-year") or "").encode("utf8") or None + },)) + + if not _pub: + flash("There was an error updating the publication details.", + "alert-danger") + return redirect(url_for( + "publications.edit_publication", publication_id=publication_id)) + + flash("Successfully updated the publication details.", + "alert-success") + return redirect_to_next({ + "uri": "publications.view_publication", + "publication_id": publication_id + }) + + +@pubbp.route("/delete/<int:publication_id>", methods=["GET", "POST"]) +@require_login +def delete_publication(publication_id: int): + """Delete a particular publication.""" + with database_connection(app.config["SQL_URI"]) as conn: + publication = fetch_publication_by_id(conn, publication_id) + linked_phenotypes=tuple(fetch_publication_phenotypes( + conn, publication_id)) + + if not bool(publication): + flash("Requested publication was not found!", "alert-warning") + return redirect(url_for('publications.index')) + + if len(linked_phenotypes) > 0: + flash("Cannot delete publication with linked phenotypes!", + "alert-warning") + return redirect(url_for( + "publications.view_publication", + publication_id=publication_id)) + + if request.method == "GET": + return render_template( + "publications/delete-publication.html", + publication=publication, + linked_phenotypes=linked_phenotypes, + publication_id=publication_id) + + delete_publications(conn, (publication,)) + flash("Deleted the publication successfully.", "alert-success") + return redirect(url_for("publications.index")) diff --git a/uploader/request_checks.py b/uploader/request_checks.py index f1d8027..84935f9 100644 --- a/uploader/request_checks.py +++ b/uploader/request_checks.py @@ -2,14 +2,20 @@ These are useful for reusability, and hence maintainability of the code. """ +import logging + +from typing import Callable from functools import wraps -from gn_libs.mysqldb import database_connection +from gn_libs.mysqldb import Connection, database_connection from flask import flash, url_for, redirect, current_app as app from uploader.species.models import species_by_id from uploader.population.models import population_by_species_and_id +logger = logging.getLogger(__name__) + + def with_species(redirect_uri: str): """Ensure the species actually exists.""" def __decorator__(function): @@ -28,7 +34,7 @@ def with_species(redirect_uri: str): "alert-danger") return redirect(url_for(redirect_uri)) except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("species_id"), exc_info=True) @@ -63,7 +69,7 @@ def with_population(species_redirect_uri: str, redirect_uri: str): "alert-danger") return select_population_uri except ValueError as _verr: - app.logger.debug( + logger.debug( "Exception converting value to integer: %s", kwargs.get("population_id"), exc_info=True) @@ -73,3 +79,45 @@ def with_population(species_redirect_uri: str, redirect_uri: str): return function(**{**kwargs, "population": population}) return __with_population__ return __decorator__ + + +def with_dataset( + species_redirect_uri: str, + population_redirect_uri: str, + redirect_uri: str, + dataset_by_id: Callable[ + [Connection, int, int, int], + dict] +): + """Ensure the dataset actually exists.""" + def __decorator__(func): + @wraps(func) + @with_population(species_redirect_uri, population_redirect_uri) + def __with_dataset__(**kwargs): + try: + _spcid = int(kwargs["species_id"]) + _popid = int(kwargs["population_id"]) + _dsetid = int(kwargs.get("dataset_id")) + select_dataset_uri = redirect(url_for( + redirect_uri, species_id=_spcid, population_id=_popid)) + if not bool(_dsetid): + flash("You need to select a valid 'dataset_id' value.", + "alert-danger") + return select_dataset_uri + with database_connection(app.config["SQL_URI"]) as conn: + dataset = dataset_by_id(conn, _spcid, _popid, _dsetid) + if not bool(dataset): + flash("You must select a valid dataset.", + "alert-danger") + return select_dataset_uri + except ValueError as _verr: + logger.debug( + "Exception converting 'dataset_id' to integer: %s", + kwargs.get("dataset_id"), + exc_info=True) + flash("Expected 'dataset_id' value to be an integer." + "alert-danger") + return select_dataset_uri + return func(**{**kwargs, "dataset": dataset}) + return __with_dataset__ + return __decorator__ diff --git a/uploader/route_utils.py b/uploader/route_utils.py new file mode 100644 index 0000000..426d7eb --- /dev/null +++ b/uploader/route_utils.py @@ -0,0 +1,92 @@ +"""Generic routing utilities.""" +import logging +from json.decoder import JSONDecodeError + +from flask import (flash, + request, + redirect, + render_template, + current_app as app) + +from gn_libs.mysqldb import database_connection + +from uploader.flask_extensions import url_for +from uploader.datautils import base64_encode_dict, base64_decode_to_dict +from uploader.population.models import (populations_by_species, + population_by_species_and_id) + +logger = logging.getLogger(__name__) + +def generic_select_population( + # pylint: disable=[too-many-arguments, too-many-positional-arguments] + species: dict, + template: str, + population_id: str, + back_to: str, + forward_to: str, + activelink: str, + error_message: str = "No such population found!" +): + """Handles common flow for 'select population' step.""" + with database_connection(app.config["SQL_URI"]) as conn: + if not bool(population_id): + return render_template( + template, + species=species, + populations=populations_by_species(conn, species["SpeciesId"]), + activelink=activelink) + + if population_id == "CREATE-POPULATION": + return redirect(url_for( + "species.populations.create_population", + species_id=species["SpeciesId"], + return_to=forward_to)) + + population = population_by_species_and_id( + conn, species["SpeciesId"], int(population_id)) + if not bool(population): + flash(error_message, "alert-danger") + return redirect(url_for(back_to, species_id=species["SpeciesId"])) + + return redirect(url_for(forward_to, + species_id=species["SpeciesId"], + population_id=population["Id"])) + + +def redirect_to_next(default: dict): + """Redirect to the next uri if specified, else redirect to default.""" + assert "uri" in default, "You must provide at least the 'uri' value." + _next = request.args.get("next") or "" + if bool(_next): + try: + next_page = base64_decode_to_dict(_next) + _uri = next_page["uri"] + next_page.pop("uri") + return redirect(url_for(_uri, **next_page)) + except (TypeError, JSONDecodeError) as _err: + logger.debug("We could not decode the next value '%s'", + next_page, + exc_info=True) + + return redirect(url_for( + default["uri"], + **{key:value for key,value in default.items() if key != "uri"})) + + +def build_next_argument(uri: str, **kwargs) -> bytes: + """Build the `next` URI argument from provided details.""" + dumps_keywords = ( + "skipkeys", "ensure_ascii", "check_circular", "allow_nan", "cls", + "indent", "separators", "default", "sort_keys") + return base64_encode_dict( + { + "uri": uri, + **{ + key: val for key,val in kwargs.items() + if key not in dumps_keywords + } + }, + **{ + key: val for key,val in kwargs.items() + if key in dumps_keywords + }) diff --git a/uploader/samples/models.py b/uploader/samples/models.py index d7d5384..1e9293f 100644 --- a/uploader/samples/models.py +++ b/uploader/samples/models.py @@ -15,11 +15,11 @@ def samples_by_species_and_population( """Fetch the samples by their species and population.""" with conn.cursor(cursorclass=DictCursor) as cursor: cursor.execute( - "SELECT iset.InbredSetId, s.* FROM InbredSet AS iset " - "INNER JOIN StrainXRef AS sxr ON iset.InbredSetId=sxr.InbredSetId " - "INNER JOIN Strain AS s ON sxr.StrainId=s.Id " - "WHERE s.SpeciesId=%(species_id)s " - "AND iset.InbredSetId=%(population_id)s", + "SELECT InbredSet.InbredSetId, Strain.* FROM InbredSet " + "INNER JOIN StrainXRef ON InbredSet.InbredSetId=StrainXRef.InbredSetId " + "INNER JOIN Strain ON StrainXRef.StrainId=Strain.Id " + "WHERE Strain.SpeciesId=%(species_id)s " + "AND InbredSet.InbredSetId=%(population_id)s", {"species_id": species_id, "population_id": population_id}) return tuple(cursor.fetchall()) @@ -34,8 +34,7 @@ def read_samples_file(filepath, separator: str, firstlineheading: bool, **kwargs else ("Name", "Name2", "Symbol", "Alias")), delimiter=separator, quotechar=kwargs.get("quotechar", '"')) - for row in reader: - yield row + yield from reader def save_samples_data(conn: mdb.Connection, diff --git a/uploader/samples/views.py b/uploader/samples/views.py index ed79101..2a09f8e 100644 --- a/uploader/samples/views.py +++ b/uploader/samples/views.py @@ -1,31 +1,31 @@ """Code regarding samples""" -import os import sys import uuid +import logging from pathlib import Path -from redis import Redis from flask import (flash, request, - url_for, redirect, Blueprint, current_app as app) -from uploader import jobs +from gn_libs import jobs +from gn_libs import sqlite3 + +from uploader import session from uploader.files import save_file +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.authorisation import require_login -from uploader.request_checks import with_population from uploader.input_validation import is_integer_input -from uploader.datautils import safe_int, order_by_family, enumerate_sequence -from uploader.population.models import population_by_id, populations_by_species +from uploader.population.models import population_by_id +from uploader.route_utils import generic_select_population +from uploader.datautils import safe_int, enumerate_sequence +from uploader.species.models import all_species, species_by_id +from uploader.request_checks import with_species, with_population from uploader.db_utils import (with_db_connection, - database_connection, - with_redis_connection) -from uploader.species.models import (all_species, - species_by_id, - order_species_by_family) + database_connection) from .models import samples_by_species_and_population @@ -40,8 +40,15 @@ def index(): if not bool(request.args.get("species_id")): return render_template( "samples/index.html", - species=order_species_by_family(all_species(conn)), + species=all_species(conn), activelink="samples") + + species_id = request.args.get("species_id") + if species_id == "CREATE-SPECIES": + return redirect(url_for( + "species.create_species", + return_to="species.populations.samples.select_population")) + species = species_by_id(conn, request.args.get("species_id")) if not bool(species): flash("No such species!", "alert-danger") @@ -52,57 +59,31 @@ def index(): @samplesbp.route("<int:species_id>/samples/select-population", methods=["GET"]) @require_login -def select_population(species_id: int): +@with_species(redirect_uri="species.populations.samples.index") +def select_population(species: dict, **kwargs):# pylint: disable=[unused-argument] """Select the population to use for the samples.""" - with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - if not bool(request.args.get("population_id")): - return render_template("samples/select-population.html", - species=species, - populations=order_by_family( - populations_by_species( - conn, - species_id), - order_key="FamilyOrder"), - activelink="samples") - - population = population_by_id(conn, request.args.get("population_id")) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - - return redirect(url_for("species.populations.samples.list_samples", - species_id=species_id, - population_id=population["Id"])) + return generic_select_population( + species, + "samples/select-population.html", + request.args.get("population_id") or "", + "species.populations.samples.select_population", + "species.populations.samples.list_samples", + "samples", + "Population not found!") @samplesbp.route("<int:species_id>/populations/<int:population_id>/samples") @require_login -def list_samples(species_id: int, population_id: int): +@with_population( + species_redirect_uri="species.populations.samples.index", + redirect_uri="species.populations.samples.select_population") +def list_samples(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument] """ List the samples in a particular population and give the ability to upload new ones. """ with database_connection(app.config["SQL_URI"]) as conn: - species = species_by_id(conn, species_id) - if not bool(species): - flash("Invalid species!", "alert-danger") - return redirect(url_for("species.populations.samples.index")) - - population = population_by_id(conn, population_id) - if not bool(population): - flash("Population not found!", "alert-danger") - return redirect(url_for( - "species.populations.samples.select_population", - species_id=species_id)) - all_samples = enumerate_sequence(samples_by_species_and_population( - conn, species_id, population_id)) + conn, species["SpeciesId"], population["Id"])) total_samples = len(all_samples) offset = max(safe_int(request.args.get("from") or 0), 0) count = int(request.args.get("count") or 20) @@ -116,22 +97,6 @@ def list_samples(species_id: int, population_id: int): activelink="list-samples") -def build_sample_upload_job(# pylint: disable=[too-many-arguments] - speciesid: int, - populationid: int, - samplesfile: Path, - separator: str, - firstlineheading: bool, - quotechar: str): - """Define the async command to run the actual samples data upload.""" - return [ - sys.executable, "-m", "scripts.insert_samples", app.config["SQL_URI"], - str(speciesid), str(populationid), str(samplesfile.absolute()), - separator, f"--redisuri={app.config['REDIS_URL']}", - f"--quotechar={quotechar}" - ] + (["--firstlineheading"] if firstlineheading else []) - - @samplesbp.route("<int:species_id>/populations/<int:population_id>/upload-samples", methods=["GET", "POST"]) @require_login @@ -173,13 +138,13 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma try: samples_file = save_file(request.files["samples_file"], - Path(app.config["UPLOAD_FOLDER"])) + Path(app.config["UPLOADS_DIRECTORY"])) except AssertionError: flash("You need to provide a file with the samples data.", "alert-error") return samples_uploads_page - firstlineheading = (request.form.get("first_line_heading") == "on") + firstlineheading = request.form.get("first_line_heading") == "on" separator = request.form.get("separator", ",") if separator == "other": @@ -190,91 +155,50 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma quotechar = (request.form.get("field_delimiter", '"') or '"') - redisuri = app.config["REDIS_URL"] - with Redis.from_url(redisuri, decode_responses=True) as rconn: - #TODO: Add a QC step here — what do we check? - # 1. Does any sample in the uploaded file exist within the database? - # If yes, what is/are its/their species and population? - # 2. If yes 1. above, provide error with notes on which species and - # populations already own the samples. - the_job = jobs.launch_job( + _jobs_db = app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"] + with sqlite3.connection(_jobs_db) as conn: + job = jobs.launch_job( jobs.initialise_job( - rconn, - jobs.jobsnamespace(), + conn, str(uuid.uuid4()), - build_sample_upload_job( - species["SpeciesId"], - population["InbredSetId"], - samples_file, + [ + sys.executable, "-m", "scripts.insert_samples", + app.config["SQL_URI"], + str(species["SpeciesId"]), + str(population["InbredSetId"]), + str(samples_file.absolute()), separator, - firstlineheading, - quotechar), + f"--quotechar={quotechar}" + ] + (["--firstlineheading"] if firstlineheading else []), "samples_upload", - app.config["JOBS_TTL_SECONDS"], - {"job_name": f"Samples Upload: {samples_file.name}"}), - redisuri, - f"{app.config['UPLOAD_FOLDER']}/job_errors") - return redirect(url_for( - "species.populations.samples.upload_status", - species_id=species_id, - population_id=population_id, - job_id=the_job["jobid"])) - - -@samplesbp.route("<int:species_id>/populations/<int:population_id>/" - "upload-samples/status/<uuid:job_id>", - methods=["GET"]) -@require_login -@with_population(species_redirect_uri="species.populations.samples.index", - redirect_uri="species.populations.samples.select_population") -def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument] - """Check on the status of a samples upload job.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if job: - status = job["status"] - if status == "success": - return render_template("samples/upload-success.html", - job=job, - species=species, - population=population,) - - if status == "error": - return redirect(url_for( - "species.populations.samples.upload_failure", job_id=job_id)) - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return redirect(url_for( - "samples.upload_failure", job_id=job_id)) - - return render_template("samples/upload-progress.html", - species=species, - population=population, - job=job) # maybe also handle this? - - return render_template("no_such_job.html", - job_id=job_id, - species=species, - population=population), 400 - -@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"]) -@require_login -def upload_failure(job_id: uuid.UUID): - """Display the errors of the samples upload failure.""" - job = with_redis_connection(lambda rconn: jobs.job( - rconn, jobs.jobsnamespace(), job_id)) - if not bool(job): - return render_template("no_such_job.html", job_id=job_id), 400 - - error_filename = Path(jobs.error_filename( - job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors")) - if error_filename.exists(): - stat = os.stat(error_filename) - if stat.st_size > 0: - return render_template("worker_failure.html", job_id=job_id) - - return render_template("samples/upload-failure.html", job=job) + extra_meta={ + "job_name": f"Samples Upload: {samples_file.name}", + "species_id": species["SpeciesId"], + "population_id": population["Id"], + "success_handler": ( + "uploader.samples.views.samples_upload_success_handler") + }, + external_id=session.logged_in_user_id()), + _jobs_db, + Path(f"{app.config['UPLOADS_DIRECTORY']}/job_errors").absolute(), + loglevel=logging.getLevelName( + app.logger.getEffectiveLevel()).lower()) + return redirect( + url_for("background-jobs.job_status", job_id=job["job_id"])) + + +def samples_upload_success_handler(job): + """Handler for background jobs: Successful upload of samples""" + return return_to_samples_list_view_handler( + job, "Samples uploaded successfully.") + + +def return_to_samples_list_view_handler(job, msg): + """Handler for background jobs: Return to list_samples page.""" + flash(msg, "alert alert-success") + return redirect(url_for( + "species.populations.samples." + "list_samples", + species_id=job["metadata"]["species_id"], + population_id=job["metadata"]["population_id"], + job_id=job["job_id"])) diff --git a/uploader/session.py b/uploader/session.py index b538187..9872ceb 100644 --- a/uploader/session.py +++ b/uploader/session.py @@ -1,12 +1,15 @@ """Deal with user sessions""" +import logging from uuid import UUID, uuid4 from datetime import datetime from typing import Any, Optional, TypedDict +from flask import session from authlib.jose import KeySet -from flask import request, session from pymonad.either import Left, Right, Either +logger = logging.getLogger(__name__) + class UserDetails(TypedDict): """Session information relating specifically to the user.""" @@ -22,8 +25,6 @@ class SessionInfo(TypedDict): session_id: UUID user: UserDetails anon_id: UUID - user_agent: str - ip_addr: str masquerade: Optional[UserDetails] auth_server_jwks: Optional[dict[str, Any]] @@ -66,9 +67,6 @@ def session_info() -> SessionInfo: "logged_in": False }, "anon_id": anon_id, - "user_agent": request.headers.get("User-Agent"), - "ip_addr": request.environ.get("HTTP_X_FORWARDED_FOR", - request.remote_addr), "masquerading": None })) @@ -77,17 +75,31 @@ def set_user_token(token: str) -> SessionInfo: """Set the user's token.""" info = session_info() return save_session_info({ - **info, "user": {**info["user"], "token": Right(token)}})#type: ignore[misc] + **info, + "user": {**info["user"], "token": Right(token), "logged_in": True} + })#type: ignore[misc] def set_user_details(userdets: UserDetails) -> SessionInfo: """Set the user details information""" - return save_session_info({**session_info(), "user": userdets})#type: ignore[misc] + info = session_info() + return save_session_info({**info, "user": {**info["user"], **userdets}})#type: ignore[misc] def user_details() -> UserDetails: """Retrieve user details.""" return session_info()["user"] + +def logged_in_user_id() -> Optional[UUID]: + """Get user id for logged in user. If user has not logged in, return None.""" + return user_token().then( + lambda _tok: user_details() + ).then( + lambda _user: Either(_user["user_id"], + (None, _user["email"] != "anon@ymous.user")) + ).either(lambda _err: None, lambda uid: uid) + + def user_token() -> Either: """Retrieve the user token.""" return session_info()["user"]["token"] diff --git a/uploader/species/models.py b/uploader/species/models.py index 51f941c..acfa51e 100644 --- a/uploader/species/models.py +++ b/uploader/species/models.py @@ -58,7 +58,8 @@ def save_species(conn: mdb.Connection, common_name: The species' common name. scientific_name; The species' scientific name. """ - genus, species_name = scientific_name.split(" ") + genus, *species_parts = scientific_name.split(" ") + species_name: str = " ".join(species_parts) families = species_families(conn) with conn.cursor() as cursor: cursor.execute("SELECT MAX(OrderId) FROM Species") @@ -68,7 +69,7 @@ def save_species(conn: mdb.Connection, "menu_name": f"{common_name} ({genus[0]}. {species_name.lower()})", "scientific_name": scientific_name, "family": family, - "family_order": families[family], + "family_order": families.get(family, 999999), "taxon_id": taxon_id, "species_order": cursor.fetchone()[0] + 5 } @@ -91,7 +92,7 @@ def save_species(conn: mdb.Connection, } -def update_species(# pylint: disable=[too-many-arguments] +def update_species(# pylint: disable=[too-many-arguments, too-many-positional-arguments] conn: mdb.Connection, species_id: int, common_name: str, @@ -116,7 +117,8 @@ def update_species(# pylint: disable=[too-many-arguments] species_order: The ordering of this species in relation to others """ with conn.cursor(cursorclass=DictCursor) as cursor: - genus, species_name = scientific_name.split(" ") + genus, *species_parts = scientific_name.split(" ") + species_name = " ".join(species_parts) species = { "species_id": species_id, "common_name": common_name, diff --git a/uploader/species/views.py b/uploader/species/views.py index fee5c75..4bfa7ae 100644 --- a/uploader/species/views.py +++ b/uploader/species/views.py @@ -1,19 +1,22 @@ """Endpoints handling species.""" +from markupsafe import escape from pymonad.either import Left, Right, Either from gn_libs.mysqldb import database_connection from flask import (flash, request, - url_for, redirect, Blueprint, current_app as app) from uploader.population import popbp from uploader.platforms import platformsbp +from uploader.flask_extensions import url_for from uploader.ui import make_template_renderer from uploader.oauth2.client import oauth2_get, oauth2_post from uploader.authorisation import require_login, require_token from uploader.datautils import order_by_family, enumerate_sequence +from uploader.population.models import (populations_by_species, + population_by_species_and_id) from .models import (all_species, save_species, @@ -40,15 +43,26 @@ def list_species(): @require_login def view_species(species_id: int): """View details of a particular species and menus to act upon it.""" + streamlined_ui = request.args.get("streamlined_ui") with database_connection(app.config["SQL_URI"]) as conn: species = species_by_id(conn, species_id) if bool(species): - return render_template("species/view-species.html", - species=species, - activelink="view-species") + population = population_by_species_and_id( + conn, species_id, request.args.get("population_id")) + if bool(population): + return redirect(url_for("species.populations.view_population", + species_id=species_id, + population_id=population["Id"])) + return render_template( + "species/view-species.html", + species=species, + activelink="view-species", + populations=populations_by_species(conn, species["SpeciesId"])) flash("Could not find a species with the given identifier.", "alert-danger") - return redirect(url_for("species.view_species")) + return redirect(url_for("base.index" + if streamlined_ui + else "species.view_species")) @speciesbp.route("/create", methods=["GET", "POST"]) @require_login @@ -62,6 +76,8 @@ def create_species(): if request.method == "GET": return render_template("species/create-species.html", families=species_families(conn), + return_to=( + request.args.get("return_to") or ""), activelink="create-species") error = False @@ -79,7 +95,7 @@ def create_species(): error = True parts = tuple(name.strip() for name in scientific_name.split(" ")) - if len(parts) != 2 or not all(bool(name) for name in parts): + if (len(parts) != 2 and len(parts) != 3) or not all(bool(name) for name in parts): flash("The scientific name you provided is invalid.", "alert-danger") error = True @@ -113,7 +129,15 @@ def create_species(): species = save_species( conn, common_name, scientific_name, family, taxon_id) - flash("Species saved successfully!", "alert-success") + flash( + f"You have successfully added species " + f"'{escape(species['scientific_name'])} " + f"({escape(species['common_name'])})'.", + "alert-success") + + return_to = request.form.get("return_to").strip() + if return_to: + return redirect(url_for(return_to, species_id=species["species_id"])) return redirect(url_for("species.view_species", species_id=species["species_id"])) diff --git a/uploader/static/css/layout-common.css b/uploader/static/css/layout-common.css new file mode 100644 index 0000000..9c9d034 --- /dev/null +++ b/uploader/static/css/layout-common.css @@ -0,0 +1,21 @@ +* { + box-sizing: border-box; +} + +body { + display: grid; + grid-gap: 1em; +} + +#header { + margin: -0.7em; /* Fill entire length of screen */ + /* Define layout for the children elements */ + display: grid; +} + +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + display: flex; +} diff --git a/uploader/static/css/layout-large.css b/uploader/static/css/layout-large.css new file mode 100644 index 0000000..c1950b1 --- /dev/null +++ b/uploader/static/css/layout-large.css @@ -0,0 +1,63 @@ +@media screen and (min-width: 20.1in) { + body { + grid-template-columns: 7fr 3fr; + } + + #header { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 1fr 9fr; + } + + #header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + + /* Content styling */ + padding-left: 1em; + } + + #header #header-nav { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; + } + + #main { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 7fr 3fr; + grid-gap: 1.5em; + } + + #main #breadcrumbs { + grid-column-start: 1; + grid-column-end: 3; + padding: 0 3px; + + margin: -0.3em -0.7em 0 -0.7em; + } + + #main #main-content { + max-width: 950px; + + grid-column-start: 1; + grid-column-end: 2; + overflow-x: auto; + } + + #main #sidebar-content { + grid-column-start: 2; + grid-column-end: 3; + padding: 1em 0 0 0; + } +} diff --git a/uploader/static/css/layout-medium.css b/uploader/static/css/layout-medium.css new file mode 100644 index 0000000..a29411d --- /dev/null +++ b/uploader/static/css/layout-medium.css @@ -0,0 +1,62 @@ +@media screen and (width > 8in) and (max-width: 20in) { + body { + grid-template-columns: 65fr 35fr; + } + + #header { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 2fr 8fr; + } + + #header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + + /* Content styling */ + padding-left: 1em; + } + + #header #header-nav { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; + } + + #main { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 7fr 3fr; + grid-gap: 5px; + } + + #main #breadcrumbs { + grid-column-start: 1; + grid-column-end: 3; + padding: 0 3px; + } + + #main #main-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + + /* Define layout for the children elements */ + max-width: 100%; + overflow-x: auto; + } + + #main #sidebar-content { + grid-column-start: 2; + grid-column-end: 3; + } +} diff --git a/uploader/static/css/layout-small.css b/uploader/static/css/layout-small.css new file mode 100644 index 0000000..87dd910 --- /dev/null +++ b/uploader/static/css/layout-small.css @@ -0,0 +1,66 @@ +@media screen and (max-width: 8in) { + body { + display: grid; + grid-template-columns: 1fr; + grid-template-rows: 1fr 90fr; + grid-gap: 1em; + } + + #header { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 1fr; + } + + #header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + + /* Content styling */ + padding-left: 1em; + } + + #header #header-nav { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + } + + #header #header-nav ul { + display: grid; + grid-template-columns: 1fr; + } + + #main { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; + display: grid; + + /* Define layout for the children elements */ + grid-template-rows: 1fr 80fr 20fr; + grid-template-columns: 1fr; + } + + #main #breadcrumbs { + grid-row-start: 1; + grid-row-end: 2; + + } + + #main #main-content { + grid-row-start: 2; + grid-row-end: 3; + overflow-x: auto; + } + + #main #sidebar-content { + grid-row-start: 3; + grid-row-end: 4; + } +} diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css index f482c1b..df50dec 100644 --- a/uploader/static/css/styles.css +++ b/uploader/static/css/styles.css @@ -1,161 +1,187 @@ +* { + box-sizing: border-box; +} + body { margin: 0.7em; - box-sizing: border-box; display: grid; - grid-template-columns: 1fr 6fr; - grid-template-rows: 5em 100%; + grid-template-columns: 2fr 8fr; grid-gap: 20px; - font-family: Georgia, Garamond, serif; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; font-style: normal; + font-size: 20px; } #header { - grid-column: 1/3; - width: 100%; - /* background: cyan; */ - padding-top: 0.5em; - border-radius: 0.5em; + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 8fr 2fr; + /* Content styling */ background-color: #336699; - border-color: #080808; color: #FFFFFF; - background-image: none; + border-radius: 3px; + min-height: 30px; } -#header .header { - font-size: 2em; - display: inline-block; - text-align: start; -} +#header #header-text { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; -#header .header-nav { - display: inline-block; - color: #FFFFFF; + /* Content styling */ + padding-left: 1em; } -#header .header-nav li { - border-width: 1px; - border-color: #FFFFFF; - vertical-align: middle; - margin: 0.2em; - border-style: solid; - border-width: 2px; - border-radius: 0.5em; - text-align: center; +#header #header-nav { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; } -#header .header-nav a { +#header #header-nav .nav li a { + /* Content styling */ color: #FFFFFF; - text-decoration: none; + background: #4477AA; + border: solid 5px #336699; + border-radius: 5px; + font-size: 0.7em; + text-align: center; + padding: 1px 7px; } #nav-sidebar { - grid-column: 1/2; - /* background: #e5e5ff; */ - padding-top: 0.5em; - border-radius: 0.5em; - font-size: 1.2em; + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 2; } -#main { - grid-column: 2/3; - width: 100%; - /* background: gray; */ +#nav-sidebar .nav li a:hover { border-radius: 0.5em; } -.pagetitle { - padding-top: 0.5em; - /* background: pink; */ +#nav-sidebar .nav .activemenu { + border-style: solid; border-radius: 0.5em; - /* background-color: #6699CC; */ - /* background-color: #77AADD; */ - background-color: #88BBEE; + border-color: #AAAAAA; + background-color: #EFEFEF; } -.pagetitle h1 { - text-align: start; - text-transform: capitalize; - padding-left: 0.25em; -} +#main { + /* Place it in the parent element */ + grid-column-start: 2; + grid-column-end: 3; -.pagetitle .breadcrumb { - background: none; + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 1fr; + grid-template-rows: 4em 100%; + grid-gap: 1em; } -.pagetitle .breadcrumb .active a { - color: #333333; +#main #pagetitle { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Content-styling */ + border-radius: 3px; + background-color: #88BBEE; } -.pagetitle .breadcrumb a { - color: #666666; +#main #pagetitle .title { + font-size: 1.4em; + text-transform: capitalize; + padding-left: 0.5em; } -.main-content { - font-size: 1.275em; +@media screen and (max-width: 20in) { + #main #all-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + max-width: 80%; + } + + #sidebar-content { + display: none; + } } -.breadcrumb { - text-transform: capitalize; +@media screen and (min-width: 20.1in) { + #main #all-content { + /* Place it in the parent element */ + grid-column-start: 1; + grid-column-end: 3; + + /* Define layout for the children elements */ + display: grid; + grid-template-columns: 7fr 3fr; + grid-gap: 1.5em; + } } -dd { - margin-left: 3em; - font-size: 0.88em; - padding-bottom: 1em; +#main #all-content .row { + margin: 0 2px; } -input[type="submit"], .btn { - text-transform: capitalize; +#main #all-content #main-content { + background: #FFFFFF; + max-width: 950px; } -.card { - margin-top: 0.3em; - border-width: 1px; - border-style: solid; - border-radius: 0.3em; - border-color: #AAAAAA; - padding: 0.5em; +#pagetitle .breadcrumb { + background: none; + text-transform: capitalize; + font-size: 0.75em; } -.activemenu { - border-style: solid; - border-radius: 0.5em; - border-color: #AAAAAA; - background-color: #EFEFEF; +#pagetitle .breadcrumb .active a { + color: #333333; } -.danger { - color: #A94442; - border-color: #DCA7A7; - background-color: #F2DEDE; +#pagetitle .breadcrumb a { + color: #666666; } .heading { border-bottom: solid #EEBB88; + text-transform: capitalize; } .subheading { padding: 1em 0 0.1em 0.5em; border-bottom: solid #88BBEE; + text-transform: capitalize; +} + +input[type="search"] { + border-radius: 5px; } -form { - margin-top: 0.3em; - background: #E5E5FF; - padding: 0.5em; - border-radius:0.5em; +.btn { + text-transform: Capitalize; } -form .form-control { - background-color: #EAEAFF; +table.dataTable thead th, table.dataTable tfoot th{ + border-right: 1px solid white; + color: white; + background-color: #369 !important; } -.sidebar-content .card .card-title { - font-size: 1.5em; +table.dataTable tbody tr.selected td { + background-color: #ffee99 !important; } -.sidebar-content .card-text table tbody td:nth-child(1) { - font-weight: bolder; +.form-group { + margin-bottom: 2em; + padding-bottom: 0.2em; + border-bottom: solid gray 1px; } diff --git a/uploader/static/css/theme.css b/uploader/static/css/theme.css new file mode 100644 index 0000000..6f5cb0c --- /dev/null +++ b/uploader/static/css/theme.css @@ -0,0 +1,102 @@ +body { + margin: 0.7em; + font-family: "Helvetica Neue", Helvetica, Arial, sans-serif; + font-style: normal; + font-size: 20px; +} + +#header { + background-color: #336699; + color: #FFFFFF; + min-height: 30px; + border-bottom: solid black 1px; +} + +#header #header-nav .nav li a { + /* Content styling */ + color: #FFFFFF; + font-size: 0.7em; + text-align: center; + padding: 1px 7px; + text-decoration: none; +} + +#main #breadcrumbs { + text-align: center; + background-color: #D5D5D5; + padding: 0 1em 0 1em; +} + +#main #breadcrumbs .breadcrumb { + padding-top: 0.5em; +} + +#main #main-content { + border-radius: 5px; + padding: 0 5px; +} + +#main #sidebar-content { + background: #FEFEFE; + + border-radius: 5px; + padding: 10px 5px; +} + +#main .row { + margin: 0 2px; +} + + +.heading { + border-bottom: solid #EEBB88; + text-transform: capitalize; +} + +.subheading { + padding: 1em 0 0.1em 0.5em; + border-bottom: solid #88BBEE; + text-transform: capitalize; +} + +label { + text-transform: Capitalize; +} + +input[type="search"] { + border-radius: 5px; +} + +.btn { + text-transform: Capitalize; +} + +table.dataTable thead th, table.dataTable tfoot th{ + border-right: 1px solid white; + color: white; + background-color: #369 !important; +} + +table.dataTable tbody tr.selected td { + background-color: #ffee99 !important; +} + +#frm-add-phenotypes .form-group { + margin-bottom: 2em; + padding-bottom: 0.2em; + border-bottom: solid #A9A9A9 1px; +} + + +.breadcrumb-item { + text-transform: Capitalize; +} + +.breadcrumb-item a { + text-decoration: none; +} + +.table thead tr th { + text-align: center; + vertical-align: middle; +} diff --git a/uploader/static/images/frontpage_banner.png b/uploader/static/images/frontpage_banner.png Binary files differnew file mode 100644 index 0000000..d25e1c9 --- /dev/null +++ b/uploader/static/images/frontpage_banner.png diff --git a/uploader/static/js/datatables.js b/uploader/static/js/datatables.js new file mode 100644 index 0000000..bfcda2a --- /dev/null +++ b/uploader/static/js/datatables.js @@ -0,0 +1,97 @@ +/** Handlers for events in datatables **/ + +var addTableLength = (menuList, lengthToAdd, dataLength) => { + if(dataLength >= lengthToAdd) { + newList = structuredClone(menuList);//menuList.slice(0, menuList.length); // shallow copy + newList.push(lengthToAdd); + return newList; + } + return menuList; +}; + +var defaultLengthMenu = (data) => { + menuList = [] + var lengths = [10, 25, 50, 100, 1000]; + if(data.length > 1000) { + lengths.push(data.length) + } + lengths.forEach((len) => { + menuList = addTableLength(menuList, len, data.length); + }); + return menuList; +}; + +var setRowCheckableProperty = (node, state) => { + /** + * Set a row's (`node`) checkbox's or radio button's checked state to the + * boolean value `state`. + **/ + if(typeof(state) == "boolean") { + var pseudoclass = state == false ? ":checked" : ":not(:checked)"; + var checkable = ( + $(node).find(`input[type="checkbox"]${pseudoclass}`)[0] + || + $(node).find(`input[type="radio"]${pseudoclass}`)[0]); + $(checkable).prop("checked", state); + } else { + throw new Error("`state` *MUST* be a boolean value.") + } +}; + +var setRowChecked = (node) => {setRowCheckableProperty(node, true);}; +var setRowUnchecked = (node) => {setRowCheckableProperty(node, false);}; + +var buildDataTable = (tableId, data = [], columns = [], userSettings = {}) => { + var defaultSettings = { + responsive: true, + layout: { + topStart: null, + topEnd: null, + bottomStart: null, + bottomEnd: null, + }, + select: true, + lengthMenu: defaultLengthMenu(data), + language: { + processing: "Processing… Please wait.", + loadingRecords: "Loading table data… Please wait.", + lengthMenu: "", + info: "" + }, + drawCallback: function (settings) { + var api = this.api(); + api.rows({selected: true}).nodes().each((node, index) => { + setRowChecked(node); + }); + api.rows({selected: false}).nodes().each((node, index) => { + setRowUnchecked(node); + }); + } + } + var theDataTable = $(tableId).DataTable({ + ...defaultSettings, + ...userSettings, + ...(data.length == 0 ? {} : {data: data}), + ...(columns.length == 0 ? {} : {columns: columns}) + }); + theDataTable.on("select", (event, datatable, type, indexes) => { + datatable + .rows(indexes) + .nodes() + .each((node, index) => { + setRowChecked(node); + }); + }); + theDataTable.on("deselect", (event, datatable, type, indexes) => { + datatable + .rows(indexes) + .nodes() + .each(function(node, index) { + setRowUnchecked(node); + }); + }); + + theDataTable.selectAll = () => {theDataTable.rows().select()}; + theDataTable.deselectAll = () => {theDataTable.rows().deselect()}; + return theDataTable; +}; diff --git a/uploader/static/js/debug.js b/uploader/static/js/debug.js new file mode 100644 index 0000000..eb01209 --- /dev/null +++ b/uploader/static/js/debug.js @@ -0,0 +1,40 @@ +/** + * The entire purpose of this function is for use to debug values inline + * without changing the flow of the code too much. + * + * This **MUST** be a non-arrow function to allow access to the `arguments` + * object. + * + * This function expects at least one argument. + * + * If more than one argument is provided, then: + * a) the last argument is considered the value, and will be returned + * b) all other arguments will be converted to string and output + * + * If only one argument is provided, it is considered the value, and will be + * returned. + * + * Zero arguments is an error condition. + **/ +function __pk__(val) { + /* Handle zero arguments */ + if (arguments.length < 1) { + throw new Error("Invalid arguments: Expected at least one argument."); + } + + msg = "/********** DEBUG **********/"; + if (arguments.length > 1) { + msg = Array.from( + arguments + ).slice( + 0, + arguments.length - 1 + ).map((val) => { + return String(val); + }).join("; ") + } + + value = arguments[arguments.length - 1]; + console.debug("/********** " + msg + " **********/", value); + return value; +} diff --git a/uploader/static/js/files.js b/uploader/static/js/files.js index 9d6bca1..7532df3 100644 --- a/uploader/static/js/files.js +++ b/uploader/static/js/files.js @@ -84,8 +84,9 @@ var errorHandler = makeResumableHandler("error"); var markResumableDragAndDropElement = (resumable, fileinput, droparea, browsebutton) => { if(resumable.support) { //Hide file input element and display drag&drop UI - add_class(fileinput, "hidden"); - remove_class(droparea, "hidden"); + add_class( + fileinput.closest(".non-resumable-elements"), "visually-hidden"); + remove_class(droparea, "visually-hidden"); // Define UI elements for browse and drag&drop resumable.assignDrop(droparea); @@ -96,7 +97,7 @@ var markResumableDragAndDropElement = (resumable, fileinput, droparea, browsebut }; -var makeResumableElement = (targeturi, fileinput, droparea, uploadbutton, filetype) => { +var makeResumableElement = (targeturi, droparea, filetype) => { var resumable = Resumable({ target: targeturi, fileType: filetype, @@ -116,3 +117,255 @@ var makeResumableElement = (targeturi, fileinput, droparea, uploadbutton, filety return resumable; }; + + +var CSVFilesMetadata = () => { + return { + "separator": $("#txt-file-separator").val(), + "comment_char": $( + "#txt-file-comment-character").val(), + "na_strings": $("#txt-file-na").val() + } +}; + + +var updatePreview = (table, filedata, formdata, numrows) => { + table.find("thead tr").remove() + table.find(".data-row").remove(); + var linenum = 0; + var tableheader = table.find("thead"); + var tablebody = table.find("tbody"); + var numheadings = 0; + var comment_chars = formdata + .comment_char + .split(" ") + .map((v) => {return v.trim();}) + .filter((v) => {return Boolean(v);}); + var navalues = formdata + .na_strings + .split(" ") + .map((v) => {return v.trim();}) + .filter((v) => {return Boolean(v);}); + filedata.forEach((line) => { + if(comment_chars.includes(line[0]) || linenum >= numrows) { + return false; + } + var row = $("<tr></tr>"); + line.split(formdata.separator) + .map((field) => { + var value = field.trim(); + if(navalues.includes(value)) { + return "[NO-VALUE]"; + } + return value; + }) + .filter((field) => { + return (field !== "" && field != undefined && field != null); + }) + .forEach((field) => { + if(linenum == 0) { + numheadings += 1; + var tablefield = $("<th></th>"); + tablefield.text(field); + row.append(tablefield); + } else { + add_class(row, "data-row"); + var tablefield = $("<td></td>"); + tablefield.text(field); + row.append(tablefield); + } + }); + + if(linenum == 0) { + tableheader.append(row); + } else { + tablebody.append(row); + } + linenum += 1; + }); + + if(table.find("tbody tr.data-row").length > 0) { + add_class(table.find(".data-row-template"), "visually-hidden"); + } else { + remove_class(table.find(".data-row-template"), "visually-hidden"); + } +}; + + +var makePreviewUpdater = (preview_table, preview_rows) => { + return (data) => { + updatePreview( + preview_table, + data, + CSVFilesMetadata(), + preview_rows); + }; +}; + + +var resumableDisplayFiles = (display_area, files) => { + files.forEach((file) => { + display_area.find(".file-display").remove(); + var display_element = display_area + .find(".file-display-template") + .clone(); + remove_class(display_element, "visually-hidden"); + remove_class(display_element, "file-display-template"); + add_class(display_element, "file-display"); + display_element.find(".filename").text(file.name + || file.fileName + || file.relativePath + || file.webkitRelativePath); + display_element.find(".filesize").text( + (file.size / (1024*1024)).toFixed(2) + "MB"); + display_element.find(".fileuniqueid").text(file.uniqueIdentifier); + display_element.find(".filemimetype").text(file.file.type); + display_area.append(display_element); + }); +}; + + +var indicateProgress = (resumable, progress_bar) => { + return () => {/*Has no event!*/ + var progress = (resumable.progress() * 100).toFixed(2); + var pbar = progress_bar.find(".progress-bar"); + remove_class(progress_bar, "visually-hidden"); + pbar.css("width", progress+"%"); + pbar.attr("aria-valuenow", progress); + pbar.text("Uploading: " + progress + "%"); + }; +}; + + +var retryUpload = (retry_button, cancel_button) => { + retry_button.on("click", (event) => { + resumable.files.forEach((file) => {file.retry();}); + add_class(retry_button, "visually-hidden"); + remove_class(cancel_button, "visually-hidden"); + add_class(browse_button, "visually-hidden"); + }); +}; + + +var cancelUpload = (cancel_button, retry_button) => { + cancel_button.on("click", (event) => { + resumable.files.forEach((file) => { + if(file.isUploading()) { + file.abort(); + } + }); + add_class(cancel_button, "visually-hidden"); + remove_class(retry_button, "visually-hidden"); + remove_class(browse_button, "visually-hidden"); + }); +}; + + +var startUpload = (browse_button, retry_button, cancel_button) => { + return (event) => { + remove_class(cancel_button, "visually-hidden"); + add_class(retry_button, "visually-hidden"); + add_class(browse_button, "visually-hidden"); + }; +}; + + +var makeFormSubmitter = function (form, processForm, num_files) { + var uploaded_files = new Set(); + + return function (new_file) { + uploaded_files.add(new_file); + if(uploaded_files.size === num_files) { + if(form.length !== 1) { + // TODO: Handle error somehow? + alert("The form was not provided. Bailing!"); + return false; + } + + $.ajax({ + "url": form.attr("action"), + "type": "POST", + "data": processForm(form[0], uploaded_files),// `uploaded_files` changes with each file added -- check this and fix. + "beforeSend": function(xhr) { + xhr.setRequestHeader("Accept", "application/json"); + }, + "processData": false, + "contentType": false, + "success": (data, textstatus, jqxhr) => { + // TODO: Redirect to endpoint that should come as part of the + // success/error message. + console.log("SUCCESS DATA: ", data); + console.log("SUCCESS STATUS: ", textstatus); + console.log("SUCCESS jqXHR: ", jqxhr); + window.location.assign(window.location.origin + data["redirect-to"]); + }, + }); + return false; + } + return false; + }; +}; + + +var uploadSuccess = (file_input_name, submitForm) => { + return (file, message) => { + submitForm({...JSON.parse(message), "file-input-name": file_input_name}); + }; +}; + + +var uploadError = (submitButton) => { + return (message, file) => { + submitButton.removeAttr("disabled"); + console.log("THE FILE:", file); + console.log("THE ERROR MESSAGE:", message); + }; +}; + +var makeResumableObject = ( + form_id, file_input_id, resumable_element_id, preview_table_id, submitForm, filetypes=["csv", "tsv", "txt"], preview_rows=5 +) => { + var the_form = $("#" + form_id); + var file_input = $("#" + file_input_id); + var submit_button = the_form.find("input[type=submit]"); + if(file_input.length != 1) { + return false; + } + var r = errorHandler( + fileSuccessHandler( + uploadStartHandler( + filesAddedHandler( + markResumableDragAndDropElement( + makeResumableElement( + the_form.attr("data-resumable-target"), + $("#" + resumable_element_id), + filetypes), + file_input, + $("#" + resumable_element_id), + $("#" + resumable_element_id + "-browse-button")), + (files) => { + // TODO: Also trigger preview! + resumableDisplayFiles( + $("#" + resumable_element_id + "-selected-files"), files); + files.forEach((file) => { + readFirstNLines( + file.file, + 100, + [makePreviewUpdater( + $("#" + preview_table_id), + preview_rows)]) + }); + }), + startUpload($("#" + resumable_element_id + "-browse-button"), + $("#" + resumable_element_id + "-retry-button"), + $("#" + resumable_element_id + "-cancel-button"))), + uploadSuccess(file_input.attr("name"), submitForm)), + uploadError(submit_button)); + + /** Setup progress indicator **/ + progressHandler( + r, + indicateProgress(r, $("#" + resumable_element_id + "-progress-bar"))); + + return r; +}; diff --git a/uploader/static/js/misc.js b/uploader/static/js/misc.js deleted file mode 100644 index cf7b39e..0000000 --- a/uploader/static/js/misc.js +++ /dev/null @@ -1,6 +0,0 @@ -"Miscellaneous functions and event-handlers" - -$(".not-implemented").click((event) => { - event.preventDefault(); - alert("This feature is not implemented yet. Please bear with us."); -}); diff --git a/uploader/static/js/populations.js b/uploader/static/js/populations.js new file mode 100644 index 0000000..111ebb7 --- /dev/null +++ b/uploader/static/js/populations.js @@ -0,0 +1,36 @@ +$(() => { + var populationsDataTable = buildDataTable( + "#tbl-select-population", + JSON.parse( + $("#tbl-select-population").attr("data-populations-list")), + [ + { + data: (apopulation) => { + return `<input type="radio" name="population_id"` + + `id="rdo_population_id_${apopulation.InbredSetId}" ` + + `value="${apopulation.InbredSetId}" ` + + `class="chk-row-select">`; + } + }, + { + searchable: true, + data: (apopulation) => { + return `${apopulation.FullName} (${apopulation.InbredSetName})`; + } + } + ], + { + select: "single", + paging: true, + scrollY: 500, + deferRender: true, + scroller: true, + scrollCollapse: true, + layout: { + topStart: "info", + topEnd: "search", + bottomStart: "pageLength", + bottomEnd: false + } + }); +}); diff --git a/uploader/static/js/pubmed.js b/uploader/static/js/pubmed.js new file mode 100644 index 0000000..f425f49 --- /dev/null +++ b/uploader/static/js/pubmed.js @@ -0,0 +1,113 @@ +var extract_details = (pubmed_id, details) => { + var months = { + "jan": "January", + "feb": "February", + "mar": "March", + "apr": "April", + "may": "May", + "jun": "June", + "jul": "July", + "aug": "August", + "sep": "September", + "oct": "October", + "nov": "November", + "dec": "December" + }; + var _date = details[pubmed_id].pubdate.split(" "); + return { + "authors": details[pubmed_id].authors.map((authobj) => { + return authobj.name; + }), + "title": details[pubmed_id].title, + "journal": details[pubmed_id].fulljournalname, + "volume": details[pubmed_id].volume, + "pages": details[pubmed_id].pages, + "month": _date.length > 1 ? (months[_date[1].toLowerCase()] || "January") : "January", + "year": _date[0], + }; +}; + +var update_publication_details = (details) => { + Object.entries(details).forEach((entry) => {; + switch(entry[0]) { + case "authors": + $("#txt-publication-authors").val(entry[1].join(", ")); + break; + case "month": + $("#select-publication-month") + .children("option") + .each((index, child) => { + console.debug(entry[1].toLowerCase()); + child.selected = child.value == entry[1].toLowerCase(); + }); + default: + $("#txt-publication-" + entry[0]).val(entry[1]); + break; + } + }); +}; + +var fetch_publication_abstract = (pubmed_id, pub_details) => { + $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi", + { + "method": "GET", + "data": { + "db": "pubmed", + "id": pubmed_id, + "rettype": "abstract", + "retmode": "xml" + }, + "success": (data, textStatus, jqXHR) => { + update_publication_details({ + ...pub_details, + ...{ + "abstract": Array.from(data + .getElementsByTagName( + "Abstract")[0] + .children) + .map((elt) => {return elt.textContent.trim();}) + .join("\r\n") + }}); + }, + "error": (jqXHR, textStatus, errorThrown) => {}, + "complete": (jqXHR, textStatus) => {}, + "dataType": "xml" + }); +}; + +var fetch_publication_details = (pubmed_id, complete_thunks) => { + error_display = $("#search-pubmed-id-error"); + error_display.text(""); + add_class(error_display, "visually-hidden"); + $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", + { + "method": "GET", + "data": {"db": "pubmed", "id": pubmed_id, "format": "json"}, + "success": (data, textStatus, jqXHR) => { + // process and update publication details + hasError = ( + Object.hasOwn(data, "error") || + Object.hasOwn(data.result[pubmed_id], "error")); + if(hasError) { + error_display.text( + "There was an error fetching a publication with " + + "the given PubMed ID! The error received " + + "was: '" + ( + data.error || + data.result[pubmed_id].error) + + "'. Please check ID you provided and try " + + "again."); + remove_class(error_display, "visually-hidden"); + } else { + fetch_publication_abstract( + pubmed_id, + extract_details(pubmed_id, data.result)); + } + }, + "error": (jqXHR, textStatus, errorThrown) => {}, + "complete": () => { + complete_thunks.forEach((thunk) => {thunk()}); + }, + "dataType": "json" + }); +}; diff --git a/uploader/static/js/species.js b/uploader/static/js/species.js new file mode 100644 index 0000000..fb0d2d2 --- /dev/null +++ b/uploader/static/js/species.js @@ -0,0 +1,34 @@ +$(() => { + var speciesDataTable = buildDataTable( + "#tbl-select-species", + JSON.parse( + $("#tbl-select-species").attr("data-species-list")), + [ + { + data: (aspecies) => { + return `<input type="radio" name="species_id"` + + `id="rdo_species_id_${aspecies.SpeciesId}" ` + + `value="${aspecies.SpeciesId}" class="chk-row-select">`; + } + }, + { + data: (aspecies) => { + return `${aspecies.FullName} (${aspecies.SpeciesName})`; + } + } + ], + { + select: "single", + paging: true, + scrollY: 500, + deferRender: true, + scroller: true, + scrollCollapse: true, + layout: { + topStart: "info", + topEnd: "search", + bottomStart: "pageLength", + bottomEnd: false + } + }); +}); diff --git a/uploader/static/js/upload_samples.js b/uploader/static/js/upload_samples.js index aed536f..1c25a1d 100644 --- a/uploader/static/js/upload_samples.js +++ b/uploader/static/js/upload_samples.js @@ -87,20 +87,20 @@ function display_preview(event) { var data_preview_table = document.getElementById("tbl:samples-preview"); remove_rows(data_preview_table); - var separator = document.getElementById("select:separator").value; + var separator = document.getElementById("select-separator").value; if(separator === "other") { - separator = document.getElementById("txt:separator").value; + separator = document.getElementById("txt-separator").value; } if(separator == "") { display_error_row(data_preview_table, "Please provide a separator."); return false; } - var delimiter = document.getElementById("txt:delimiter").value; + var delimiter = document.getElementById("txt-delimiter").value; - var firstlineheading = document.getElementById("chk:heading").checked; + var firstlineheading = document.getElementById("chk-heading").checked; - var fileelement = document.getElementById("file:samples"); + var fileelement = document.getElementById("file-samples"); var preview_data = JSON.parse( fileelement.getAttribute("data-preview-content") || "[]"); if(preview_data.length == 0) { @@ -115,18 +115,18 @@ function display_preview(event) { delimiter)); } -document.getElementById("chk:heading").addEventListener( +document.getElementById("chk-heading").addEventListener( "change", display_preview); -document.getElementById("select:separator").addEventListener( +document.getElementById("select-separator").addEventListener( "change", display_preview); -document.getElementById("txt:separator").addEventListener( +document.getElementById("txt-separator").addEventListener( "keyup", display_preview); -document.getElementById("txt:delimiter").addEventListener( +document.getElementById("txt-delimiter").addEventListener( "keyup", display_preview); -document.getElementById("file:samples").addEventListener( +document.getElementById("file-samples").addEventListener( "change", (event) => { read_first_n_lines(event, - document.getElementById("file:samples"), + document.getElementById("file-samples"), 30, - document.getElementById("chk:heading").checked); + document.getElementById("chk-heading").checked); }); diff --git a/uploader/static/js/urls.js b/uploader/static/js/urls.js new file mode 100644 index 0000000..e3fb7c6 --- /dev/null +++ b/uploader/static/js/urls.js @@ -0,0 +1,26 @@ +function baseURL() { + return new URL(`${window.location.protocol}//${window.location.host}`); +}; + +function buildURLFromCurrentURL(pathname, searchParams = new URLSearchParams()) { + var uri = baseURL(); + uri.pathname=pathname; + var _search = new URLSearchParams(window.location.search); + searchParams.forEach(function(value, key) { + _search.set(key, value); + }); + uri.search = _search.toString(); + return uri +}; + +function deleteSearchParams(url, listOfParams = []) { + _params = new URLSearchParams(url.search); + listOfParams.forEach(function(paramName) { + _params.delete(paramName); + }); + + + newUrl = new URL(url.toString()); + newUrl.search = _params.toString(); + return newUrl; +} diff --git a/uploader/static/js/utils.js b/uploader/static/js/utils.js index 045dd47..62d3662 100644 --- a/uploader/static/js/utils.js +++ b/uploader/static/js/utils.js @@ -8,3 +8,31 @@ function trigger_change_event(element) { evt = new Event("change"); element.dispatchEvent(evt); } + + +var remove_class = (element, classvalue) => { + new_classes = (element.attr("class") || "").split(" ").map((val) => { + return val.trim(); + }).filter((val) => { + return ((val !== classvalue) && + (val !== "")) + }).join(" "); + + if(new_classes === "") { + element.removeAttr("class"); + } else { + element.attr("class", new_classes); + } +}; + + +var add_class = (element, classvalue) => { + remove_class(element, classvalue); + element.attr("class", + ((element.attr("class") || "") + " " + classvalue).trim()); +}; + +$(".not-implemented").click((event) => { + event.preventDefault(); + alert("This feature is not implemented yet. Please bear with us."); +}); diff --git a/uploader/templates/background-jobs/base.html b/uploader/templates/background-jobs/base.html new file mode 100644 index 0000000..7201207 --- /dev/null +++ b/uploader/templates/background-jobs/base.html @@ -0,0 +1,10 @@ +{%extends "base.html"%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.list_jobs')}}"> + background jobs + </a> +</li> +{%endblock%} diff --git a/uploader/templates/background-jobs/delete-job.html b/uploader/templates/background-jobs/delete-job.html new file mode 100644 index 0000000..242c775 --- /dev/null +++ b/uploader/templates/background-jobs/delete-job.html @@ -0,0 +1,61 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}"> + summary + </a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">background jobs: delete?</h2> + + <p class="text-danger">Are you sure you want to delete the job below?</p> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + <form id="frm-delete-job" + method="POST" + action="{{url_for('background-jobs.delete_single', job_id=job.job_id)}}"> + <div class="row"> + <div class="col"> + <input type="submit" + class="btn btn-info" + value="cancel" + name="btn-confirm-delete" /> + </div> + <div class="col"> + <input type="submit" + class="btn btn-danger" + value="delete" + name="btn-confirm-delete" /> + </div> + </div> + </form> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>Confirm whether or not you want to delete job + <strong>{{job.job_id}}</strong>.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/background-jobs/job-status.html b/uploader/templates/background-jobs/job-status.html new file mode 100644 index 0000000..2e75c6d --- /dev/null +++ b/uploader/templates/background-jobs/job-status.html @@ -0,0 +1,45 @@ +{%extends "background-jobs/base.html"%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5" /> +{%endblock%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">job status</h2> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + <div class="col"> + <a href="{{url_for('background-jobs.stop_job', job_id=job.job_id)}}" + title="Stop/Kill this job." + class="btn btn-danger">stop job</a> + </div> +</div> + +<div class="row"> + <h3 class="subheading">STDOUT</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stdout"]}}</pre> + </div> +</div> + +<div class="row"> + <h3 class="subheading">STDERR</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stderr"]}}</pre> + </div> +</div> + +{%endblock%} diff --git a/uploader/templates/background-jobs/job-summary.html b/uploader/templates/background-jobs/job-summary.html new file mode 100644 index 0000000..ef9ef6c --- /dev/null +++ b/uploader/templates/background-jobs/job-summary.html @@ -0,0 +1,75 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}"> + summary + </a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">background jobs: summary</h2> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + {%if view_under_construction%} + <div class="col"> + <a href="#" + class="btn btn-info not-implemented" + title="Update the expiry date and time for this job.">update expiry</a> + </div> + + {%if job.metadata.status in ("stopped",)%} + <div class="col"> + <a href="#" + class="btn btn-warning not-implemented" + title="Create a new copy of this job, and run the copy.">Run Copy</a> + </div> + {%endif%} + {%endif%} + + <div class="col"> + <a href="{{url_for('background-jobs.delete_single', job_id=job.job_id)}}" + class="btn btn-danger" + title="Delete this job.">delete</a> + </div> +</div> + +<div class="row"> + <h3 class="subheading">Script Errors and Logging</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stderr"]}}</pre> + </div> +</div> + +<div class="row"> + <h3 class="subheading">Script Output</h3> + <div style="max-width: 40em; overflow: scroll"> + <pre>{{job["stdout"]}}</pre> + </div> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>This page shows the results of running job '{{job.job_id}}'.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/background-jobs/list-jobs.html b/uploader/templates/background-jobs/list-jobs.html new file mode 100644 index 0000000..c16b850 --- /dev/null +++ b/uploader/templates/background-jobs/list-jobs.html @@ -0,0 +1,79 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"><h2 class="heading">Background Jobs</h2></div> + +<div class="row"> + <div class="table-responsive"> + <table class="table"> + <thead> + <tr class="table-primary"> + <th>Type</th> + <th>Created</th> + <th title="Date and time past which the job's details will be deleted from the system."> + Expires</th> + <th>Status</th> + <th>Actions</th> + </tr> + </thead> + + <tbody> + {%for job in jobs%} + <tr> + <td>{{job.metadata["job-type"]}}</td> + <td>{{display_datetime(job.created)}}</td> + <td title="Date and time past which the job's details will be deleted from the system."> + {{display_datetime(job.expires)}} + </td> + <td {%if job.metadata.status == "completed"%} + class="fw-bold text-capitalize text-success" + {%elif job.metadata.status == "error"%} + class="fw-bold text-capitalize text-danger" + {%elif job.metadata.status == "stopped"%} + class="fw-bold text-capitalize text-warning" + {%else%} + class="fw-bold text-capitalize text-info" + {%endif%}> + <div> + {{job.metadata.status}} + </div> + </td> + <td> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}" + class="btn btn-info" + title="View more detailed information about this job."> + view summary</a> + </td> + </tr> + {%else%} + <tr> + <td colspan="5"> + You do not have any jobs you have run in the background.</td> + </tr> + {%endfor%} + </tbody> + </table> + </div> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>The table lists the jobs that are running in the background, that you + started.</p> + <p>You can use the tools provided on this page to manage the jobs, and to view + each job's details.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/background-jobs/macro-display-job-details.html b/uploader/templates/background-jobs/macro-display-job-details.html new file mode 100644 index 0000000..82e33c0 --- /dev/null +++ b/uploader/templates/background-jobs/macro-display-job-details.html @@ -0,0 +1,29 @@ +{%macro display_job_details(job, display_datetime)%} +<table class="table"> + <thead> + </thead> + + <tbody> + <tr> + <th class="table-primary">Job ID</th> + <td>{{job.job_id}}</td> + </tr> + <tr> + <th class="table-primary">Type</th> + <td>{{job.metadata["job-type"]}}</td> + </tr> + <tr> + <th class="table-primary">Created</th> + <td>{{display_datetime(job.created)}}</td> + </tr> + <tr> + <th class="table-primary">Expires</th> + <td>{{display_datetime(job.expires)}}</td> + </tr> + <tr> + <th class="table-primary">Status</th> + <td>{{job.metadata.status}}</td> + </tr> + </tbody> +</table> +{%endmacro%} diff --git a/uploader/templates/background-jobs/stop-job.html b/uploader/templates/background-jobs/stop-job.html new file mode 100644 index 0000000..fc190ac --- /dev/null +++ b/uploader/templates/background-jobs/stop-job.html @@ -0,0 +1,61 @@ +{%extends "background-jobs/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "background-jobs/macro-display-job-details.html" import display_job_details%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('background-jobs.job_summary', job_id=job.job_id)}}"> + summary + </a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">background jobs: stop?</h2> + + <p class="text-danger">Are you sure you want to stop the job below?</p> + + {{display_job_details(job, display_datetime)}} +</div> + +<div class="row"> + <form id="frm-stop-job" + method="POST" + action="{{url_for('background-jobs.stop_job', job_id=job.job_id)}}"> + <div class="row"> + <div class="col"> + <input type="submit" + class="btn btn-info" + value="cancel" + name="btn-confirm-stop" /> + </div> + <div class="col"> + <input type="submit" + class="btn btn-danger" + value="stop" + name="btn-confirm-stop" /> + </div> + </div> + </form> +</div> +{%endblock%} + + +{%block sidebarcontents%} +<div class="row"> + <h6 class="subheading">What is this?</h6> +</div> +<div class="row"> + <p>Confirm whether or not you want to stop job + <strong>{{job.job_id}}</strong>.</p> +</div> +{{super()}} +{%endblock%} diff --git a/uploader/templates/base.html b/uploader/templates/base.html index 3a8ef16..ae4ecef 100644 --- a/uploader/templates/base.html +++ b/uploader/templates/base.html @@ -8,126 +8,114 @@ <meta name="viewport" content="width=device-width, initial-scale=1.0" /> {%block extrameta%}{%endblock%} - <title>GN Uploader: {%block title%}{%endblock%}</title> + <title>Data Upload and Quality Control: {%block title%}{%endblock%}</title> <link rel="stylesheet" type="text/css" href="{{url_for('base.bootstrap', filename='css/bootstrap.min.css')}}" /> <link rel="stylesheet" type="text/css" - href="{{url_for('base.bootstrap', - filename='css/bootstrap-theme.min.css')}}" /> - <link rel="stylesheet" type="text/css" href="/static/css/styles.css" /> + href="{{url_for('base.datatables', + filename='css/dataTables.bootstrap5.min.css')}}" /> + <link rel="stylesheet" type="text/css" href="/static/css/layout-common.css" /> + <link rel="stylesheet" type="text/css" href="/static/css/layout-large.css" /> + <link rel="stylesheet" type="text/css" href="/static/css/layout-medium.css" /> + <link rel="stylesheet" type="text/css" href="/static/css/layout-small.css" /> + <link rel="stylesheet" type="text/css" href="/static/css/theme.css" /> {%block css%}{%endblock%} </head> <body> - <header id="header" class="container-fluid"> - <div class="row"> - <span class="header col-lg-9">GeneNetwork Data Quality Control and Upload</span> - <nav class="header-nav col-lg-3"> - <ul class="nav justify-content-end"> - <li> - {%if user_logged_in()%} - <a href="{{url_for('oauth2.logout')}}" - title="Log out of the system">{{user_email()}} — Log Out</a> - {%else%} - <a href="{{authserver_authorise_uri()}}" - title="Log in to the system">Log In</a> - {%endif%} - </li> - </ul> - </nav> + <header id="header"> + <span id="header-text">GeneNetwork</span> + <nav id="header-nav"> + <ul class="nav"> + {%if user_logged_in()%} + <li> + <a href="{{url_for('background-jobs.list_jobs')}}" + title="User's background jobs."> + <!-- https://icons.getbootstrap.com/icons/back/ --> + <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-back" viewBox="0 0 16 16"> + <path d="M0 2a2 2 0 0 1 2-2h8a2 2 0 0 1 2 2v2h2a2 2 0 0 1 2 2v8a2 2 0 0 1-2 2H6a2 2 0 0 1-2-2v-2H2a2 2 0 0 1-2-2zm2-1a1 1 0 0 0-1 1v8a1 1 0 0 0 1 1h8a1 1 0 0 0 1-1V2a1 1 0 0 0-1-1z"/> + </svg> + Background jobs + </a> + </li> + + <li> + <a href="{{url_for('oauth2.logout')}}" + title="Log out of the system"> + <!-- https://icons.getbootstrap.com/icons/file-person/ --> + <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-file-person" viewBox="0 0 16 16"> + <path d="M12 1a1 1 0 0 1 1 1v10.755S12 11 8 11s-5 1.755-5 1.755V2a1 1 0 0 1 1-1zM4 0a2 2 0 0 0-2 2v12a2 2 0 0 0 2 2h8a2 2 0 0 0 2-2V2a2 2 0 0 0-2-2z"/> + <path d="M8 10a3 3 0 1 0 0-6 3 3 0 0 0 0 6"/> + </svg> + <span class="glyphicon glyphicon-user"></span> + Sign Out ({{user_email()}})</a> + </li> + {%else%} + <li> + <a href="{{authserver_authorise_uri()}}" + title="Log in to the system">Sign In</a> + {%endif%} + </li> + </ul> + </nav> </header> - <aside id="nav-sidebar" class="container-fluid"> - <ul class="nav flex-column"> - <li {%if activemenu=="home"%}class="activemenu"{%endif%}> - <a href="/" >Home</a></li> - <li {%if activemenu=="species"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.list_species')}}" - title="View and manage species information.">Species</a></li> - <li {%if activemenu=="platforms"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.platforms.index')}}" - title="View and manage species platforms.">Sequencing Platforms</a></li> - <li {%if activemenu=="populations"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.populations.index')}}" - title="View and manage species populations.">Populations</a></li> - <li {%if activemenu=="samples"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.populations.samples.index')}}" - title="Upload population samples.">Samples</a></li> - <li {%if activemenu=="genotypes"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.populations.genotypes.index')}}" - title="Upload Genotype data.">Genotype Data</a></li> - <!-- - TODO: Maybe include menus here for managing studies and dataset or - maybe have the studies/datasets managed under their respective - sections, e.g. "Publish*" studies/datasets under the "Phenotypes" - section, "ProbeSet*" studies/datasets under the "Expression Data" - sections, etc. - --> - <li {%if activemenu=="phenotypes"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.populations.phenotypes.index')}}" - title="Upload phenotype data.">Phenotype Data</a></li> - <li {%if activemenu=="expression-data"%}class="activemenu"{%endif%}> - <a href="{{url_for('species.populations.expression-data.index')}}" - title="Upload expression data." - class="not-implemented">Expression Data</a></li> - <li {%if activemenu=="individuals"%}class="activemenu"{%endif%}> - <a href="#" - class="not-implemented" - title="Upload individual data.">Individual Data</a></li> - <li {%if activemenu=="rna-seq"%}class="activemenu"{%endif%}> - <a href="#" - class="not-implemented" - title="Upload RNA-Seq data.">RNA-Seq Data</a></li> - <li {%if activemenu=="async-jobs"%}class="activemenu"{%endif%}> - <a href="#" - class="not-implemented" - title="View and manage the backgroud jobs you have running"> - Background Jobs</a></li> - </ul> - </aside> - - <main id="main" class="main container-fluid"> - - <div class="pagetitle row"> - <h1>GN Uploader: {%block pagetitle%}{%endblock%}</h1> - <nav> - <ol class="breadcrumb"> - <li {%if activelink is not defined or activelink=="home"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('base.index')}}">Home</a> - </li> - {%block lvl1_breadcrumbs%}{%endblock%} - </ol> - </nav> - </div> - - <div class="row"> - <div class="container-fluid"> - <div class="col-md-8 main-content"> - {%block contents%}{%endblock%} - </div> - <div class="sidebar-content col-md-4"> - {%block sidebarcontents%}{%endblock%} - </div> + + <main id="main" class="main"> + <nav id="breadcrumbs" aria-label="breadcrumb"> + <ol class="breadcrumb"> + {%block breadcrumbs%} + <li class="breadcrumb-item"> + <a href="{{url_for('base.index')}}">Home</a></li> + {%endblock%} + </ol> + </nav> + + <div id="main-content"> + {%block contents%}{%endblock%} + </div> + + <div id="sidebar-content"> + {%block sidebarcontents%}{%endblock%} </div> - </div> </main> + + <script type="text/javascript" src="/static/js/debug.js"></script> + <!-- + Core dependencies + --> <script src="{{url_for('base.jquery', filename='jquery.min.js')}}"></script> <script src="{{url_for('base.bootstrap', filename='js/bootstrap.min.js')}}"></script> - <script type="text/javascript" src="/static/js/misc.js"></script> - {%block javascript%}{%endblock%} - </body> + <!-- + DataTables dependencies + --> + <script type="text/javascript" + src="{{url_for('base.datatables', + filename='js/dataTables.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='scroller/js/dataTables.scroller.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='buttons/js/dataTables.buttons.min.js')}}"></script> + <script type="text/javascript" + src="{{url_for('base.datatables_extensions', + filename='select/js/dataTables.select.min.js')}}"></script> + <!-- + local dependencies + --> + <script type="text/javascript" src="/static/js/utils.js"></script> + <script type="text/javascript" src="/static/js/datatables.js"></script> + {%block javascript%}{%endblock%} + </body> </html> diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html index 64b1a9a..9cff09d 100644 --- a/uploader/templates/cli-output.html +++ b/uploader/templates/cli-output.html @@ -1,7 +1,7 @@ {%macro cli_output(job, stream)%} <h4 class="subheading">{{stream | upper}} Output</h4> -<div class="cli-output" style="max-height: 10em; overflow: auto;"> +<div class="cli-output" style="overflow: auto;"> <pre>{{job.get(stream, "")}}</pre> </div> diff --git a/uploader/templates/flash_messages.html b/uploader/templates/flash_messages.html index b7af178..b42e64e 100644 --- a/uploader/templates/flash_messages.html +++ b/uploader/templates/flash_messages.html @@ -1,11 +1,11 @@ {%macro flash_all_messages()%} {%with messages = get_flashed_messages(with_categories=true)%} {%if messages:%} -<ul> +<div> {%for category, message in messages:%} - <li class="{{category}}">{{message}}</li> + <div class="alert {{category}}">{{message}}</div> {%endfor%} -</ul> +</div> {%endif%} {%endwith%} {%endmacro%} @@ -13,13 +13,13 @@ {%macro flash_messages(filter_class)%} {%with messages = get_flashed_messages(with_categories=true)%} {%if messages:%} -<ul> +<div> {%for category, message in messages:%} {%if filter_class in category%} - <li class="{{category}}">{{message}}</li> + <div class="alert {{category}}">{{message}}</div> {%endif%} {%endfor%} -</ul> +</div> {%endif%} {%endwith%} {%endmacro%} diff --git a/uploader/templates/genotypes/add-genotypes-records-base.html b/uploader/templates/genotypes/add-genotypes-records-base.html new file mode 100644 index 0000000..bf3812f --- /dev/null +++ b/uploader/templates/genotypes/add-genotypes-records-base.html @@ -0,0 +1,39 @@ +{%extends "genotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Genotypes{%endblock%} + +{%block pagetitle%}Genotypes{%endblock%} + +{%block contents%} + +<div class="row"> + <form id="frm-add-genotypes-records" + method="POST" + enctype="multipart/form-data" + action="{{url_for( + 'species.populations.genotypes.add_genotype_records', + species_id=species.SpeciesId, population_id=population.Id, + dataset_id=dataset.Id)}}" + data-resumable-target="{{url_for('files.resumable_upload_post')}}"> + <legend>Add New Genotype Records</legend> + + {%block frm_add_genotypes_records_elements%}{%endblock%} + + <div class="form-group"> + <input type="submit" + value="upload genotypes" + class="btn btn-primary" /> + </div> + </form> +</div> + +<div class="row"> + <h2 class="heading" id="page-documentation">Help</h2> + {%block page_documentation%}{%endblock%} +</div> +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/genotypes/add-genotypes-records-csv.html b/uploader/templates/genotypes/add-genotypes-records-csv.html new file mode 100644 index 0000000..6ebc005 --- /dev/null +++ b/uploader/templates/genotypes/add-genotypes-records-csv.html @@ -0,0 +1,147 @@ +{%extends "genotypes/add-genotypes-records-base.html"%} +{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%} +{%from "macro-csv-fields.html" import display_csv_fields%} +{%from "macro-csv-fields.html" import add_csv_fields_event_handlers%} +{%from "macro-csv-fields.html" import display_csv_fields_documentation%} +{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%} + +{%block frm_add_genotypes_records_elements%} +<div class="form-text help-block"> + <p>You can add new genotype records here.</p> +</div> + +<div class="border rounded p-3 mb-3" role="group" aria-labelledby="file-inputs-group-label"> + <div id="file-inputs-group-label" class="fw-bold mb-2">File Details</div> + {{display_csv_fields()}} + + <div class="form-group"> + <div class="non-resumable-elements"> + <div class="row mb-3"> + <label for="finput-genotypes-records-file" class="col-sm-2 col-form-label"> + genotypes records</label> + <div class="col-sm-10"> + <input id="finput-genotypes-records-file" + name="genotypes-records-file" + class="form-control" + type="file" + data-preview-table="tbl-preview-geno-records" + required="required" /> + </div> + <span class="form-text text-muted"> + Provide a file that contains only the genotypes records, + <a href="#docs-file-genotypes-records-csv" + title="Documentation of the genotypes records file format."> + the documentation for the expected format of the file</a>.</span> + </div> + </div> + {{display_resumable_elements( + "resumable-genotypes-records-file", + "Genotypes records", + '<p>Drag and drop the CSV file here, that contains the genotype records you + want to add.</p> + + <p>Please see the + <a href="#docs-file-genotypes-records" + title="Documentation of the genotypes records data file format."> + "Genotypes records" documentation</a> section below for more + information on the expected format of the file provided here.</p>')}} + {{display_preview_table("tbl-preview-geno-records", "genotypes records")}} + </div> +</div> +{%endblock%} + +{%block page_documentation%} +{{super()}} + +<h3 class="sub-heading">CSV file metadata</h3> +{{display_csv_fields_documentation()}} +{%endblock%} + +{%block javascript%} +{{super()}} + +<script src="{{url_for('base.node_modules', + filename='resumablejs/resumable.js')}}"></script> +<script src="/static/js/files.js"></script> + +{{add_csv_fields_event_handlers()}} + +<script type="text/javascript"> + $(function(evt) { + var NUM_READ_LINES = 100; + var NUM_PREVIEW_ROWS = 10; + + $("#finput-genotypes-records-file").on("change", function(event) { + readFirstNLines( + event.target.files[0], + NUM_READ_LINES, + [makePreviewUpdater($("#tbl-preview-geno-records"))]); + }); + + var r = makeResumableObject( + form_id="frm-add-genotypes-records", + file_input_id="finput-genotypes-records-file", + resumable_element_id="resumable-genotypes-records-file", + preview_table_id="tbl-preview-geno-records", + makeFormSubmitter( + $("#frm-add-genotypes-records"), + function(form, uploaded_files) { + var formdata = new FormData(form); + uploaded_files.forEach((msg) => { + formdata.delete(msg["file-input-name"]); + formdata.append(msg["file-input-name"], JSON.stringify({ + "uploaded-file": msg["uploaded-file"], + "original-name": msg["original-name"] + })); + }); + formdata.append("resumable-upload", "true"); + return formdata; + }, + 1), + filetypes=["csv", "tsv", "txt", "geno"], + preview_rows=NUM_PREVIEW_ROWS); + + var handler_update_previews = function(event) { + var preview_table = $("#tbl-preview-geno-records"); + var file_input = $("#finput-genotypes-records-file"); + if(file_input[0].files.length > 0) { + readFirstNLines( + file_input[0].files[0], + NUM_READ_LINES, + [makePreviewUpdater(preview_table, NUM_PREVIEW_ROWS)]); + } + + if(typeof(r) !== "undefined") { + if(r.files.length > 0) { + readFirstNLines( + r.files[0].file, + NUM_READ_LINES, + [makePreviewUpdater(preview_table, NUM_PREVIEW_ROWS)]); + } + } + }; + + [ + "#txt-file-separator", + "#txt-file-comment-character", + "#txt-file-na" + ].forEach((elementid) => { + $(elementid).on("change", handler_update_previews); + }); + + $("#frm-add-genotypes-records input[type=submit]").on("click", function(event) { + event.preventDefault(); + var submit_button = event.target; + submit_button.setAttribute("disabled", "disabled"); + r.upload(); + + try { + var filename = r.files[0].name; + } catch (error) { + window.alert("You MUST provide a file before attempting to upload."); + submit_button.removeAttribute("disabled"); + } + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/genotypes/base.html b/uploader/templates/genotypes/base.html index 7d61312..c2abc63 100644 --- a/uploader/templates/genotypes/base.html +++ b/uploader/templates/genotypes/base.html @@ -1,23 +1,23 @@ {%extends "populations/base.html"%} +{%from "populations/macro-display-population-card.html" import display_sui_population_card%} +{%from "genotypes/macro-display-dataset-card.html" import display_dataset_card%} -{%block lvl3_breadcrumbs%} -<li {%if activelink=="genotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - {%if population is mapping%} - <a href="{{url_for('species.populations.genotypes.list_genotypes', - species_id=species.SpeciesId, - population_id=population.Id)}}"> - {%if dataset is defined and dataset is mapping%} - {{dataset.Name}} - {%else%} - Genotypes - {%endif%}</a> - {%else%} - <a href="{{url_for('species.populations.genotypes.index')}}">Genotypes</a> - {%endif%} +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('species.populations.genotypes.index', + species_id=species['SpeciesId'], + population_id=population['Id'])}}"> + genotype + </a> </li> -{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} + + +{%block sidebarcontents%} +{%if dataset is defined and dataset is not none%} +{{display_dataset_card(species, population, dataset)}} +{%else%} +{{display_sui_population_card(species, population)}} +{%endif%} {%endblock%} diff --git a/uploader/templates/genotypes/create-dataset.html b/uploader/templates/genotypes/create-dataset.html index 10331c1..7f435a1 100644 --- a/uploader/templates/genotypes/create-dataset.html +++ b/uploader/templates/genotypes/create-dataset.html @@ -35,13 +35,13 @@ id="txt-geno-dataset-name" name="geno-dataset-name" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" + readonly="readonly" /> <small class="form-text text-muted"> <p>This is a short representative, but constrained name for the genotype - dataset.<br /> - The field will only accept letters ('A-Za-z'), numbers (0-9), hyphens - and underscores. Any other character will cause the name to be - rejected.</p></small> + dataset. It is used internally by GeneNetwork.</p> + </small> </div> <div class="form-group"> @@ -50,7 +50,8 @@ id="txt-geno-dataset-fullname" name="geno-dataset-fullname" required="required" - class="form-control" /> + class="form-control" + value="{{population.Name}} Genotypes" /> <small class="form-text text-muted"> <p>This is a longer, more descriptive name for your dataset.</p></small> </div> @@ -61,7 +62,8 @@ <input type="text" id="txt-geno-dataset-shortname" name="geno-dataset-shortname" - class="form-control" /> + class="form-control" + value="{{population.Name}}Geno" /> <small class="form-text text-muted"> <p>A short name for your dataset. If you leave this field blank, the short name will be set to the same value as the diff --git a/uploader/templates/genotypes/index.html b/uploader/templates/genotypes/index.html index e749f5a..1c3483d 100644 --- a/uploader/templates/genotypes/index.html +++ b/uploader/templates/genotypes/index.html @@ -1,28 +1,200 @@ {%extends "genotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-select-species.html" import select_species_form%} {%block title%}Genotypes{%endblock%} {%block pagetitle%}Genotypes{%endblock%} - {%block contents%} {{flash_all_messages()}} + +{%if dataset is defined and dataset is not none%} + +<div class="row"> + <h2>Genotype Data</h2> + + <div class="row"> + <div class="col"> + <p> + <a href="{{url_for( + 'species.populations.genotypes.add_genotype_records', + species_id=species.SpeciesId, population_id=population.Id, + dataset_id=dataset.Id)}}" + class="btn btn-primary"> + Add genotype records + </a> + </p> + </div> + </div> + + <div class="table-responsive"> + <table id="tbl-genotype-records" class="table compact stripe cell-border"> + <thead> + <tr> + <th title="">#</th> + <th title="">Index</th> + <th title="Locus of marker on the chromosome">Locus</th> + <th title="Chromosome">Chr</th> + <th title="Physical location of marker in centimorgans">cM</th> + <th title="Physical location of marker in megabasepairs">Mb</th> + {%for sample in samples%} + <th title="Data for sample {{sample}}">{{sample}}</th> + {%endfor%} + </tr> + </thead> + + <tbody> + {%for record in genotype_records%} + <tr> + <td> + <input type="checkbox" + id="chk-geno-record-{{record.Id}}" + name="geno_record_id" + value="{{record.Id}}" /> + </td> + <td>{{record.index}}</td> + <td>{{record.Name}}</td> + <td>{{record.Chr}}</td> + <td>{{record.cM}}</td> + <td>{{record.Mb}}</td> + {%for sample in samples%} + <td>{{record.data[sample]}}</td> + {%endfor%} + </tr> + {%else%} + <tr> + <td colspan="6" class="text-info"> + There are no records + </td> + </tr> + {%endfor%} + </tbody> + </table> + </div> +</div> + <div class="row"> - <p> - This section allows you to upload genotype information for your experiments, - in the case that you have not previously done so. - </p> - <p> - We'll need to link the genotypes to the species and population, so do please - go ahead and select those in the next two steps. - </p> + <h2>Genotype Encoding</h2> + <p>The numerical values in the table above are mapped from the following allele symbols:</p> + + <table class="table"> + <thead> + <tr> + <th>Allele Type</th> + <th>Allele Symbol</th> + <th>Mapped To</th> + </tr> + </thead> + + <tbody> + {%for row in genocode%} + <tr> + <td {%if row.AlleleType == 'mat'%} + title="Maternal allele" + {%elif row.AlleleType == "pat"%} + title="Paternal allele" + {%elif row.AlleleType == "het"%} + title="Heterozygous allele" + {%else%} + title="Unknown allele" + {%endif%}> + {{row.AlleleType}}</td> + <td>{{row.AlleleSymbol}}</td> + <td>{{row.DatabaseValue if row.DatabaseValue is not none}}</td> + </tr> + {%else%} + <tr> + <td colspan="3" class="text-info"> + There is no genotype encoding defined for this data. + </td> + </tr> + {%endfor%} + </tbody> + </table> </div> +{%else%} + <div class="row"> - {{select_species_form(url_for("species.populations.genotypes.index"), - species)}} + <p>We need to create a dataset to hold the genotype information for this + species/population, before we can proceed to upload the genotype data.</p> + <p>Please click the button below to create the dataset.</p> + + <div class="col"> + <a href="{{url_for('species.populations.genotypes.create_dataset', species_id=species.SpeciesId, population_id=population.Id)}}" + class="btn btn-primary">create genotype dataset</a> + </div> </div> + +{%endif%} + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var genoRecordsUrl = "{{url_for('species.populations.genotypes.index', species_id=species.SpeciesId, population_id=population.Id)}}"; + + var dtGenotypeRecords = false; + fetch(genoRecordsUrl, { + method: "POST", + headers: { + "Accept": "application/json", + "Content-Type": "application/json" + }, + body: JSON.stringify({}) + }) + .then(response => response.json()) + .then(recordsData => { + var records = recordsData.genotype_records; + var samples = recordsData.samples_order; + var columns = [ + { + data: function(record) { + return `<input type="checkbox"` + + `id="chk-geno-record-` + record.Id + `"` + + `name="geno_record_id"` + + `value="` + record.Id + `"` + + ` />`; + } + }, + {data: "index"}, + {data: "Name"}, + {data: "Chr"}, + {data: "cM"}, + {data: "Mb"} + ].concat(samples.map((sample) => { + return {data: (record) => record.data[sample]}; + })); + + dtGenotypeRecords = buildDataTable( + "#tbl-genotype-records", + [], + columns, + { + serverSide: true, + ajax: { + url: genoRecordsUrl, + dataSrc: "genotype_records", + recordsTotal: "total_genotype_records", + recordsFiltered: "fetched_genotype_records" + }, + paging: true, + scroller: true, + scrollY: "50vh", + scrollCollapse: false, + layout: { + top: "info", + topStart: null, + topEnd: null, + bottom: null, + bottomStart: null, + bottomEnd: null + } + }); + }); + }); +</script> {%endblock%} diff --git a/uploader/templates/genotypes/list-genotypes.html b/uploader/templates/genotypes/list-genotypes.html deleted file mode 100644 index 0f074fd..0000000 --- a/uploader/templates/genotypes/list-genotypes.html +++ /dev/null @@ -1,149 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "populations/macro-display-population-card.html" import display_population_card%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - -{%block lvl4_breadcrumbs%} -<li {%if activelink=="list-genotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.genotypes.list_genotypes', - species_id=species.SpeciesId, - population_id=population.Id)}}">List genotypes</a> -</li> -{%endblock%} - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <h2>Genetic Markers</h2> - <p>There are a total of {{total_markers}} currently registered genetic markers - for the "{{species.FullName}}" species. You can click - <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="View genetic markers for species '{{species.FullName}}"> - this link to view the genetic markers - </a>. - </p> -</div> - -<div class="row"> - <h2>Genotype Encoding</h2> - <p> - The genotype encoding used for the "{{population.FullName}}" population from - the "{{species.FullName}}" species is as shown in the table below. - </p> - <table class="table"> - - <thead> - <tr> - <th>Allele Type</th> - <th>Allele Symbol</th> - <th>Allele Value</th> - </tr> - </thead> - - <tbody> - {%for row in genocode%} - <tr> - <td>{{row.AlleleType}}</td> - <td>{{row.AlleleSymbol}}</td> - <td>{{row.DatabaseValue if row.DatabaseValue is not none else "NULL"}}</td> - </tr> - {%else%} - <tr> - <td colspan="7" class="text-info"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no explicit genotype encoding defined for this population. - </td> - </tr> - {%endfor%} - </tbody> - </table> - - {%if genocode | length < 1%} - <a href="#add-genotype-encoding" - title="Add a genotype encoding system for this population" - class="btn btn-primary not-implemented"> - add genotype encoding - </a> - {%endif%} -</div> - -<div class="row text-danger"> - <h3>Some Important Concepts to Consider/Remember</h3> - <ul> - <li>Reference vs. Non-reference alleles</li> - <li>In <em>GenoCode</em> table, items are ordered by <strong>InbredSet</strong></li> - </ul> - <h3>Possible references</h3> - <ul> - <li>https://mr-dictionary.mrcieu.ac.uk/term/genotype/</li> - <li>https://www.ncbi.nlm.nih.gov/pmc/articles/PMC7363099/</li> - </ul> -</div> - -<div class="row"> - <h2>Genotype Datasets</h2> - - <p>The genotype data is organised under various genotype datasets. You can - click on the link for the relevant dataset to view a little more information - about it.</p> - - {%if dataset is not none%} - <table class="table"> - <thead> - <tr> - <th>Name</th> - <th>Full Name</th> - </tr> - </thead> - - <tbody> - <tr> - <td>{{dataset.Name}}</td> - <td><a href="{{url_for('species.populations.genotypes.view_dataset', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}" - title="View details regarding and manage dataset '{{dataset.FullName}}'"> - {{dataset.FullName}}</a></td> - </tr> - </tbody> - </table> - {%else%} - <p class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no genotype dataset defined for this population. - </p> - <p> - <a href="{{url_for('species.populations.genotypes.create_dataset', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="Create a new genotype dataset for the '{{population.FullName}}' population for the '{{species.FullName}}' species." - class="btn btn-primary"> - create new genotype dataset</a></p> - {%endif%} -</div> -<div class="row text-warning"> - <p> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>NOTE</strong>: Currently the GN2 (and related) system(s) expect a - single genotype dataset. If there is more than one, the system apparently - fails in unpredictable ways. - </p> - <p>Fix this to allow multiple datasets, each with a different assembly from - all the rest.</p> -</div> -{%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/genotypes/list-markers.html b/uploader/templates/genotypes/list-markers.html index a705ae3..22189c7 100644 --- a/uploader/templates/genotypes/list-markers.html +++ b/uploader/templates/genotypes/list-markers.html @@ -1,20 +1,18 @@ {%extends "genotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} {%block title%}Genotypes: List Markers{%endblock%} {%block pagetitle%}Genotypes: List Markers{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="list-markers"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> <a href="{{url_for('species.populations.genotypes.list_markers', - species_id=species.SpeciesId, - population_id=population.Id)}}">List markers</a> + species_id=species['SpeciesId'], + population_id=population['Id'])}}"> + markers + </a> </li> {%endblock%} @@ -59,7 +57,7 @@ <table class="table"> <thead> <tr> - <th title="">#</th> + <th title="">Index</th> <th title="">Marker Name</th> <th title="Chromosome">Chr</th> <th title="Physical location of the marker in megabasepairs"> @@ -99,7 +97,3 @@ </div> {%endif%} {%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} diff --git a/uploader/templates/genotypes/macro-display-dataset-card.html b/uploader/templates/genotypes/macro-display-dataset-card.html new file mode 100644 index 0000000..2b197d4 --- /dev/null +++ b/uploader/templates/genotypes/macro-display-dataset-card.html @@ -0,0 +1,24 @@ +{%from "populations/macro-display-population-card.html" import display_sui_population_card%} + +{%macro display_dataset_card(species, population, dataset)%} +{{display_sui_population_card(species, population)}} +<div class="row"> + <table class="table"> + <caption>Current genotype dataset</caption> + <tbody> + <tr> + <th>Name</th> + <td>{{dataset.Name}}</td> + </tr> + <tr> + <th>Full Name</th> + <td>{{dataset.FullName}}</td> + </tr> + <tr> + <th>Short Name</th> + <td>{{dataset.ShortName}}</td> + </tr> + </tbody> + </table> +</div> +{%endmacro%} diff --git a/uploader/templates/genotypes/select-population.html b/uploader/templates/genotypes/select-population.html deleted file mode 100644 index 7c81943..0000000 --- a/uploader/templates/genotypes/select-population.html +++ /dev/null @@ -1,31 +0,0 @@ -{%extends "genotypes/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} -{%from "populations/macro-select-population.html" import select_population_form%} - -{%block title%}Genotypes{%endblock%} - -{%block pagetitle%}Genotypes{%endblock%} - - -{%block contents%} -{{flash_all_messages()}} - -<div class="row"> - <p> - You have indicated that you intend to upload the genotypes for species - '{{species.FullName}}'. We now just require the population for your - experiment/study, and you should be good to go. - </p> -</div> - -<div class="row"> - {{select_population_form(url_for("species.populations.genotypes.select_population", - species_id=species.SpeciesId), - populations)}} -</div> -{%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} diff --git a/uploader/templates/genotypes/view-dataset.html b/uploader/templates/genotypes/view-dataset.html index e7ceb36..d95a8e3 100644 --- a/uploader/templates/genotypes/view-dataset.html +++ b/uploader/templates/genotypes/view-dataset.html @@ -1,21 +1,17 @@ {%extends "genotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Genotypes: View Dataset{%endblock%} {%block pagetitle%}Genotypes: View Dataset{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="view-dataset"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> <a href="{{url_for('species.populations.genotypes.view_dataset', species_id=species.SpeciesId, population_id=population.Id, - dataset_id=dataset.Id)}}">view dataset</a> + dataset_id=dataset.Id)}}">dataset</a> </li> {%endblock%} @@ -50,12 +46,9 @@ <div class="row"> <h2>Genotype Data</h2> - <p class="text-danger"> - Provide link to enable uploading of genotype data here.</p> + <div class="col" style="margin-bottom: 3px;"> + <a href="#" class="btn btn-primary not-implemented">upload genotypes</a> + </div> </div> {%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/index.html b/uploader/templates/index.html index d6f57eb..6e9c777 100644 --- a/uploader/templates/index.html +++ b/uploader/templates/index.html @@ -1,99 +1,170 @@ {%extends "base.html"%} {%from "flash_messages.html" import flash_all_messages%} +{%from "macro-forms.html" import add_http_feature_flags%} {%block title%}Home{%endblock%} {%block pagetitle%}Home{%endblock%} +{%block extra_breadcrumbs%}{%endblock%} + {%block contents%} +{%macro add_form_buttons()%} +<div class="row form-buttons"> + <div class="col"> + <input type="submit" + class="btn btn-primary" + value="use selected species" /> + </div> + <div class="col"> + <a href="{{url_for('species.create_species', return_to='base.index')}}" + class="btn btn-outline-primary" + title="Add a new species to Genenetwork.">add a new Species</a> + </div> +</div> +{%endmacro%} + +<div class="row">{{flash_all_messages()}}</div> + +{%if user_logged_in()%} + +<div class="row"> + <ul class="nav nav-tabs" id="index-actions"> + <li class="nav-item presentation"> + <button class="nav-link active" + id="upload-data-tab" + data-bs-toggle="tab" + data-bs-target="#upload-data-content" + type="button" + role="tab" + aria-controls="upload-data-content" + aria-selected="false">Upload Data</button></li> + <li class="nav-item presentation"> + <button class="nav-link" + id="publications-tab" + data-bs-toggle="tab" + data-bs-target="#publications-content" + type="button" + role="tab" + aria-controls="publications-content" + aria-selected="true">Publications</button></li> + </ul> +</div> + +<div class="row"> + <div class="tab-content" id="upload-data-tabs-content"> + <div class="tab-pane fade show active" + id="upload-data-content" + role="tabpanel" + aria-labelledby="upload-data-content-tab"> + <h2 class="heading">Species</h2> + + <p>Select the species you want to work with.</p> + + <form method="GET" action="{{url_for('base.index')}}" class="form-horizontal"> + {{add_http_feature_flags()}} + + {{add_form_buttons()}} + + {%if species | length != 0%} + <div style="margin-top:1em;"> + <table id="tbl-select-species" class="table compact stripe" + data-species-list='{{species | tojson}}'> + <thead> + <tr> + <th></th> + <th>Species Name</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </div> + + {%else%} + + <label class="control-label" for="rdo-cant-find-species"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + There are no species to select from. Create the first one.</label> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary col-sm-offset-1" + value="continue" /> + </div> + + {%endif%} + + {{add_form_buttons()}} + + </form> + </div> + + <div class="tab-pane fade" + id="publications-content" + role="tabpanel" + aria-labelledby="publications-content-tab"> + <p>You can view, edit, and delete existing publications, as well as add + new ones, by clicking the button below.</p> + + <a href="{{url_for('publications.index')}}" + title="Manage publications." + class="btn btn-primary">manage publications</a> + </div> + </div> +</div> + +{%else%} + +<div class="row"> + <img src="/static/images/frontpage_banner.png" + alt="Banner image showing the process flow a user would follow." /> +</div> + +<div class="row"> + <p>The GeneNetwork Uploader (gn-uploader) lets you easily add new data to the + GeneNetwork System. It automatically checks your data for quality and walks + you through fixing any issues before submission.</p> +</div> + <div class="row"> - {{flash_all_messages()}} - <div class="explainer"> - <p>Welcome to the <strong>GeneNetwork Data Quality Control and Upload System</strong>. This system is provided to help in uploading your data onto GeneNetwork where you can do analysis on it.</p> - - <p>The sections below provide an overview of what features the menu items on - the left provide to you. Please peruse the information to get a good - big-picture understanding of what the system provides you and how to get - the most out of it.</p> - - {%block extrapageinfo%}{%endblock%} - - <h2>Species</h2> - - <p>The GeneNetwork service provides datasets and tools for doing genetic - studies — from - <a href="{{gn2server_intro}}" - target="_blank" - title="GeneNetwork introduction — opens in a new tab."> - its introduction</a>: - - <blockquote class="blockquote"> - <p>GeneNetwork is a group of linked data sets and tools used to study - complex networks of genes, molecules, and higher order gene function - and phenotypes. …</p> - </blockquote> - </p> - - <p>With this in mind, it follows that the data in the system is centered - aroud a variety of species. The <strong>species section</strong> will - list the currently available species in the system, and give you the - ability to add new ones, if the one you want to work on does not currently - exist on GeneNetwork</p> - - <h2>Populations</h2> - - <p>Your studies will probably focus on a particular subset of the entire - species you are interested in – your population.</p> - <p>Populations are a way to organise the species data so as to link data to - specific know populations for a particular species, e.g. The BXD - population of mice (Mus musculus)</p> - <p>In older GeneNetwork documentation, you might run into the term - <em>InbredSet</em>. Should you run into it, it is a term that we've - deprecated that essentially just means the population.</p> - - <h2>Samples</h2> - - <p>These are the samples or individuals (sometimes cases) that were involved - in the experiment, and from whom the data was derived.</p> - - <h2>Genotype Data</h2> - - <p>This section will allow you to view and upload the genetic markers for - your species, and the genotype encodings used for your particular - population.</p> - <p>While, technically, genetic markers relate to the species in general, and - not to a particular population, the data (allele information) itself - relates to the particular population it was generated from – - specifically, to the actual individuals used in the experiment.</p> - <p>This is the reason why the genotype data information comes under the - population, and will check for the prior existence of the related - samples/individuals before attempting an upload of your data.</p> - - <h2>Expression Data</h2> - - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>TODO</strong>: Document this …</p> - - <h2>Phenotype Data</h2> - - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>TODO</strong>: Document this …</p> - - <h2>Individual Data</h2> - - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>TODO</strong>: Document this …</p> - - <h2>RNA-Seq Data</h2> - - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - <strong>TODO</strong>: Document this …</p> + <div class="col"> + <a href="{{authserver_authorise_uri()}}" + title="Sign in to the system" + class="btn btn-primary">Sign in</a> </div> </div> +{%endif%} + +{%endblock%} + + + +{%block sidebarcontents%} +{%if view_under_construction%} +<div class="row"> + <p>The data in Genenetwork is related to one species or another. Use the form + provided to select from existing species, or click on the + "Create a New Species" button if you cannot find the species you want to + work with.</p> +</div> +<div class="row"> + <form id="frm-quick-navigation"> + <legend>Quick Navigation</legend> + <div class="form-group"> + <label for="fqn-species-id">Species</label> + <select name="species_id"> + <option value="">Select species</option> + </select> + </div> + </form> +</div> +{%endif%} +{%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> {%endblock%} diff --git a/uploader/templates/jobs/job-error.html b/uploader/templates/jobs/job-error.html new file mode 100644 index 0000000..b3015fc --- /dev/null +++ b/uploader/templates/jobs/job-error.html @@ -0,0 +1,17 @@ +{%extends "base.html"%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Background Jobs: Error{%endblock%} + +{%block pagetitle%}Background Jobs: Error{%endblock%} + +{%block contents%} + +<h1>Background Jobs: Error</h1> +<p>Job <strong>{{job["job_id"]}}</strong> failed!</p> +<p>The error details are in the "STDERR" section below.</p> + +<h2>STDERR</h2> +<pre>{{job["stderr"]}}</pre> +{%endblock%} diff --git a/uploader/templates/jobs/job-not-found.html b/uploader/templates/jobs/job-not-found.html new file mode 100644 index 0000000..a71e66f --- /dev/null +++ b/uploader/templates/jobs/job-not-found.html @@ -0,0 +1,11 @@ +{%extends "base.html"%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} +<p>Could not find job with ID: {{job_id}}</p> +{%endblock%} diff --git a/uploader/templates/jobs/job-status.html b/uploader/templates/jobs/job-status.html new file mode 100644 index 0000000..83c02fd --- /dev/null +++ b/uploader/templates/jobs/job-status.html @@ -0,0 +1,24 @@ +{%extends "base.html"%} + +{%from "flash_messages.html" import flash_all_messages%} + +{%block extrameta%} +<meta http-equiv="refresh" content="5" /> +{%endblock%} + +{%block title%}Background Jobs{%endblock%} + +{%block pagetitle%}Background Jobs{%endblock%} + +{%block contents%} + +<p>Status: {{job["metadata"]["status"]}}</p> +<p>Job Type: {{job["metadata"]["job-type"]}}</p> + +<h2>STDOUT</h2> +<pre>{{job["stdout"]}}</pre> + +<h2>STDERR</h2> +<pre>{{job["stderr"]}}</pre> + +{%endblock%} diff --git a/uploader/templates/login.html b/uploader/templates/login.html deleted file mode 100644 index 1f71416..0000000 --- a/uploader/templates/login.html +++ /dev/null @@ -1,11 +0,0 @@ -{%extends "index.html"%} - -{%block title%}Data Upload{%endblock%} - -{%block pagetitle%}log in{%endblock%} - -{%block extrapageinfo%} -<p class="text-dark text-primary"> - You <strong>do need to be logged in</strong> to upload data onto this system. - Please do that by clicking the "Log In" button at the top of the page.</p> -{%endblock%} diff --git a/uploader/templates/macro-csv-fields.html b/uploader/templates/macro-csv-fields.html new file mode 100644 index 0000000..b58a6d0 --- /dev/null +++ b/uploader/templates/macro-csv-fields.html @@ -0,0 +1,139 @@ +{%macro display_csv_fields()%} +<div class="form-group"> + <div class="row mb-3"> + <label for="txt-file-separator" class="col-sm-3 col-form-label"> + File Separator</label> + <div class="col-sm-9"> + <div class="input-group"> + <input id="txt-file-separator" + name="file-separator" + type="text" + value="	" + class="form-control" + maxlength="1" /> + <span class="input-group-btn"> + <button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button> + </span> + </div> + </div> + <span class="form-text text-muted"> + Provide the character that separates the fields in your file(s). It should + be the same character for all files (if more than one is provided).<br /> + A tab character will be assumed if you leave this field blank. See + <a href="#docs-file-separator" + title="Documentation for file-separator characters"> + documentation for more information</a>. + </span> + </div> +</div> + +<div class="form-group"> + <div class="row mb-3"> + <label for="txt-file-comment-character" class="col-sm-3 col-form-label">File Comment-Characters</label> + <div class="col-sm-9"> + <div class="input-group"> + <input id="txt-file-comment-character" + name="file-comment-character" + type="text" + value="#" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-reset-file-comment-character" class="btn btn-info"> + Reset Default</button> + </span> + </div> + </div> + <span class="form-text text-muted"> + This specifies that lines that begin with the character(s) provided will be + considered comment lines and ignored in their entirety. See + <a href="#docs-file-comment-character" + title="Documentation for comment characters"> + documentation for more information</a>. + </span> + </div> +</div> + +<div class="form-group"> + <div class="row mb-3"> + <label for="txt-file-na" class="col-sm-3 col-form-label">File "No-Value" Indicators</label> + <div class="col-sm-9"> + <div class="input-group"> + <input id="txt-file-na" + name="file-na" + type="text" + value="- NA N/A" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-reset-file-na" class="btn btn-info">Reset Default</button> + </span> + </div> + </div> + <span class="form-text text-muted"> + This specifies strings in your file indicate that there is no value for a + particular cell (a cell is where a column and row intersect). Provide a + space-separated list of strings if you have more than one way of + indicating no values. See + <a href="#docs-file-na" title="Documentation for no-value fields"> + documentation for more information</a>.</span> + </div> +</div> +{%endmacro%} + + +{%macro display_csv_fields_documentation()%} +<dl> + <dt id="docs-file-separator">File separator</dt> + <dd>The files you provide should be character-separated value (CSV) files. + We need to know what character you used to separate the values in your + file. Some common ones are the Tab character, the comma, etc.<br /> + Providing that information makes it possible for the system to parse and + process your files correctly.<br> + <strong>NOTE:</strong> All the files you upload MUST use the same + separator.</dd> + + <dt id="docs-file-comment-character">Comment characters</dt> + <dd>We support use of comment lines in your files. We only support one type + of comment style, the <em>line comment</em>.<br /> + This mean the comment begins at the start of the line, and the end of that + line indicates the end of that comment. If you have a really long comment, + then you need to break it across multiple lines, marking each line a + comment line.<br /> + The "comment character" is the character at the start of the line that + indicates that the line is a line comment.<br /> + You can provide more than one comment character, separated by spaces.</dd> + + <dt id="docs-file-na">No-Value indicator(s)</dt> + <dd>Data in the real world is messy, and in some cases, entirely absent. You + need to indicate, in your files, that a particular field did not have a + value, and once you do that, you then need to let the system know how you + mark such fields. Common ways of indicating "empty values" are, leaving + the field blank, using a character such as '-', or using strings like + "NA", "N/A", "NULL", etc.<br /> + Providing this information will help with parsing and processing such + no-value fields the correct way.</dd> +</dl> +{%endmacro%} + + +{%macro add_csv_fields_event_handlers()%} +<script type="text/javascript"> + $(function(evt) { + /* The reset buttons */ + $("#btn-reset-file-separator").on("click", (event) => { + event.preventDefault(); + $("#txt-file-separator").val("\t"); + $("#txt-file-separator").trigger("change"); + }); + $("#btn-reset-file-comment-character").on("click", (event) => { + event.preventDefault(); + $("#txt-file-comment-character").val("#"); + $("#txt-file-comment-character").trigger("change"); + }); + $("#btn-reset-file-na").on("click", (event) => { + event.preventDefault(); + $("#txt-file-na").val("- NA N/A"); + $("#txt-file-na").trigger("change"); + }); + }); +</script> +{%endmacro%} diff --git a/uploader/templates/macro-forms.html b/uploader/templates/macro-forms.html new file mode 100644 index 0000000..0ccab32 --- /dev/null +++ b/uploader/templates/macro-forms.html @@ -0,0 +1,9 @@ +{%macro add_http_feature_flags()%} +{%for flag in http_feature_flags():%} +{%if (request.args.get(flag) or request.form.get(flag) or ""):%} +<input type="hidden" + name="{{flag}}" + value="{{(request.args.get(flag) or request.form.get(flag))}}" /> +{%endif%} +{%endfor%} +{%endmacro%} diff --git a/uploader/templates/macro-step-indicator.html b/uploader/templates/macro-step-indicator.html new file mode 100644 index 0000000..ac0be77 --- /dev/null +++ b/uploader/templates/macro-step-indicator.html @@ -0,0 +1,15 @@ +{%macro step_indicator(step, width=100)%} +<svg width="{{width}}" height="{{width}}" xmlns="http://www.w3.org/2000/svg"> + <circle cx="{{0.5*width}}" + cy="{{0.5*width}}" + r="{{0.5*width}}" + fill="#E5E5FF" /> + <text x="{{0.5*width}}" + y="{{0.6*width}}" + font-size="{{0.2*width}}" + text-anchor="middle" + fill="#555555"> + Step {{step}} + </text> +</svg> +{%endmacro%} diff --git a/uploader/templates/phenotypes/add-phenotypes-base.html b/uploader/templates/phenotypes/add-phenotypes-base.html index 97b55f2..3207129 100644 --- a/uploader/templates/phenotypes/add-phenotypes-base.html +++ b/uploader/templates/phenotypes/add-phenotypes-base.html @@ -1,26 +1,13 @@ {%extends "phenotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} {%from "macro-table-pagination.html" import table_pagination%} -{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} {%block title%}Phenotypes{%endblock%} {%block pagetitle%}Phenotypes{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="add-phenotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.phenotypes.add_phenotypes', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}">Add Phenotypes</a> -</li> -{%endblock%} - {%block contents%} +{{super()}} {{flash_all_messages()}} <div class="row"> @@ -42,111 +29,29 @@ {%block frm_add_phenotypes_elements%}{%endblock%} - <div class="checkbox"> - <label> - <input id="chk-published" type="checkbox" name="published?" /> - These phenotypes are published</label> - </div> - - <fieldset id="fldset-publication-info" class="hidden"> - <legend>Publication Information</legend> - <div class="form-group"> - <label for="txt-pubmed-id" class="form-label">Pubmed ID</label> - <div class="input-group"> - <input id="txt-pubmed-id" name="pubmed-id" type="text" - class="form-control" /> - <span class="input-group-btn"> - <button id="btn-search-pubmed-id" class="btn btn-info">Search</button> - </span> - </div> - <span id="search-pubmed-id-error" - class="form-text text-muted text-danger hidden"> - </span><br /> - <span class="form-text text-muted"> - Enter your publication's PubMed ID above and click "Search" to search - for some (or all) of the publication details requested below. - </span> - </div> - - <div class="form-group"> - <label for="txt-publication-authors" class="form-label">Authors</label> - <input id="txt-publication-authors" name="publication-authors" - type="text" class="form-control" /> - <span class="form-text text-muted"> - Enter the authors in the following format …</span> - </div> - - <div class="form-group"> - <label for="txt-publication-title" class="form-label"> - Publication Title</label> - <input id="txt-publication-title" name="publication-title" type="text" - class="form-control" /> - <span class="form-text text-muted"> - Enter your publication's title.</span> - </div> - - <div class="form-group"> - <label for="txt-publication-abstract" class="form-label"> - Publication Abstract</label> - <textarea id="txt-publication-abstract" name="publication-abstract" - class="form-control" rows="10"></textarea> - <span class="form-text text-muted"> - Enter the abstract for your publication.</span> - </div> - - <div class="form-group"> - <label for="txt-publication-journal" class="form-label">Journal</label> - <input id="txt-publication-journal" name="journal" type="text" - class="form-control" /> - <span class="form-text text-muted"> - Enter the name of the journal where your work was published.</span> - </div> - - <div class="form-group"> - <label for="txt-publication-volume" class="form-label">Volume</label> - <input id="txt-publication-volume" name="publication-volume" type="text" - class="form-control" /> - <span class="form-text text-muted"> - Enter the volume in the following format …</span> - </div> - - <div class="form-group"> - <label for="txt-publication-pages" class="form-label">Pages</label> - <input id="txt-publication-pages" name="publication-pages" type="text" - class="form-control" /> - <span class="form-text text-muted"> - Enter the journal volume where your work was published.</span> - </div> - - <div class="form-group"> - <label for="select-publication-month" class="form-label"> - Publication Month</label> - <select id="select-publication-month" name="publication-month" - class="form-control"> - {%for month in monthnames%} - <option value="{{month | lower}}" - {%if current_month | lower == month | lower%} - selected="selected" - {%endif%}>{{month | capitalize}}</option> - {%endfor%} - </select> - <span class="form-text text-muted"> - Select the month when the work was published. - <span class="text-danger"> - This cannot be before, say 1600 and cannot be in the future!</span></span> - </div> - - <div class="form-group"> - <label for="txt-publication-year" class="form-label">Publication Year</label> - <input id="txt-publication-year" name="publication-year" type="text" - class="form-control" value="{{current_year}}" /> - <span class="form-text text-muted"> - Enter the year your work was published. - <span class="text-danger"> - This cannot be before, say 1600 and cannot be in the future!</span> - </span> - </div> - </fieldset> + <h4>Publication Information</h4> + <input type="hidden" name="publication-id" id="txt-publication-id" /> + <span class="form-text text-muted"> + Select a publication for your data. <br /> + Can't find a publication you can use? Go ahead and + <a href="{{url_for( + 'publications.create_publication', + return_to='species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">create a new publication</a>.</span> + <table id="tbl-select-publication" class="table compact stripe"> + <thead> + <tr> + <th>Index</th> + <th>PubMed ID</th> + <th>Title</th> + <th>Authors</th> + </tr> + </thead> + + <tbody></tbody> + </table> <div class="form-group"> <input type="submit" @@ -163,167 +68,82 @@ {%endblock%} -{%block javascript%} -<script type="text/javascript"> - var remove_class = (element, classvalue) => { - new_classes = (element.attr("class") || "").split(" ").map((val) => { - return val.trim(); - }).filter((val) => { - return ((val !== classvalue) && - (val !== "")) - }).join(" "); - - if(new_classes === "") { - element.removeAttr("class"); - } else { - element.attr("class", new_classes); - } - }; - - var add_class = (element, classvalue) => { - remove_class(element, classvalue); - element.attr("class", (element.attr("class") || "") + " " + classvalue); - }; - $("#chk-published").on("click", (event) => { - pub_details = $("#fldset-publication-info") - if(event.target.checked) { - // display the publication details - remove_class(pub_details, "hidden"); - } else { - // hide the publication details - add_class(pub_details, "hidden"); - } - }); - - var extract_details = (pubmed_id, details) => { - var months = { - "jan": "January", - "feb": "February", - "mar": "March", - "apr": "April", - "may": "May", - "jun": "June", - "jul": "July", - "aug": "August", - "sep": "September", - "oct": "October", - "nov": "November", - "dec": "December" - }; - var _date = details[pubmed_id].pubdate.split(" "); - return { - "authors": details[pubmed_id].authors.map((authobj) => { - return authobj.name; - }), - "title": details[pubmed_id].title, - "journal": details[pubmed_id].fulljournalname, - "volume": details[pubmed_id].volume, - "pages": details[pubmed_id].pages, - "month": _date.length > 1 ? months[_date[1].toLowerCase()] : "jan", - "year": _date[0], - }; - }; - var update_publication_details = (details) => { - Object.entries(details).forEach((entry) => {; - switch(entry[0]) { - case "authors": - $("#txt-publication-authors").val(entry[1].join(", ")); - break; - case "month": - $("#select-publication-month") - .children("option") - .each((index, child) => { - child.selected = child.value == entry[1].toLowerCase(); - }); - default: - $("#txt-publication-" + entry[0]).val(entry[1]); - break; - } +{%block javascript%} +<script type="text/javascript"> + $(function() { + var publicationsDataTable = buildDataTable( + "#tbl-select-publication", + [], + [ + {data: "index"}, + { + searchable: true, + data: (pub) => { + if(pub.PubMed_ID) { + return `<a href="https://pubmed.ncbi.nlm.nih.gov/` + + `${pub.PubMed_ID}/" target="_blank" ` + + `title="Link to publication on NCBI. This will ` + + `open in a new tab.">` + + `${pub.PubMed_ID}</a>`; + } + return ""; + } + }, + { + searchable: true, + data: (pub) => { + var title = "⸻"; + if(pub.Title) { + title = pub.Title + } + return title; + } + }, + { + searchable: true, + data: (pub) => { + authors = pub.Authors.split(",").map( + (item) => {return item.trim();}); + if(authors.length > 1) { + return authors[0] + ", et. al."; + } + return authors[0]; + } + } + ], + { + serverSide: true, + ajax: { + url: "/publications/list", + dataSrc: "publications" + }, + select: "single", + paging: true, + scrollY: 700, + deferRender: true, + scroller: true, + scrollCollapse: true, + layout: { + topStart: "info", + topEnd: "search" + } + }); + publicationsDataTable.on("select", (event, datatable, type, indexes) => { + indexes.forEach((element, index, thearray) => { + let row = datatable.row(element).node(); + console.debug(datatable.row(element).data()); + $("#frm-add-phenotypes #txt-publication-id").val( + datatable.row(element).data().Id); + }); + }); + publicationsDataTable.on("deselect", (event, datatable, type, indexes) => { + indexes.forEach((element, index, thearray) => { + let row = datatable.row(element).node(); + $("#frm-add-phenotypes #txt-publication-id").val(null); + }); }); - }; - - var fetch_publication_abstract = (pubmed_id, pub_details) => { - $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi", - { - "method": "GET", - "data": { - "db": "pubmed", - "id": pubmed_id, - "rettype": "abstract", - "retmode": "xml" - }, - "success": (data, textStatus, jqXHR) => { - update_publication_details({ - ...pub_details, - ...{ - "abstract": Array.from(data - .getElementsByTagName( - "Abstract")[0] - .children) - .map((elt) => {return elt.textContent.trim();}) - .join("\r\n") - }}); - }, - "error": (jqXHR, textStatus, errorThrown) => {}, - "complete": (jqXHR, textStatus) => {}, - "dataType": "xml" - }); - }; - - var fetch_publication_details = (pubmed_id, complete_thunks) => { - error_display = $("#search-pubmed-id-error"); - error_display.text(""); - add_class(error_display, "hidden"); - $.ajax("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi", - { - "method": "GET", - "data": {"db": "pubmed", "id": pubmed_id, "format": "json"}, - "success": (data, textStatus, jqXHR) => { - // process and update publication details - hasError = ( - Object.hasOwn(data, "error") || - Object.hasOwn(data.result[pubmed_id], "error")); - if(hasError) { - error_display.text( - "There was an error fetching a publication with " + - "the given PubMed ID! The error received " + - "was: '" + ( - data.error || - data.result[pubmed_id].error) + - "'. Please check ID you provided and try " + - "again."); - remove_class(error_display, "hidden"); - } else { - fetch_publication_abstract( - pubmed_id, - extract_details(pubmed_id, data.result)); - } - }, - "error": (jqXHR, textStatus, errorThrown) => {}, - "complete": () => { - complete_thunks.forEach((thunk) => {thunk()}); - }, - "dataType": "json" - }); - }; - - $("#btn-search-pubmed-id").on("click", (event) => { - event.preventDefault(); - var search_button = event.target; - var pubmed_id = $("#txt-pubmed-id").val().trim(); - remove_class($("#txt-pubmed-id").parent(), "has-error"); - if(pubmed_id == "") { - add_class($("#txt-pubmed-id").parent(), "has-error"); - return false; - } - - search_button.disabled = true; - // Fetch publication details - fetch_publication_details(pubmed_id, - [() => {search_button.disabled = false;}]); - return false; }); </script> diff --git a/uploader/templates/phenotypes/add-phenotypes-raw-files.html b/uploader/templates/phenotypes/add-phenotypes-raw-files.html index d9a8424..79556cf 100644 --- a/uploader/templates/phenotypes/add-phenotypes-raw-files.html +++ b/uploader/templates/phenotypes/add-phenotypes-raw-files.html @@ -1,7 +1,6 @@ {%extends "phenotypes/add-phenotypes-base.html"%} {%from "flash_messages.html" import flash_all_messages%} {%from "macro-table-pagination.html" import table_pagination%} -{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} {%from "phenotypes/macro-display-preview-table.html" import display_preview_table%} {%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%} @@ -9,19 +8,6 @@ {%block pagetitle%}Phenotypes{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="add-phenotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.phenotypes.add_phenotypes', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}">Add Phenotypes</a> -</li> -{%endblock%} - {%block frm_add_phenotypes_documentation%} <p>This page will allow you to upload all the separate files that make up your phenotypes. Here, you will have to upload each separate file individually. If @@ -35,8 +21,7 @@ {%endblock%} {%block frm_add_phenotypes_elements%} -<fieldset id="fldset-file-metadata"> - <legend>File(s) Metadata</legend> + <h4>File(s) Metadata</h4> <div class="form-group"> <label for="txt-file-separator" class="form-label">File Separator</label> <div class="input-group"> @@ -103,114 +88,205 @@ <a href="#docs-file-na" title="Documentation for no-value fields"> documentation for more information</a>.</span> </div> -</fieldset> -<fieldset id="fldset-data-files"> <legend>Data File(s)</legend> - <div class="form-group non-resumable-elements"> - <label for="finput-phenotype-descriptions" class="form-label"> - Phenotype Descriptions</label> - <input id="finput-phenotype-descriptions" - name="phenotype-descriptions" - class="form-control" - type="file" - data-preview-table="tbl-preview-pheno-desc" - required="required" /> - <span class="form-text text-muted"> - Provide a file that contains only the phenotype descriptions, - <a href="#docs-file-phenotype-description" - title="Documentation of the phenotype data file format."> - the documentation for the expected format of the file</a>.</span> - </div> - - {{display_resumable_elements( - "resumable-phenotype-descriptions", - "phenotype descriptions", - '<p>You can drop a CSV file that contains the phenotype descriptions here, - or you can click the "Browse" button (below and to the right) to select it - from your computer.</p> - <p>The CSV file must conform to some standards, as documented in the - <a href="#docs-file-phenotype-description" - title="Documentation of the phenotype data file format."> - "Phenotypes Descriptions" documentation</a> section below.</p>')}} - - - <div class="form-group non-resumable-elements"> - <label for="finput-phenotype-data" class="form-label">Phenotype Data</label> - <input id="finput-phenotype-data" - name="phenotype-data" - class="form-control" - type="file" - data-preview-table="tbl-preview-pheno-data" - required="required" /> - <span class="form-text text-muted"> - Provide a file that contains only the phenotype data. See - <a href="#docs-file-phenotype-data" - title="Documentation of the phenotype data file format."> - the documentation for the expected format of the file</a>.</span> - </div> + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-descriptions-transposed" + name="phenotype-descriptions-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-descriptions-transposed" + class="form-check-label"> + Description file transposed?</label> + </div> + + <div class="non-resumable-elements"> + <label for="finput-phenotype-descriptions" class="form-label"> + Phenotype Descriptions</label> + <input id="finput-phenotype-descriptions" + name="phenotype-descriptions" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-desc" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the phenotype descriptions, + <a href="#docs-file-phenotype-description" + title="Documentation of the phenotype data file format."> + the documentation for the expected format of the file</a>.</span> + </div> + {{display_resumable_elements( + "resumable-phenotype-descriptions", + "phenotype descriptions", + '<p>Drag and drop the CSV file that contains the descriptions of your + phenotypes here.</p> + + <p>The CSV file should be a matrix of + <strong>phenotypes × descriptions</strong> i.e. The first column + contains the phenotype names/identifiers whereas the first row is a list + of metadata fields like, "description", "units", etc.</p> + + <p>If the format is transposed (i.e. + <strong>descriptions × phenotypes</strong>) select the checkbox above. + </p> + + <p>Please see the + <a href="#docs-file-phenotype-description" + title="Documentation of the phenotype data file format."> + "Phenotypes Descriptions" documentation</a> section below for more + information on the expected format of the file provided here.</p>')}} + {{display_preview_table( + "tbl-preview-pheno-desc", "phenotype descriptions")}} + </div> - {{display_resumable_elements( - "resumable-phenotype-data", - "phenotype data", - '<p>You can drop a CSV file that contains the phenotype data here, - or you can click the "Browse" button (below and to the right) to select it - from your computer.</p> - <p>The CSV file must conform to some standards, as documented in the - <a href="#docs-file-phenotype-data" - title="Documentation of the phenotype data file format."> - "Phenotypes Data" documentation</a> section below.</p>')}} + + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-data-transposed" + name="phenotype-data-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-data-transposed" class="form-check-label"> + Data file transposed?</label> + </div> + + <div class="non-resumable-elements"> + <label for="finput-phenotype-data" class="form-label">Phenotype Data</label> + <input id="finput-phenotype-data" + name="phenotype-data" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-data" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the phenotype data. See + <a href="#docs-file-phenotype-data" + title="Documentation of the phenotype data file format."> + the documentation for the expected format of the file</a>.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-data", + "phenotype data", + '<p>Drag and drop a CSV file that contains the phenotypes numerical data + here. You can click the "Browse" button (below and to the right) to + select the file from your computer.</p> + + <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.</p> + + <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>) + select the checkbox above.</p> + <p>Please see the + <a href="#docs-file-phenotype-data" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below for more information + on the expected format for the file provided here.</p>')}} + {{display_preview_table("tbl-preview-pheno-data", "phenotype data")}} + </div> + {%if population.Family in families_with_se_and_n%} - <div class="form-group non-resumable-elements"> - <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label> - <input id="finput-phenotype-se" - name="phenotype-se" - class="form-control" - type="file" - data-preview-table="tbl-preview-pheno-se" - required="required" /> - <span class="form-text text-muted"> - Provide a file that contains only the standard errors for the phenotypes, - computed from the data above.</span> - </div> - {{display_resumable_elements( - "resumable-phenotype-se", - "standard errors", - '<p>You can drop a CSV file that contains the computed standard-errors data - here, or you can click the "Browse" button (below and to the right) to - select it from your computer.</p> - <p>The CSV file must conform to some standards, as documented in the - <a href="#docs-file-phenotype-se" - title="Documentation of the phenotype data file format."> - "Phenotypes Data" documentation</a> section below.</p>')}} + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-se-transposed" + name="phenotype-se-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-se-transposed" class="form-check-label"> + Standard-Errors file transposed?</label> + </div> + <div class="group non-resumable-elements"> + <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label> + <input id="finput-phenotype-se" + name="phenotype-se" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-se" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the standard errors for the phenotypes, + computed from the data above.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-se", + "standard errors", + '<p>Drag and drop a CSV file that contains the phenotypes standard-errors + data here. You can click the "Browse" button (below and to the right) to + select the file from your computer.</p> + + <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.</p> + + <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>) + select the checkbox above.</p> + + <p>Please see the + <a href="#docs-file-phenotype-se" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below for more information + on the expected format of the file provided here.</p>')}} + + {{display_preview_table("tbl-preview-pheno-se", "standard errors")}} + </div> - - <div class="form-group non-resumable-elements"> - <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label> - <input id="finput-phenotype-n" - name="phenotype-n" - class="form-control" - type="file" - data-preview-table="tbl-preview-pheno-n" - required="required" /> - <span class="form-text text-muted"> - Provide a file that contains only the number of samples/individuals used in - the computation of the standard errors above.</span> - </div> - {{display_resumable_elements( - "resumable-phenotype-n", - "number of samples/individuals", - '<p>You can drop a CSV file that contains the number of samples/individuals - used in computation of the standard-errors here, or you can click the - "Browse" button (below and to the right) to select it from your computer. - </p> - <p>The CSV file must conform to some standards, as documented in the - <a href="#docs-file-phenotype-n" - title="Documentation of the phenotype data file format."> - "Phenotypes Data" documentation</a> section below.</p>')}} -</fieldset> + + <div class="form-group"> + <div class="form-check"> + <input id="chk-phenotype-n-transposed" + name="phenotype-n-transposed" + type="checkbox" + class="form-check-input" + style="border: solid #8EABF0" /> + <label for="chk-phenotype-n-transposed" class="form-check-label"> + Counts file transposed?</label> + </div> + <div class="non-resumable-elements"> + <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label> + <input id="finput-phenotype-n" + name="phenotype-n" + class="form-control" + type="file" + data-preview-table="tbl-preview-pheno-n" + required="required" /> + <span class="form-text text-muted"> + Provide a file that contains only the number of samples/individuals used in + the computation of the standard errors above.</span> + </div> + + {{display_resumable_elements( + "resumable-phenotype-n", + "number of samples/individuals", + '<p>Drag and drop a CSV file that contains the samples\' phenotypes counts + data here. You can click the "Browse" button (below and to the right) to + select the file from your computer.</p> + + <p>The CSV should be a matrix of <strong>samples × phenotypes</strong>, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.</p> + + <p>If the format is transposed (i.e <strong>phenotypes × samples</strong>) + select the checkbox above.</p> + + <p>Please see the + <a href="#docs-file-phenotype-se" + title="Documentation of the phenotype data file format."> + "Phenotypes Data" documentation</a> section below for more information + on the expected format of the file provided here.</p>')}} + + {{display_preview_table("tbl-preview-pheno-n", "number of samples/individuals")}} + </div> {%endif%} {%endblock%} @@ -322,15 +398,15 @@ <span id="docs-file-phenotype-data"></span> <span id="docs-file-phenotype-se"></span> <span id="docs-file-phenotype-n"></span> - <p>The data is a matrix of <em>phenotypes × individuals</em>, e.g.</p> + <p>The data is a matrix of <em>samples(or individuals) × phenotypes</em>, e.g.</p> <code> # num-cases: 2549 # num-phenos: 13 - id,IND001,IND002,IND003,IND004,…<br /> - pheno10001,61.400002,54.099998,483,49.799999,…<br /> - pheno10002,49,50.099998,403,45.5,…<br /> - pheno10003,62.5,53.299999,501,62.900002,…<br /> - pheno10004,53.099998,55.099998,403,NA,…<br /> + id,pheno10001,pheno10002,pheno10003,pheno10004,53.099998,…<br /> + IND001,61.400002,49,62.5,55.099998,…<br /> + IND002,54.099998,50.099998,53.299999,55.099998,…<br /> + IND003,483,403,501,403,…<br /> + IND004,49.799999,45.5,62.900002,NA,…<br /> ⋮<br /></code> <p>where <code>IND001,IND002,IND003,IND004,…</code> are the @@ -345,16 +421,6 @@ {%endblock%} -{%block sidebarcontents%} -{{display_preview_table("tbl-preview-pheno-desc", "descriptions")}} -{{display_preview_table("tbl-preview-pheno-data", "data")}} -{%if population.Family in families_with_se_and_n%} -{{display_preview_table("tbl-preview-pheno-se", "standard errors")}} -{{display_preview_table("tbl-preview-pheno-n", "number of samples")}} -{%endif%} -{{display_pheno_dataset_card(species, population, dataset)}} -{%endblock%} - {%block more_javascript%} <script src="{{url_for('base.node_modules', @@ -399,7 +465,7 @@ .map((field) => { var value = field.trim(); if(navalues.includes(value)) { - return "⋘NUL⋙"; + return "[NO-VALUE]"; } return value; }) @@ -429,9 +495,9 @@ }); if(table.find("tbody tr.data-row").length > 0) { - add_class(table.find(".data-row-template"), "hidden"); + add_class(table.find(".data-row-template"), "visually-hidden"); } else { - remove_class(table.find(".data-row-template"), "hidden"); + remove_class(table.find(".data-row-template"), "visually-hidden"); } }; @@ -467,13 +533,24 @@ Object.entries(preview_tables_to_elements_map).forEach((mapentry) => { var preview_table = $(mapentry[0]); var file_input = $(mapentry[1]); - if(file_input.length === 1) { + if(file_input[0].files.length > 0) { readFirstNLines( file_input[0].files[0], 10, [makePreviewUpdater(preview_table)]); } }); + + if(typeof(resumables) !== "undefined") { + resumables.forEach((resumable) => { + if(resumable.files.length > 0) { + readFirstNLines( + resumable.files[0].file, + 10, + [makePreviewUpdater(resumable.preview_table)]); + } + }); + } }; [ @@ -506,7 +583,7 @@ var display_element = display_area .find(".file-display-template") .clone(); - remove_class(display_element, "hidden"); + remove_class(display_element, "visually-hidden"); remove_class(display_element, "file-display-template"); add_class(display_element, "file-display"); display_element.find(".filename").text(file.name @@ -526,7 +603,7 @@ return () => {/*Has no event!*/ var progress = (resumable.progress() * 100).toFixed(2); var pbar = progress_bar.find(".progress-bar"); - remove_class(progress_bar, "hidden"); + remove_class(progress_bar, "visually-hidden"); pbar.css("width", progress+"%"); pbar.attr("aria-valuenow", progress); pbar.text("Uploading: " + progress + "%"); @@ -536,9 +613,9 @@ var retryUpload = (retry_button, cancel_button) => { retry_button.on("click", (event) => { resumable.files.forEach((file) => {file.retry();}); - add_class(retry_button, "hidden"); - remove_class(cancel_button, "hidden"); - add_class(browse_button, "hidden"); + add_class(retry_button, "visually-hidden"); + remove_class(cancel_button, "visually-hidden"); + add_class(browse_button, "visually-hidden"); }); }; @@ -549,18 +626,18 @@ file.abort(); } }); - add_class(cancel_button, "hidden"); - remove_class(retry_button, "hidden"); - remove_class(browse_button, "hidden"); + add_class(cancel_button, "visually-hidden"); + remove_class(retry_button, "visually-hidden"); + remove_class(browse_button, "visually-hidden"); }); }; var startUpload = (browse_button, retry_button, cancel_button) => { return (event) => { - remove_class(cancel_button, "hidden"); - add_class(retry_button, "hidden"); - add_class(browse_button, "hidden"); + remove_class(cancel_button, "visually-hidden"); + add_class(retry_button, "visually-hidden"); + add_class(browse_button, "visually-hidden"); }; }; @@ -574,6 +651,7 @@ })); }); formdata.append("resumable-upload", "true"); + formdata.append("publication-id", $("#txt-publication-id").val()); return formdata; } @@ -600,10 +678,12 @@ console.log("SUCCESS DATA: ", data); console.log("SUCCESS STATUS: ", textstatus); console.log("SUCCESS jqXHR: ", jqxhr); + window.location.assign(window.location.origin + data["redirect-to"]); }, }); return false; } + return false; }; var uploadSuccess = (file_input_name) => { @@ -637,10 +717,8 @@ markResumableDragAndDropElement( makeResumableElement( the_form.attr("data-resumable-target"), - file_input.parent(), $("#" + resumable_element_id), - submit_button, - ["csv", "tsv"]), + ["csv", "tsv", "txt"]), file_input.parent(), $("#" + resumable_element_id), $("#" + resumable_element_id + "-browse-button")), @@ -676,13 +754,20 @@ ["frm-add-phenotypes", "finput-phenotype-se", "resumable-phenotype-se", "tbl-preview-pheno-se"], ["frm-add-phenotypes", "finput-phenotype-n", "resumable-phenotype-n", "tbl-preview-pheno-n"], ].map((row) => { - return makeResumableObject(row[0], row[1], row[2], row[3]); + r = makeResumableObject(row[0], row[1], row[2], row[3]); + r.preview_table = $("#" + row[3]); + return r; }).filter((val) => { return Boolean(val); }); $("#frm-add-phenotypes input[type=submit]").on("click", (event) => { event.preventDefault(); + console.debug(); + if ($("#txt-publication-id").val() == "") { + alert("You MUST provide a publication for the phenotypes."); + return false; + } // TODO: Check all the relevant files exist // TODO: Verify that files are not duplicated var filenames = []; diff --git a/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html index 898fc0c..4afd6ab 100644 --- a/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html +++ b/uploader/templates/phenotypes/add-phenotypes-with-rqtl2-bundle.html @@ -1,25 +1,11 @@ {%extends "phenotypes/add-phenotypes-base.html"%} {%from "flash_messages.html" import flash_all_messages%} {%from "macro-table-pagination.html" import table_pagination%} -{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} {%block title%}Phenotypes{%endblock%} {%block pagetitle%}Phenotypes{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="add-phenotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.phenotypes.add_phenotypes', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}">Add Phenotypes</a> -</li> -{%endblock%} - {%block frm_add_phenotypes_documentation%} <p>Select the zip file bundle containing information on the phenotypes you wish to upload, then click the "Upload Phenotypes" button below to @@ -201,7 +187,3 @@ <em>phenotypes × individuals</em>.</p> </div> {%endblock%} - -{%block sidebarcontents%} -{{display_pheno_dataset_card(species, population, dataset)}} -{%endblock%} diff --git a/uploader/templates/phenotypes/base.html b/uploader/templates/phenotypes/base.html index adbc012..5959422 100644 --- a/uploader/templates/phenotypes/base.html +++ b/uploader/templates/phenotypes/base.html @@ -1,19 +1,27 @@ {%extends "populations/base.html"%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_sui_pheno_dataset_card%} -{%block lvl3_breadcrumbs%} -<li {%if activelink=="phenotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - {%if dataset is mapping%} +{%block breadcrumbs%} +{{super()}} +{%if dataset%} +<li class="breadcrumb-item"> <a href="{{url_for('species.populations.phenotypes.view_dataset', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}">{{dataset.Name}}</a> - {%else%} - <a href="{{url_for('species.populations.phenotypes.index')}}">Phenotypes</a> - {%endif%} + species_id=species['SpeciesId'], + population_id=population['Id'], + dataset_id=dataset['Id'])}}"> + {{dataset["Name"]}} + </a> </li> -{%block lvl4_breadcrumbs%}{%endblock%} +{%endif%} +{%endblock%} + +{%block contents%} +<div class="row"> + <h2 class="heading">{{dataset.FullName}} ({{dataset.Name}})</h2> +</div> +{%endblock%} + + +{%block sidebarcontents%} +{{display_sui_pheno_dataset_card(species, population, dataset)}} {%endblock%} diff --git a/uploader/templates/phenotypes/bulk-edit-upload.html b/uploader/templates/phenotypes/bulk-edit-upload.html new file mode 100644 index 0000000..d0f38f5 --- /dev/null +++ b/uploader/templates/phenotypes/bulk-edit-upload.html @@ -0,0 +1,62 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +<div class="row"> + <p>Upload the edited file you downloaded and edited.</p> +</div> + +<div class="row"> + <form id="frm-bulk-edit-upload" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_upload_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + enctype="multipart/form-data"> + + <div class="form-group row"> + <label for="file-upload-bulk-edit-upload" + class="form-label col-form-label col-sm-2"> + Edited File</label> + <div class="col-sm-10"> + <input id="file-upload-bulk-edit-upload" + name="file-upload-bulk-edit-upload" + class="form-control" + type="file" + accept="text/tab-separated-values" + required="required" /> + </div> + </div> + + <input type="submit" class="btn btn-primary" + value="upload to edit" /> + + </form> +</div> +{%endblock%} + + +{%block javascript%} +{%endblock%} diff --git a/uploader/templates/phenotypes/confirm-delete-phenotypes.html b/uploader/templates/phenotypes/confirm-delete-phenotypes.html new file mode 100644 index 0000000..3cf6e65 --- /dev/null +++ b/uploader/templates/phenotypes/confirm-delete-phenotypes.html @@ -0,0 +1,196 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Delete Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="view-dataset"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"><h2>Delete Phenotypes</h2></div> + +{%if phenotypes | length > 0%} +<div class="row"> + <p>You have requested to delete the following phenotypes:</p> +</div> + +<div class="row"> + <div class="col"> + <a id="btn-select-all-phenotypes" + href="#" + class="btn btn-info" + title="Select all phenotypes">select all</a> + </div> + <div class="col"> + <a id="btn-deselect-all-phenotypes" + href="#" + class="btn btn-warning" + title="Deselect all phenotypes">deselect all</a> + </div> +</div> + +<div class="row"> + <table id="tbl-delete-phenotypes" class="table"> + <thead> + <tr> + <th>Index</th> + <th>Record ID</th> + <th>Description</th> + </tr> + </thead> + <tbody> + {%for phenotype in phenotypes%} + <tr> + <td> + <input id="chk-xref-id-{{phenotype.xref_id}}" + name="xref_ids" + type="checkbox" + value="{{phenotype.xref_id}}" + class="chk-row-select" /> + </td> + <td>{{phenotype.xref_id}}</td> + <td>{{phenotype.Post_publication_description or + phenotype.Pre_publication_description or + phenotype.original_description}}</td> + </tr> + {%endfor%} + </tbody> + </table> +</div> + +<div class="row"> + <form id="frm-delete-phenotypes-selected" + method="POST" + action="{{url_for('species.populations.phenotypes.delete_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}"> + <div class="row"> + <div class="col"> + <input class="btn btn-info" + type="submit" + title="Cancel delete and return to dataset page." + name="action" + value="cancel" /></div> + <div class="col"> + <input id="btn-delete-phenotypes-selected" + class="btn btn-danger" + type="submit" + title="Delete the selected phenotypes from this dataset." + name="action" + value="delete" /> + </div> + </div> + </form> +</div> +{%else%} +<div class="row"> + <p>You did not select any phenotypes to delete. Delete everything?</p> +</div> + +<div class="row"> + <form id="frm-delete-phenotypes-all" + method="POST" + action="{{url_for('species.populations.phenotypes.delete_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}"> + <div class="form-check"> + <input class="form-check-input" + type="checkbox" + name="confirm_delete_all_phenotypes" + id="chk-confirm-delete-all-phenotypes" /> + <label class="form-check-label" + for="chk-confirm-delete-all-phenotypes"> + delete all phenotypes?</label> + </div> + + <div class="row"> + <div class="col"> + <input class="btn btn-info" + type="submit" + title="Cancel delete and return to dataset page." + name="action" + value="cancel" /></div> + <div class="col"> + <input class="btn btn-danger" + type="submit" + title="Delete all phenotypes in this dataset." + name="action" + value="delete" /> + </div> + </div> + </form> +</div> +{%endif%} + +{%endblock%} + +{%block javascript%} +<script type="text/javascript"> + $(function() { + var dt = buildDataTable( + "#tbl-delete-phenotypes", + data=[], + columns=[], + userSettings={ + responsive: true, + select: { + style: "os", + info: false + }, + initComplete: function(setting, json) { + var api = this.api(); + api.rows().select(); + api.rows({selected: true}).nodes().each((node, index) => { + setRowChecked(node); + }); + } + }); + + $("#btn-select-all-phenotypes").on("click", function(event) { + dt.selectAll(); + }); + + $("#btn-deselect-all-phenotypes").on("click", function(event) { + dt.deselectAll(); + }); + + $("#btn-delete-phenotypes-selected").on("click", function(event) { + event.preventDefault(); + form = $("#frm-delete-phenotypes-selected"); + form.find(".dynamically-added-element").remove(); + dt.rows({selected: true}).nodes().each(function(node, index) { + var xref_id = $(node) + .find('input[type="checkbox"]:checked') + .val(); + var chk = $('<input type="checkbox">'); + chk.attr("class", "dynamically-added-element"); + chk.attr("value", xref_id); + chk.attr("name", "xref_ids"); + chk.attr("style", "display: none"); + chk.prop("checked", true); + form.append(chk); + }); + form.append( + $('<input type="hidden" name="action" value="delete" />')); + form.submit(); + }) + }); +</script> +{%endblock%} + diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html index 93de92f..9963953 100644 --- a/uploader/templates/phenotypes/create-dataset.html +++ b/uploader/templates/phenotypes/create-dataset.html @@ -42,16 +42,17 @@ <input type="text" name="dataset-name" id="txt-dataset-name" - value="{{original_formdata.get('dataset-name') or (population.InbredSetCode + 'Publish')}}" + value="{{original_formdata.get('dataset-name') or (population.Name + 'Publish')}}" {%if errors["dataset-name"] is defined%} class="form-control danger" {%else%} class="form-control" {%endif%} - required="required" /> + required="required" + readonly="readonly" /> <small class="form-text text-muted"> <p>A short representative name for the dataset.</p> - <p>Recommended: Use the population code and append "Publish" at the end. + <p>Recommended: Use the population name and append "Publish" at the end. <br />This field will only accept names composed of letters ('A-Za-z'), numbers (0-9), hyphens and underscores.</p> </small> @@ -66,7 +67,7 @@ <input id="txt-dataset-fullname" name="dataset-fullname" type="text" - value="{{original_formdata.get('dataset-fullname', '')}}" + value="{{original_formdata.get('dataset-fullname', '') or population.Name + ' Phenotypes'}}" {%if errors["dataset-fullname"] is defined%} class="form-control danger" {%else%} @@ -74,8 +75,10 @@ {%endif%} required="required" /> <small class="form-text text-muted"> - <p>A longer, descriptive name for the dataset — useful for humans. - </p></small> + <p>A longer, descriptive name for the dataset. The name is meant for use + by humans, and therefore, it should be clear what the dataset contains + from the name.</p> + </small> </div> <div class="form-group"> @@ -84,7 +87,7 @@ name="dataset-shortname" type="text" class="form-control" - value="{{original_formdata.get('dataset-shortname') or (population.InbredSetCode + ' Publish')}}" /> + value="{{original_formdata.get('dataset-shortname') or (population.Name + 'Publish')}}" /> <small class="form-text text-muted"> <p>An optional, short name for the dataset. <br /> If this is not provided, it will default to the value provided for the diff --git a/uploader/templates/phenotypes/edit-phenotype.html b/uploader/templates/phenotypes/edit-phenotype.html new file mode 100644 index 0000000..1b3ee9d --- /dev/null +++ b/uploader/templates/phenotypes/edit-phenotype.html @@ -0,0 +1,208 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "populations/macro-display-population-card.html" import display_population_card%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="edit-phenotype"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <h2 class="heading">edit phenotype data</h2> + <p>The forms provided in this page help you update the data for the + phenotypes, and the publication information for the phenotype, + respectively.</p> +</div> + +<div class="row"> + <h3 class="subheading">Basic metadata</h3> + <form name="frm-phenotype-basic-metadata" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}"> + <input type="hidden" name="phenotype-id" value="{{phenotype.Id}}" /> + <div class="form-group"> + <label for="txt-pre-publication-description" + class="control-label col-sm-2">Pre-Publication Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-pre-publication-description" + name="pre-publication-description" + class="form-control" + value="{{phenotype['Pre_publication_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-pre-publication-abbreviation" + class="control-label col-sm-2">Pre-Publication Abbreviation</label> + <div class="col-sm-10"> + <input type="text" + id="txt-pre-publication-abbreviation" + name="pre-publication-abbreviation" + class="form-control" + value="{{phenotype['Pre_publication_abbreviation'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-post-publication-description" + class="control-label col-sm-2">Post-Publication Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-post-publication-description" + name="post-publication-description" + class="form-control" + value="{{phenotype['Post_publication_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-post-publication-abbreviation" + class="control-label col-sm-2">Post-Publication Abbreviation</label> + <div class="col-sm-10"> + <input type="text" + id="txt-post-publication-abbreviation" + name="post-publication-abbreviation" + class="form-control" + value="{{phenotype['Post_publication_abbreviation'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-original-description" + class="control-label col-sm-2">Original Description</label> + <div class="col-sm-10"> + <input type="text" + id="txt-original-description" + name="original-description" + class="form-control" + value="{{phenotype['Original_description'] or ''}}" /> + </div> + </div> + + <div class="form-group"> + <label for="txt-units" + class="control-label col-sm-2">units</label> + <div class="col-sm-10"> + <input type="text" + id="txt-units" + name="units" + class="form-control" + required="required" + value="{{phenotype['Units']}}" /> + </div> + </div> + + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary" + value="update basic metadata"> + </div> + </div> + </form> +</div> + + +<div class="row"> + <h3 class="subheading">phenotype data</h3> + <form id="frm-edit-phenotype-data" + class="form-horizontal" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}"> + <div style="max-height: 23.37em;overflow-y: scroll;"> + <table class="table table-striped table-responsive table-form-table"> + <thead style="position: sticky; top: 0;"> + <tr> + <th>Index</th> + <th>Sample</th> + <th>Value</th> + {%if population.Family in families_with_se_and_n%} + <th>Standard-Error</th> + <th>Number of Samples</th> + {%endif%} + </tr> + </thead> + + <tbody> + {%for item in phenotype.data%} + <tr> + <td>{{loop.index}}</td> + <td>{{item.StrainName}}</td> + <td> + <input type="text" + name="value-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.value}}" + class="form-control" /> + <input type="hidden" + name="value-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.value}}" /></td> + {%if population.Family in families_with_se_and_n%} + <td> + <input type="text" + name="se-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.error or ''}}" + data-original-value="{{item.error or ''}}" + class="form-control" /> + <input type="hidden" + name="se-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.error or ''}}" /></td> + <td> + <input type="text" + name="n-new::{{item.DataId}}::{{item.StrainId}}" + value="{{item.count or ''}}" + data-original-value="{{item.count or "-"}}" + class="form-control" /> + <input type="hidden" + name="n-original::{{item.DataId}}::{{item.StrainId}}" + value="{{item.count or ''}}" /></td> + {%endif%} + </tr> + {%endfor%} + </tbody> + </table> + </div> + <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> + <input type="submit" + name="submit" + class="btn btn-primary" + value="update data" /> + </div> + </div> + </form> +</div> + +{%endblock%} + +{%block sidebarcontents%} +{{display_population_card(species, population)}} +{%endblock%} diff --git a/uploader/templates/phenotypes/index.html b/uploader/templates/phenotypes/index.html index 0c691e6..689c28e 100644 --- a/uploader/templates/phenotypes/index.html +++ b/uploader/templates/phenotypes/index.html @@ -11,16 +11,11 @@ {{flash_all_messages()}} <div class="row"> - <p>This section deals with phenotypes that - <span class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - … what are the characteristics of these phenotypes? …</span></p> - <p>Select the species to begin the process of viewing/uploading data about - your phenotypes</p> + {{select_species_form(url_for("species.populations.phenotypes.index"), species)}} </div> +{%endblock%} -<div class="row"> - {{select_species_form(url_for("species.populations.phenotypes.index"), - species)}} -</div> + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> {%endblock%} diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html index 6f43d22..951907f 100644 --- a/uploader/templates/phenotypes/job-status.html +++ b/uploader/templates/phenotypes/job-status.html @@ -2,7 +2,6 @@ {%from "cli-output.html" import cli_output%} {%from "flash_messages.html" import flash_all_messages%} {%from "macro-table-pagination.html" import table_pagination%} -{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} {%block extrameta%} {%if job and job.status not in ("success", "completed:success", "error", "completed:error")%} @@ -14,23 +13,13 @@ {%block pagetitle%}Phenotypes{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="add-phenotypes"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.phenotypes.add_phenotypes', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}">View Datasets</a> -</li> -{%endblock%} - {%block contents%} {%if job%} -<h4 class="subheading">Progress</h4> +<div class="row"> + <h2 class="heading">{{dataset.FullName}} ({{dataset.Name}})</h2> + <h3 class="subheading">upload progress</h3> +</div> <div class="row" style="overflow:scroll;"> <p><strong>Process Status:</strong> {{job.status}}</p> {%if metadata%} @@ -62,8 +51,12 @@ {%if job.status in ("completed:success", "success")%} <p> {%if errors | length == 0%} - <a href="#" - class="not-implemented btn btn-primary" + <a href="{{url_for('species.populations.phenotypes.review_job_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" + class="btn btn-primary" title="Continue to process data">Continue</a> {%else%} <span class="text-muted" @@ -76,14 +69,29 @@ {%endif%} </div> -<h4 class="subheading">Errors</h4> +<h3 class="subheading">upload errors</h3> +{%if errors | length == 0 %} <div class="row" style="max-height: 20em; overflow: scroll;"> - {%if errors | length == 0 %} <p class="text-info"> <span class="glyphicon glyphicon-info-sign"></span> No errors found so far </p> - {%else%} +</div> +{%else%} +{%if errors | length > 0%} +<div class="row"> + <div class="col"> + <a href="{{url_for('species.populations.phenotypes.download_errors', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + job_id=job_id)}}" + class="btn btn-info" + title="Download the errors as a CSV file.">download errors CSV</a> + </div> +</div> +{%endif%} +<div class="row" style="max-height: 20em; overflow: scroll;"> <table class="table table-responsive"> <thead style="position: sticky; top: 0; background: white;"> <tr> @@ -101,7 +109,7 @@ <td>{{error.filename}}</td> <td>{{error.rowtitle}}</td> <td>{{error.coltitle}}</td> - <td>{%if error.cellvalue | length > 25%} + <td>{%if error.cellvalue is not none and error.cellvalue | length > 25%} {{error.cellvalue[0:24]}}… {%else%} {{error.cellvalue}} @@ -118,7 +126,8 @@ {%endfor%} </tbody> </table> - {%endif%} +</div> +{%endif%} </div> <div class="row"> @@ -145,7 +154,3 @@ </div> {%endif%} {%endblock%} - -{%block sidebarcontents%} -{{display_pheno_dataset_card(species, population, dataset)}} -{%endblock%} diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html index 2eaf43a..2cf2c7f 100644 --- a/uploader/templates/phenotypes/list-datasets.html +++ b/uploader/templates/phenotypes/list-datasets.html @@ -48,9 +48,12 @@ </tbody> </table> {%else%} - <p class="text-warning"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - There is no dataset for this population!</p> + <p>Phenotypes need to go into a dataset. We do not currently have a dataset + for species <strong>'{{species["FullName"]}} ({{species["Name"]}})'</strong> + phenotypes.</p> + + <p>Do, please, create a new dataset by clicking on the "Create Dataset" button + below and following the prompts/instructions.</p> <p><a href="{{url_for('species.populations.phenotypes.create_dataset', species_id=species.SpeciesId, population_id=population.Id)}}" diff --git a/uploader/templates/phenotypes/load-phenotypes-success.html b/uploader/templates/phenotypes/load-phenotypes-success.html new file mode 100644 index 0000000..1fb0e61 --- /dev/null +++ b/uploader/templates/phenotypes/load-phenotypes-success.html @@ -0,0 +1,26 @@ +{%extends "phenotypes/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block contents%} +{{super()}} + +<div class="row"> + <p>You have successfully loaded + <!-- maybe indicate the number of phenotypes here? -->your + new phenotypes into the database.</p> + <!-- TODO: Maybe notify user that they have sole access. --> + <!-- TODO: Maybe provide a link to go to GeneNetwork to view the data. --> + <p>View your data + <a href="{{search_page_uri}}" + target="_blank">on GeneNetwork2</a>. + You might need to login to GeneNetwork2 to view specific traits.</p> +</div> +{%endblock%} + + +{%block more_javascript%}{%endblock%} diff --git a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html index 11b108b..641421d 100644 --- a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html +++ b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html @@ -1,4 +1,4 @@ -{%from "populations/macro-display-population-card.html" import display_population_card%} +{%from "populations/macro-display-population-card.html" import display_population_card, display_sui_population_card%} {%macro display_pheno_dataset_card(species, population, dataset)%} {{display_population_card(species, population)}} @@ -29,3 +29,29 @@ </div> </div> {%endmacro%} + +{%macro display_sui_pheno_dataset_card(species, population, dataset)%} +{{display_sui_population_card(species, population)}} + +<div class="row"> + <table class="table"> + <caption>Current dataset</caption> + <tbody> + <tr> + <th>Name</th> + <td>{{dataset.Name}}</td> + </tr> + + <tr> + <th>Full Name</th> + <td>{{dataset.FullName}}</td> + </tr> + + <tr> + <th>Short Name</th> + <td>{{dataset.ShortName}}</td> + </tr> + </tbody> + </table> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/macro-display-preview-table.html b/uploader/templates/phenotypes/macro-display-preview-table.html index f54c53e..6dffe9f 100644 --- a/uploader/templates/phenotypes/macro-display-preview-table.html +++ b/uploader/templates/phenotypes/macro-display-preview-table.html @@ -1,21 +1,11 @@ {%macro display_preview_table(tableid, filetype)%} -<div class="card" style="max-width: 676px;"> - <div class="card-body"> - <h5 class="card-title">Phenotypes '{{filetype | title}}' File Preview</h5> - <div class="card-text" style="overflow: scroll;"> - <table id="{{tableid}}" class="table table-condensed table-responsive"> - <thead> - <tr> - </tr> - <tbody> - <tr> - <td class="data-row-template text-info"> - Provide a phenotype '{{filetype | lower}}' file to preview. - </td> - </tr> - </tbody> - </table> - </div> - </div> +<div class="table-responsive" + style="max-width:39.2em;border-radius:5px;border: solid 1px;overflow-x: scroll;"> + <h5>{{filetype | title}}: File Preview</h5> + <table id="{{tableid}}" class="table"> + <thead><tr></tr></thead> + + <tbody></tbody> + </table> </div> {%endmacro%} diff --git a/uploader/templates/phenotypes/macro-display-resumable-elements.html b/uploader/templates/phenotypes/macro-display-resumable-elements.html new file mode 100644 index 0000000..ed14ea5 --- /dev/null +++ b/uploader/templates/phenotypes/macro-display-resumable-elements.html @@ -0,0 +1,60 @@ +{%macro display_resumable_elements(id, title, help)%} +<div id="{{id}}" + class="resumable-elements visually-hidden" + style="background:#D4D4EE;border-radius: 5px;;padding: 1em;border-left: solid #B2B2CC 1px;border-bottom: solid #B2B2CC 2px;margin-top:0.3em;"> + <strong style="line-height: 1.2em;">{{title | title}}</strong> + + <span class="form-text text-muted">{{help | safe}}</span> + + <div id="{{id}}-selected-files" + class="resumable-selected-files" + style="display:flex;flex-direction:row;flex-wrap: wrap;justify-content:space-around;gap:10px 20px;"> + <div class="panel panel-info file-display-template visually-hidden"> + <div class="panel-heading filename">The Filename Goes Here!</div> + <div class="panel-body"> + <ul> + <li> + <strong>Name</strong>: + <span class="filename">the file's name</span></li> + + <li><strong>Size</strong>: <span class="filesize">0 MB</span></li> + + <li> + <strong>Unique Identifier</strong>: + <span class="fileuniqueid">brrr</span></li> + + <li> + <strong>Mime</strong>: + <span class="filemimetype">text/csv</span></li> + </ul> + </div> + </div> + </div> + + <a id="{{id}}-browse-button" + class="resumable-browse-button btn btn-info" + href="#{{id}}" + style="margin-left: 80%;">Browse</a> + + <div id="{{id}}-progress-bar" class="progress visually-hidden"> + <div class="progress-bar" + role="progress-bar" + aria-valuenow="60" + aria-valuemin="0" + aria-valuemax="100" + style="width: 0%;"> + Uploading: 60% + </div> + </div> + + <div id="{{id}}-cancel-resume-buttons"> + <a id="{{id}}-resume-button" + class="resumable-resume-button btn btn-info visually-hidden" + href="#">resume upload</a> + + <a id="{{id}}-cancel-button" + class="resumable-cancel-button btn btn-danger visually-hidden" + href="#">cancel upload</a> + </div> +</div> +{%endmacro%} diff --git a/uploader/templates/phenotypes/review-job-data.html b/uploader/templates/phenotypes/review-job-data.html new file mode 100644 index 0000000..0e8f119 --- /dev/null +++ b/uploader/templates/phenotypes/review-job-data.html @@ -0,0 +1,152 @@ +{%extends "phenotypes/base.html"%} +{%from "cli-output.html" import cli_output%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%} + +{%block extrameta%} +{%if not job%} +<meta http-equiv="refresh" + content="20; url={{url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" /> +{%endif%} +{%endblock%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block lvl4_breadcrumbs%} +<li {%if activelink=="add-phenotypes"%} + class="breadcrumb-item active" + {%else%} + class="breadcrumb-item" + {%endif%}> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">View Datasets</a> +</li> +{%endblock%} + +{%block contents%} + +{%if job%} +<div class="row"> + <h3 class="heading">Data Review</h3> + <p class="text-info"><strong> + The data has <em>NOT</em> been added/saved yet. Review the details below + and click "Continue" to save the data.</strong></p> + <p>The “<strong>{{dataset.FullName}}</strong>” dataset from the + “<strong>{{population.FullName}}</strong>” population of the + species “<strong>{{species.SpeciesName}} ({{species.FullName}})</strong>” + will be updated as follows:</p> + + <ul> + {%if publication%} + <li>All {{summary.get("pheno", {}).get("total-data-rows", "0")}} phenotypes + are linked to the following publication: + <ul> + <li><strong>Publication Title:</strong> + {{publication.Title or "—"}}</li> + <li><strong>Author(s):</strong> + {{publication.Authors or "—"}}</li> + </ul> + </li> + {%endif%} + {%for ftype in ("phenocovar", "pheno", "phenose", "phenonum")%} + {%if summary.get(ftype, False)%} + <li>A total of {{summary[ftype]["number-of-files"]}} files will be processed + adding {%if ftype == "phenocovar"%}(possibly){%endif%} + {{summary[ftype]["total-data-rows"]}} new + {%if ftype == "phenocovar"%} + phenotypes + {%else%} + {{summary[ftype]["description"]}} rows + {%endif%} + to the database. + </li> + {%endif%} + {%endfor%} + </ul> +</div> + +<div class="row"> + + <form id="frm-review-phenotype-data" + method="POST" + action="{{url_for('species.populations.phenotypes.load_data_to_database', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}"> + <input type="hidden" name="data-qc-job-id" value="{{job.jobid}}" /> + <div class="form-group"> + <label for="txt-data-name">data name</label> + <input type="text" + id="txt-data-name" + class="form-control" + name="data_name" + title="A short, descriptive name for this data." + placeholder="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + value="{{user.email}} - {{dataset.Name}} - {{timestamp}}" + required="required"> + <span class="form-text text-muted"> + This is a short, descriptive name for the data. It is useful to humans, + enabling them identify what traits each data "resource" wraps around. + </span> + </div> + + {%if view_under_construction%} + <div class="form-group"> + <label for="txt-data-description">data description</label> + <textarea id="txt-data-description" + class="form-control" + name="data_description" + title="A longer description for this data." + rows="5"></textarea> + <span class="form-text text-muted"> + </span> + </div> + {%endif%} + + <button type="submit" class="btn btn-primary">continue</button> + </form> + +</div> +{%else%} +<div class="row"> + <h4 class="subheading">Invalid Job</h3> + <p class="text-danger"> + Could not find a job with the ID: <strong>{{job_id}}.</p> + <p>You will be redirected in + <span id="countdown-element" class="text-info">20</span> second(s)</p> + <p class="text-muted"> + <small> + If you are not redirected, please + <a href="{{url_for( + 'species.populations.phenotypes.view_dataset', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}">click here</a> to continue + </small> + </p> +</div> +{%endif%} +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(document).ready(function() { + var countdown = 20; + var countdown_element = $("#countdown-element"); + if(countdown_element.length === 1) { + intv = window.setInterval(function() { + countdown = countdown - 1; + countdown_element.html(countdown); + }, 1000); + } + }); +</script> +{%endblock%} diff --git a/uploader/templates/phenotypes/select-population.html b/uploader/templates/phenotypes/select-population.html index eafd4a7..48c19b1 100644 --- a/uploader/templates/phenotypes/select-population.html +++ b/uploader/templates/phenotypes/select-population.html @@ -11,18 +11,16 @@ {%block contents%} {{flash_all_messages()}} -<div class="row"> - <p>Select the population for your phenotypes to view and manage the phenotype - datasets that relate to it.</p> -</div> <div class="row"> - {{select_population_form(url_for("species.populations.phenotypes.select_population", - species_id=species.SpeciesId), - populations)}} + {{select_population_form(url_for("species.populations.phenotypes.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index 66de5d8..fc84757 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -1,7 +1,6 @@ {%extends "phenotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} {%from "macro-table-pagination.html" import table_pagination%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Phenotypes{%endblock%} @@ -24,73 +23,226 @@ {{flash_all_messages()}} <div class="row"> - <p>The basic dataset details are:</p> - - <table class="table"> - <thead> - <tr> - <th>Name</th> - <th>Full Name</th> - <th>Short Name</th> - </tr> - </thead> + <h2>Phenotype Data</h2> - <tbody> - <tr> - <td>{{dataset.Name}}</td> - <td>{{dataset.FullName}}</td> - <td>{{dataset.ShortName}}</td> - </tr> - </tbody> - </table> + <p>Click on any of the phenotypes in the table below to view and edit that + phenotype's data.</p> + <p>Use the search to filter through all the phenotypes and find specific + phenotypes of interest.</p> </div> <div class="row"> - <p><a href="{{url_for('species.populations.phenotypes.add_phenotypes', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id)}}" - title="Add a bunch of phenotypes" - class="btn btn-primary">Add phenotypes</a></p> -</div> + <div class="col"> + <a href="{{url_for('species.populations.phenotypes.add_phenotypes', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id)}}" + title="Add a bunch of phenotypes" + class="btn btn-primary">Add phenotypes</a> + </div> -<div class="row"> - <h2>Phenotype Data</h2> + <div class="col"> + <form id="frm-recompute-phenotype-means" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.recompute_means', + species_id=species['SpeciesId'], + population_id=population['Id'], + dataset_id=dataset['Id'])}}" + class="d-flex flex-row align-items-center flex-wrap" + style="display: inline;"> + <input type="submit" + title="Compute/Recompute the means for all phenotypes." + class="btn btn-info" + value="compute means" + id="submit-frm-recompute-phenotype-means" /> + </form> + </div> - <p>This dataset has a total of {{phenotype_count}} phenotypes.</p> + <div class="col"> + <form id="frm-run-qtlreaper" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.rerun_qtlreaper', + species_id=species['SpeciesId'], + population_id=population['Id'], + dataset_id=dataset['Id'])}}" + class="d-flex flex-row align-items-center flex-wrap" + style="display: inline;"> + <input type="submit" + title="Run/Rerun QTLReaper." + class="btn btn-info" + value="run QTLReaper" + id="submit-frm-rerun-qtlreaper" /> + </form> + </div> - {{table_pagination(start_from, count, phenotype_count, url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id), "phenotypes")}} + <div class="col"> + <form id="frm-delete-phenotypes" + method="POST" + action="{{url_for( + 'species.populations.phenotypes.delete_phenotypes', + species_id=species['SpeciesId'], + population_id=population['Id'], + dataset_id=dataset['Id'])}}"> + <input type="submit" + class="btn btn-danger" + id="btn-delete-phenotypes" + title="Delete phenotypes from this dataset. If no phenotypes are selected in the table, this will delete ALL the phenotypes." + value="delete phenotypes" /> + </form> + </div> +</div> - <table class="table"> +<div class="row" style="margin-top: 0.5em;"> + <table id="tbl-phenotypes-list" class="table compact stripe cell-border"> <thead> <tr> - <th>#</th> + <th></th> + <th>Index</th> <th>Record</th> <th>Description</th> </tr> </thead> - <tbody> - {%for pheno in phenotypes%} - <tr> - <td>{{pheno.sequence_number}}</td> - <td><a href="{{url_for('species.populations.phenotypes.view_phenotype', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id, - xref_id=pheno['pxr.Id'])}}" - title="View phenotype details"> - {{pheno.InbredSetCode}}_{{pheno["pxr.Id"]}}</a></td> - <td>{{pheno.Post_publication_description or pheno.Pre_publication_abbreviation or pheno.Original_description}}</td> - </tr> - {%else%} - <tr><td colspan="5"></td></tr> - {%endfor%} - </tbody> + <tbody></tbody> </table> </div> {%endblock%} -{%block sidebarcontents%} -{{display_population_card(species, population)}} + +{%block javascript%} +<script type="text/javascript" src="/static/js/urls.js"></script> + +<script type="text/javascript"> + $(function() { + var species_id = {{species.SpeciesId}}; + var population_id = {{population.Id}}; + var dataset_id = {{dataset.Id}}; + var dataset_name = "{{dataset.Name}}"; + var data = {{phenotypes | tojson}}; + + var dtPhenotypesList = buildDataTable( + "#tbl-phenotypes-list", + data, + [ + { + data: function(pheno) { + return `<input type="checkbox" name="selected-phenotypes" ` + + `id="chk-selected-phenotypes-${pheno.InbredSetCode}_${pheno.xref_id}" ` + + `value="${pheno.InbredSetCode}_${pheno.xref_id}" ` + + `class="chk-row-select" />` + } + }, + {data: "sequence_number"}, + { + data: function(pheno, type, set, meta) { + var spcs_id = {{species.SpeciesId}}; + var pop_id = {{population.Id}}; + var dtst_id = {{dataset.Id}}; + var url = buildURLFromCurrentURL( + (`/species/${spcs_id}` + + `/populations/${pop_id}` + + `/phenotypes/datasets/${dtst_id}` + + `/phenotype/${pheno.xref_id}`)); + return `<a href="${url.toString()}" target="_blank">` + + `${pheno.InbredSetCode}_${pheno.xref_id}` + + `</a>`; + }, + title: "Record", + visible: true, + searchable: true + }, + { + data: function(pheno) { + return (pheno.Post_publication_description || + pheno.Original_description || + pheno.Pre_publication_description); + }, + title: "Description", + visible: true, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Title; + }, + title: "Publication Title", + visible: false, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Authors; + }, + title: "Authors", + visible: false, + searchable: true + } + ], + { + select: "multi+shift", + layout: { + top1Start: { + pageLength: { + text: "Show _MENU_ of _TOTAL_" + } + }, + topStart: "info", + top1End: null + }, + rowId: function(pheno) { + return `${pheno.InbredSetCode}_${pheno.xref_id}`; + } + }); + + + $("#submit-frm-rerun-qtlreaper").on( + "click", + function(event) { + // (Re)run the QTLReaper script for selected phenotypes. + event.preventDefault(); + var form = $("#frm-run-qtlreaper"); + form.find(".dynamically-added-element").remove(); + dtPhenotypesList.rows({selected: true}).nodes().each((node, index) => { + _cloned = $(node).find(".chk-row-select").clone(); + _cloned.removeAttr("id"); + _cloned.removeAttr("class"); + _cloned.attr("style", "display: none;"); + _cloned.attr("data-type", "dynamically-added-element"); + _cloned.attr("class", "dynamically-added-element checkbox"); + _cloned.prop("checked", true); + form.append(_cloned); + }); + form.submit(); + }); + + $("#btn-delete-phenotypes").on( + "click", + function(event) { + // Collect selected phenotypes for deletion, if any. + event.preventDefault(); + form = $("#frm-delete-phenotypes"); + form.find(".dynamically-added-element").remove(); + $("#tbl-phenotypes-list") + .DataTable() + .rows({selected: true}). + nodes().each(function(node, index) { + var parts = $(node) + .find(".chk-row-select") + .val() + .split("_"); + var xref_id = parts[parts.length - 1].trim(); + var chk = $('<input type="checkbox">'); + chk.attr("class", "dynamically-added-element"); + chk.attr("value", xref_id); + chk.attr("name", "xref_ids"); + chk.attr("style", "display: none"); + chk.prop("checked", true); + form.append(chk); + }); + form.submit(); + }); + }); +</script> {%endblock%} diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html index 99bb8e5..a59949e 100644 --- a/uploader/templates/phenotypes/view-phenotype.html +++ b/uploader/templates/phenotypes/view-phenotype.html @@ -1,31 +1,18 @@ {%extends "phenotypes/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Phenotypes{%endblock%} {%block pagetitle%}Phenotypes{%endblock%} -{%block lvl4_breadcrumbs%} -<li {%if activelink=="view-phenotype"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.phenotypes.view_phenotype', - species_id=species.SpeciesId, - population_id=population.Id, - dataset_id=dataset.Id, - xref_id=xref_id)}}">View Datasets</a> -</li> -{%endblock%} - {%block contents%} {{flash_all_messages()}} <div class="row"> - <div class="panel panel-default"> - <div class="panel-heading"><strong>Basic Phenotype Details</strong></div> + <div class="card"> + <div class="card-header"> + <h5 class="card-title">Basic Phenotype Details</h5> + </div> <table class="table"> <tbody> @@ -34,11 +21,7 @@ <td>{{phenotype.Post_publication_description or phenotype.Pre_publication_abbreviation or phenotype.Original_description}} </tr> <tr> - <td><strong>Cross-Reference ID</strong></td> - <td>{{phenotype.xref_id}}</td> - </tr> - <tr> - <td><strong>Collation</strong></td> + <td><strong>Database</strong></td> <td>{{dataset.FullName}}</td> </tr> <tr> @@ -47,38 +30,71 @@ </tr> </tbody> </table> + </div> +</div> - <form action="#edit-delete-phenotype" - method="POST" - id="frm-delete-phenotype"> +<div class="row" style="margin-top:5px;"> + <div class="card"> + <div class="card-header"> + <h5 class="card-title">Publication Details</h5> + </div> - <input type="hidden" name="species_id" value="{{species.SpeciesId}}" /> - <input type="hidden" name="population_id" value="{{population.Id}}" /> - <input type="hidden" name="dataset_id" value="{{dataset.Id}}" /> - <input type="hidden" name="phenotype_id" value="{{phenotype.Id}}" /> + <div class="card-body"> + <table class="table"> + <tbody> + <tr> + {%for key in ("PubMed_ID", "Authors", "Title", "Journal"):%} + <tr> + <td><strong>{{key}}</strong></td> + <td>{{publication.get(key, "")}}</td> + </tr> + {%else%} + <tr> + <td colspan="2" class="text-muted"> + <span class="glyphicon glyphicon-exclamation-sign"></span> + No publication data found. + </td> + </tr> + {%endfor%} + </tr> + </tbody> + </table> + <div style="text-align: right;"> + <a href="{{url_for('publications.edit_publication', publication_id=publication.Id, next=next)}}" + class="btn btn-info">edit</a> + <a href="#" class="btn btn-danger not-implemented">change</a> + </div> + </div> + </div> +</div> - <div class="btn-group btn-group-justified"> - <div class="btn-group"> - {%if "group:resource:edit-resource" in privileges%} - <input type="submit" - title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values." - class="btn btn-primary not-implemented" - value="edit" /> - {%endif%} - </div> - <div class="btn-group"></div> - <div class="btn-group"> - {%if "group:resource:delete-resource" in privileges%} - <input type="submit" - title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype." - class="btn btn-danger not-implemented" - value="delete" /> - {%endif%} - </div> - </div> - </form> +{%if "group:resource:edit-resource" in privileges +or "group:resource:delete-resource" in privileges%} +<div class="row"> + <div class="btn-group btn-group-justified"> + <div class="btn-group"> + {%if "group:resource:edit-resource" in privileges%} + <a href="{{url_for('species.populations.phenotypes.edit_phenotype_data', + species_id=species.SpeciesId, + population_id=population.Id, + dataset_id=dataset.Id, + xref_id=xref_id)}}" + title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values." + class="btn btn-primary">Edit</a> + {%endif%} + </div> + <div class="btn-group"></div> + <div class="btn-group"> + {%if "group:resource:delete-resource" in privileges%} + <a href="#" + title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype." + class="btn btn-danger not-implemented" + disabled="disabled">delete</a> + {%endif%} + </div> </div> </div> +{%endif%} <div class="row"> <div class="panel panel-default"> @@ -87,12 +103,13 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th>Index</th> <th>Sample</th> <th>Value</th> - <th>Symbol</th> + {%if has_se%} <th>SE</th> <th>N</th> + {%endif%} </tr> </thead> @@ -102,9 +119,10 @@ <td>{{loop.index}}</td> <td>{{item.StrainName}}</td> <td>{{item.value}}</td> - <td>{{item.Symbol or "-"}}</td> + {%if has_se%} <td>{{item.error or "-"}}</td> <td>{{item.count or "-"}}</td> + {%endif%} </tr> {%endfor%} </tbody> @@ -120,7 +138,3 @@ </div> {%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/platforms/base.html b/uploader/templates/platforms/base.html index dac965f..d4f3686 100644 --- a/uploader/templates/platforms/base.html +++ b/uploader/templates/platforms/base.html @@ -1,13 +1,17 @@ {%extends "species/base.html"%} +{%from "species/macro-display-species-card.html" import display_sui_species_card%} -{%block lvl3_breadcrumbs%} -<li {%if activelink=="platforms"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.platforms.index')}}"> - Sequencing Platforms</a> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('species.platforms.list_platforms', + species_id=species['SpeciesId'])}}"> + Platforms + </a> </li> -{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} + + +{%block sidebarcontents%} +{{display_sui_species_card(species)}} {%endblock%} diff --git a/uploader/templates/platforms/create-platform.html b/uploader/templates/platforms/create-platform.html index 0866d5e..3a62472 100644 --- a/uploader/templates/platforms/create-platform.html +++ b/uploader/templates/platforms/create-platform.html @@ -1,19 +1,15 @@ {%extends "platforms/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} {%block title%}Platforms — Create Platforms{%endblock%} -{%block pagetitle%}Platforms — Create Platforms{%endblock%} - -{%block lvl3_breadcrumbs%} -<li {%if activelink=="create-platform"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> <a href="{{url_for('species.platforms.create_platform', - species_id=species.SpeciesId)}}">create platform</a> + species_id=species['SpeciesId'])}}"> + Create + </a> </li> {%endblock%} @@ -118,7 +114,3 @@ </form> </div> {%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} diff --git a/uploader/templates/platforms/index.html b/uploader/templates/platforms/index.html index 35b6464..555b444 100644 --- a/uploader/templates/platforms/index.html +++ b/uploader/templates/platforms/index.html @@ -19,3 +19,7 @@ {{select_species_form(url_for("species.platforms.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/platforms/list-platforms.html b/uploader/templates/platforms/list-platforms.html index 718dd1d..db14745 100644 --- a/uploader/templates/platforms/list-platforms.html +++ b/uploader/templates/platforms/list-platforms.html @@ -1,6 +1,5 @@ {%extends "platforms/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-display-species-card.html" import display_species_card%} {%block title%}Platforms — List Platforms{%endblock%} @@ -58,7 +57,7 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th></th> <th>Platform Name</th> <th><a href="https://www.ncbi.nlm.nih.gov/geo/browse/?view=platforms&tax={{species.TaxonomyId}}" title="Gene Expression Omnibus: Platforms section" @@ -87,7 +86,3 @@ {%endif%} </div> {%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} diff --git a/uploader/templates/populations/base.html b/uploader/templates/populations/base.html index 9db8083..24cacc2 100644 --- a/uploader/templates/populations/base.html +++ b/uploader/templates/populations/base.html @@ -1,18 +1,20 @@ {%extends "species/base.html"%} +{%from "populations/macro-display-population-card.html" import display_sui_population_card%} -{%block lvl2_breadcrumbs%} -<li {%if activelink=="populations"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - {%if population is mapping%} +{%block breadcrumbs%} +{{super()}} +{%if population%} +<li class="breadcrumb-item"> <a href="{{url_for('species.populations.view_population', - species_id=species.SpeciesId, - population_id=population.Id)}}">{{population.Name}}</a> - {%else%} - <a href="{{url_for('species.populations.index')}}">Populations</a> - {%endif%} + species_id=species['SpeciesId'], + population_id=population['Id'])}}"> + {{population["Name"]}} + </a> </li> -{%block lvl3_breadcrumbs%}{%endblock%} +{%endif%} +{%endblock%} + + +{%block sidebarcontents%} +{{display_sui_population_card(species, population)}} {%endblock%} diff --git a/uploader/templates/populations/create-population.html b/uploader/templates/populations/create-population.html index b05ce37..d5359f5 100644 --- a/uploader/templates/populations/create-population.html +++ b/uploader/templates/populations/create-population.html @@ -1,20 +1,16 @@ {%extends "populations/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-select-species.html" import select_species_form%} -{%from "species/macro-display-species-card.html" import display_species_card%} {%block title%}Create Population{%endblock%} {%block pagetitle%}Create Population{%endblock%} -{%block lvl3_breadcrumbs%} -<li {%if activelink=="create-population"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> <a href="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}">create population</a> + species_id=species['SpeciesId'])}}"> + create population</a> </li> {%endblock%} @@ -37,12 +33,15 @@ <div class="row"> <form method="POST" action="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}"> + species_id=species.SpeciesId, + return_to=return_to)}}"> <legend>Create Population</legend> {{flash_all_messages()}} + <input type="hidden" name="return_to" value="{{return_to}}"> + <div {%if errors.population_fullname%} class="form-group has-error" {%else%} @@ -107,9 +106,12 @@ value="{{error_values.population_code or ''}}" class="form-control" /> <small class="form-text text-muted"> - <p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-sign"></span> - What is this field is for? Confirm with Arthur and the rest. + <p class="form-text text-muted"> + This is a 3-character code for your population, that is prepended to + the phenotype identifiers. e.g. For the "BXD Family" population, the + code is "BXD" and therefore, the phenotype identifiers for the + population look like the following examples: <em>BXD_10148</em>, + <em>BXD_10180</em>, <em>BXD_10197</em>, etc. </p> </small> </div> @@ -148,24 +150,35 @@ {%else%} class="form-group" {%endif%}> - <label for="select-population-family" class="form-label">Family</label> - <select id="select-population-family" - name="population_family" - class="form-control" - required="required"> - <option value="">Please select a family</option> + <label for="txt-population-family" class="form-label">Family</label> + <input type="text" + id="txt-population-family" + name="population_family" + class="form-control" + list="families-list" /> + <datalist id="families-list"> {%for family in families%} - <option value="{{family}}" - {%if error_values.population_family == family%} - selected="selected" - {%endif%}>{{family}}</option> + <option value="{{family}}">{{family}}</option> {%endfor%} - </select> + </datalist> <small class="form-text text-muted"> <p> - This is a rough grouping of the populations in GeneNetwork into lists - of common types of populations. - </p> + This is <strong>optional</strong> metadata. It is used to group + populations into "families" for presentation in the menus. + {%if families | length > 0%} + Examples of currently existing families are: + <ul> + {%for family in families[0:7]%} + <li>{{family}}</li> + {%endfor%} + <li>etc.</li> + </ul> + {%endif%} + + You can + {%if families|length>0%} select from existing families, or {%endif%} + create a new family by typing in the input box above. You can also + leave the family blank.</p> </small> </div> @@ -246,7 +259,3 @@ </form> </div> {%endblock%} - -{%block sidebarcontents%} -{{display_species_card(species)}} -{%endblock%} diff --git a/uploader/templates/populations/index.html b/uploader/templates/populations/index.html index 4354e02..d2bee77 100644 --- a/uploader/templates/populations/index.html +++ b/uploader/templates/populations/index.html @@ -22,3 +22,7 @@ {{select_species_form(url_for("species.populations.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/populations/list-populations.html b/uploader/templates/populations/list-populations.html index 7c7145f..a092e34 100644 --- a/uploader/templates/populations/list-populations.html +++ b/uploader/templates/populations/list-populations.html @@ -51,10 +51,10 @@ <caption>Populations for {{species.FullName}}</caption> <thead> <tr> - <th>#</th> + <th></th> <th>Name</th> <th>Full Name</th> - <th>Description</th> + <th>Information</th> </tr> </thead> @@ -71,7 +71,10 @@ </a> </td> <td>{{population.FullName}}</td> - <td>{{population.Description}}</td> + <td><a href="https://info.genenetwork.org/species/source.php?SpeciesName={{species.Name}}&InbredSetName={{population.Name}}" + title="Link to detailed information on this population." + class="btn btn-info" + target="_blank">info</a></td> </tr> {%else%} <tr> diff --git a/uploader/templates/populations/macro-display-population-card.html b/uploader/templates/populations/macro-display-population-card.html index 79f7925..f3040ea 100644 --- a/uploader/templates/populations/macro-display-population-card.html +++ b/uploader/templates/populations/macro-display-population-card.html @@ -1,4 +1,4 @@ -{%from "species/macro-display-species-card.html" import display_species_card%} +{%from "species/macro-display-species-card.html" import display_species_card,display_sui_species_card%} {%macro display_population_card(species, population)%} {{display_species_card(species)}} @@ -33,14 +33,47 @@ <td>Family</td> <td>{{population.Family}}</td> </tr> - - <tr> - <td>Description</td> - <td>{{(population.Description or "")[0:500]}}…</td> - </tr> </tbody> </table> </div> </div> </div> {%endmacro%} + + +{%macro display_sui_population_card(species, population)%} +{{display_sui_species_card(species)}} +{%if population%} +<div class="row"> + <table class="table"> + <caption>Current population</caption> + <tbody> + <tr> + <th>Name</th> + <td>{{population.Name}}</td> + </tr> + + <tr> + <th>Full Name</th> + <td>{{population.FullName}}</td> + </tr> + + <tr> + <th>Code</th> + <td>{{population.InbredSetCode}}</td> + </tr> + + <tr> + <th>Genetic Type</th> + <td>{{population.GeneticType}}</td> + </tr> + + <tr> + <th>Family</th> + <td>{{population.Family}}</td> + </tr> + </tbody> + </table> +</div> +{%endif%} +{%endmacro%} diff --git a/uploader/templates/populations/macro-select-population.html b/uploader/templates/populations/macro-select-population.html index af4fd3a..14b0510 100644 --- a/uploader/templates/populations/macro-select-population.html +++ b/uploader/templates/populations/macro-select-population.html @@ -1,30 +1,52 @@ -{%macro select_population_form(form_action, populations)%} -<form method="GET" action="{{form_action}}"> - <legend>Select Population</legend> - - <div class="form-group"> - <label for="select-population" class="form-label">Select Population</label> - <select id="select-population" - name="population_id" - class="form-control" - required="required"> - <option value="">Select Population</option> - {%for family in populations%} - <optgroup {%if family[0][1] is not none%} - label="{{family[0][1]}}" - {%else%} - label="Undefined" - {%endif%}> - {%for population in family[1]%} - <option value="{{population.Id}}">{{population.FullName}}</option> - {%endfor%} - </optgroup> - {%endfor%} - </select> +{%from "macro-step-indicator.html" import step_indicator%} + +{%macro select_population_form(form_action, species, populations)%} +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("2")}} What population do you want to work with?</h2> + + {%if populations | length != 0%} + + <p class="form-text">Search for, and select the population from the table + below and click "Continue"</p> + + <div class="radio"> + <label class="control-label" for="rdo-cant-find-population"> + <input type="radio" id="rdo-cant-find-population" + name="population_id" value="CREATE-POPULATION" /> + I cannot find the population I want — create it! + </label> + </div> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> + </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-population" class="table compact stripe" + data-populations-list='{{populations | tojson}}'> + <thead> + <tr> + <th></th> + <th>Population</th> + </tr> + </thead> + + <tbody></tbody> + </table> </div> - <div class="form-group"> - <input type="submit" value="Select" class="btn btn-primary" /> + {%else%} + <p class="form-text"> + There are no populations currently defined for {{species['FullName']}} + ({{species['SpeciesName']}}).<br /> + Click "Continue" to create the first!</p> + <input type="hidden" name="population_id" value="CREATE-POPULATION" /> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" value="continue" class="btn btn-primary" /> </div> + {%endif%} + </form> {%endmacro%} diff --git a/uploader/templates/populations/view-population.html b/uploader/templates/populations/view-population.html index b23caeb..6da4cd7 100644 --- a/uploader/templates/populations/view-population.html +++ b/uploader/templates/populations/view-population.html @@ -1,102 +1,135 @@ {%extends "populations/base.html"%} -{%from "flash_messages.html" import flash_all_messages%} -{%from "species/macro-select-species.html" import select_species_form%} -{%from "species/macro-display-species-card.html" import display_species_card%} - -{%block title%}Populations{%endblock%} - -{%block pagetitle%}Populations{%endblock%} - -{%block lvl3_breadcrumbs%} -<li {%if activelink=="view-population"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.view_population', - species_id=species.SpeciesId, - population_id=population.InbredSetId)}}">view</a> -</li> -{%endblock%} - +{%from "macro-step-indicator.html" import step_indicator%} +{%from "populations/macro-display-population-card.html" import display_sui_population_card%} {%block contents%} <div class="row"> - <h2>Population Details</h2> - - {{flash_all_messages()}} - - <dl> - <dt>Name</dt> - <dd>{{population.Name}}</dd> - - <dt>FullName</dt> - <dd>{{population.FullName}}</dd> - - <dt>Code</dt> - <dd>{{population.InbredSetCode}}</dd> - - <dt>Genetic Type</dt> - <dd>{{population.GeneticType}}</dd> - - <dt>Family</dt> - <dd>{{population.Family}}</dd> - - <dt>Description</dt> - <dd><pre>{{population.Description or "-"}}</pre></dd> - </dl> + <h2 class="heading">Population: {{population.FullName}} ({{population.Name}})</h2> </div> <div class="row"> - … maybe provide a way to organise populations in the same family here … + <ul class="nav nav-tabs" id="population-actions"> + <li class="nav-item presentation"> + <button class="nav-link" + id="samples-tab" + data-bs-toggle="tab" + data-bs-target="#samples-content" + type="button" + role="tab" + aria-controls="samples-content" + aria-selected="true">Samples</button></li> + <li class="nav-item presentation"> + <button class="nav-link active" + id="phenotypes-tab" + data-bs-toggle="tab" + data-bs-target="#phenotypes-content" + type="button" + role="tab" + aria-controls="phenotypes-content" + aria-selected="false">Phenotypes</button></li> + {%if view_under_construction%} + <li class="nav-item presentation"> + <button class="nav-link" + id="genotypes-tab" + data-bs-toggle="tab" + data-bs-target="#genotypes-content" + type="button" + role="tab" + aria-controls="genotypes-content" + aria-selected="false">Genotypes</button></li> + <li class="nav-item presentation"> + <button class="nav-link" + id="expression-data-tab" + data-bs-toggle="tab" + data-bs-target="#expression-data-content" + type="button" + role="tab" + aria-controls="expression-data-content" + aria-selected="false">Expression-Data</button></li> + {%endif%} + </ul> </div> <div class="row"> - <h3>Actions</h3> - - <p> - Click any of the following links to use this population in performing the - subsequent operations. - </p> - - <nav class="nav"> - <ul> - <li> - <a href="{{url_for('species.populations.samples.list_samples', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="Manage samples: Add new or delete existing."> - manage samples</a> - </li> - <li> - <a href="{{url_for('species.populations.genotypes.list_genotypes', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="Manage genotypes for {{species.FullName}}">Manage Genotypes</a> - </li> - <li> - <a href="{{url_for('species.populations.phenotypes.list_datasets', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="Manage phenotype data.">manage phenotype data</a> - </li> - <li> - <a href="#" title="Manage expression data" - class="not-implemented">manage expression data</a> - </li> - <li> - <a href="#" title="Manage individual data" - class="not-implemented">manage individual data</a> - </li> - <li> - <a href="#" title="Manage RNA-Seq data" - class="not-implemented">manage RNA-Seq data</a> - </li> - </ul> - </nav> + <div class="tab-content" id="populations-tabs-content"> + <div class="tab-pane fade" + id="samples-content" + role="tabpanel" + aria-labelledby="samples-content-tab"> + <p>Think of a <strong>"sample"</strong> as say a single case or individual + in the experiment. It could even be a single strain (where applicable). + These are, effectively, identifiers for the organisms (plants, animals, + etc) that your data is collected from, and is about. + </p> + + <p>The samples should be uploaded before any of the other types of data + (genotype, phenotype, expression, etc.), or be bundled together with + them, since they all need references to the samples.</p> + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="View and upload samples for population '{{population['Name']}}'" + class="btn btn-primary">manage samples</a> + </div> + + <div class="tab-pane fade show active" + id="phenotypes-content" + role="tabpanel" + aria-labelledby="phenotypes-content-tab"> + + <div class="row" style="margin-top: 1em;"> + <p>Phenotype data measures the actual observable traits or + characteristics of an organism e.g. physical appearance, biochemical + properties, development, behaviour and/or disease states.</p> + <p>This section enables you to view existing and/or upload new phenotype + data.</p> + + <div class="row"> + <div class="col"> + <a href="{{url_for( + 'species.populations.phenotypes.list_datasets', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="View and upload phenotype traits" + class="btn btn-primary">manage phenotypes</a> + </div> + </div> + </div> + </div> + + <div class="tab-pane fade" + id="genotypes-content" + role="tabpanel" + aria-labelledby="genotypes-content-tab"> + <p>Genotype data records specific genetic variations (e.g. single + nucleotide polymorphisms (SNPs)) present at particular locations in an + individual's (see "Samples" section) DNA.</p> + <p>Click the button below to view existing and/or upload new genotype data + for this population.</p> + <a href="{{url_for('species.populations.genotypes.index', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Upload genotype information for the '{{population.FullName}}' population of the '{{species.FullName}}' species." + class="btn btn-primary">manage genotypes</a> + </div> + <div class="tab-pane fade" id="expression-data-content" role="tabpanel" aria-labelledby="expression-data-content-tab"> + <p>Expression data is data measuring how much genes are turned on or active.</p> + <a href="#" title="" class="btn btn-primary">manage expression data</a> + </div> + </div> </div> {%endblock%} {%block sidebarcontents%} -{{display_species_card(species)}} +<div class="row"> + <p>Each tab presents a feature that's available at the population level. + Select the tab that allows you to continue with your task.</p> +</div> +{{super()}} +{%endblock%} + + + + +{%block javascript%} {%endblock%} diff --git a/uploader/templates/publications/base.html b/uploader/templates/publications/base.html new file mode 100644 index 0000000..de0a350 --- /dev/null +++ b/uploader/templates/publications/base.html @@ -0,0 +1,9 @@ +{%extends "base.html"%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('publications.index')}}" + title="Manage publications">Publications</a> +</li> +{%endblock%} diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html new file mode 100644 index 0000000..da5889e --- /dev/null +++ b/uploader/templates/publications/create-publication.html @@ -0,0 +1,203 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}View Publication{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('publications.create_publication', **get_args)}}" + title="Manage publications">create publication</a> +</li> +{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <form id="frm-create-publication" + method="POST" + action="{{url_for('publications.create_publication', **get_args)}}" + class="form-horizontal"> + + <div class="row mb-3"> + <label for="txt-pubmed-id" class="col-sm-2 col-form-label"> + PubMed ID</label> + <div class="col-sm-10"> + <div class="input-group"> + <input type="text" + id="txt-pubmed-id" + name="pubmed-id" + class="form-control"/> + <div class="input-group-text"> + <button class="btn btn-outline-primary" + id="btn-search-pubmed-id">search</button> + </div> + </div> + <span id="search-pubmed-id-error" + class="form-text text-muted text-danger visually-hidden"> + </span> + <span class="form-text text-muted">This is the publication's ID on + <a href="https://pubmed.ncbi.nlm.nih.gov/" + title="Link to NCBI's PubMed service">NCBI's Pubmed Service</a> + </span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-title" class="col-sm-2 col-form-label"> + Title</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-title" + name="publication-title" + class="form-control" /> + <span class="form-text text-muted">Provide the publication's title here.</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-authors" class="col-sm-2 col-form-label"> + Authors</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-authors" + name="publication-authors" + required="required" + class="form-control" /> + <span class="form-text text-muted"> + A publication <strong>MUST</strong> have an author. You <em>must</em> + provide a value for the authors field. + </span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-journal" class="col-sm-2 col-form-label"> + Journal</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-journal" + name="publication-journal" + class="form-control" /> + <span class="form-text text-muted">Provide the name journal where the + publication was done, here.</span> + </div> + </div> + + <div class="row mb-3"> + <label for="select-publication-month" + class="col-sm-2 col-form-label"> + Month</label> + <div class="col-sm-4"> + <select class="form-select" + id="select-publication-month" + name="publication-month"> + <option value="">Select a month</option> + <option {%if current_month | lower == "january"%}selected="selected"{%endif%}value="january">January</option> + <option {%if current_month | lower == "february"%}selected="selected"{%endif%}value="february">February</option> + <option {%if current_month | lower == "march"%}selected="selected"{%endif%}value="march">March</option> + <option {%if current_month | lower == "april"%}selected="selected"{%endif%}value="april">April</option> + <option {%if current_month | lower == "may"%}selected="selected"{%endif%}value="may">May</option> + <option {%if current_month | lower == "june"%}selected="selected"{%endif%}value="june">June</option> + <option {%if current_month | lower == "july"%}selected="selected"{%endif%}value="july">July</option> + <option {%if current_month | lower == "august"%}selected="selected"{%endif%}value="august">August</option> + <option {%if current_month | lower == "september"%}selected="selected"{%endif%}value="september">September</option> + <option {%if current_month | lower == "october"%}selected="selected"{%endif%}value="october">October</option> + <option {%if current_month | lower == "november"%}selected="selected"{%endif%}value="november">November</option> + <option {%if current_month | lower == "december"%}selected="selected"{%endif%}value="december">December</option> + </select> + <span class="form-text text-muted">Month of publication</span> + </div> + + <label for="txt-publication-year" + class="col-sm-2 col-form-label"> + Year</label> + <div class="col-sm-4"> + <input type="number" + id="txt-publication-year" + name="publication-year" + class="form-control" + min="1960" + max="{{current_year}}" + value="{{current_year or ''}}" + required="required" /> + <span class="form-text text-muted">Year of publication</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-volume" + class="col-sm-2 col-form-label"> + Volume</label> + <div class="col-sm-4"> + <input type="text" + id="txt-publication-volume" + name="publication-volume" + class="form-control"> + <span class="form-text text-muted">Journal volume</span> + </div> + + <label for="txt-publication-pages" + class="col-sm-2 col-form-label"> + Pages</label> + <div class="col-sm-4"> + <input type="text" + id="txt-publication-pages" + name="publication-pages" + class="form-control" /> + <span class="form-text text-muted">Journal pages for the publication</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-abstract" class="col-sm-2 col-form-label">Abstract</label> + <div class="col-sm-10"> + <textarea id="txt-publication-abstract" + name="publication-abstract" + class="form-control" + rows="7"></textarea> + </div> + </div> + + <div class="row"> + <div class="col"> + <input type="submit" + class="btn btn-primary" + value="create publication" /> + </div> + <div class="col"> + <input type="reset" class="btn btn-danger" value="reset form" /> + </div> + </div> + +</form> +</div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript" src="/static/js/pubmed.js"></script> +<script type="text/javascript"> + $(function() { + $("#btn-search-pubmed-id").on("click", (event) => { + event.preventDefault(); + var search_button = event.target; + var pubmed_id = $("#txt-pubmed-id").val().trim(); + remove_class($("#txt-pubmed-id").parent(), "has-error"); + if(pubmed_id == "") { + add_class($("#txt-pubmed-id").parent(), "has-error"); + return false; + } + + search_button.disabled = true; + // Fetch publication details + fetch_publication_details(pubmed_id, + [() => {search_button.disabled = false;}]); + return false; + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/publications/delete-publication-success.html b/uploader/templates/publications/delete-publication-success.html new file mode 100644 index 0000000..53a44ec --- /dev/null +++ b/uploader/templates/publications/delete-publication-success.html @@ -0,0 +1,18 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}View Publication{%endblock%} + +{%block pagetitle%}View Publication{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() {}); +</script> +{%endblock%} diff --git a/uploader/templates/publications/delete-publication.html b/uploader/templates/publications/delete-publication.html new file mode 100644 index 0000000..a9c8c7c --- /dev/null +++ b/uploader/templates/publications/delete-publication.html @@ -0,0 +1,95 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Delete Publication{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('publications.delete_publication', + publication_id=publication.Id)}}" + title="Manage publications">delete publication</a> +</li> +{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} +<div class="row"> + <p>You are about to delete the publication with the following details:</p> +</div> + +<div class="row"> + <table class="table"> + <tr> + <th>Linked Phenotypes</th> + <td>{{linked_phenotypes | count}}</td> + </tr> + <tr> + <th>PubMed</th> + <td> + {%if publication.PubMed_ID%} + <a href="https://pubmed.ncbi.nlm.nih.gov/{{publication.PubMed_ID}}/" + target="_blank">{{publication.PubMed_ID}}</a> + {%else%} + — + {%endif%} + </td> + </tr> + <tr> + <th>Title</th> + <td>{{publication.Title or "—"}}</td> + </tr> + <tr> + <th>Authors</th> + <td>{{publication.Authors or "—"}}</td> + </tr> + <tr> + <th>Journal</th> + <td>{{publication.Journal or "—"}}</td> + </tr> + <tr> + <th>Published</th> + <td>{{publication.Month or ""}} {{publication.Year or "—"}}</td> + </tr> + <tr> + <th>Volume</th> + <td>{{publication.Volume or "—"}}</td> + </tr> + <tr> + <th>Pages</th> + <td>{{publication.Pages or "—"}}</td> + </tr> + <tr> + <th>Abstract</th> + <td> + {%for line in (publication.Abstract or "—").replace("\r\n", "<br />").replace("\n", "<br />").split("<br />")%} + <p>{{line}}</p> + {%endfor%} + </td> + </tr> + </table> +</div> + +<div class="row"> + <p>If you are sure that is what you want, click the button below to delete the + publication</p> + <p class="form-text text-small"> + <small>You will not be able to recover the data if you click + delete below.</small></p> + + <form action="{{url_for('publications.delete_publication', publication_id=publication_id)}}" + method="POST"> + <div class="form-group"> + <input type="submit" value="delete" class="btn btn-danger" /> + </div> + </form> +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() {}); +</script> +{%endblock%} diff --git a/uploader/templates/publications/edit-publication.html b/uploader/templates/publications/edit-publication.html new file mode 100644 index 0000000..314a78c --- /dev/null +++ b/uploader/templates/publications/edit-publication.html @@ -0,0 +1,203 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Edit Publication{%endblock%} + +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('publications.edit_publication', + publication_id=publication.Id)}}" + title="Edit the publication's details">edit publication</a> +</li> +{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <form id="frm-create-publication" + method="POST" + action="{{url_for('publications.edit_publication', + publication_id=publication_id, + next=request.args.get('next', ''))}}" + class="form-horizontal"> + + <div class="row mb-3"> + <label for="txt-pubmed-id" class="col-sm-2 col-form-label"> + PubMed ID</label> + <div class="col-sm-10"> + <div class="input-group"> + <input type="text" + id="txt-pubmed-id" + name="pubmed-id" + value="{{publication.PubMed_ID or ''}}" + class="form-control" /> + <div class="input-group-text"> + <button class="btn btn-outline-primary" + id="btn-search-pubmed-id">search</button> + </div> + </div> + <span id="search-pubmed-id-error" + class="form-text text-muted text-danger visually-hidden"> + </span> + <span class="form-text text-muted">This is the publication's ID on + <a href="https://pubmed.ncbi.nlm.nih.gov/" + title="Link to NCBI's PubMed service">NCBI's Pubmed Service</a> + </span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-title" class="col-sm-2 col-form-label"> + Title</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-title" + name="publication-title" + value="{{publication.Title}}" + class="form-control" /> + <span class="form-text text-muted">Provide the publication's title here.</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-authors" class="col-sm-2 col-form-label"> + Authors</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-authors" + name="publication-authors" + value="{{publication.Authors}}" + required="required" + class="form-control" /> + <span class="form-text text-muted"> + A publication <strong>MUST</strong> have an author. You <em>must</em> + provide a value for the authors field. + </span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-journal" class="col-sm-2 col-form-label"> + Journal</label> + <div class="col-sm-10"> + <input type="text" + id="txt-publication-journal" + name="publication-journal" + value="{{publication.Journal}}" + class="form-control" /> + <span class="form-text text-muted">Provide the name journal where the + publication was done, here.</span> + </div> + </div> + + <div class="row mb-3"> + <label for="select-publication-month" + class="col-sm-2 col-form-label"> + Month</label> + <div class="col-sm-4"> + <select class="form-control" + id="select-publication-month" + name="publication-month"> + <option value="">Select a month</option> + {%for month in ("january", "february", "march", "april", "may", "june", "july", "august", "september", "october", "november", "december"):%} + <option value="{{month}}" + {%if publication.Month | lower == month %} + selected="selected" + {%endif%}> + {{month | title}} + </option> + {%endfor%} + </select> + <span class="form-text text-muted">Month of publication</span> + </div> + + <label for="txt-publication-year" + class="col-sm-2 col-form-label"> + Year</label> + <div class="col-sm-4"> + <input type="number" + id="txt-publication-year" + name="publication-year" + value="{{publication.Year}}" + class="form-control" + min="1960" /> + <span class="form-text text-muted">Year of publication</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-publication-volume" + class="col-sm-2 col-form-label"> + Volume</label> + <div class="col-sm-4"> + <input type="text" + id="txt-publication-volume" + name="publication-volume" + value="{{publication.Volume}}" + class="form-control"> + <span class="form-text text-muted">Journal volume</span> + </div> + + <label for="txt-publication-pages" + class="col-sm-2 col-form-label"> + Pages</label> + <div class="col-sm-4"> + <input type="text" + id="txt-publication-pages" + name="publication-pages" + value="{{publication.Pages}}" + class="form-control" /> + <span class="form-text text-muted">Journal pages for the publication</span> + </div> + </div> + + <div class="row mb-3"> + <label for="txt-abstract" class="col-sm-2 col-form-label">Abstract</label> + <div class="col-sm-10"> + <textarea id="txt-publication-abstract" + name="publication-abstract" + class="form-control" + rows="7">{{publication.Abstract or ""}}</textarea> + </div> + </div> + + <div class="row mb-3"> + <div class="col-sm-2"></div> + <div class="col-sm-8"> + <input type="submit" class="btn btn-primary" value="Save" /> + <input type="reset" class="btn btn-danger" /> + </div> + </div> + +</form> +</div> + +{%endblock%} + + +{%block javascript%} +<script type="text/javascript" src="/static/js/pubmed.js"></script> +<script type="text/javascript"> + $(function() { + $("#btn-search-pubmed-id").on("click", (event) => { + event.preventDefault(); + var search_button = event.target; + var pubmed_id = $("#txt-pubmed-id").val().trim(); + remove_class($("#txt-pubmed-id").parent(), "has-error"); + if(pubmed_id == "") { + add_class($("#txt-pubmed-id").parent(), "has-error"); + return false; + } + + search_button.disabled = true; + // Fetch publication details + fetch_publication_details(pubmed_id, + [() => {search_button.disabled = false;}]); + return false; + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/publications/index.html b/uploader/templates/publications/index.html new file mode 100644 index 0000000..eb2e81b --- /dev/null +++ b/uploader/templates/publications/index.html @@ -0,0 +1,109 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}Publications{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row" style="padding-bottom: 1em;"> + <div class="col"> + <a href="{{url_for('publications.create_publication')}}" + class="btn btn-primary" + title="Create a new publication."> + add new publication</a> + </div> +</div> + +<div class="row"> + <p>Click on the title to view more details or to edit the information for that + publication.</p> +</div> + +<div class="row"> + <table id="tbl-list-publications" class="table compact stripe"> + <thead> + <tr> + <th>Index</th> + <th>PubMed ID</th> + <th>Title</th> + <th>Authors</th> + </tr> + </thead> + + <tbody></tbody> + </table> +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript" src="/static/js/urls.js"></script> + +<script type="text/javascript"> + $(function() { + var publicationsDataTable = buildDataTable( + "#tbl-list-publications", + [], + [ + {data: "index"}, + { + searchable: true, + data: (pub) => { + if(pub.PubMed_ID) { + return `<a href="https://pubmed.ncbi.nlm.nih.gov/` + + `${pub.PubMed_ID}/" target="_blank" ` + + `title="Link to publication on NCBI.">` + + `${pub.PubMed_ID}</a>`; + } + return ""; + } + }, + { + searchable: true, + data: (pub) => { + var title = "⸻"; + if(pub.Title) { + title = pub.Title + } + url=buildURLFromCurrentURL( + `/publications/view/${pub.Id}`); + return `<a href="${url}" target="_blank" ` + + `title="Link to view publication details">` + + `${title}</a>`; + } + }, + { + searchable: true, + data: (pub) => { + authors = pub.Authors.split(",").map( + (item) => {return item.trim();}); + if(authors.length > 1) { + return authors[0] + ", et. al."; + } + return authors[0]; + } + } + ], + { + serverSide: true, + ajax: { + url: "/publications/list", + dataSrc: "publications" + }, + scrollY: 700, + scroller: true, + scrollCollapse: true, + paging: true, + deferRender: true, + layout: { + topStart: "info", + topEnd: "search", + bottomStart: "pageLength", + bottomEnd: false + } + }); + }); +</script> +{%endblock%} diff --git a/uploader/templates/publications/view-publication.html b/uploader/templates/publications/view-publication.html new file mode 100644 index 0000000..01ccf1e --- /dev/null +++ b/uploader/templates/publications/view-publication.html @@ -0,0 +1,80 @@ +{%extends "publications/base.html"%} +{%from "flash_messages.html" import flash_all_messages%} + +{%block title%}View Publication{%endblock%} + + +{%block contents%} +{{flash_all_messages()}} + +<div class="row"> + <table class="table"> + <tr> + <th>Linked Phenotypes</th> + <td>{{linked_phenotypes | count}}</td> + </tr> + <tr> + <th>PubMed</th> + <td> + {%if publication.PubMed_ID%} + <a href="https://pubmed.ncbi.nlm.nih.gov/{{publication.PubMed_ID}}/" + target="_blank">{{publication.PubMed_ID}}</a> + {%else%} + — + {%endif%} + </td> + </tr> + <tr> + <th>Title</th> + <td>{{publication.Title or "—"}}</td> + </tr> + <tr> + <th>Authors</th> + <td>{{publication.Authors or "—"}}</td> + </tr> + <tr> + <th>Journal</th> + <td>{{publication.Journal or "—"}}</td> + </tr> + <tr> + <th>Published</th> + <td>{{publication.Month or ""}} {{publication.Year or "—"}}</td> + </tr> + <tr> + <th>Volume</th> + <td>{{publication.Volume or "—"}}</td> + </tr> + <tr> + <th>Pages</th> + <td>{{publication.Pages or "—"}}</td> + </tr> + <tr> + <th>Abstract</th> + <td> + {%for line in (publication.Abstract or "—").replace("\r\n", "<br />").replace("\n", "<br />").split("<br />")%} + <p>{{line}}</p> + {%endfor%} + </td> + </tr> + </table> +</div> + +<div class="row"> + <div> + <a href="{{url_for('publications.edit_publication', publication_id=publication.Id)}}" + title="Edit details for this publication." + class="btn btn-primary">Edit</a> + {%if linked_phenotypes | length == 0%} + <a href="{{url_for('publications.delete_publication', publication_id=publication.Id)}}" + class="btn btn-danger">delete</a> + {%endif%} + </div> +</div> +{%endblock%} + + +{%block javascript%} +<script type="text/javascript"> + $(function() {}); +</script> +{%endblock%} diff --git a/uploader/templates/samples/base.html b/uploader/templates/samples/base.html index 291782b..7fd5020 100644 --- a/uploader/templates/samples/base.html +++ b/uploader/templates/samples/base.html @@ -1,12 +1,25 @@ {%extends "populations/base.html"%} +{%from "populations/macro-display-population-card.html" import display_sui_population_card%} -{%block lvl3_breadcrumbs%} -<li {%if activelink=="samples"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.samples.index')}}">Samples</a> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> + <a href="{{url_for('species.populations.samples.list_samples', + species_id=species['SpeciesId'], + population_id=population.Id)}}"> + Samples + </a> </li> -{%block lvl4_breadcrumbs%}{%endblock%} +{%endblock%} + +{%block contents%} +<div class="row"> + <h2 class="heading">{{population.FullName}} ({{population.Name}})</h2> +</div> +{%endblock%} + + + +{%block sidebarcontents%} +{{display_sui_population_card(species, population)}} {%endblock%} diff --git a/uploader/templates/samples/index.html b/uploader/templates/samples/index.html index ee4a63e..ee98734 100644 --- a/uploader/templates/samples/index.html +++ b/uploader/templates/samples/index.html @@ -17,3 +17,7 @@ {{select_species_form(url_for("species.populations.samples.index"), species)}} </div> {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/species.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/list-samples.html b/uploader/templates/samples/list-samples.html index 13e5cec..3aac984 100644 --- a/uploader/templates/samples/list-samples.html +++ b/uploader/templates/samples/list-samples.html @@ -1,40 +1,34 @@ {%extends "samples/base.html"%} {%from "flash_messages.html" import flash_all_messages%} {%from "populations/macro-select-population.html" import select_population_form%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Samples — List Samples{%endblock%} -{%block pagetitle%}Samples — List Samples{%endblock%} - -{%block lvl4_breadcrumbs%} -<li {%if activelink=="list-samples"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.populations.samples.list_samples', - species_id=species.SpeciesId, - population_id=population.Id)}}">List</a> -</li> -{%endblock%} - {%block contents%} -{{flash_all_messages()}} +{{super()}} <div class="row"> - <p> - You selected the population "{{population.FullName}}" from the - "{{species.FullName}}" species. - </p> + <h3 class="subheading">manage samples</h3> + {{flash_all_messages()}} +</div> + +<div class="row"> + <div class="col"> + <a href="{{url_for('species.populations.samples.upload_samples', + species_id=species.SpeciesId, + population_id=population.Id)}}" + title="Add samples for population '{{population.FullName}}' from species + '{{species.FullName}}'." + class="btn btn-primary">add new samples</a> + </div> </div> {%if samples | length > 0%} <div class="row"> <p> - This population already has <strong>{{total_samples}}</strong> - samples/individuals entered. You can explore the list of samples in this - population in the table below. + Population "{{population.FullName}} ({{population.Name}})" already has + <strong>{{total_samples}}</strong> samples/individuals entered. You can + explore the list of samples in the table below. </p> </div> @@ -73,7 +67,7 @@ <table class="table"> <thead> <tr> - <th>#</th> + <th></th> <th>Name</th> <th>Auxilliary Name</th> <th>Symbol</th> @@ -93,40 +87,12 @@ {%endfor%} </tbody> </table> - - <p> - <a href="#" - title="Add samples for population '{{population.FullName}}' from species - '{{species.FullName}}'." - class="btn btn-danger"> - delete all samples - </a> - </p> </div> - {%else%} - <div class="row"> - <p> - There are no samples entered for this population. Do please go ahead and add - the samples for this population by clicking on the button below. - </p> - - <p> - <a href="{{url_for('species.populations.samples.upload_samples', - species_id=species.SpeciesId, - population_id=population.Id)}}" - title="Add samples for population '{{population.FullName}}' from species - '{{species.FullName}}'." - class="btn btn-primary"> - add samples - </a> - </p> + <p>There are no samples entered for this population. Click the "Add Samples" + button above, to add some new samples.</p> </div> {%endif%} {%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/samples/select-population.html b/uploader/templates/samples/select-population.html index f437780..1cc7573 100644 --- a/uploader/templates/samples/select-population.html +++ b/uploader/templates/samples/select-population.html @@ -12,28 +12,15 @@ {{flash_all_messages()}} <div class="row"> - <p>You have selected "{{species.FullName}}" as the species that your data relates to.</p> - <p>Next, we need information regarding the population your data relates to. Do please select the population from the existing ones below</p> -</div> - -<div class="row"> {{select_population_form( - url_for("species.populations.samples.select_population", species_id=species.SpeciesId), - populations)}} -</div> - -<div class="row"> - <p> - If you cannot find the population your data relates to in the drop-down - above, you might want to - <a href="{{url_for('species.populations.create_population', - species_id=species.SpeciesId)}}" - title="Create a new population for species '{{species.FullName}},"> - add a new population to GeneNetwork</a> - instead. + url_for("species.populations.samples.select_population", species_id=species.SpeciesId), species, populations)}} </div> {%endblock%} {%block sidebarcontents%} {{display_species_card(species)}} {%endblock%} + +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> +{%endblock%} diff --git a/uploader/templates/samples/upload-failure.html b/uploader/templates/samples/upload-failure.html index 458ab55..75192ec 100644 --- a/uploader/templates/samples/upload-failure.html +++ b/uploader/templates/samples/upload-failure.html @@ -1,6 +1,5 @@ {%extends "base.html"%} {%from "cli-output.html" import cli_output%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Samples Upload Failure{%endblock%} @@ -15,7 +14,7 @@ <h3>Debugging Information</h3> <ul> - <li><strong>job id</strong>: {{job.job_id}}</li> + <li><strong>job id</strong>: {{job.jobid}}</li> <li><strong>status</strong>: {{job.status}}</li> <li><strong>job type</strong>: {{job["job-type"]}}</li> </ul> @@ -31,7 +30,3 @@ {{cli_output(job, "stderr")}} </div> {%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/samples/upload-progress.html b/uploader/templates/samples/upload-progress.html index 677d457..38f931b 100644 --- a/uploader/templates/samples/upload-progress.html +++ b/uploader/templates/samples/upload-progress.html @@ -1,6 +1,5 @@ {%extends "samples/base.html"%} {%from "cli-output.html" import cli_output%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block extrameta%} <meta http-equiv="refresh" content="5"> @@ -25,7 +24,3 @@ </div> {%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/samples/upload-samples.html b/uploader/templates/samples/upload-samples.html index 25d3290..1f665a3 100644 --- a/uploader/templates/samples/upload-samples.html +++ b/uploader/templates/samples/upload-samples.html @@ -1,21 +1,16 @@ {%extends "samples/base.html"%} {%from "flash_messages.html" import flash_all_messages%} -{%from "populations/macro-select-population.html" import select_population_form%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Samples — Upload Samples{%endblock%} -{%block pagetitle%}Samples — Upload Samples{%endblock%} - -{%block lvl4_breadcrumbs%} -<li {%if activelink=="uploade-samples"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> +{%block breadcrumbs%} +{{super()}} +<li class="breadcrumb-item"> <a href="{{url_for('species.populations.samples.upload_samples', - species_id=species.SpeciesId, - population_id=population.Id)}}">List</a> + species_id=species['SpeciesId'], + population_id=population.Id)}}"> + Upload + </a> </li> {%endblock%} @@ -23,35 +18,6 @@ {{flash_all_messages()}} <div class="row"> - <p> - You can now upload the samples for the "{{population.FullName}}" population - from the "{{species.FullName}}" species here. - </p> - <p> - Upload a <strong>character-separated value (CSV)</strong> file that contains - details about your samples. The CSV file should have the following fields: - <dl> - <dt>Name</dt> - <dd>The primary name/identifier for the sample/individual.</dd> - - <dt>Name2</dt> - <dd>A secondary name for the sample. This can simply be the same as - <strong>Name</strong> above. This field <strong>MUST</strong> contain a - value.</dd> - - <dt>Symbol</dt> - <dd>A symbol for the sample. This can be a strain name, e.g. 'BXD60' for - species that have strains. This field can be left empty for species like - Humans that do not have strains..</dd> - - <dt>Alias</dt> - <dd>An alias for the sample. Can be an empty field, or take on the same - value as that of the Symbol.</dd> - </dl> - </p> -</div> - -<div class="row"> <form id="form-samples" method="POST" action="{{url_for('species.populations.samples.upload_samples', @@ -65,14 +31,17 @@ <div class="form-group"> <label for="file-samples" class="form-label">select file</label> - <input type="file" name="samples_file" id="file:samples" - accept="text/csv, text/tab-separated-values" + <input type="file" name="samples_file" id="file-samples" + accept="text/csv, text/tab-separated-values, text/plain" class="form-control" /> + <small class="form-text text-muted"> + See the <a href="#docs-samples-upload">documentation below</a> for + details on expected file format.</small> </div> <div class="form-group"> - <label for="select:separator" class="form-label">field separator</label> - <select id="select:separator" + <label for="select-separator" class="form-label">field separator</label> + <select id="select-separator" name="separator" required="required" class="form-control"> @@ -83,7 +52,7 @@ <option value=";">Semicolon</option> <option value="other">Other</option> </select> - <input id="txt:separator" + <input id="txt-separator" type="text" name="other_separator" class="form-control" /> @@ -95,11 +64,11 @@ </div> <div class="form-group form-check"> - <input id="chk:heading" + <input id="chk-heading" type="checkbox" name="first_line_heading" class="form-check-input" /> - <label for="chk:heading" class="form-check-label"> + <label for="chk-heading" class="form-check-label"> first line is a heading?</label> <small class="form-text text-muted"> Select this if the first line in your file contains headings for the @@ -108,8 +77,8 @@ </div> <div class="form-group"> - <label for="txt:delimiter" class="form-label">field delimiter</label> - <input id="txt:delimiter" + <label for="txt-delimiter" class="form-label">field delimiter</label> + <input id="txt-delimiter" type="text" name="field_delimiter" maxlength="1" @@ -149,10 +118,34 @@ </tbody> </table> </div> -{%endblock%} -{%block sidebarcontents%} -{{display_population_card(species, population)}} + + +<div class="row" id="docs-samples-upload"> + <h3 class="subheading">File Format</h3> + <p> + Upload a <strong>character-separated value (CSV)</strong> file that contains + details about your samples. The CSV file should have the following fields: + <dl> + <dt>Name</dt> + <dd>The primary name/identifier for the sample/individual.</dd> + + <dt>Name2</dt> + <dd>A secondary name for the sample. This can simply be the same as + <strong>Name</strong> above. This field <strong>MUST</strong> contain a + value.</dd> + + <dt>Symbol</dt> + <dd>A symbol for the sample. This can be a strain name, e.g. 'BXD60' for + species that have strains. This field can be left empty for species like + Humans that do not have strains..</dd> + + <dt>Alias</dt> + <dd>An alias for the sample. Can be an empty field, or take on the same + value as that of the Symbol.</dd> + </dl> + </p> +</div> {%endblock%} {%block javascript%} diff --git a/uploader/templates/samples/upload-success.html b/uploader/templates/samples/upload-success.html index 881d466..d6318e9 100644 --- a/uploader/templates/samples/upload-success.html +++ b/uploader/templates/samples/upload-success.html @@ -1,6 +1,5 @@ {%extends "samples/base.html"%} {%from "cli-output.html" import cli_output%} -{%from "populations/macro-display-population-card.html" import display_population_card%} {%block title%}Job Status{%endblock%} @@ -30,7 +29,3 @@ </div> {%endblock%} - -{%block sidebarcontents%} -{{display_population_card(species, population)}} -{%endblock%} diff --git a/uploader/templates/species/base.html b/uploader/templates/species/base.html index f64f72b..3be79f0 100644 --- a/uploader/templates/species/base.html +++ b/uploader/templates/species/base.html @@ -1,17 +1,12 @@ {%extends "base.html"%} -{%block lvl1_breadcrumbs%} -<li {%if activelink=="species"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - {%if species is mapping%} - <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}"> - {{species.Name}}</a> - {%else%} - <a href="{{url_for('species.list_species')}}">Species</a> - {%endif%} +{%block breadcrumbs%} +{{super()}} +{%if species%} +<li class="breadcrumb-item"> + <a href="{{url_for('species.view_species', species_id=species['SpeciesId'])}}"> + {{species["Name"]|title}} + </a> </li> -{%block lvl2_breadcrumbs%}{%endblock%} +{%endif%} {%endblock%} diff --git a/uploader/templates/species/create-species.html b/uploader/templates/species/create-species.html index 0d0bedf..138dbaa 100644 --- a/uploader/templates/species/create-species.html +++ b/uploader/templates/species/create-species.html @@ -19,72 +19,88 @@ <div class="row"> <form id="frm-create-species" method="POST" - action="{{url_for('species.create_species')}}"> + action="{{url_for('species.create_species', return_to=return_to)}}" + class="form-horizontal"> <legend>Create Species</legend> {{flash_all_messages()}} + <input type="hidden" name="return_to" value="{{return_to}}"> + <div class="form-group"> - <label for="txt-taxonomy-id" class="form-label"> + <label for="txt-taxonomy-id" class="control-label col-sm-2"> Taxonomy ID</label> - <div class="input-group"> - <input id="txt-taxonomy-id" - name="species_taxonomy_id" - type="text" - class="form-control" /> - <span class="input-group-btn"> - <button id="btn-search-taxonid" class="btn btn-info">Search</button> - </span> + <div class="col-sm-10"> + <div class="input-group"> + <input id="txt-taxonomy-id" + name="species_taxonomy_id" + type="text" + class="form-control" /> + <span class="input-group-btn"> + <button id="btn-search-taxonid" class="btn btn-info">Search</button> + </span> + </div> + <small class="form-text text-small text-muted"> + Use + <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/taxonomyhome.html/" + title="NCBI's Taxonomy Browser homepage" + target="_blank"> + NCBI's Taxonomy Browser homepage</a> to search for the species you + want. If the species exists on NCBI, they will have a Taxonomy ID. Copy + that Taxonomy ID to this field, and click "Search" to auto-fill the + details.<br /> + This field is optional.</small> </div> - <small class="form-text text-small text-muted">Provide the taxonomy ID for - your species that can be used to link to external sites like NCBI. Enter - the taxonomy ID and click "Search" to auto-fill the form with data. - <br /> - While it is recommended to provide a value for this field, doing so is - optional. - </small> </div> <div class="form-group"> - <label for="txt-species-name" class="form-label">Common Name</label> - <input id="txt-species-name" - name="common_name" - type="text" - class="form-control" - required="required" /> - <small class="form-text text-muted">Provide the common, possibly - non-scientific name for the species here, e.g. Human, Mouse, etc.</small> + <label for="txt-species-name" class="control-label col-sm-2">Common Name</label> + <div class="col-sm-10"> + <input id="txt-species-name" + name="common_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the day-to-day term used by + laymen, e.g. Mouse (instead of Mus musculus), round worm (instead of + Ascaris lumbricoides), etc.<br /> + For species without this, just enter the scientific name. + </small> + </div> </div> <div class="form-group"> - <label for="txt-species-scientific" class="form-label"> + <label for="txt-species-scientific" class="control-label col-sm-2"> Scientific Name</label> - <input id="txt-species-scientific" - name="scientific_name" - type="text" - class="form-control" - required="required" /> - <small class="form-text text-muted">Provide the scientific name for the - species you are creating, e.g. Homo sapiens, Mus musculus, etc.</small> + <div class="col-sm-10"> + <input id="txt-species-scientific" + name="scientific_name" + type="text" + class="form-control" + required="required" /> + <small class="form-text text-muted">This is the scientific name for the + species e.g. Homo sapiens, Mus musculus, etc.</small> + </div> </div> <div class="form-group"> - <label for="select-species-family" class="form-label">Family</label> - <select id="select-species-family" - name="species_family" - required="required" - class="form-control"> - <option value="">Please select a grouping</option> - {%for family in families%} - <option value="{{family}}">{{family}}</option> - {%endfor%} - </select> - <small class="form-text text-muted"> - This is a generic grouping for the species that determines under which - grouping the species appears in the GeneNetwork menus</small> + <label for="select-species-family" class="control-label col-sm-2">Family</label> + <div class="col-sm-10"> + <select id="select-species-family" + name="species_family" + required="required" + class="form-control"> + <option value="ungrouped">I do not know what to pick</option> + {%for family in families%} + <option value="{{family}}">{{family}}</option> + {%endfor%} + </select> + <small class="form-text text-muted"> + This is a rough grouping of the species.</small> + </div> </div> - <div class="form-group"> + <div class="col-sm-offset-2 col-sm-10"> <input type="submit" value="create new species" class="btn btn-primary" /> @@ -113,7 +129,7 @@ } msg = ( "Request to '${uri}' failed with message '${textStatus}'. " - + "Please try again later, or fill the details manually."); + + "Please try again later, or fill the details manually."); alert(msg); console.error(msg, data, textStatus); return false; diff --git a/uploader/templates/species/list-species.html b/uploader/templates/species/list-species.html index 85c9d40..64084b0 100644 --- a/uploader/templates/species/list-species.html +++ b/uploader/templates/species/list-species.html @@ -29,7 +29,7 @@ <caption>Available Species</caption> <thead> <tr> - <th>#</td> + <th></td> <th title="A common, layman's name for the species.">Common Name</th> <th title="The scientific name for the species">Organism Name</th> <th title="An identifier for the species in the NCBI taxonomy database"> diff --git a/uploader/templates/species/macro-display-species-card.html b/uploader/templates/species/macro-display-species-card.html index 166c7b9..30c564f 100644 --- a/uploader/templates/species/macro-display-species-card.html +++ b/uploader/templates/species/macro-display-species-card.html @@ -20,3 +20,32 @@ </div> </div> {%endmacro%} + + +{%macro display_sui_species_card(species)%} +<div class="row"> + <table class="table"> + <caption>Current Species</caption> + <tbody> + <tr> + <th>Name</th> + <td>{{species["Name"] | title}}</td> + </tr> + <tr> + <th>Scientific</th> + <td>{{species["FullName"]}}</td> + </tr> + {%if species["TaxonomyId"]%} + <tr> + <th>Taxonomy ID</th> + <td> + <a href="https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?id={{species.TaxonomyId}}" + title="NCBI's Taxonomy Browser page for {{species.Name}}"> + {{species.TaxonomyId}}</a> + </td> + </tr> + </tbody> + {%endif%} + </table> +</div> +{%endmacro%} diff --git a/uploader/templates/species/macro-select-species.html b/uploader/templates/species/macro-select-species.html index dd086c0..3714ae4 100644 --- a/uploader/templates/species/macro-select-species.html +++ b/uploader/templates/species/macro-select-species.html @@ -1,36 +1,59 @@ +{%from "macro-step-indicator.html" import step_indicator%} + {%macro select_species_form(form_action, species)%} -{%if species | length > 0%} -<form method="GET" action="{{form_action}}"> - <div class="form-group"> - <label for="select-species" class="form-label">Species</label> - <select id="select-species" - name="species_id" - class="form-control" - required="required"> - <option value="">Select Species</option> - {%for group in species%} - {{group}} - <optgroup {%if group[0][1] is not none%} - label="{{group[0][1].capitalize()}}" - {%else%} - label="Undefined" - {%endif%}> - {%for aspecies in group[1]%} - <option value="{{aspecies.SpeciesId}}">{{aspecies.MenuName}}</option> - {%endfor%} - </optgroup> - {%endfor%} - </select> +<form method="GET" action="{{form_action}}" class="form-horizontal"> + + <h2>{{step_indicator("1")}} What species do you want to work with?</h2> + + {%if species | length != 0%} + + <p class="form-text">Search for, and select the species from the table below + and click "Continue"</p> + + <div class="radio"> + <label for="rdo-cant-find-species" + style="font-weight: 1;"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + I could not find the species I want (create it). + </label> </div> - <div class="form-group"> - <input type="submit" value="Select" class="btn btn-primary" /> + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary" + value="continue" /> </div> + + <div style="margin-top:3em;"> + <table id="tbl-select-species" class="table compact stripe" + data-species-list='{{species | tojson}}'> + <div class=""> + <thead> + <tr> + <th></th> + <th>Species Name</th> + </tr> + </thead> + + <tbody></tbody> + </table> + </div> + + {%else%} + + <label class="control-label" for="rdo-cant-find-species"> + <input id="rdo-cant-find-species" type="radio" name="species_id" + value="CREATE-SPECIES" /> + There are no species to select from. Create the first one.</label> + + <div class="col-sm-offset-10 col-sm-2"> + <input type="submit" + class="btn btn-primary col-sm-offset-1" + value="continue" /> + </div> + + {%endif%} + </form> -{%else%} -<p class="text-danger"> - <span class="glyphicon glyphicon-exclamation-mark"></span> - We could not find species to select from! -</p> -{%endif%} {%endmacro%} diff --git a/uploader/templates/species/view-species.html b/uploader/templates/species/view-species.html index 2d02f7e..81608fc 100644 --- a/uploader/templates/species/view-species.html +++ b/uploader/templates/species/view-species.html @@ -1,90 +1,127 @@ {%extends "species/base.html"%} {%from "flash_messages.html" import flash_all_messages%} +{%from "macro-forms.html" import add_http_feature_flags%} +{%from "macro-step-indicator.html" import step_indicator%} +{%from "species/macro-display-species-card.html" import display_sui_species_card%} {%block title%}View Species{%endblock%} -{%block pagetitle%}View Species{%endblock%} +{%macro add_form_buttons()%} +<div class="row form-buttons"> + <div class="col"> + <input type="submit" + value="use selected population" + class="btn btn-primary" /> + </div> + + <div class="col"> + <a href="{{url_for('species.populations.create_population', + species_id=species.SpeciesId, + return_to='species.view_species')}}" + title="Create a new population for species '{{species.Name}}'." + class="btn btn-outline-info"> + Create a new population + </a> + </div> +</div> +{%endmacro%} -{%block lvl2_breadcrumbs%} -<li {%if activelink=="view-species"%} - class="breadcrumb-item active" - {%else%} - class="breadcrumb-item" - {%endif%}> - <a href="{{url_for('species.view_species', species_id=species.SpeciesId)}}">View</a> -</li> -{%endblock%} {%block contents%} -{{flash_all_messages()}} <div class="row"> - <h2>Details on species {{species.FullName}}</h2> + <h2 class="heading">{{species.FullName}} ({{species.Name}})</h2> +</div> - <dl> - <dt>Common Name</dt> - <dd>{{species.SpeciesName}}</dd> +<div class "row"> + <ul class="nav nav-tabs" id="species-actions"> + <li class="nav-item presentation"> + <button class="nav-link active" + id="populations-tab" + data-bs-toggle="tab" + data-bs-target="#populations-content" + type="button" + role="tab" + aria-controls="populations-content" + aria-selected="true">Populations</button> + </li> + <li class="nav-item presentation"> + <button class="nav-link" + id="sequencing-platforms-tab" + data-bs-toggle="tab" + data-bs-target="#sequencing-platforms-content" + type="button" + role="tab" + aria-controls="sequencing-platforms-content" + aria-selected="true">Sequencing Platforms</button> + </li> + </ul> +</div> - <dt>Scientific Name</dt> - <dd>{{species.FullName}}</dd> +<div class="row"> + <div class="tab-content" id="species-tabs-content"> + <div class="tab-pane fade show active" + id="populations-content" + role="tabpanel" + aria-labelledby="populations-content-tab"> + <p>Data belonging to a particular species is further divided into one or more + populations for easier handling. Please select the population you want to work + with.</p> - <dt>Taxonomy ID</dt> - <dd>{{species.TaxonomyId}}</dd> - </dl> + <form method="GET" + action="{{url_for('species.view_species', species_id=species.SpeciesId)}}" + class="form-horizontal"> + {{add_http_feature_flags()}} + {{add_form_buttons()}} - <h3>Actions</h3> + {%if populations | length != 0%} + <div style="margin-top:0.3em;"> + <table id="tbl-select-population" class="table compact stripe" + data-populations-list='{{populations | tojson}}'> + <thead> + <tr> + <th></th> + <th>Population</th> + </tr> + </thead> - <p> - You can proceed to perform any of the following actions for species - {{species.FullName}} - </p> + <tbody></tbody> + </table> + </div> - <ol> - <li> - <a href="{{url_for('species.populations.list_species_populations', - species_id=species.SpeciesId)}}" - title="Create/Edit populations for {{species.FullName}}"> - Manage populations</a> - </li> - <li> - <a href="{{url_for('species.platforms.list_platforms', - species_id=species.SpeciesId)}}" - title="Create/Edit sequencing platforms for {{species.FullName}}"> - Manage sequencing platforms</a> - </li> - </ol> + {%else%} + <p class="form-text"> + There are no populations currently defined for {{species['FullName']}} + ({{species['SpeciesName']}}).</p> + {%endif%} - + {{add_form_buttons()}} + + </form> + </div> + <div class="tab-pane fade" + id="sequencing-platforms-content" + role="tabpanel" + aria-labelledby="sequencing-platforms-content-tab"> + <p>Upload and manage the sequencing platforms for species + '{{species.Name | title}} ({{species.FullName}})' + <a href="{{url_for('species.platforms.list_platforms', + species_id=species.SpeciesId)}}" + title="Manage sequencing platforms for {{species.Name}}">here</a>. + </p> + </div> + </div> </div> {%endblock%} {%block sidebarcontents%} -<div class="card"> - <div class="card-body"> - <h5 class="card-title">Species Extras</h5> - <div class="card-text"> - <p>Some extra internal-use details (mostly for UI concerns on GeneNetwork)</p> - <p> - <small> - If you do not understand what the following are about, simply ignore them - — - They have no bearing whatsoever on your data, or its analysis. - </small> - </p> - <dl> - <dt>Family</dt> - <dd>{{species.Family}}</dd> +<div class="row"> + <p>You can manage species' populations and sequencing platforms here. Select + the tab for the feature you wish to continue working on.</p> +</div> +{{display_sui_species_card(species)}} +{%endblock%} - <dt>FamilyOrderId</dt> - <dd>{{species.FamilyOrderId}}</dd> - <dt>OrderId</dt> - <dd>{{species.OrderId}}</dd> - </dl> - </div> - <a href="{{url_for('species.edit_species_extra', - species_id=species.SpeciesId)}}" - class="card-link" - title="Edit the species' internal-use details.">Edit</a> - </div> -</div> +{%block javascript%} +<script type="text/javascript" src="/static/js/populations.js"></script> {%endblock%} diff --git a/uploader/ui.py b/uploader/ui.py index 1994056..41791c7 100644 --- a/uploader/ui.py +++ b/uploader/ui.py @@ -1,5 +1,5 @@ """Utilities to handle the UI""" -from flask import render_template as flask_render_template +from uploader.flask_extensions import render_template as flask_render_template def make_template_renderer(default): """Render template for species.""" |
