aboutsummaryrefslogtreecommitdiff
path: root/uploader
diff options
context:
space:
mode:
Diffstat (limited to 'uploader')
-rw-r--r--uploader/default_settings.py7
-rw-r--r--uploader/files.py23
-rw-r--r--uploader/input_validation.py44
-rw-r--r--uploader/jobs.py2
-rw-r--r--uploader/phenotypes/models.py28
-rw-r--r--uploader/phenotypes/views.py180
-rw-r--r--uploader/population/views.py27
-rw-r--r--uploader/samples/__init__.py1
-rw-r--r--uploader/samples/views.py90
-rw-r--r--uploader/static/css/styles.css34
-rw-r--r--uploader/templates/macro-table-pagination.html26
-rw-r--r--uploader/templates/phenotypes/add-phenotypes.html231
-rw-r--r--uploader/templates/phenotypes/create-dataset.html106
-rw-r--r--uploader/templates/phenotypes/list-datasets.html6
-rw-r--r--uploader/templates/phenotypes/macro-display-pheno-dataset-card.html31
-rw-r--r--uploader/templates/phenotypes/view-dataset.html20
-rw-r--r--uploader/templates/phenotypes/view-phenotype.html34
-rw-r--r--uploader/templates/populations/macro-display-population-card.html42
-rw-r--r--uploader/templates/species/macro-display-species-card.html18
19 files changed, 818 insertions, 132 deletions
diff --git a/uploader/default_settings.py b/uploader/default_settings.py
index 26fe665..1acb247 100644
--- a/uploader/default_settings.py
+++ b/uploader/default_settings.py
@@ -2,15 +2,12 @@
The default configuration file. The values here should be overridden in the
actual configuration file used for the production and staging systems.
"""
-
-import os
-
-LOG_LEVEL = os.getenv("LOG_LEVEL", "WARNING")
+LOG_LEVEL = "WARNING"
SECRET_KEY = b"<Please! Please! Please! Change This!>"
UPLOAD_FOLDER = "/tmp/qc_app_files"
REDIS_URL = "redis://"
JOBS_TTL_SECONDS = 1209600 # 14 days
-GNQC_REDIS_PREFIX="GNQC"
+GNQC_REDIS_PREFIX="gn-uploader"
SQL_URI = ""
GN2_SERVER_URL = "https://genenetwork.org/"
diff --git a/uploader/files.py b/uploader/files.py
index b163612..d37a53e 100644
--- a/uploader/files.py
+++ b/uploader/files.py
@@ -1,7 +1,9 @@
"""Utilities to deal with uploaded files."""
import hashlib
from pathlib import Path
+from typing import Iterator
from datetime import datetime
+
from flask import current_app
from werkzeug.utils import secure_filename
@@ -21,6 +23,27 @@ def save_file(fileobj: FileStorage, upload_dir: Path) -> Path:
fileobj.save(filepath)
return filepath
+
def fullpath(filename: str):
"""Get a file's full path. This makes use of `flask.current_app`."""
return Path(current_app.config["UPLOAD_FOLDER"], filename).absolute()
+
+
+def chunked_binary_read(filepath: Path, chunksize: int = 2048) -> Iterator:
+ """Read a file in binary mode in chunks."""
+ with open(filepath, "rb") as inputfile:
+ while True:
+ data = inputfile.read(chunksize)
+ if data != b"":
+ yield data
+ continue
+ break
+
+
+def sha256_digest_over_file(filepath: Path) -> str:
+ """Compute the sha256 digest over a file's contents."""
+ filehash = hashlib.sha256()
+ for chunk in chunked_binary_read(filepath):
+ filehash.update(chunk)
+
+ return filehash.hexdigest()
diff --git a/uploader/input_validation.py b/uploader/input_validation.py
index 9abe742..627c69e 100644
--- a/uploader/input_validation.py
+++ b/uploader/input_validation.py
@@ -1,14 +1,19 @@
"""Input validation utilities"""
+import re
+import json
+import base64
from typing import Any
def is_empty_string(value: str) -> bool:
"""Check whether as string is empty"""
return (isinstance(value, str) and value.strip() == "")
+
def is_empty_input(value: Any) -> bool:
"""Check whether user provided an empty value."""
return (value is None or is_empty_string(value))
+
def is_integer_input(value: Any) -> bool:
"""
Check whether user provided a value that can be parsed into an integer.
@@ -25,3 +30,42 @@ def is_integer_input(value: Any) -> bool:
__is_int__(value, 10)
or __is_int__(value, 8)
or __is_int__(value, 16))))
+
+
+def is_valid_representative_name(repr_name: str) -> bool:
+ """
+ Check whether the given representative name is a valid according to our rules.
+
+ Parameters
+ ----------
+ repr_name: a string of characters.
+
+ Checks For
+ ----------
+ * The name MUST start with an alphabet [a-zA-Z]
+ * The name MUST end with an alphabet [a-zA-Z] or number [0-9]
+ * The name MUST be composed of alphabets [a-zA-Z], numbers [0-9],
+ underscores (_) and/or hyphens (-).
+
+ Returns
+ -------
+ Boolean indicating whether or not the name is valid.
+ """
+ pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]$")
+ return bool(pattern.match(repr_name))
+
+
+def encode_errors(errors: tuple[tuple[str, str], ...], form) -> bytes:
+ """Encode form errors into base64 string."""
+ return base64.b64encode(
+ json.dumps({
+ "errors": dict(errors),
+ "original_formdata": dict(form)
+ }).encode("utf8"))
+
+
+def decode_errors(errorstr) -> dict[str, dict]:
+ """Decode errors from base64 string"""
+ if not bool(errorstr):
+ return {"errors": {}, "original_formdata": {}}
+ return json.loads(base64.b64decode(errorstr.encode("utf8")).decode("utf8"))
diff --git a/uploader/jobs.py b/uploader/jobs.py
index 21889da..4a3fc80 100644
--- a/uploader/jobs.py
+++ b/uploader/jobs.py
@@ -10,7 +10,7 @@ from typing import Union, Optional
from redis import Redis
from flask import current_app as app
-JOBS_PREFIX = "JOBS"
+JOBS_PREFIX = "jobs"
class JobNotFound(Exception):
"""Raised if we try to retrieve a non-existent job."""
diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py
index be970ac..9324601 100644
--- a/uploader/phenotypes/models.py
+++ b/uploader/phenotypes/models.py
@@ -1,6 +1,7 @@
"""Database and utility functions for phenotypes."""
from typing import Optional
from functools import reduce
+from datetime import datetime
import MySQLdb as mdb
from MySQLdb.cursors import Cursor, DictCursor
@@ -202,3 +203,30 @@ def phenotypes_data(conn: mdb.Connection,
cursor.execute(_query, (population_id, dataset_id))
debug_query(cursor)
return tuple(dict(row) for row in cursor.fetchall())
+
+
+def save_new_dataset(cursor: Cursor,
+ population_id: int,
+ dataset_name: str,
+ dataset_fullname: str,
+ dataset_shortname: str) -> dict:
+ """Create a new phenotype dataset."""
+ params = {
+ "population_id": population_id,
+ "dataset_name": dataset_name,
+ "dataset_fullname": dataset_fullname,
+ "dataset_shortname": dataset_shortname,
+ "created": datetime.now().date().isoformat(),
+ "public": 2,
+ "confidentiality": 0,
+ "users": None
+ }
+ cursor.execute(
+ "INSERT INTO PublishFreeze(Name, FullName, ShortName, CreateTime, "
+ "public, InbredSetId, confidentiality, AuthorisedUsers) "
+ "VALUES(%(dataset_name)s, %(dataset_fullname)s, %(dataset_shortname)s, "
+ "%(created)s, %(public)s, %(population_id)s, %(confidentiality)s, "
+ "%(users)s)",
+ params)
+ debug_query(cursor)
+ return {**params, "Id": cursor.lastrowid}
diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py
index 63e0b84..02e8078 100644
--- a/uploader/phenotypes/views.py
+++ b/uploader/phenotypes/views.py
@@ -1,6 +1,13 @@
"""Views handling ('classical') phenotypes."""
+import sys
+import uuid
+import json
+from pathlib import Path
from functools import wraps
+from redis import Redis
+from requests.models import Response
+from MySQLdb.cursors import DictCursor
from flask import (flash,
request,
url_for,
@@ -9,6 +16,12 @@ from flask import (flash,
render_template,
current_app as app)
+# from r_qtl import r_qtl2 as rqtl2
+from r_qtl import r_qtl2_qc as rqc
+from r_qtl import exceptions as rqe
+
+from uploader import jobs
+from uploader.files import save_file#, fullpath
from uploader.oauth2.client import oauth2_post
from uploader.authorisation import require_login
from uploader.db_utils import database_connection
@@ -18,10 +31,14 @@ from uploader.request_checks import with_species, with_population
from uploader.datautils import safe_int, order_by_family, enumerate_sequence
from uploader.population.models import (populations_by_species,
population_by_species_and_id)
+from uploader.input_validation import (encode_errors,
+ decode_errors,
+ is_valid_representative_name)
from .models import (dataset_by_id,
phenotype_by_id,
phenotypes_count,
+ save_new_dataset,
dataset_phenotypes,
datasets_by_population)
@@ -170,6 +187,8 @@ def view_dataset(# pylint: disable=[unused-argument]
offset=start_at,
limit=count),
start=start_at+1),
+ start_from=start_at,
+ count=count,
activelink="view-dataset")
@@ -190,6 +209,31 @@ def view_phenotype(# pylint: disable=[unused-argument]
**kwargs
):
"""View an individual phenotype from the dataset."""
+ def __render__(privileges):
+ return render_template(
+ "phenotypes/view-phenotype.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ phenotype=phenotype_by_id(conn,
+ species["SpeciesId"],
+ population["Id"],
+ dataset["Id"],
+ xref_id),
+ privileges=(privileges
+ ### For demo! Do not commit this part
+ + ("group:resource:edit-resource",
+ "group:resource:delete-resource",)
+ ### END: For demo! Do not commit this part
+ ),
+ activelink="view-phenotype")
+
+ def __fail__(error):
+ if isinstance(error, Response) and error.json() == "No linked resource!":
+ return __render__(tuple())
+ return make_either_error_handler(
+ "There was an error fetching the roles and privileges.")(error)
+
with database_connection(app.config["SQL_URI"]) as conn:
return oauth2_post(
"/auth/resource/phenotypes/individual/linked-resource",
@@ -203,20 +247,122 @@ def view_phenotype(# pylint: disable=[unused-argument]
lambda resource: tuple(
privilege["privilege_id"] for role in resource["roles"]
for privilege in role["privileges"])
- ).then(
- lambda privileges: render_template(
- "phenotypes/view-phenotype.html",
- species=species,
- population=population,
- dataset=dataset,
- phenotype=phenotype_by_id(conn,
- species["SpeciesId"],
- population["Id"],
- dataset["Id"],
- xref_id),
- privileges=privileges,
- activelink="view-phenotype")
- ).either(
- make_either_error_handler(
- "There was an error fetching the roles and privileges."),
- lambda resp: resp)
+ ).then(__render__).either(__fail__, lambda resp: resp)
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets/create",
+ methods=["GET", "POST"])
+@require_login
+@with_population(
+ species_redirect_uri="species.populations.phenotypes.index",
+ redirect_uri="species.populations.phenotypes.select_population")
+def create_dataset(species: dict, population: dict, **kwargs):# pylint: disable=[unused-argument]
+ """Create a new phenotype dataset."""
+ with (database_connection(app.config["SQL_URI"]) as conn,
+ conn.cursor(cursorclass=DictCursor) as cursor):
+ if request.method == "GET":
+ return render_template("phenotypes/create-dataset.html",
+ activelink="create-dataset",
+ species=species,
+ population=population,
+ **decode_errors(
+ request.args.get("error_values", "")))
+
+ form = request.form
+ _errors: tuple[tuple[str, str], ...] = tuple()
+ if not is_valid_representative_name(
+ (form.get("dataset-name") or "").strip()):
+ _errors = _errors + (("dataset-name", "Invalid dataset name."),)
+
+ if not bool((form.get("dataset-fullname") or "").strip()):
+ _errors = _errors + (("dataset-fullname",
+ "You must provide a value for 'Full Name'."),)
+
+ if bool(_errors) > 0:
+ return redirect(url_for(
+ "species.populations.phenotypes.create_dataset",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ error_values=encode_errors(_errors, form)))
+
+ dataset_shortname = (
+ form["dataset-shortname"] or form["dataset-name"]).strip()
+ _pheno_dataset = save_new_dataset(
+ cursor,
+ population["Id"],
+ form["dataset-name"].strip(),
+ form["dataset-fullname"].strip(),
+ dataset_shortname)
+ return redirect(url_for("species.populations.phenotypes.list_datasets",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"]))
+
+
+@phenotypesbp.route(
+ "<int:species_id>/populations/<int:population_id>/phenotypes/datasets"
+ "/<int:dataset_id>/add-phenotypes",
+ methods=["GET", "POST"])
+@require_login
+@with_dataset(
+ species_redirect_uri="species.populations.phenotypes.index",
+ population_redirect_uri="species.populations.phenotypes.select_population",
+ redirect_uri="species.populations.phenotypes.list_datasets")
+def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# pylint: disable=[unused-argument, too-many-locals]
+ """Add one or more phenotypes to the dataset."""
+ add_phenos_uri = redirect(url_for(
+ "species.populations.phenotypes.add_phenotypes",
+ species_id=species["SpeciesId"],
+ population_id=population["Id"],
+ dataset_id=dataset["Id"]))
+ _redisuri = app.config["REDIS_URL"]
+ _sqluri = app.config["SQL_URI"]
+ with (Redis.from_url(_redisuri, decode_responses=True) as rconn,
+ # database_connection(_sqluri) as conn,
+ # conn.cursor(cursorclass=DictCursor) as cursor
+ ):
+ if request.method == "GET":
+ return render_template("phenotypes/add-phenotypes.html",
+ species=species,
+ population=population,
+ dataset=dataset,
+ activelink="add-phenotypes")
+
+ try:
+ ## Handle huge files here...
+ phenobundle = save_file(request.files["phenotypes-bundle"],
+ Path(app.config["UPLOAD_FOLDER"]))
+ rqc.validate_bundle(phenobundle)
+ except AssertionError as _aerr:
+ app.logger.debug("File upload error!", exc_info=True)
+ flash("Expected a zipped bundle of files with phenotypes' "
+ "information.",
+ "alert-danger")
+ return add_phenos_uri
+ except rqe.RQTLError as rqtlerr:
+ app.logger.debug("Bundle validation error!", exc_info=True)
+ flash("R/qtl2 Error: " + " ".join(rqtlerr.args), "alert-danger")
+ return add_phenos_uri
+
+ _jobid = uuid.uuid4()
+ _namespace = jobs.jobsnamespace()
+ _ttl_seconds = app.config["JOBS_TTL_SECONDS"]
+ _job = jobs.initialise_job(
+ rconn,
+ _namespace,
+ str(_jobid),
+ [sys.executable, "-m", "scripts.rqtl2.phenotypes_qc", _sqluri,
+ _redisuri, _namespace, str(_jobid), str(species["SpeciesId"]),
+ str(population["Id"]), str(dataset["Id"]), "--redisexpiry",
+ str(_ttl_seconds)], "phenotype_qc", _ttl_seconds,
+ {"job-metadata": json.dumps({
+ "speciesid": species["SpeciesId"],
+ "populationid": population["Id"],
+ "datasetid": dataset["Id"],
+ "bundle": str(phenobundle.absolute())})})
+ # jobs.launch_job(
+ # _job,
+ # redisuri,
+ # f"{app.config['UPLOAD_FOLDER']}/job_errors")
+
+ raise NotImplementedError("Please implement this...")
diff --git a/uploader/population/views.py b/uploader/population/views.py
index 3638453..36201ba 100644
--- a/uploader/population/views.py
+++ b/uploader/population/views.py
@@ -1,5 +1,4 @@
"""Views dealing with populations/inbredsets"""
-import re
import json
import base64
@@ -21,6 +20,7 @@ from uploader.datautils import enumerate_sequence
from uploader.phenotypes.views import phenotypesbp
from uploader.expression_data.views import exprdatabp
from uploader.monadic_requests import make_either_error_handler
+from uploader.input_validation import is_valid_representative_name
from uploader.species.models import (all_species,
species_by_id,
order_species_by_family)
@@ -73,29 +73,6 @@ def list_species_populations(species_id: int):
activelink="list-populations")
-def valid_population_name(population_name: str) -> bool:
- """
- Check whether the given name is a valid population name.
-
- Parameters
- ----------
- population_name: a string of characters.
-
- Checks For
- ----------
- * The name MUST start with an alphabet [a-zA-Z]
- * The name MUST end with an alphabet [a-zA-Z] or number [0-9]
- * The name MUST be composed of alphabets [a-zA-Z], numbers [0-9],
- underscores (_) and/or hyphens (-).
-
- Returns
- -------
- Boolean indicating whether or not the name is valid.
- """
- pattern = re.compile(r"^[a-zA-Z]+[a-zA-Z0-9_-]*[a-zA-Z0-9]$")
- return bool(pattern.match(population_name))
-
-
@popbp.route("/<int:species_id>/populations/create", methods=["GET", "POST"])
@require_login
def create_population(species_id: int):
@@ -139,7 +116,7 @@ def create_population(species_id: int):
errors = errors + (("population_name",
"You must provide a name for the population!"),)
- if not valid_population_name(population_name):
+ if not is_valid_representative_name(population_name):
errors = errors + ((
"population_name",
"The population name can only contain letters, numbers, "
diff --git a/uploader/samples/__init__.py b/uploader/samples/__init__.py
new file mode 100644
index 0000000..1bd6d2d
--- /dev/null
+++ b/uploader/samples/__init__.py
@@ -0,0 +1 @@
+"""Samples package. Handle samples uploads and editing."""
diff --git a/uploader/samples/views.py b/uploader/samples/views.py
index 9ba1df8..ed79101 100644
--- a/uploader/samples/views.py
+++ b/uploader/samples/views.py
@@ -3,11 +3,8 @@ import os
import sys
import uuid
from pathlib import Path
-from typing import Iterator
-import MySQLdb as mdb
from redis import Redis
-from MySQLdb.cursors import DictCursor
from flask import (flash,
request,
url_for,
@@ -19,19 +16,16 @@ from uploader import jobs
from uploader.files import save_file
from uploader.ui import make_template_renderer
from uploader.authorisation import require_login
+from uploader.request_checks import with_population
from uploader.input_validation import is_integer_input
-from uploader.datautils import order_by_family, enumerate_sequence
-from uploader.db_utils import (
- with_db_connection,
- database_connection,
- with_redis_connection)
+from uploader.datautils import safe_int, order_by_family, enumerate_sequence
+from uploader.population.models import population_by_id, populations_by_species
+from uploader.db_utils import (with_db_connection,
+ database_connection,
+ with_redis_connection)
from uploader.species.models import (all_species,
species_by_id,
order_species_by_family)
-from uploader.population.models import(save_population,
- population_by_id,
- populations_by_species,
- population_by_species_and_id)
from .models import samples_by_species_and_population
@@ -110,9 +104,7 @@ def list_samples(species_id: int, population_id: int):
all_samples = enumerate_sequence(samples_by_species_and_population(
conn, species_id, population_id))
total_samples = len(all_samples)
- offset = int(request.args.get("from") or 0)
- if offset < 0:
- offset = 0
+ offset = max(safe_int(request.args.get("from") or 0), 0)
count = int(request.args.get("count") or 20)
return render_template("samples/list-samples.html",
species=species,
@@ -233,53 +225,41 @@ def upload_samples(species_id: int, population_id: int):#pylint: disable=[too-ma
"upload-samples/status/<uuid:job_id>",
methods=["GET"])
@require_login
-def upload_status(species_id: int, population_id: int, job_id: uuid.UUID):
+@with_population(species_redirect_uri="species.populations.samples.index",
+ redirect_uri="species.populations.samples.select_population")
+def upload_status(species: dict, population: dict, job_id: uuid.UUID, **kwargs):# pylint: disable=[unused-argument]
"""Check on the status of a samples upload job."""
- with database_connection(app.config["SQL_URI"]) as conn:
- species = species_by_id(conn, species_id)
- if not bool(species):
- flash("You must provide a valid species.", "alert-danger")
- return redirect(url_for("species.populations.samples.index"))
+ job = with_redis_connection(lambda rconn: jobs.job(
+ rconn, jobs.jobsnamespace(), job_id))
+ if job:
+ status = job["status"]
+ if status == "success":
+ return render_template("samples/upload-success.html",
+ job=job,
+ species=species,
+ population=population,)
- population = population_by_species_and_id(
- conn, species_id, population_id)
- if not bool(population):
- flash("You must provide a valid population.", "alert-danger")
+ if status == "error":
return redirect(url_for(
- "species.populations.samples.select_population",
- species_id=species_id))
+ "species.populations.samples.upload_failure", job_id=job_id))
- job = with_redis_connection(lambda rconn: jobs.job(
- rconn, jobs.jobsnamespace(), job_id))
- if job:
- status = job["status"]
- if status == "success":
- return render_template("samples/upload-success.html",
- job=job,
- species=species,
- population=population,)
-
- if status == "error":
+ error_filename = Path(jobs.error_filename(
+ job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
+ if error_filename.exists():
+ stat = os.stat(error_filename)
+ if stat.st_size > 0:
return redirect(url_for(
- "species.populations.samples.upload_failure", job_id=job_id))
+ "samples.upload_failure", job_id=job_id))
- error_filename = Path(jobs.error_filename(
- job_id, f"{app.config['UPLOAD_FOLDER']}/job_errors"))
- if error_filename.exists():
- stat = os.stat(error_filename)
- if stat.st_size > 0:
- return redirect(url_for(
- "samples.upload_failure", job_id=job_id))
-
- return render_template("samples/upload-progress.html",
- species=species,
- population=population,
- job=job) # maybe also handle this?
-
- return render_template("no_such_job.html",
- job_id=job_id,
+ return render_template("samples/upload-progress.html",
species=species,
- population=population), 400
+ population=population,
+ job=job) # maybe also handle this?
+
+ return render_template("no_such_job.html",
+ job_id=job_id,
+ species=species,
+ population=population), 400
@samplesbp.route("/upload/failure/<uuid:job_id>", methods=["GET"])
@require_login
diff --git a/uploader/static/css/styles.css b/uploader/static/css/styles.css
index 574f53e..f482c1b 100644
--- a/uploader/static/css/styles.css
+++ b/uploader/static/css/styles.css
@@ -125,3 +125,37 @@ input[type="submit"], .btn {
border-color: #AAAAAA;
background-color: #EFEFEF;
}
+
+.danger {
+ color: #A94442;
+ border-color: #DCA7A7;
+ background-color: #F2DEDE;
+}
+
+.heading {
+ border-bottom: solid #EEBB88;
+}
+
+.subheading {
+ padding: 1em 0 0.1em 0.5em;
+ border-bottom: solid #88BBEE;
+}
+
+form {
+ margin-top: 0.3em;
+ background: #E5E5FF;
+ padding: 0.5em;
+ border-radius:0.5em;
+}
+
+form .form-control {
+ background-color: #EAEAFF;
+}
+
+.sidebar-content .card .card-title {
+ font-size: 1.5em;
+}
+
+.sidebar-content .card-text table tbody td:nth-child(1) {
+ font-weight: bolder;
+}
diff --git a/uploader/templates/macro-table-pagination.html b/uploader/templates/macro-table-pagination.html
new file mode 100644
index 0000000..292c531
--- /dev/null
+++ b/uploader/templates/macro-table-pagination.html
@@ -0,0 +1,26 @@
+{%macro table_pagination(start_at, page_count, total_count, base_uri, name)%}
+{%set ns = namespace(forward_uri=base_uri, back_uri=base_uri)%}
+{%set ns.forward_uri="brr"%}
+ <div class="row">
+ <div class="col-md-2" style="text-align: start;">
+ {%if start_at > 0%}
+ <a href="{{base_uri +
+ '?start_at='+((start_at-page_count)|string) +
+ '&count='+(page_count|string)}}">
+ <span class="glyphicon glyphicon-backward"></span>
+ Previous
+ </a>
+ {%endif%}
+ </div>
+ <div class="col-md-8" style="text-align: center;">
+ Displaying {{name}} {{start_at+1}} to {{start_at+page_count if start_at+page_count < total_count else total_count}} of {{total_count}}</div>
+ <div class="col-md-2" style="text-align: end;">
+ {%if start_at + page_count < total_count%}
+ <a href="{{base_uri +
+ '?start_at='+((start_at+page_count)|string) +
+ '&count='+(page_count|string)}}">
+ Next<span class="glyphicon glyphicon-forward"></span></a>
+ {%endif%}
+ </div>
+ </div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/add-phenotypes.html b/uploader/templates/phenotypes/add-phenotypes.html
new file mode 100644
index 0000000..196bc69
--- /dev/null
+++ b/uploader/templates/phenotypes/add-phenotypes.html
@@ -0,0 +1,231 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="add-phenotypes"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">View Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <form id="frm-add-phenotypes"
+ method="POST"
+ enctype="multipart/form-data"
+ action="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}">
+ <legend>Add New Phenotypes</legend>
+
+ <div class="form-text help-block">
+ <p>Select the zip file bundle containing information on the phenotypes you
+ wish to upload, then click the "Upload Phenotypes" button below to
+ upload the data.</p>
+ <p>See the <a href="#section-file-formats">File Formats</a> section below
+ to get an understanding of what is expected of the bundle files you
+ upload.</p>
+ <p><strong>This will not update any existing phenotypes!</strong></p>
+ </div>
+
+ <div class="form-group">
+ <label for="finput-phenotypes-bundle" class="form-label">
+ Phenotypes Bundle</label>
+ <input type="file"
+ id="finput-phenotypes-bundle"
+ name="phenotypes-bundle"
+ accept="application/zip, .zip"
+ required="required"
+ class="form-control" />
+ </div>
+
+ <div class="form-group">
+ <input type="submit"
+ value="upload phenotypes"
+ class="btn btn-primary" />
+ </div>
+ </form>
+</div>
+
+<div class="row">
+ <h2 class="heading" id="section-file-formats">File Formats</h2>
+ <p>We accept an extended form of the
+ <a href="https://kbroman.org/qtl2/assets/vignettes/input_files.html#format-of-the-data-files"
+ title="R/qtl2 software input file format documentation">
+ input files' format used with the R/qtl2 software</a> as a single ZIP
+ file</p>
+ <p>The files that are used for this feature are:
+ <ul>
+ <li>the <em>control</em> file</li>
+ <li><em>pheno</em> file(s)</li>
+ <li><em>phenocovar</em> file(s)</li>
+ <li><em>phenose</em> files(s)</li>
+ </ul>
+ </p>
+ <p>Other files within the bundle will be ignored, for this feature.</p>
+ <p>The following section will detail the expectations for each of the
+ different file types within the uploaded ZIP file bundle for phenotypes:</p>
+
+ <h3 class="subheading">Control File</h3>
+ <p>There <strong>MUST be <em>one, and only one</em></strong> file that acts
+ as the control file. This file can be:
+ <ul>
+ <li>a <em>JSON</em> file, or</li>
+ <li>a <em>YAML</em> file.</li>
+ </ul>
+ </p>
+
+ <p>The control file is useful for defining things about the bundle such as:</p>
+ <ul>
+ <li>The field separator value (default: <code>sep: ','</code>). There can
+ only ever be one field separator and it <strong>MUST</strong> be the same
+ one for <strong>ALL</strong> files in the bundle.</li>
+ <li>The comment character (default: <code>comment.char: '#'</code>). Any
+ line that starts with this character will be considered a comment line and
+ be ignored in its entirety.</li>
+ <li>Code for missing values (default: <code>na.strings: 'NA'</code>). You
+ can specify more than one code to indicate missing values, e.g.
+ <code>{…, "na.strings": ["NA", "N/A", "-"], …}</code></li>
+ </ul>
+
+ <h3 class="subheading"><em>pheno</em> File(s)</h3>
+ <p>These files are the main data files. You must have at least one of these
+ files in your bundle for it to be valid for this step.</p>
+ <p>The data is a matrix of <em>individuals × phenotypes</em> by default, as
+ below:<br />
+ <code>
+ id,10001,10002,10003,10004,…<br />
+ BXD1,61.400002,54.099998,483,49.799999,…<br />
+ BXD2,49,50.099998,403,45.5,…<br />
+ BXD5,62.5,53.299999,501,62.900002,…<br />
+ BXD6,53.099998,55.099998,403,NA,…<br />
+ ⋮<br /></code>
+ </p>
+ <p>If the <code>pheno_transposed</code> value is set to <code>True</code>,
+ then the data will be a <em>phenotypes × individuals</em> matrix as in the
+ example below:<br />
+ <code>
+ id,BXD1,BXD2,BXD5,BXD6,…<br />
+ 10001,61.400002,49,62.5,53.099998,…<br />
+ 10002,54.099998,50.099998,53.299999,55.099998,…<br />
+ 10003,483,403,501,403,…<br />
+ 10004,49.799999,45.5,62.900002,NA,…<br />
+ ⋮
+ </code>
+ </p>
+
+
+ <h3 class="subheading"><em>phenocovar</em> File(s)</h3>
+ <p>At least one phenotypes metadata file with the metadata values such as
+ descriptions, PubMed Identifier, publication titles (if present), etc.</p>
+ <p>The data in this/these file(s) is a matrix of
+ <em>phenotypes × phenotypes-covariates</em>. The first column is always the
+ phenotype names/identifiers — same as in the R/qtl2 format.</p>
+ <p><em>phenocovar</em> files <strong>should never be transposed</strong>!</p>
+ <p>This file <strong>MUST</strong> be present in the bundle, and have data for
+ the bundle to be considered valid by our system for this step.<br />
+ In addition to that, the following are the fields that <strong>must be
+ present</strong>, and
+ have values, in the file before the file is considered valid:
+ <ul>
+ <li><em>description</em>: A description for each phenotype. Useful
+ for users to know what the phenotype is about.</li>
+ <li><em>units</em>: The units of measurement for the phenotype,
+ e.g. milligrams for brain weight, centimetres/millimetres for
+ tail-length, etc.</li>
+ </ul></p>
+
+ <p>The following <em>optional</em> fields can also be provided:
+ <ul>
+ <li><em>pubmedid</em>: A PubMed Identifier for the publication where
+ the phenotype is published. If this field is not provided, the system will
+ assume your phenotype is not published.</li>
+ </ul>
+ </p>
+ <p>These files will be marked up in the control file with the
+ <code>phenocovar</code> key, as in the examples below:
+ <ol>
+ <li>JSON: single file<br />
+ <code>{<br />
+ &nbsp;&nbsp;⋮,<br />
+ &nbsp;&nbsp;"phenocovar": "your_covariates_file.csv",<br />
+ &nbsp;&nbsp;⋮<br />
+ }
+ </code>
+ </li>
+ <li>JSON: multiple files<br />
+ <code>{<br />
+ &nbsp;&nbsp;⋮,<br />
+ &nbsp;&nbsp;"phenocovar": [<br />
+ &nbsp;&nbsp;&nbsp;&nbsp;"covariates_file_01.csv",<br />
+ &nbsp;&nbsp;&nbsp;&nbsp;"covariates_file_01.csv",<br />
+ &nbsp;&nbsp;&nbsp;&nbsp;⋮<br />
+ &nbsp;&nbsp;],<br />
+ &nbsp;&nbsp;⋮<br />
+ }
+ </code>
+ </li>
+ <li>YAML: single file or<br />
+ <code>
+ ⋮<br />
+ phenocovar: your_covariates_file.csv<br />
+ ⋮
+ </code>
+ </li>
+ <li>YAML: multiple files<br />
+ <code>
+ ⋮<br />
+ phenocovar:<br />
+ - covariates_file_01.csv<br />
+ - covariates_file_02.csv<br />
+ - covariates_file_03.csv<br />
+ …<br />
+ ⋮
+ </code>
+ </li>
+ </ol>
+ </p>
+
+ <h3 class="subheading"><em>phenose</em> and <em>phenonum</em> File(s)</h3>
+ <p>These are extensions to the R/qtl2 standard, i.e. these types ofs file are
+ not supported by the original R/qtl2 file format</p>
+ <p>We use these files to upload the standard errors (<em>phenose</em>) when
+ the data file (<em>pheno</em>) is average data. In that case, the
+ <em>phenonum</em> file(s) contains the number of individuals that were
+ involved when computing the averages.</p>
+ <p>Both types of files are matrices of <em>individuals × phenotypes</em> by
+ default. Like the related <em>pheno</em> files, if
+ <code>pheno_transposed: True</code>, then the file will be a matrix of
+ <em>phenotypes × individuals</em>.</p>
+</div>
+
+<div class="row text-warning">
+ <h3 class="subheading">Notes for Devs (well… Fred, really.)</h3>
+ <p>Use the following resources for automated retrieval of certain data</p>
+ <ul>
+ <li><a href="https://www.ncbi.nlm.nih.gov/pmc/tools/developers/"
+ title="NCBI APIs: Retrieve articles' metadata etc.">
+ NCBI APIS</a></li>
+ </ul>
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_pheno_dataset_card(species, population, dataset)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/create-dataset.html b/uploader/templates/phenotypes/create-dataset.html
new file mode 100644
index 0000000..93de92f
--- /dev/null
+++ b/uploader/templates/phenotypes/create-dataset.html
@@ -0,0 +1,106 @@
+{%extends "phenotypes/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Phenotypes{%endblock%}
+
+{%block pagetitle%}Phenotypes{%endblock%}
+
+{%block lvl4_breadcrumbs%}
+<li {%if activelink=="create-dataset"%}
+ class="breadcrumb-item active"
+ {%else%}
+ class="breadcrumb-item"
+ {%endif%}>
+ <a href="{{url_for('species.populations.phenotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}">Create Datasets</a>
+</li>
+{%endblock%}
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+ <p>Create a new phenotype dataset.</p>
+</div>
+
+<div class="row">
+ <form id="frm-create-pheno-dataset"
+ action="{{url_for('species.populations.phenotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ method="POST">
+
+ <div class="form-group">
+ <label class="form-label" for="txt-dataset-name">Name</label>
+ {%if errors["dataset-name"] is defined%}
+ <small class="form-text text-muted danger">
+ <p>{{errors["dataset-name"]}}</p></small>
+ {%endif%}
+ <input type="text"
+ name="dataset-name"
+ id="txt-dataset-name"
+ value="{{original_formdata.get('dataset-name') or (population.InbredSetCode + 'Publish')}}"
+ {%if errors["dataset-name"] is defined%}
+ class="form-control danger"
+ {%else%}
+ class="form-control"
+ {%endif%}
+ required="required" />
+ <small class="form-text text-muted">
+ <p>A short representative name for the dataset.</p>
+ <p>Recommended: Use the population code and append "Publish" at the end.
+ <br />This field will only accept names composed of
+ letters ('A-Za-z'), numbers (0-9), hyphens and underscores.</p>
+ </small>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt-dataset-fullname">Full Name</label>
+ {%if errors["dataset-fullname"] is defined%}
+ <small class="form-text text-muted danger">
+ <p>{{errors["dataset-fullname"]}}</p></small>
+ {%endif%}
+ <input id="txt-dataset-fullname"
+ name="dataset-fullname"
+ type="text"
+ value="{{original_formdata.get('dataset-fullname', '')}}"
+ {%if errors["dataset-fullname"] is defined%}
+ class="form-control danger"
+ {%else%}
+ class="form-control"
+ {%endif%}
+ required="required" />
+ <small class="form-text text-muted">
+ <p>A longer, descriptive name for the dataset &mdash; useful for humans.
+ </p></small>
+ </div>
+
+ <div class="form-group">
+ <label class="form-label" for="txt-dataset-shortname">Short Name</label>
+ <input id="txt-dataset-shortname"
+ name="dataset-shortname"
+ type="text"
+ class="form-control"
+ value="{{original_formdata.get('dataset-shortname') or (population.InbredSetCode + ' Publish')}}" />
+ <small class="form-text text-muted">
+ <p>An optional, short name for the dataset. <br />
+ If this is not provided, it will default to the value provided for the
+ <strong>Name</strong> field above.</p></small>
+ </div>
+
+ <div class="form-group">
+ <input type="submit"
+ class="btn btn-primary"
+ value="create phenotype dataset" />
+ </div>
+
+ </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/phenotypes/list-datasets.html b/uploader/templates/phenotypes/list-datasets.html
index 360fd2c..2eaf43a 100644
--- a/uploader/templates/phenotypes/list-datasets.html
+++ b/uploader/templates/phenotypes/list-datasets.html
@@ -51,8 +51,10 @@
<p class="text-warning">
<span class="glyphicon glyphicon-exclamation-sign"></span>
There is no dataset for this population!</p>
- <p><a href="#"
- class="not-implemented btn btn-primary"
+ <p><a href="{{url_for('species.populations.phenotypes.create_dataset',
+ species_id=species.SpeciesId,
+ population_id=population.Id)}}"
+ class="btn btn-primary"
title="Create a new phenotype dataset.">create dataset</a></p>
{%endif%}
</div>
diff --git a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html
new file mode 100644
index 0000000..11b108b
--- /dev/null
+++ b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html
@@ -0,0 +1,31 @@
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%macro display_pheno_dataset_card(species, population, dataset)%}
+{{display_population_card(species, population)}}
+
+<div class="card">
+ <div class="card-body">
+ <h5 class="card-title">Phenotypes' Dataset</h5>
+ <div class="card-text">
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Name</td>
+ <td>{{dataset.Name}}</td>
+ </tr>
+
+ <tr>
+ <td>Full Name</td>
+ <td>{{dataset.FullName}}</td>
+ </tr>
+
+ <tr>
+ <td>Short Name</td>
+ <td>{{dataset.ShortName}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+</div>
+{%endmacro%}
diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html
index e2ccb60..b136bb6 100644
--- a/uploader/templates/phenotypes/view-dataset.html
+++ b/uploader/templates/phenotypes/view-dataset.html
@@ -1,5 +1,6 @@
{%extends "phenotypes/base.html"%}
{%from "flash_messages.html" import flash_all_messages%}
+{%from "macro-table-pagination.html" import table_pagination%}
{%from "populations/macro-display-population-card.html" import display_population_card%}
{%block title%}Phenotypes{%endblock%}
@@ -36,10 +37,7 @@
<tbody>
<tr>
- <td><a href="{{url_for('species.populations.phenotypes.view_dataset',
- species_id=species.SpeciesId,
- population_id=population.Id,
- dataset_id=dataset.Id)}}">{{dataset.Name}}</a></td>
+ <td>{{dataset.Name}}</td>
<td>{{dataset.FullName}}</td>
<td>{{dataset.ShortName}}</td>
</tr>
@@ -48,12 +46,20 @@
</div>
<div class="row">
+ <p><a href="{{url_for('species.populations.phenotypes.add_phenotypes',
+ species_id=species.SpeciesId,
+ population_id=population.Id,
+ dataset_id=dataset.Id)}}"
+ title="Add a bunch of phenotypes"
+ class="btn btn-primary">Add phenotypes</a></p>
+</div>
+
+<div class="row">
<h2>Phenotype Data</h2>
<p>This dataset has a total of {{phenotype_count}} phenotypes.</p>
- <p class="text-warning">
- <span class="glyphicon glyphicon-exclamation-sign"></span>
- Display pagination controls here &hellip;</p>
+
+ {{table_pagination(start_from, count, phenotype_count, url_for('species.populations.phenotypes.view_dataset', species_id=species.SpeciesId, population_id=population.Id, dataset_id=dataset.Id), "phenotypes")}}
<table class="table">
<thead>
diff --git a/uploader/templates/phenotypes/view-phenotype.html b/uploader/templates/phenotypes/view-phenotype.html
index 03935f5..99bb8e5 100644
--- a/uploader/templates/phenotypes/view-phenotype.html
+++ b/uploader/templates/phenotypes/view-phenotype.html
@@ -41,8 +41,42 @@
<td><strong>Collation</strong></td>
<td>{{dataset.FullName}}</td>
</tr>
+ <tr>
+ <td><strong>Units</strong></td>
+ <td>{{phenotype.Units}}</td>
+ </tr>
</tbody>
</table>
+
+ <form action="#edit-delete-phenotype"
+ method="POST"
+ id="frm-delete-phenotype">
+
+ <input type="hidden" name="species_id" value="{{species.SpeciesId}}" />
+ <input type="hidden" name="population_id" value="{{population.Id}}" />
+ <input type="hidden" name="dataset_id" value="{{dataset.Id}}" />
+ <input type="hidden" name="phenotype_id" value="{{phenotype.Id}}" />
+
+ <div class="btn-group btn-group-justified">
+ <div class="btn-group">
+ {%if "group:resource:edit-resource" in privileges%}
+ <input type="submit"
+ title="Edit the values for the phenotype. This is meant to be used when you need to update only a few values."
+ class="btn btn-primary not-implemented"
+ value="edit" />
+ {%endif%}
+ </div>
+ <div class="btn-group"></div>
+ <div class="btn-group">
+ {%if "group:resource:delete-resource" in privileges%}
+ <input type="submit"
+ title="Delete the entire phenotype. This is useful when you need to change data for most or all of the fields for this phenotype."
+ class="btn btn-danger not-implemented"
+ value="delete" />
+ {%endif%}
+ </div>
+ </div>
+ </form>
</div>
</div>
diff --git a/uploader/templates/populations/macro-display-population-card.html b/uploader/templates/populations/macro-display-population-card.html
index e68f8e3..79f7925 100644
--- a/uploader/templates/populations/macro-display-population-card.html
+++ b/uploader/templates/populations/macro-display-population-card.html
@@ -7,25 +7,39 @@
<div class="card-body">
<h5 class="card-title">Population</h5>
<div class="card-text">
- <dl>
- <dt>Name</dt>
- <dd>{{population.Name}}</dd>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Name</td>
+ <td>{{population.Name}}</td>
+ </tr>
- <dt>Full Name</dt>
- <dd>{{population.FullName}}</dd>
+ <tr>
+ <td>Full Name</td>
+ <td>{{population.FullName}}</td>
+ </tr>
- <dt>Code</dt>
- <dd>{{population.InbredSetCode}}</dd>
+ <tr>
+ <td>Code</td>
+ <td>{{population.InbredSetCode}}</td>
+ </tr>
- <dt>Genetic Type</dt>
- <dd>{{population.GeneticType}}</dd>
+ <tr>
+ <td>Genetic Type</td>
+ <td>{{population.GeneticType}}</td>
+ </tr>
- <dt>Family</dt>
- <dd>{{population.Family}}</dd>
+ <tr>
+ <td>Family</td>
+ <td>{{population.Family}}</td>
+ </tr>
- <dt>Description</dt>
- <dd>{{population.Description or "-"}}</dd>
- </dl>
+ <tr>
+ <td>Description</td>
+ <td>{{(population.Description or "")[0:500]}}&hellip;</td>
+ </tr>
+ </tbody>
+ </table>
</div>
</div>
</div>
diff --git a/uploader/templates/species/macro-display-species-card.html b/uploader/templates/species/macro-display-species-card.html
index 857c0f0..166c7b9 100644
--- a/uploader/templates/species/macro-display-species-card.html
+++ b/uploader/templates/species/macro-display-species-card.html
@@ -3,13 +3,19 @@
<div class="card-body">
<h5 class="card-title">Species</h5>
<div class="card-text">
- <dl>
- <dt>Common Name</dt>
- <dd>{{species.SpeciesName}}</dd>
+ <table class="table">
+ <tbody>
+ <tr>
+ <td>Common Name</td>
+ <td>{{species.SpeciesName}}</td>
+ </tr>
- <dt>Scientific Name</dt>
- <dd>{{species.FullName}}</dd>
- </dl>
+ <tr>
+ <td>Scientific Name</td>
+ <td>{{species.FullName}}</td>
+ </tr>
+ </tbody>
+ </table>
</div>
</div>
</div>
generated by cgit v1.2.3 (git 2.44.0) at 2024-11-19 16:03:21 +0000