diff options
| -rw-r--r-- | scripts/load_phenotypes_to_db.py | 49 | ||||
| -rw-r--r-- | scripts/rqtl2/phenotypes_qc.py | 7 | ||||
| -rw-r--r-- | uploader/__init__.py | 9 | ||||
| -rw-r--r-- | uploader/default_settings.py | 3 | ||||
| -rw-r--r-- | uploader/phenotypes/models.py | 30 | ||||
| -rw-r--r-- | uploader/publications/models.py | 14 | ||||
| -rw-r--r-- | uploader/publications/views.py | 6 | ||||
| -rw-r--r-- | uploader/templates/phenotypes/view-dataset.html | 26 | ||||
| -rw-r--r-- | uploader/templates/publications/create-publication.html | 31 |
9 files changed, 120 insertions, 55 deletions
diff --git a/scripts/load_phenotypes_to_db.py b/scripts/load_phenotypes_to_db.py index 9b70fed..31eb715 100644 --- a/scripts/load_phenotypes_to_db.py +++ b/scripts/load_phenotypes_to_db.py @@ -7,6 +7,7 @@ import logging import argparse from pathlib import Path from zipfile import ZipFile +from datetime import datetime from typing import Any, Iterable from urllib.parse import urljoin from functools import reduce, partial @@ -204,6 +205,8 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments dataset, xrefdata): """Grant the user access to their data.""" + logger.info("Updating authorisation for the data.") + logger.debug("Resource details for the authorisation: %s", resource_details) authserver, token = auth_details _tries = 0 _delay = 1 @@ -215,14 +218,14 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments return urljoin(authserver, endpoint) def __fetch_user_details__(): - logger.debug("… Fetching user details") + logger.info("… Fetching user details") return mrequests.get( authserveruri("/auth/user/"), headers=headers ) def __link_data__(user): - logger.debug("… linking uploaded data to user's group") + logger.info("… linking uploaded data to user's group") return mrequests.post( authserveruri("/auth/data/link/phenotype"), headers=headers, @@ -245,7 +248,7 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments }).then(lambda ld_results: (user, ld_results)) def __fetch_phenotype_category_details__(user, linkeddata): - logger.debug("… fetching phenotype category details") + logger.info("… fetching phenotype category details") return mrequests.get( authserveruri("/auth/resource/categories"), headers=headers @@ -258,7 +261,7 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments ) def __create_resource__(user, linkeddata, category): - logger.debug("… creating authorisation resource object") + logger.info("… creating authorisation resource object") return mrequests.post( authserveruri("/auth/resource/create"), headers=headers, @@ -269,7 +272,7 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments }).then(lambda cr_results: (user, linkeddata, cr_results)) def __attach_data_to_resource__(user, linkeddata, resource): - logger.debug("… attaching data to authorisation resource object") + logger.info("… attaching data to authorisation resource object") return mrequests.post( authserveruri("/auth/resource/data/link"), headers=headers, @@ -286,8 +289,8 @@ def update_auth(# pylint: disable=[too-many-locals,too-many-positional-arguments # This is hacky. If the auth already exists, something went wrong # somewhere. # This needs investigation to recover correctly. - logger.info( - "The authorisation for the data was already set up.") + logger.error( + "Error: The authorisation for the data was already set up.") return 0 logger.error("ERROR: Updating the authorisation for the data failed.") logger.debug( @@ -459,6 +462,25 @@ if __name__ == "__main__": logging.getLogger("uploader.phenotypes.models").setLevel(log_level) + def __parse_resource_details__(meta) -> dict: + """Parse out details regarding the wrapper resource from the metadata.""" + _key_mappings_ = { + # allow both 'data_*' and 'data*' for the metadata. + "data_description": "description", + "datadescription": "description" + } + return { + "resource_name": meta.get( + "dataname", + meta.get("data_name", + "Unnamed phenotypes - " + datetime.now().isoformat())), + "resource_metadata": { + rkey: meta[mkey] + for mkey, rkey in _key_mappings_.items() if mkey in meta + } + } + + def main(): """Entry-point for this script.""" args = parse_args() @@ -515,19 +537,6 @@ if __name__ == "__main__": logger.info("Updating authorisation.") _job_metadata = job["metadata"] - def __parse_resource_details__(meta) -> dict: - _key_mappings_ = { - "data_description": "description", - } - return { - "resource_name": _job_metadata["dataname"], - "resource_metadata": { - rkey: meta[mkey] - for mkey, rkey in _key_mappings_.items() - if mkey in meta - } - } - return update_auth((_job_metadata["authserver"], _job_metadata["token"]), __parse_resource_details__(_job_metadata), diff --git a/scripts/rqtl2/phenotypes_qc.py b/scripts/rqtl2/phenotypes_qc.py index 72d6c83..084c876 100644 --- a/scripts/rqtl2/phenotypes_qc.py +++ b/scripts/rqtl2/phenotypes_qc.py @@ -198,7 +198,7 @@ def qc_phenocovar_file( "-", "-", (f"File {filepath.name} is missing the {heading} heading " - "in the header line."))),) + "in the header row/line."))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount @@ -312,8 +312,9 @@ def qc_pheno_file(# pylint: disable=[too-many-locals, too-many-arguments, too-ma "header row", "-", ", ".join(_absent), - ("The following phenotype names do not exist in any of the " - f"provided phenocovar files: ({', '.join(_absent)})"))),) + ("The following trait names/identifiers do not exist in any of " + "the provided descriptions/covariates files: " + f"({', '.join(_absent)})"))),) def collect_errors(errors_and_linecount, line): _errs, _lc = errors_and_linecount diff --git a/uploader/__init__.py b/uploader/__init__.py index e00c726..afaa78d 100644 --- a/uploader/__init__.py +++ b/uploader/__init__.py @@ -131,13 +131,8 @@ def create_app(config: Optional[dict] = None): default_timeout=int(app.config["SESSION_FILESYSTEM_CACHE_TIMEOUT"])) setup_logging(app) - setup_modules_logging(app.logger, ( - "uploader.base_routes", - "uploader.flask_extensions", - "uploader.publications.models", - "uploader.publications.datatables", - "uploader.phenotypes.models", - "uploader.phenotypes.views")) + setup_modules_logging( + app.logger, tuple(app.config.get("LOGGABLE_MODULES", []))) # setup jinja2 symbols app.add_template_global(user_logged_in) diff --git a/uploader/default_settings.py b/uploader/default_settings.py index 6381a67..04e1c0a 100644 --- a/uploader/default_settings.py +++ b/uploader/default_settings.py @@ -39,3 +39,6 @@ JWKS_DELETION_AGE_DAYS = 14 # Days (from creation) to keep a JWK around before d ## --- Feature flags --- FEATURE_FLAGS_HTTP: list[str] = [] + +## --- Modules for which to log output --- +LOGGABLE_MODULES: list[str] = [] diff --git a/uploader/phenotypes/models.py b/uploader/phenotypes/models.py index 3946a0f..3d656d2 100644 --- a/uploader/phenotypes/models.py +++ b/uploader/phenotypes/models.py @@ -96,20 +96,34 @@ def dataset_phenotypes(# pylint: disable=[too-many-arguments, too-many-positiona xref_ids: tuple[int, ...] = tuple() ) -> tuple[dict, ...]: """Fetch the actual phenotypes.""" - _query = ( - "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, ist.InbredSetCode " + _narrow_by_ids = ( + f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" + if len(xref_ids) > 0 else "") + _narrow_by_limit = ( + f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + _pub_query = ( + "SELECT pub.* " + "FROM PublishXRef AS pxr " + "INNER JOIN Publication AS pub ON pxr.PublicationId=pub.Id " + "WHERE pxr.InbredSetId=%s") + _narrow_by_ids + _pheno_query = (( + "SELECT pheno.*, pxr.Id AS xref_id, pxr.InbredSetId, pxr.PublicationId, " + "ist.InbredSetCode " "FROM Phenotype AS pheno " "INNER JOIN PublishXRef AS pxr ON pheno.Id=pxr.PhenotypeId " "INNER JOIN PublishFreeze AS pf ON pxr.InbredSetId=pf.InbredSetId " "INNER JOIN InbredSet AS ist ON pf.InbredSetId=ist.Id " - "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + ( - f" AND pxr.Id IN ({', '.join(['%s'] * len(xref_ids))})" - if len(xref_ids) > 0 else "") + ( - f" LIMIT {limit} OFFSET {offset}" if bool(limit) else "") + "WHERE pxr.InbredSetId=%s AND pf.Id=%s") + + _narrow_by_ids + + _narrow_by_limit) with conn.cursor(cursorclass=DictCursor) as cursor: - cursor.execute(_query, (population_id, dataset_id) + xref_ids) + cursor.execute(_pub_query, (population_id,) + xref_ids) debug_query(cursor, logger) - return tuple(dict(row) for row in cursor.fetchall()) + _pubs = {row["Id"]: dict(row) for row in cursor.fetchall()} + cursor.execute(_pheno_query, (population_id, dataset_id) + xref_ids) + debug_query(cursor, logger) + return tuple({**dict(row), "publication": _pubs[row["PublicationId"]]} + for row in cursor.fetchall()) def __phenotype_se__(cursor: BaseCursor, xref_id, dataids_and_strainids): diff --git a/uploader/publications/models.py b/uploader/publications/models.py index dcfa02b..d913144 100644 --- a/uploader/publications/models.py +++ b/uploader/publications/models.py @@ -101,6 +101,20 @@ def fetch_publication_by_id(conn: Connection, publication_id: int) -> dict: return dict(_res) if _res else {} +def fetch_publications_by_ids( + conn: Connection, publications_ids: tuple[int, ...] +) -> tuple[dict, ...]: + """Fetch publications with the given IDs.""" + if len(publications_ids) == 0: + return tuple() + + with conn.cursor(cursorclass=DictCursor) as cursor: + paramstr = ", ".join(["%s"] * len(publications_ids)) + cursor.execute(f"SELECT * FROM Publication WHERE Id IN ({paramstr})", + tuple(publications_ids)) + return tuple(dict(row) for row in cursor.fetchall()) + + def fetch_publication_phenotypes( conn: Connection, publication_id: int) -> Iterable[dict]: """Fetch all phenotypes linked to this publication.""" diff --git a/uploader/publications/views.py b/uploader/publications/views.py index d9eb294..89e9f5d 100644 --- a/uploader/publications/views.py +++ b/uploader/publications/views.py @@ -1,5 +1,6 @@ """Endpoints for publications""" import json +import datetime from gn_libs.mysqldb import database_connection from flask import ( @@ -89,9 +90,12 @@ def create_publication(): } if request.method == "GET": + now = datetime.datetime.now() return render_template( "publications/create-publication.html", - get_args=_get_args) + get_args=_get_args, + current_year=now.year, + current_month=now.strftime("%B")) form = request.form authors = form.get("publication-authors").encode("utf8") if authors is None or authors == "": diff --git a/uploader/templates/phenotypes/view-dataset.html b/uploader/templates/phenotypes/view-dataset.html index 3bb2586..fc84757 100644 --- a/uploader/templates/phenotypes/view-dataset.html +++ b/uploader/templates/phenotypes/view-dataset.html @@ -148,14 +148,36 @@ return `<a href="${url.toString()}" target="_blank">` + `${pheno.InbredSetCode}_${pheno.xref_id}` + `</a>`; - } + }, + title: "Record", + visible: true, + searchable: true }, { data: function(pheno) { return (pheno.Post_publication_description || pheno.Original_description || pheno.Pre_publication_description); - } + }, + title: "Description", + visible: true, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Title; + }, + title: "Publication Title", + visible: false, + searchable: true + }, + { + data: function(pheno) { + return pheno.publication.Authors; + }, + title: "Authors", + visible: false, + searchable: true } ], { diff --git a/uploader/templates/publications/create-publication.html b/uploader/templates/publications/create-publication.html index fb0127d..da5889e 100644 --- a/uploader/templates/publications/create-publication.html +++ b/uploader/templates/publications/create-publication.html @@ -91,22 +91,22 @@ class="col-sm-2 col-form-label"> Month</label> <div class="col-sm-4"> - <select class="form-control" + <select class="form-select" id="select-publication-month" name="publication-month"> <option value="">Select a month</option> - <option value="january">January</option> - <option value="february">February</option> - <option value="march">March</option> - <option value="april">April</option> - <option value="may">May</option> - <option value="june">June</option> - <option value="july">July</option> - <option value="august">August</option> - <option value="september">September</option> - <option value="october">October</option> - <option value="november">November</option> - <option value="december">December</option> + <option {%if current_month | lower == "january"%}selected="selected"{%endif%}value="january">January</option> + <option {%if current_month | lower == "february"%}selected="selected"{%endif%}value="february">February</option> + <option {%if current_month | lower == "march"%}selected="selected"{%endif%}value="march">March</option> + <option {%if current_month | lower == "april"%}selected="selected"{%endif%}value="april">April</option> + <option {%if current_month | lower == "may"%}selected="selected"{%endif%}value="may">May</option> + <option {%if current_month | lower == "june"%}selected="selected"{%endif%}value="june">June</option> + <option {%if current_month | lower == "july"%}selected="selected"{%endif%}value="july">July</option> + <option {%if current_month | lower == "august"%}selected="selected"{%endif%}value="august">August</option> + <option {%if current_month | lower == "september"%}selected="selected"{%endif%}value="september">September</option> + <option {%if current_month | lower == "october"%}selected="selected"{%endif%}value="october">October</option> + <option {%if current_month | lower == "november"%}selected="selected"{%endif%}value="november">November</option> + <option {%if current_month | lower == "december"%}selected="selected"{%endif%}value="december">December</option> </select> <span class="form-text text-muted">Month of publication</span> </div> @@ -119,7 +119,10 @@ id="txt-publication-year" name="publication-year" class="form-control" - min="1960" /> + min="1960" + max="{{current_year}}" + value="{{current_year or ''}}" + required="required" /> <span class="form-text text-muted">Year of publication</span> </div> </div> |