diff options
Diffstat (limited to 'uploader')
| -rw-r--r-- | uploader/jobs.py | 34 | ||||
| -rw-r--r-- | uploader/phenotypes/views.py | 14 | ||||
| -rw-r--r-- | uploader/templates/cli-output.html | 4 | ||||
| -rw-r--r-- | uploader/templates/phenotypes/add-phenotypes.html | 2 | ||||
| -rw-r--r-- | uploader/templates/phenotypes/job-status.html | 71 |
5 files changed, 118 insertions, 7 deletions
diff --git a/uploader/jobs.py b/uploader/jobs.py index 4a3fc80..e86ee05 100644 --- a/uploader/jobs.py +++ b/uploader/jobs.py @@ -1,6 +1,8 @@ """Handle jobs""" import os import sys +import uuid +import json import shlex import subprocess from uuid import UUID, uuid4 @@ -10,6 +12,8 @@ from typing import Union, Optional from redis import Redis from flask import current_app as app +from functional_tools import take + JOBS_PREFIX = "jobs" class JobNotFound(Exception): @@ -128,3 +132,33 @@ def update_stdout_stderr(rconn: Redis, contents = thejob.get(stream, '') new_contents = contents + bytes_read.decode("utf-8") rconn.hset(name=job_key(rprefix, jobid), key=stream, value=new_contents) + + +def job_errors( + rconn: Redis, + prefix: str, + job_id: Union[str, uuid.UUID], + count: int = 100 +) -> list: + """Fetch job errors""" + return take( + ( + json.loads(error) + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:errors:*") + for error in rconn.lrange(key, 0, -1)), + count) + + +def job_files_metadata( + rconn: Redis, + prefix: str, + job_id: Union[str, uuid.UUID] +) -> dict: + """Get the metadata for specific job file.""" + return { + key.split(":")[-1]: { + **rconn.hgetall(key), + "filetype": key.split(":")[-3] + } + for key in rconn.keys(f"{prefix}:{str(job_id)}:*:metadata*") + } diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index b8c0e93..a664ba9 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -395,10 +395,14 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p species_redirect_uri="species.populations.phenotypes.index", population_redirect_uri="species.populations.phenotypes.select_population", redirect_uri="species.populations.phenotypes.list_datasets") -def job_status(species: dict, population: dict, dataset: dict, job_id: uuid, **kwargs): +def job_status( + species: dict, + population: dict, + dataset: dict, + job_id: uuid.UUID, + **kwargs +):# pylint: disable=[unused-argument] """Retrieve current status of a particular phenotype QC job.""" - from uploader.debug import __pk__ - with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn: try: job = jobs.job(rconn, jobs.jobsnamespace(), str(job_id)) @@ -410,4 +414,8 @@ def job_status(species: dict, population: dict, dataset: dict, job_id: uuid, **k dataset=dataset, job_id=job_id, job=job, + errors=jobs.job_errors( + rconn, jobs.jobsnamespace(), job['jobid']), + metadata=jobs.job_files_metadata( + rconn, jobs.jobsnamespace(), job['jobid']), activelink="add-phenotypes") diff --git a/uploader/templates/cli-output.html b/uploader/templates/cli-output.html index 33fb73b..64b1a9a 100644 --- a/uploader/templates/cli-output.html +++ b/uploader/templates/cli-output.html @@ -1,7 +1,7 @@ {%macro cli_output(job, stream)%} -<h4>{{stream | upper}} Output</h4> -<div class="cli-output"> +<h4 class="subheading">{{stream | upper}} Output</h4> +<div class="cli-output" style="max-height: 10em; overflow: auto;"> <pre>{{job.get(stream, "")}}</pre> </div> diff --git a/uploader/templates/phenotypes/add-phenotypes.html b/uploader/templates/phenotypes/add-phenotypes.html index 196bc69..9e368e1 100644 --- a/uploader/templates/phenotypes/add-phenotypes.html +++ b/uploader/templates/phenotypes/add-phenotypes.html @@ -40,7 +40,7 @@ <p>See the <a href="#section-file-formats">File Formats</a> section below to get an understanding of what is expected of the bundle files you upload.</p> - <p><strong>This will not update any existing phenotypes!</strong></p> + <p><strong class="text-warning">This will not update any existing phenotypes!</strong></p> </div> <div class="form-group"> diff --git a/uploader/templates/phenotypes/job-status.html b/uploader/templates/phenotypes/job-status.html index d531a71..30316b5 100644 --- a/uploader/templates/phenotypes/job-status.html +++ b/uploader/templates/phenotypes/job-status.html @@ -30,8 +30,32 @@ {%block contents%} {%if job%} +<h4 class="subheading">Progress</h4> <div class="row"> - <p><strong>Status:</strong> {{job.status}}</p> + <p><strong>Process Status:</strong> {{job.status}}</p> + {%if metadata%} + <table class="table"> + <thead> + <tr> + <th>File</th> + <th>Status</th> + <th>Lines Processed</th> + <th>Total Errors</th> + </tr> + </thead> + + <tbody> + {%for file,meta in metadata.items()%} + <tr> + <td>{{file}}</td> + <td>{{meta.status}}</td> + <td>{{meta.linecount}}</td> + <td>{{meta["total-errors"]}}</td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} {%if job.status in ("completed:success", "success")%} <p><a href="#" class="not-implemented btn btn-primary" @@ -40,6 +64,50 @@ {%endif%} </div> +<h4 class="subheading">Errors</h4> +<div class="row" style="max-height: 20em; overflow: auto;"> + {%if errors | length == 0 %} + <p class="text-info"> + <span class="glyphicon glyphicon-info-sign"></span> + No errors found so far + </p> + {%else%} + <table class="table"> + <thead> + <tr> + <th>File</th> + <th>Row</th> + <th>Column</th> + <th>Value</th> + <th>Message</th> + </thead> + + <tbody style="font-size: 0.9em;"> + {%for error in errors%} + <tr> + <td>{{error.filename}}</td> + <td>{{error.rowtitle}}</td> + <td>{{error.coltitle}}</td> + <td>{%if error.cellvalue | length > 25%} + {{error.cellvalue[0:24]}}… + {%else%} + {{error.cellvalue}} + {%endif%} + </td> + <td> + {%if error.message | length > 250 %} + {{error.message[0:249]}}… + {%else%} + {{error.message}} + {%endif%} + </td> + </tr> + {%endfor%} + </tbody> + </table> + {%endif%} +</div> + <div class="row"> {{cli_output(job, "stdout")}} </div> @@ -47,6 +115,7 @@ <div class="row"> {{cli_output(job, "stderr")}} </div> + {%else%} <div class="row"> <h3 class="text-danger">No Such Job</h3> |