aboutsummaryrefslogtreecommitdiff
path: root/uploader
diff options
context:
space:
mode:
Diffstat (limited to 'uploader')
-rw-r--r--uploader/__init__.py11
-rw-r--r--uploader/authorisation.py2
-rw-r--r--uploader/background_jobs.py27
-rw-r--r--uploader/jobs.py9
-rw-r--r--uploader/monadic_requests.py16
-rw-r--r--uploader/platforms/models.py3
-rw-r--r--uploader/route_utils.py3
7 files changed, 48 insertions, 23 deletions
diff --git a/uploader/__init__.py b/uploader/__init__.py
index b986c81..97b9af5 100644
--- a/uploader/__init__.py
+++ b/uploader/__init__.py
@@ -3,13 +3,17 @@ import os
import sys
import logging
from pathlib import Path
+from typing import Optional
from flask import Flask, request
-from flask_session import Session
+
from cachelib import FileSystemCache
from gn_libs import jobs as gnlibs_jobs
+from flask_session import Session
+
+
from uploader.oauth2.client import user_logged_in, authserver_authorise_uri
from . import session
@@ -66,12 +70,15 @@ def setup_modules_logging(app_logger):
logging.getLogger("uploader.publications.models").setLevel(loglevel)
-def create_app(config: dict = {}):
+def create_app(config: Optional[dict] = None):
"""The application factory.
config: dict
Useful to override settings in the settings files and environment
especially in environments such as testing."""
+ if config is None:
+ config = {}
+
app = Flask(__name__)
### BEGIN: Application configuration
diff --git a/uploader/authorisation.py b/uploader/authorisation.py
index bc950d8..3cf3585 100644
--- a/uploader/authorisation.py
+++ b/uploader/authorisation.py
@@ -48,7 +48,7 @@ def require_token(func: Callable) -> Callable:
"""
def __invalid_token__(_whatever):
logging.debug("==========> Failure log: %s", _whatever)
- raise Exception(
+ raise Exception(# pylint: disable=[broad-exception-raised]
"You attempted to access a feature of the system that requires "
"authorisation. Unfortunately, we could not verify you have the "
"appropriate authorisation to perform the action you requested. "
diff --git a/uploader/background_jobs.py b/uploader/background_jobs.py
index 09ea0c0..dc9f837 100644
--- a/uploader/background_jobs.py
+++ b/uploader/background_jobs.py
@@ -1,11 +1,10 @@
+"""Generic views and utilities to handle background jobs."""
import uuid
-import logging
import importlib
from typing import Callable
from functools import partial
from flask import (
- request,
url_for,
redirect,
Response,
@@ -29,8 +28,11 @@ def __default_error_handler__(job: dict) -> Response:
def register_handlers(
job_type: str,
success_handler: HandlerType,
+ # pylint: disable=[redefined-outer-name]
error_handler: HandlerType = __default_error_handler__
+ # pylint: disable=[redefined-outer-name]
) -> str:
+ """Register success and error handlers for each job type."""
if not bool(app.config.get("background-jobs")):
app.config["background-jobs"] = {}
@@ -49,19 +51,19 @@ def register_job_handlers(job: str):
_parts = absolute_function_path.split(".")
app.logger.debug("THE PARTS ARE: %s", _parts)
assert len(_parts) > 1, f"Invalid path: {absolute_function_path}"
- function = _parts[-1]
module = importlib.import_module(f".{_parts[-2]}",
package=".".join(_parts[0:-2]))
return getattr(module, _parts[-1])
metadata = job["metadata"]
if metadata["success_handler"]:
- success_handler = __load_handler__(metadata["success_handler"])
+ _success_handler = __load_handler__(metadata["success_handler"])
try:
- error_handler = __load_handler__(metadata["error_handler"])
- except Exception as _exc:
- error_handler = __default_error_handler__
- register_handlers(metadata["job-type"], success_handler, error_handler)
+ _error_handler = __load_handler__(metadata["error_handler"])
+ except Exception as _exc:# pylint: disable=[broad-exception-caught]
+ _error_handler = __default_error_handler__
+ register_handlers(
+ metadata["job-type"], _success_handler, _error_handler)
def handler(job: dict, handler_type: str) -> HandlerType:
@@ -74,7 +76,7 @@ def handler(job: dict, handler_type: str) -> HandlerType:
).get(handler_type)
if bool(_handler):
return _handler(job)
- raise Exception(
+ raise Exception(# pylint: disable=[broad-exception-raised]
f"No '{handler_type}' handler registered for job type: {_job_type}")
@@ -98,8 +100,8 @@ def job_status(job_id: uuid.UUID):
if status == "completed":
return success_handler(job)
- return render_template(f"jobs/job-status.html", job=job)
- except JobNotFound as jnf:
+ return render_template("jobs/job-status.html", job=job)
+ except JobNotFound as _jnf:
return render_template(
"jobs/job-not-found.html",
job_id=job_id)
@@ -108,9 +110,10 @@ def job_status(job_id: uuid.UUID):
@background_jobs_bp.route("/error/<uuid:job_id>")
@require_login
def job_error(job_id: uuid.UUID):
+ """Handle job errors in a generic manner."""
with sqlite3.connection(app.config["ASYNCHRONOUS_JOBS_SQLITE_DB"]) as conn:
try:
job = jobs.job(conn, job_id, fulldetails=True)
return render_template("jobs/job-error.html", job=job)
- except JobNotFound as jnf:
+ except JobNotFound as _jnf:
return render_template("jobs/job-not-found.html", job_id=job_id)
diff --git a/uploader/jobs.py b/uploader/jobs.py
index e86ee05..5968c03 100644
--- a/uploader/jobs.py
+++ b/uploader/jobs.py
@@ -41,7 +41,8 @@ def error_filename(jobid, error_dir):
"Compute the path of the file where errors will be dumped."
return f"{error_dir}/job_{jobid}.error"
-def initialise_job(# pylint: disable=[too-many-arguments]
+def initialise_job(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
rconn: Redis, rprefix: str, jobid: str, command: list, job_type: str,
ttl_seconds: int = 86400, extra_meta: Optional[dict] = None) -> dict:
"Initialise a job 'object' and put in on redis"
@@ -54,7 +55,8 @@ def initialise_job(# pylint: disable=[too-many-arguments]
name=job_key(rprefix, jobid), time=timedelta(seconds=ttl_seconds))
return the_job
-def build_file_verification_job(#pylint: disable=[too-many-arguments]
+def build_file_verification_job(
+ #pylint: disable=[too-many-arguments, too-many-positional-arguments]
redis_conn: Redis,
dburi: str,
redisuri: str,
@@ -77,7 +79,8 @@ def build_file_verification_job(#pylint: disable=[too-many-arguments]
"filename": os.path.basename(filepath), "percent": 0
})
-def data_insertion_job(# pylint: disable=[too-many-arguments]
+def data_insertion_job(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
redis_conn: Redis, filepath: str, filetype: str, totallines: int,
speciesid: int, platformid: int, datasetid: int, databaseuri: str,
redisuri: str, ttl_seconds: int) -> dict:
diff --git a/uploader/monadic_requests.py b/uploader/monadic_requests.py
index f1f5c77..eda42d0 100644
--- a/uploader/monadic_requests.py
+++ b/uploader/monadic_requests.py
@@ -59,6 +59,11 @@ def get(url, params=None, **kwargs) -> Either:
:rtype: pymonad.either.Either
"""
+ timeout = kwargs.get("timeout")
+ kwargs = {key: val for key,val in kwargs.items() if key != "timeout"}
+ if timeout is None:
+ timeout = (9.13, 20)
+
try:
resp = requests.get(url, params=params, **kwargs)
if resp.status_code in SUCCESS_CODES:
@@ -76,6 +81,11 @@ def post(url, data=None, json=None, **kwargs) -> Either:
:rtype: pymonad.either.Either
"""
+ timeout = kwargs.get("timeout")
+ kwargs = {key: val for key,val in kwargs.items() if key != "timeout"}
+ if timeout is None:
+ timeout = (9.13, 20)
+
try:
resp = requests.post(url, data=data, json=json, **kwargs)
if resp.status_code in SUCCESS_CODES:
@@ -95,10 +105,10 @@ def make_either_error_handler(msg):
try:
_data = error.json()
except Exception as _exc:
- raise Exception(error.content) from _exc
- raise Exception(_data)
+ raise Exception(error.content) from _exc# pylint: disable=[broad-exception-raised]
+ raise Exception(_data)# pylint: disable=[broad-exception-raised]
app.logger.debug("\n\n%s\n\n", msg)
- raise Exception(error)
+ raise Exception(error)# pylint: disable=[broad-exception-raised]
return __fail__
diff --git a/uploader/platforms/models.py b/uploader/platforms/models.py
index a859371..0dd9368 100644
--- a/uploader/platforms/models.py
+++ b/uploader/platforms/models.py
@@ -56,7 +56,8 @@ def platform_by_species_and_id(
return None
-def save_new_platform(# pylint: disable=[too-many-arguments]
+def save_new_platform(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
cursor: Cursor,
species_id: int,
geo_platform: str,
diff --git a/uploader/route_utils.py b/uploader/route_utils.py
index 18eadda..ce718fb 100644
--- a/uploader/route_utils.py
+++ b/uploader/route_utils.py
@@ -6,7 +6,8 @@ from gn_libs.mysqldb import database_connection
from uploader.population.models import (populations_by_species,
population_by_species_and_id)
-def generic_select_population(# pylint: disable=[too-many-arguments]
+def generic_select_population(
+ # pylint: disable=[too-many-arguments, too-many-positional-arguments]
species: dict,
template: str,
population_id: str,
generated by cgit v1.2.3 (git 2.47.1) at 2025-06-16 08:25:21 +0000