From c9af16ce37ccfbf75183ed0214e3a52a9d91dcc7 Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Mon, 16 Oct 2023 14:41:39 +0300 Subject: Indicate progress for data insert. --- qc_app/dbinsert.py | 36 ++++++++++++---------- qc_app/jobs.py | 7 +++-- qc_app/templates/continue_from_create_dataset.html | 4 +-- qc_app/templates/continue_from_create_study.html | 4 +-- qc_app/templates/dbupdate_hidden_fields.html | 3 ++ qc_app/templates/final_confirmation.html | 2 +- qc_app/templates/insert_progress.html | 16 ++++++++-- qc_app/templates/select_dataset.html | 4 +-- qc_app/templates/select_platform.html | 1 + qc_app/templates/select_study.html | 6 ++-- 10 files changed, 53 insertions(+), 30 deletions(-) diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py index 32043a7..e9dc3ff 100644 --- a/qc_app/dbinsert.py +++ b/qc_app/dbinsert.py @@ -134,8 +134,9 @@ def select_platform(): gchips = genechips() return render_template( "select_platform.html", filename=filename, - filetype=job["filetype"], default_species=default_species, - species=species(), genechips=gchips[default_species], + filetype=job["filetype"], totallines=int(job["currentline"]), + default_species=default_species, species=species(), + genechips=gchips[default_species], genechips_data=json.dumps(gchips)) return render_error(f"File '{filename}' no longer exists.") return render_error(f"Job '{job_id}' no longer exists.") @@ -159,8 +160,9 @@ def select_study(): organise_groups_by_family, groups_by_species(speciesid), {}) return render_template( "select_study.html", filename=form["filename"], - filetype=form["filetype"], species=speciesid, genechipid=genechipid, - studies=the_studies, groups=the_groups, tissues = tissues(), + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studies=the_studies, + groups=the_groups, tissues = tissues(), selected_group=int(form.get("inbredsetid", -13)), selected_tissue=int(form.get("tissueid", -13))) except AssertionError as aserr: @@ -198,8 +200,8 @@ def create_study(): return render_template( "continue_from_create_study.html", filename=form["filename"], filetype=form["filetype"], - species=form["species"], genechipid=form["genechipid"], - studyid=new_studyid) + totallines=form["totallines"], species=form["species"], + genechipid=form["genechipid"], studyid=new_studyid) except AssertionError as aserr: flash(f"Missing data: {aserr.args[0]}", "alert-error") return redirect(url_for("dbinsert.select_study"), code=307) @@ -252,9 +254,10 @@ def select_dataset(): datasets = datasets_by_study(studyid) return render_template( "select_dataset.html", filename=form["filename"], - filetype=form["filetype"], species=form["species"], - genechipid=form["genechipid"], studyid=studyid, datasets=datasets, - avgmethods=averaging_methods(), datascales=dataset_datascales()) + filetype=form["filetype"], totallines=form["totallines"], + species=form["species"], genechipid=form["genechipid"], + studyid=studyid, datasets=datasets, avgmethods=averaging_methods(), + datascales=dataset_datascales()) except AssertionError as aserr: return render_error(f"Missing data: {aserr.args[0]}") @@ -295,7 +298,8 @@ def create_dataset(): "continue_from_create_dataset.html", filename=form["filename"], filetype=form["filetype"], species=form["species"], genechipid=form["genechipid"], - studyid=form["studyid"], datasetid=new_datasetid) + studyid=form["studyid"], datasetid=new_datasetid, + totallines=form["totallines"]) except AssertionError as aserr: flash(f"Missing data {aserr.args[0]}", "alert-error") return redirect(url_for("dbinsert.select_dataset"), code=307) @@ -343,9 +347,9 @@ def final_confirmation(): datasetid=form["datasetid"] return render_template( "final_confirmation.html", filename=form["filename"], - filetype=form["filetype"], species=speciesid, genechipid=genechipid, - studyid=studyid, datasetid=datasetid, - the_species=selected_keys( + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studyid=studyid, + datasetid=datasetid, the_species=selected_keys( species_by_id(speciesid), ("SpeciesName", "Name", "MenuName")), platform=selected_keys( platform_by_id(genechipid), @@ -378,9 +382,9 @@ def insert_data(): with Redis.from_url(redisurl, decode_responses=True) as rconn: job = jobs.launch_job( jobs.data_insertion_job( - rconn, filepath, form.get("filetype"), - form.get("species"), form.get("genechipid"), - form.get("datasetid"), app.config["SQL_URI"], redisurl, + rconn, filepath, form["filetype"], form["totallines"], + form["species"], form["genechipid"], form["datasetid"], + app.config["SQL_URI"], redisurl, app.config["JOBS_TTL_SECONDS"]), redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") diff --git a/qc_app/jobs.py b/qc_app/jobs.py index dc9ba92..5f351d4 100644 --- a/qc_app/jobs.py +++ b/qc_app/jobs.py @@ -41,9 +41,9 @@ def build_file_verification_job( }) def data_insertion_job(# pylint: disable=[too-many-arguments] - redis_conn: Redis, filepath: str, filetype: str, speciesid: int, - platformid: int, datasetid: int, databaseuri: str, redisuri: str, - ttl_seconds: int) -> dict: + redis_conn: Redis, filepath: str, filetype: str, totallines: int, + speciesid: int, platformid: int, datasetid: int, databaseuri: str, + redisuri: str, ttl_seconds: int) -> dict: "Build a data insertion job" command = [ sys.executable, "-m", "scripts.insert_data", filetype, filepath, @@ -52,6 +52,7 @@ def data_insertion_job(# pylint: disable=[too-many-arguments] return __init_job__( redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, { "filename": os.path.basename(filepath), "filetype": filetype, + "totallines": totallines }) def launch_job(the_job: dict, redisurl: str, error_dir): diff --git a/qc_app/templates/continue_from_create_dataset.html b/qc_app/templates/continue_from_create_dataset.html index 1e493c5..4358b11 100644 --- a/qc_app/templates/continue_from_create_dataset.html +++ b/qc_app/templates/continue_from_create_dataset.html @@ -27,7 +27,7 @@ continue with new dataset {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, datasetid=datasetid)}} + studyid=studyid, datasetid=datasetid, totallines=totallines)}}
Select from existing dataset {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, datasetid=datasetid)}} + studyid=studyid, datasetid=datasetid, totallines=totallines)}}
continue with new study {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}}
Select from existing study {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}}
+{%if kwargs.get("totallines")%} + +{%endif%} {%if kwargs.get("species"):%} {%endif%} diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html index 018f8d6..b08e1f5 100644 --- a/qc_app/templates/final_confirmation.html +++ b/qc_app/templates/final_confirmation.html @@ -38,7 +38,7 @@
{{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid,datasetid=datasetid)}} + studyid=studyid,datasetid=datasetid, totallines=totallines)}}
diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html index 52beaa4..0e1a9e0 100644 --- a/qc_app/templates/insert_progress.html +++ b/qc_app/templates/insert_progress.html @@ -13,9 +13,21 @@ {{job_status}}: {{message}}
+{%if job.get("stdout", "").split("\n\n") | length < 3 %} +{%set lines = 0%} +{%else%} +{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %} +{%endif%} +{%set totallines = job.get("totallines", lines+3) | int %} +{%if totallines > 1000 %} +{%set fraction = ((lines*1000)/totallines) %} +{%else%} +{%set fraction = (lines/totallines)%} +{%endif%} + -{{job["percent"]}} -{{"%.2f" | format(job['percent'] | float)}}% +{{fraction*100}} +{{"%.2f" | format(fraction * 100 | float)}}%
{{stdout_output(job)}} diff --git a/qc_app/templates/select_dataset.html b/qc_app/templates/select_dataset.html index a16fc75..a21957d 100644 --- a/qc_app/templates/select_dataset.html +++ b/qc_app/templates/select_dataset.html @@ -16,7 +16,7 @@ choose existing dataset {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}}
@@ -49,7 +49,7 @@ create new dataset {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}} {%with messages = get_flashed_messages(with_categories=true)%} {%if messages:%} diff --git a/qc_app/templates/select_platform.html b/qc_app/templates/select_platform.html index 3ceae01..dadf779 100644 --- a/qc_app/templates/select_platform.html +++ b/qc_app/templates/select_platform.html @@ -9,6 +9,7 @@ id="select-platform-form" data-genechips="{{genechips_data}}"> +
diff --git a/qc_app/templates/select_study.html b/qc_app/templates/select_study.html index d8bdcf0..36d1299 100644 --- a/qc_app/templates/select_study.html +++ b/qc_app/templates/select_study.html @@ -15,7 +15,8 @@ id="select-platform-form" data-genechips="{{genechips_data}}" class="two-col-sep-col1"> Select from existing study - {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}} + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}}
@@ -50,7 +51,8 @@ {%endif%} {%endwith%} Create new study - {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}} + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}}
-- cgit v1.2.3