aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorFrederick Muriuki Muriithi2023-10-16 14:41:39 +0300
committerFrederick Muriuki Muriithi2023-10-16 14:41:39 +0300
commitc9af16ce37ccfbf75183ed0214e3a52a9d91dcc7 (patch)
tree1f6e2ae055e40fe326883cb120dc9788c2673e39
parent0d5dc5489837957e3fab513b9dcc550eddfe53ca (diff)
downloadgn-uploader-c9af16ce37ccfbf75183ed0214e3a52a9d91dcc7.tar.gz
Indicate progress for data insert.
-rw-r--r--qc_app/dbinsert.py36
-rw-r--r--qc_app/jobs.py7
-rw-r--r--qc_app/templates/continue_from_create_dataset.html4
-rw-r--r--qc_app/templates/continue_from_create_study.html4
-rw-r--r--qc_app/templates/dbupdate_hidden_fields.html3
-rw-r--r--qc_app/templates/final_confirmation.html2
-rw-r--r--qc_app/templates/insert_progress.html16
-rw-r--r--qc_app/templates/select_dataset.html4
-rw-r--r--qc_app/templates/select_platform.html1
-rw-r--r--qc_app/templates/select_study.html6
10 files changed, 53 insertions, 30 deletions
diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py
index 32043a7..e9dc3ff 100644
--- a/qc_app/dbinsert.py
+++ b/qc_app/dbinsert.py
@@ -134,8 +134,9 @@ def select_platform():
gchips = genechips()
return render_template(
"select_platform.html", filename=filename,
- filetype=job["filetype"], default_species=default_species,
- species=species(), genechips=gchips[default_species],
+ filetype=job["filetype"], totallines=int(job["currentline"]),
+ default_species=default_species, species=species(),
+ genechips=gchips[default_species],
genechips_data=json.dumps(gchips))
return render_error(f"File '{filename}' no longer exists.")
return render_error(f"Job '{job_id}' no longer exists.")
@@ -159,8 +160,9 @@ def select_study():
organise_groups_by_family, groups_by_species(speciesid), {})
return render_template(
"select_study.html", filename=form["filename"],
- filetype=form["filetype"], species=speciesid, genechipid=genechipid,
- studies=the_studies, groups=the_groups, tissues = tissues(),
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=speciesid, genechipid=genechipid, studies=the_studies,
+ groups=the_groups, tissues = tissues(),
selected_group=int(form.get("inbredsetid", -13)),
selected_tissue=int(form.get("tissueid", -13)))
except AssertionError as aserr:
@@ -198,8 +200,8 @@ def create_study():
return render_template(
"continue_from_create_study.html",
filename=form["filename"], filetype=form["filetype"],
- species=form["species"], genechipid=form["genechipid"],
- studyid=new_studyid)
+ totallines=form["totallines"], species=form["species"],
+ genechipid=form["genechipid"], studyid=new_studyid)
except AssertionError as aserr:
flash(f"Missing data: {aserr.args[0]}", "alert-error")
return redirect(url_for("dbinsert.select_study"), code=307)
@@ -252,9 +254,10 @@ def select_dataset():
datasets = datasets_by_study(studyid)
return render_template(
"select_dataset.html", filename=form["filename"],
- filetype=form["filetype"], species=form["species"],
- genechipid=form["genechipid"], studyid=studyid, datasets=datasets,
- avgmethods=averaging_methods(), datascales=dataset_datascales())
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=form["species"], genechipid=form["genechipid"],
+ studyid=studyid, datasets=datasets, avgmethods=averaging_methods(),
+ datascales=dataset_datascales())
except AssertionError as aserr:
return render_error(f"Missing data: {aserr.args[0]}")
@@ -295,7 +298,8 @@ def create_dataset():
"continue_from_create_dataset.html",
filename=form["filename"], filetype=form["filetype"],
species=form["species"], genechipid=form["genechipid"],
- studyid=form["studyid"], datasetid=new_datasetid)
+ studyid=form["studyid"], datasetid=new_datasetid,
+ totallines=form["totallines"])
except AssertionError as aserr:
flash(f"Missing data {aserr.args[0]}", "alert-error")
return redirect(url_for("dbinsert.select_dataset"), code=307)
@@ -343,9 +347,9 @@ def final_confirmation():
datasetid=form["datasetid"]
return render_template(
"final_confirmation.html", filename=form["filename"],
- filetype=form["filetype"], species=speciesid, genechipid=genechipid,
- studyid=studyid, datasetid=datasetid,
- the_species=selected_keys(
+ filetype=form["filetype"], totallines=form["totallines"],
+ species=speciesid, genechipid=genechipid, studyid=studyid,
+ datasetid=datasetid, the_species=selected_keys(
species_by_id(speciesid), ("SpeciesName", "Name", "MenuName")),
platform=selected_keys(
platform_by_id(genechipid),
@@ -378,9 +382,9 @@ def insert_data():
with Redis.from_url(redisurl, decode_responses=True) as rconn:
job = jobs.launch_job(
jobs.data_insertion_job(
- rconn, filepath, form.get("filetype"),
- form.get("species"), form.get("genechipid"),
- form.get("datasetid"), app.config["SQL_URI"], redisurl,
+ rconn, filepath, form["filetype"], form["totallines"],
+ form["species"], form["genechipid"], form["datasetid"],
+ app.config["SQL_URI"], redisurl,
app.config["JOBS_TTL_SECONDS"]),
redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors")
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index dc9ba92..5f351d4 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -41,9 +41,9 @@ def build_file_verification_job(
})
def data_insertion_job(# pylint: disable=[too-many-arguments]
- redis_conn: Redis, filepath: str, filetype: str, speciesid: int,
- platformid: int, datasetid: int, databaseuri: str, redisuri: str,
- ttl_seconds: int) -> dict:
+ redis_conn: Redis, filepath: str, filetype: str, totallines: int,
+ speciesid: int, platformid: int, datasetid: int, databaseuri: str,
+ redisuri: str, ttl_seconds: int) -> dict:
"Build a data insertion job"
command = [
sys.executable, "-m", "scripts.insert_data", filetype, filepath,
@@ -52,6 +52,7 @@ def data_insertion_job(# pylint: disable=[too-many-arguments]
return __init_job__(
redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, {
"filename": os.path.basename(filepath), "filetype": filetype,
+ "totallines": totallines
})
def launch_job(the_job: dict, redisurl: str, error_dir):
diff --git a/qc_app/templates/continue_from_create_dataset.html b/qc_app/templates/continue_from_create_dataset.html
index 1e493c5..4358b11 100644
--- a/qc_app/templates/continue_from_create_dataset.html
+++ b/qc_app/templates/continue_from_create_dataset.html
@@ -27,7 +27,7 @@
<legend>continue with new dataset</legend>
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid, datasetid=datasetid)}}
+ studyid=studyid, datasetid=datasetid, totallines=totallines)}}
<fieldset>
<input type="submit"
@@ -44,7 +44,7 @@
<legend>Select from existing dataset</legend>
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid, datasetid=datasetid)}}
+ studyid=studyid, datasetid=datasetid, totallines=totallines)}}
<fieldset>
<input type="submit"
diff --git a/qc_app/templates/continue_from_create_study.html b/qc_app/templates/continue_from_create_study.html
index fecb19b..3684f57 100644
--- a/qc_app/templates/continue_from_create_study.html
+++ b/qc_app/templates/continue_from_create_study.html
@@ -27,7 +27,7 @@
<legend>continue with new study</legend>
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid)}}
+ studyid=studyid, totallines=totallines)}}
<fieldset>
<input type="submit"
@@ -44,7 +44,7 @@
<legend>Select from existing study</legend>
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid)}}
+ studyid=studyid, totallines=totallines)}}
<fieldset>
<input type="submit"
diff --git a/qc_app/templates/dbupdate_hidden_fields.html b/qc_app/templates/dbupdate_hidden_fields.html
index 5a95cbb..ccbc299 100644
--- a/qc_app/templates/dbupdate_hidden_fields.html
+++ b/qc_app/templates/dbupdate_hidden_fields.html
@@ -4,6 +4,9 @@
<input type="hidden" name="filename" value="{{filename}}" />
<input type="hidden" name="filetype" value="{{filetype}}" />
+{%if kwargs.get("totallines")%}
+<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" />
+{%endif%}
{%if kwargs.get("species"):%}
<input type="hidden" name="species" value="{{kwargs['species']}}" />
{%endif%}
diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html
index 018f8d6..b08e1f5 100644
--- a/qc_app/templates/final_confirmation.html
+++ b/qc_app/templates/final_confirmation.html
@@ -38,7 +38,7 @@
<form method="POST" action="{{url_for('dbinsert.insert_data')}}">
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid,datasetid=datasetid)}}
+ studyid=studyid,datasetid=datasetid, totallines=totallines)}}
<fieldset>
<input type="submit" class="btn btn-main" value="confirm" />
</fieldset>
diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html
index 52beaa4..0e1a9e0 100644
--- a/qc_app/templates/insert_progress.html
+++ b/qc_app/templates/insert_progress.html
@@ -13,9 +13,21 @@
<label for="job_status">status:</label>
<span>{{job_status}}: {{message}}</span><br />
+{%if job.get("stdout", "").split("\n\n") | length < 3 %}
+{%set lines = 0%}
+{%else%}
+{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %}
+{%endif%}
+{%set totallines = job.get("totallines", lines+3) | int %}
+{%if totallines > 1000 %}
+{%set fraction = ((lines*1000)/totallines) %}
+{%else%}
+{%set fraction = (lines/totallines)%}
+{%endif%}
+
<label for="job_{{job_id}}">inserting: </label>
-<progress id="job_{{job_id}}" value="{{(job['percent'] | float) / 100}}">{{job["percent"]}}</progress>
-<span>{{"%.2f" | format(job['percent'] | float)}}%</span>
+<progress id="jobs_{{job_id}}" value="{{(fraction)}}">{{fraction*100}}</progress>
+<span>{{"%.2f" | format(fraction * 100 | float)}}%</span><br />
{{stdout_output(job)}}
diff --git a/qc_app/templates/select_dataset.html b/qc_app/templates/select_dataset.html
index a16fc75..a21957d 100644
--- a/qc_app/templates/select_dataset.html
+++ b/qc_app/templates/select_dataset.html
@@ -16,7 +16,7 @@
<legend>choose existing dataset</legend>
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid)}}
+ studyid=studyid, totallines=totallines)}}
<fieldset>
<label for="datasetid" class="form-col-1">dataset:</label>
@@ -49,7 +49,7 @@
<legend>create new dataset</legend>
{{hidden_fields(
filename, filetype, species=species, genechipid=genechipid,
- studyid=studyid)}}
+ studyid=studyid, totallines=totallines)}}
{%with messages = get_flashed_messages(with_categories=true)%}
{%if messages:%}
diff --git a/qc_app/templates/select_platform.html b/qc_app/templates/select_platform.html
index 3ceae01..dadf779 100644
--- a/qc_app/templates/select_platform.html
+++ b/qc_app/templates/select_platform.html
@@ -9,6 +9,7 @@
id="select-platform-form" data-genechips="{{genechips_data}}">
<input type="hidden" name="filename" value="{{filename}}" />
<input type="hidden" name="filetype" value="{{filetype}}" />
+ <input type="hidden" name="totallines" value="{{totallines}}" />
<fieldset>
<label for="species" class="form-col-1">species</label>
diff --git a/qc_app/templates/select_study.html b/qc_app/templates/select_study.html
index d8bdcf0..36d1299 100644
--- a/qc_app/templates/select_study.html
+++ b/qc_app/templates/select_study.html
@@ -15,7 +15,8 @@
id="select-platform-form" data-genechips="{{genechips_data}}"
class="two-col-sep-col1">
<legend>Select from existing study</legend>
- {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}}
+ {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+ totallines=totallines)}}
<fieldset>
<label for="study" class="form-col-1">study:</label>
@@ -50,7 +51,8 @@
{%endif%}
{%endwith%}
<legend>Create new study</legend>
- {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}}
+ {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+ totallines=totallines)}}
<fieldset>
<label for="studyname" class="form-col-1">name:</label>