Indicate progress for data insert.

10 files changed, 53 insertions, 30 deletions

diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py
index 32043a7..e9dc3ff 100644
--- a/qc_app/dbinsert.py
+++ b/qc_app/dbinsert.py
@@ -134,8 +134,9 @@ def select_platform():
                 gchips = genechips()
                 return render_template(
                     "select_platform.html", filename=filename,
-                    filetype=job["filetype"], default_species=default_species,
-                    species=species(), genechips=gchips[default_species],
+                    filetype=job["filetype"], totallines=int(job["currentline"]),
+                    default_species=default_species, species=species(),
+                    genechips=gchips[default_species],
                     genechips_data=json.dumps(gchips))
             return render_error(f"File '{filename}' no longer exists.")
         return render_error(f"Job '{job_id}' no longer exists.")
@@ -159,8 +160,9 @@ def select_study():
             organise_groups_by_family, groups_by_species(speciesid), {})
         return render_template(
             "select_study.html", filename=form["filename"],
-            filetype=form["filetype"], species=speciesid, genechipid=genechipid,
-            studies=the_studies, groups=the_groups, tissues = tissues(),
+            filetype=form["filetype"], totallines=form["totallines"],
+            species=speciesid, genechipid=genechipid, studies=the_studies,
+            groups=the_groups, tissues = tissues(),
             selected_group=int(form.get("inbredsetid", -13)),
             selected_tissue=int(form.get("tissueid", -13)))
     except AssertionError as aserr:
@@ -198,8 +200,8 @@ def create_study():
                 return render_template(
                     "continue_from_create_study.html",
                     filename=form["filename"], filetype=form["filetype"],
-                    species=form["species"], genechipid=form["genechipid"],
-                    studyid=new_studyid)
+                    totallines=form["totallines"], species=form["species"],
+                    genechipid=form["genechipid"], studyid=new_studyid)
     except AssertionError as aserr:
         flash(f"Missing data: {aserr.args[0]}", "alert-error")
         return redirect(url_for("dbinsert.select_study"), code=307)
@@ -252,9 +254,10 @@ def select_dataset():
         datasets = datasets_by_study(studyid)
         return render_template(
             "select_dataset.html", filename=form["filename"],
-            filetype=form["filetype"], species=form["species"],
-            genechipid=form["genechipid"], studyid=studyid, datasets=datasets,
-            avgmethods=averaging_methods(), datascales=dataset_datascales())
+            filetype=form["filetype"], totallines=form["totallines"],
+            species=form["species"], genechipid=form["genechipid"],
+            studyid=studyid, datasets=datasets, avgmethods=averaging_methods(),
+            datascales=dataset_datascales())
     except AssertionError as aserr:
         return render_error(f"Missing data: {aserr.args[0]}")
 
@@ -295,7 +298,8 @@ def create_dataset():
                     "continue_from_create_dataset.html",
                     filename=form["filename"], filetype=form["filetype"],
                     species=form["species"], genechipid=form["genechipid"],
-                    studyid=form["studyid"], datasetid=new_datasetid)
+                    studyid=form["studyid"], datasetid=new_datasetid,
+                    totallines=form["totallines"])
     except AssertionError as aserr:
         flash(f"Missing data {aserr.args[0]}", "alert-error")
         return redirect(url_for("dbinsert.select_dataset"), code=307)
@@ -343,9 +347,9 @@ def final_confirmation():
         datasetid=form["datasetid"]
         return render_template(
             "final_confirmation.html", filename=form["filename"],
-            filetype=form["filetype"], species=speciesid, genechipid=genechipid,
-            studyid=studyid, datasetid=datasetid,
-            the_species=selected_keys(
+            filetype=form["filetype"], totallines=form["totallines"],
+            species=speciesid, genechipid=genechipid, studyid=studyid,
+            datasetid=datasetid, the_species=selected_keys(
                 species_by_id(speciesid), ("SpeciesName", "Name", "MenuName")),
             platform=selected_keys(
                 platform_by_id(genechipid),
@@ -378,9 +382,9 @@ def insert_data():
             with Redis.from_url(redisurl, decode_responses=True) as rconn:
                 job = jobs.launch_job(
                     jobs.data_insertion_job(
-                        rconn, filepath, form.get("filetype"),
-                        form.get("species"), form.get("genechipid"),
-                        form.get("datasetid"), app.config["SQL_URI"], redisurl,
+                        rconn, filepath, form["filetype"], form["totallines"],
+                        form["species"], form["genechipid"], form["datasetid"],
+                        app.config["SQL_URI"], redisurl,
                         app.config["JOBS_TTL_SECONDS"]),
                     redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors")
 
diff --git a/qc_app/jobs.py b/qc_app/jobs.py
index dc9ba92..5f351d4 100644
--- a/qc_app/jobs.py
+++ b/qc_app/jobs.py
@@ -41,9 +41,9 @@ def build_file_verification_job(
         })
 
 def data_insertion_job(# pylint: disable=[too-many-arguments]
-        redis_conn: Redis, filepath: str, filetype: str, speciesid: int,
-        platformid: int, datasetid: int, databaseuri: str, redisuri: str,
-        ttl_seconds: int) -> dict:
+        redis_conn: Redis, filepath: str, filetype: str, totallines: int,
+        speciesid: int, platformid: int, datasetid: int, databaseuri: str,
+        redisuri: str, ttl_seconds: int) -> dict:
     "Build a data insertion job"
     command = [
         sys.executable, "-m", "scripts.insert_data", filetype, filepath,
@@ -52,6 +52,7 @@ def data_insertion_job(# pylint: disable=[too-many-arguments]
     return __init_job__(
         redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, {
             "filename": os.path.basename(filepath), "filetype": filetype,
+            "totallines": totallines
         })
 
 def launch_job(the_job: dict, redisurl: str, error_dir):
diff --git a/qc_app/templates/continue_from_create_dataset.html b/qc_app/templates/continue_from_create_dataset.html
index 1e493c5..4358b11 100644
--- a/qc_app/templates/continue_from_create_dataset.html
+++ b/qc_app/templates/continue_from_create_dataset.html
@@ -27,7 +27,7 @@
     <legend>continue with new dataset</legend>
     {{hidden_fields(
     filename, filetype, species=species, genechipid=genechipid,
-    studyid=studyid, datasetid=datasetid)}}
+    studyid=studyid, datasetid=datasetid, totallines=totallines)}}
 
     <fieldset>
       <input type="submit"
@@ -44,7 +44,7 @@
     <legend>Select from existing dataset</legend>
     {{hidden_fields(
     filename, filetype, species=species, genechipid=genechipid,
-    studyid=studyid, datasetid=datasetid)}}
+    studyid=studyid, datasetid=datasetid, totallines=totallines)}}
 
     <fieldset>
       <input type="submit"
diff --git a/qc_app/templates/continue_from_create_study.html b/qc_app/templates/continue_from_create_study.html
index fecb19b..3684f57 100644
--- a/qc_app/templates/continue_from_create_study.html
+++ b/qc_app/templates/continue_from_create_study.html
@@ -27,7 +27,7 @@
     <legend>continue with new study</legend>
     {{hidden_fields(
     filename, filetype, species=species, genechipid=genechipid,
-    studyid=studyid)}}
+    studyid=studyid, totallines=totallines)}}
 
     <fieldset>
       <input type="submit"
@@ -44,7 +44,7 @@
     <legend>Select from existing study</legend>
     {{hidden_fields(
     filename, filetype, species=species, genechipid=genechipid,
-    studyid=studyid)}}
+    studyid=studyid, totallines=totallines)}}
 
     <fieldset>
       <input type="submit"
diff --git a/qc_app/templates/dbupdate_hidden_fields.html b/qc_app/templates/dbupdate_hidden_fields.html
index 5a95cbb..ccbc299 100644
--- a/qc_app/templates/dbupdate_hidden_fields.html
+++ b/qc_app/templates/dbupdate_hidden_fields.html
@@ -4,6 +4,9 @@
 
 <input type="hidden" name="filename" value="{{filename}}" />
 <input type="hidden" name="filetype" value="{{filetype}}" />
+{%if kwargs.get("totallines")%}
+<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" />
+{%endif%}
 {%if kwargs.get("species"):%}
 <input type="hidden" name="species" value="{{kwargs['species']}}" />
 {%endif%}
diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html
index 018f8d6..b08e1f5 100644
--- a/qc_app/templates/final_confirmation.html
+++ b/qc_app/templates/final_confirmation.html
@@ -38,7 +38,7 @@
 <form method="POST" action="{{url_for('dbinsert.insert_data')}}">
   {{hidden_fields(
   filename, filetype, species=species, genechipid=genechipid,
-  studyid=studyid,datasetid=datasetid)}}
+  studyid=studyid,datasetid=datasetid, totallines=totallines)}}
   <fieldset>
     <input type="submit" class="btn btn-main" value="confirm" />
   </fieldset>
diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html
index 52beaa4..0e1a9e0 100644
--- a/qc_app/templates/insert_progress.html
+++ b/qc_app/templates/insert_progress.html
@@ -13,9 +13,21 @@
 <label for="job_status">status:</label>
 <span>{{job_status}}: {{message}}</span><br />
 
+{%if job.get("stdout", "").split("\n\n") | length < 3 %}
+{%set lines = 0%}
+{%else%}
+{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %}
+{%endif%}
+{%set totallines = job.get("totallines", lines+3) | int %}
+{%if totallines > 1000 %}
+{%set fraction = ((lines*1000)/totallines) %}
+{%else%}
+{%set fraction = (lines/totallines)%}
+{%endif%}
+
 <label for="job_{{job_id}}">inserting: </label>
-<progress id="job_{{job_id}}" value="{{(job['percent'] | float) / 100}}">{{job["percent"]}}</progress>
-<span>{{"%.2f" | format(job['percent'] | float)}}%</span>
+<progress id="jobs_{{job_id}}" value="{{(fraction)}}">{{fraction*100}}</progress>
+<span>{{"%.2f" | format(fraction * 100 | float)}}%</span><br />
 
 
 {{stdout_output(job)}}
diff --git a/qc_app/templates/select_dataset.html b/qc_app/templates/select_dataset.html
index a16fc75..a21957d 100644
--- a/qc_app/templates/select_dataset.html
+++ b/qc_app/templates/select_dataset.html
@@ -16,7 +16,7 @@
     <legend>choose existing dataset</legend>
     {{hidden_fields(
     filename, filetype, species=species, genechipid=genechipid,
-    studyid=studyid)}}
+    studyid=studyid, totallines=totallines)}}
 
     <fieldset>
       <label for="datasetid" class="form-col-1">dataset:</label>
@@ -49,7 +49,7 @@
     <legend>create new dataset</legend>
     {{hidden_fields(
     filename, filetype, species=species, genechipid=genechipid,
-    studyid=studyid)}}
+    studyid=studyid, totallines=totallines)}}
 
     {%with messages = get_flashed_messages(with_categories=true)%}
     {%if messages:%}
diff --git a/qc_app/templates/select_platform.html b/qc_app/templates/select_platform.html
index 3ceae01..dadf779 100644
--- a/qc_app/templates/select_platform.html
+++ b/qc_app/templates/select_platform.html
@@ -9,6 +9,7 @@
       id="select-platform-form" data-genechips="{{genechips_data}}">
   <input type="hidden" name="filename" value="{{filename}}" />
   <input type="hidden" name="filetype" value="{{filetype}}" />
+  <input type="hidden" name="totallines" value="{{totallines}}" />
 
   <fieldset>
     <label for="species" class="form-col-1">species</label>
diff --git a/qc_app/templates/select_study.html b/qc_app/templates/select_study.html
index d8bdcf0..36d1299 100644
--- a/qc_app/templates/select_study.html
+++ b/qc_app/templates/select_study.html
@@ -15,7 +15,8 @@
 	id="select-platform-form" data-genechips="{{genechips_data}}"
 	class="two-col-sep-col1">
     <legend>Select from existing study</legend>
-    {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}}
+    {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+    totallines=totallines)}}
 
     <fieldset>
       <label for="study" class="form-col-1">study:</label>
@@ -50,7 +51,8 @@
     {%endif%}
     {%endwith%}
     <legend>Create new study</legend>
-    {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}}
+    {{hidden_fields(filename, filetype, species=species, genechipid=genechipid,
+    totallines=totallines)}}
 
     <fieldset>
       <label for="studyname" class="form-col-1">name:</label>