diff options
| -rw-r--r-- | qc_app/dbinsert.py | 36 | ||||
| -rw-r--r-- | qc_app/jobs.py | 7 | ||||
| -rw-r--r-- | qc_app/templates/continue_from_create_dataset.html | 4 | ||||
| -rw-r--r-- | qc_app/templates/continue_from_create_study.html | 4 | ||||
| -rw-r--r-- | qc_app/templates/dbupdate_hidden_fields.html | 3 | ||||
| -rw-r--r-- | qc_app/templates/final_confirmation.html | 2 | ||||
| -rw-r--r-- | qc_app/templates/insert_progress.html | 16 | ||||
| -rw-r--r-- | qc_app/templates/select_dataset.html | 4 | ||||
| -rw-r--r-- | qc_app/templates/select_platform.html | 1 | ||||
| -rw-r--r-- | qc_app/templates/select_study.html | 6 |
10 files changed, 53 insertions, 30 deletions
diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py index 32043a7..e9dc3ff 100644 --- a/qc_app/dbinsert.py +++ b/qc_app/dbinsert.py @@ -134,8 +134,9 @@ def select_platform(): gchips = genechips() return render_template( "select_platform.html", filename=filename, - filetype=job["filetype"], default_species=default_species, - species=species(), genechips=gchips[default_species], + filetype=job["filetype"], totallines=int(job["currentline"]), + default_species=default_species, species=species(), + genechips=gchips[default_species], genechips_data=json.dumps(gchips)) return render_error(f"File '{filename}' no longer exists.") return render_error(f"Job '{job_id}' no longer exists.") @@ -159,8 +160,9 @@ def select_study(): organise_groups_by_family, groups_by_species(speciesid), {}) return render_template( "select_study.html", filename=form["filename"], - filetype=form["filetype"], species=speciesid, genechipid=genechipid, - studies=the_studies, groups=the_groups, tissues = tissues(), + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studies=the_studies, + groups=the_groups, tissues = tissues(), selected_group=int(form.get("inbredsetid", -13)), selected_tissue=int(form.get("tissueid", -13))) except AssertionError as aserr: @@ -198,8 +200,8 @@ def create_study(): return render_template( "continue_from_create_study.html", filename=form["filename"], filetype=form["filetype"], - species=form["species"], genechipid=form["genechipid"], - studyid=new_studyid) + totallines=form["totallines"], species=form["species"], + genechipid=form["genechipid"], studyid=new_studyid) except AssertionError as aserr: flash(f"Missing data: {aserr.args[0]}", "alert-error") return redirect(url_for("dbinsert.select_study"), code=307) @@ -252,9 +254,10 @@ def select_dataset(): datasets = datasets_by_study(studyid) return render_template( "select_dataset.html", filename=form["filename"], - filetype=form["filetype"], species=form["species"], - genechipid=form["genechipid"], studyid=studyid, datasets=datasets, - avgmethods=averaging_methods(), datascales=dataset_datascales()) + filetype=form["filetype"], totallines=form["totallines"], + species=form["species"], genechipid=form["genechipid"], + studyid=studyid, datasets=datasets, avgmethods=averaging_methods(), + datascales=dataset_datascales()) except AssertionError as aserr: return render_error(f"Missing data: {aserr.args[0]}") @@ -295,7 +298,8 @@ def create_dataset(): "continue_from_create_dataset.html", filename=form["filename"], filetype=form["filetype"], species=form["species"], genechipid=form["genechipid"], - studyid=form["studyid"], datasetid=new_datasetid) + studyid=form["studyid"], datasetid=new_datasetid, + totallines=form["totallines"]) except AssertionError as aserr: flash(f"Missing data {aserr.args[0]}", "alert-error") return redirect(url_for("dbinsert.select_dataset"), code=307) @@ -343,9 +347,9 @@ def final_confirmation(): datasetid=form["datasetid"] return render_template( "final_confirmation.html", filename=form["filename"], - filetype=form["filetype"], species=speciesid, genechipid=genechipid, - studyid=studyid, datasetid=datasetid, - the_species=selected_keys( + filetype=form["filetype"], totallines=form["totallines"], + species=speciesid, genechipid=genechipid, studyid=studyid, + datasetid=datasetid, the_species=selected_keys( species_by_id(speciesid), ("SpeciesName", "Name", "MenuName")), platform=selected_keys( platform_by_id(genechipid), @@ -378,9 +382,9 @@ def insert_data(): with Redis.from_url(redisurl, decode_responses=True) as rconn: job = jobs.launch_job( jobs.data_insertion_job( - rconn, filepath, form.get("filetype"), - form.get("species"), form.get("genechipid"), - form.get("datasetid"), app.config["SQL_URI"], redisurl, + rconn, filepath, form["filetype"], form["totallines"], + form["species"], form["genechipid"], form["datasetid"], + app.config["SQL_URI"], redisurl, app.config["JOBS_TTL_SECONDS"]), redisurl, f"{app.config['UPLOAD_FOLDER']}/job_errors") diff --git a/qc_app/jobs.py b/qc_app/jobs.py index dc9ba92..5f351d4 100644 --- a/qc_app/jobs.py +++ b/qc_app/jobs.py @@ -41,9 +41,9 @@ def build_file_verification_job( }) def data_insertion_job(# pylint: disable=[too-many-arguments] - redis_conn: Redis, filepath: str, filetype: str, speciesid: int, - platformid: int, datasetid: int, databaseuri: str, redisuri: str, - ttl_seconds: int) -> dict: + redis_conn: Redis, filepath: str, filetype: str, totallines: int, + speciesid: int, platformid: int, datasetid: int, databaseuri: str, + redisuri: str, ttl_seconds: int) -> dict: "Build a data insertion job" command = [ sys.executable, "-m", "scripts.insert_data", filetype, filepath, @@ -52,6 +52,7 @@ def data_insertion_job(# pylint: disable=[too-many-arguments] return __init_job__( redis_conn, str(uuid4()), command, "data-insertion", ttl_seconds, { "filename": os.path.basename(filepath), "filetype": filetype, + "totallines": totallines }) def launch_job(the_job: dict, redisurl: str, error_dir): diff --git a/qc_app/templates/continue_from_create_dataset.html b/qc_app/templates/continue_from_create_dataset.html index 1e493c5..4358b11 100644 --- a/qc_app/templates/continue_from_create_dataset.html +++ b/qc_app/templates/continue_from_create_dataset.html @@ -27,7 +27,7 @@ <legend>continue with new dataset</legend> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, datasetid=datasetid)}} + studyid=studyid, datasetid=datasetid, totallines=totallines)}} <fieldset> <input type="submit" @@ -44,7 +44,7 @@ <legend>Select from existing dataset</legend> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid, datasetid=datasetid)}} + studyid=studyid, datasetid=datasetid, totallines=totallines)}} <fieldset> <input type="submit" diff --git a/qc_app/templates/continue_from_create_study.html b/qc_app/templates/continue_from_create_study.html index fecb19b..3684f57 100644 --- a/qc_app/templates/continue_from_create_study.html +++ b/qc_app/templates/continue_from_create_study.html @@ -27,7 +27,7 @@ <legend>continue with new study</legend> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}} <fieldset> <input type="submit" @@ -44,7 +44,7 @@ <legend>Select from existing study</legend> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}} <fieldset> <input type="submit" diff --git a/qc_app/templates/dbupdate_hidden_fields.html b/qc_app/templates/dbupdate_hidden_fields.html index 5a95cbb..ccbc299 100644 --- a/qc_app/templates/dbupdate_hidden_fields.html +++ b/qc_app/templates/dbupdate_hidden_fields.html @@ -4,6 +4,9 @@ <input type="hidden" name="filename" value="{{filename}}" /> <input type="hidden" name="filetype" value="{{filetype}}" /> +{%if kwargs.get("totallines")%} +<input type="hidden" name="totallines" value="{{kwargs['totallines']}}" /> +{%endif%} {%if kwargs.get("species"):%} <input type="hidden" name="species" value="{{kwargs['species']}}" /> {%endif%} diff --git a/qc_app/templates/final_confirmation.html b/qc_app/templates/final_confirmation.html index 018f8d6..b08e1f5 100644 --- a/qc_app/templates/final_confirmation.html +++ b/qc_app/templates/final_confirmation.html @@ -38,7 +38,7 @@ <form method="POST" action="{{url_for('dbinsert.insert_data')}}"> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid,datasetid=datasetid)}} + studyid=studyid,datasetid=datasetid, totallines=totallines)}} <fieldset> <input type="submit" class="btn btn-main" value="confirm" /> </fieldset> diff --git a/qc_app/templates/insert_progress.html b/qc_app/templates/insert_progress.html index 52beaa4..0e1a9e0 100644 --- a/qc_app/templates/insert_progress.html +++ b/qc_app/templates/insert_progress.html @@ -13,9 +13,21 @@ <label for="job_status">status:</label> <span>{{job_status}}: {{message}}</span><br /> +{%if job.get("stdout", "").split("\n\n") | length < 3 %} +{%set lines = 0%} +{%else%} +{%set lines = (job.get("stdout", "").split("\n\n") | length / 3) %} +{%endif%} +{%set totallines = job.get("totallines", lines+3) | int %} +{%if totallines > 1000 %} +{%set fraction = ((lines*1000)/totallines) %} +{%else%} +{%set fraction = (lines/totallines)%} +{%endif%} + <label for="job_{{job_id}}">inserting: </label> -<progress id="job_{{job_id}}" value="{{(job['percent'] | float) / 100}}">{{job["percent"]}}</progress> -<span>{{"%.2f" | format(job['percent'] | float)}}%</span> +<progress id="jobs_{{job_id}}" value="{{(fraction)}}">{{fraction*100}}</progress> +<span>{{"%.2f" | format(fraction * 100 | float)}}%</span><br /> {{stdout_output(job)}} diff --git a/qc_app/templates/select_dataset.html b/qc_app/templates/select_dataset.html index a16fc75..a21957d 100644 --- a/qc_app/templates/select_dataset.html +++ b/qc_app/templates/select_dataset.html @@ -16,7 +16,7 @@ <legend>choose existing dataset</legend> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}} <fieldset> <label for="datasetid" class="form-col-1">dataset:</label> @@ -49,7 +49,7 @@ <legend>create new dataset</legend> {{hidden_fields( filename, filetype, species=species, genechipid=genechipid, - studyid=studyid)}} + studyid=studyid, totallines=totallines)}} {%with messages = get_flashed_messages(with_categories=true)%} {%if messages:%} diff --git a/qc_app/templates/select_platform.html b/qc_app/templates/select_platform.html index 3ceae01..dadf779 100644 --- a/qc_app/templates/select_platform.html +++ b/qc_app/templates/select_platform.html @@ -9,6 +9,7 @@ id="select-platform-form" data-genechips="{{genechips_data}}"> <input type="hidden" name="filename" value="{{filename}}" /> <input type="hidden" name="filetype" value="{{filetype}}" /> + <input type="hidden" name="totallines" value="{{totallines}}" /> <fieldset> <label for="species" class="form-col-1">species</label> diff --git a/qc_app/templates/select_study.html b/qc_app/templates/select_study.html index d8bdcf0..36d1299 100644 --- a/qc_app/templates/select_study.html +++ b/qc_app/templates/select_study.html @@ -15,7 +15,8 @@ id="select-platform-form" data-genechips="{{genechips_data}}" class="two-col-sep-col1"> <legend>Select from existing study</legend> - {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}} + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}} <fieldset> <label for="study" class="form-col-1">study:</label> @@ -50,7 +51,8 @@ {%endif%} {%endwith%} <legend>Create new study</legend> - {{hidden_fields(filename, filetype, species=species, genechipid=genechipid)}} + {{hidden_fields(filename, filetype, species=species, genechipid=genechipid, + totallines=totallines)}} <fieldset> <label for="studyname" class="form-col-1">name:</label> |