Implement UI for dataset selection

As part of updating the database with the new data, there is a need to
select the appropriate dataset that the data belongs to, and this
commit provides the UI to assist the user do that.

10 files changed, 289 insertions, 1 deletions

diff --git a/etc/default_config.py b/etc/default_config.py
index 76b6b43..fcb17d9 100644
--- a/etc/default_config.py
+++ b/etc/default_config.py
@@ -10,3 +10,4 @@ SECRET_KEY = b"<Please! Please! Please! Change This!>"
 UPLOAD_FOLDER = "/tmp/qc_app_files"
 REDIS_URL = "redis://"
 JOBS_TTL_SECONDS = 1209600 # 14 days
+GN3_URL="http://localhost:8080"
diff --git a/mypy.ini b/mypy.ini
index 74e2d91..1f63c34 100644
--- a/mypy.ini
+++ b/mypy.ini
@@ -19,4 +19,7 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 
 [mypy-jsonpickle.*]
+ignore_missing_imports = True
+
+[mypy-requests.*]
 ignore_missing_imports = True
\ No newline at end of file
diff --git a/qc_app/__init__.py b/qc_app/__init__.py
index 08b56c9..6b760b9 100644
--- a/qc_app/__init__.py
+++ b/qc_app/__init__.py
@@ -6,6 +6,7 @@ from flask import Flask
 
 from .entry import entrybp
 from .parse import parsebp
+from .dbinsert import dbinsertbp
 
 def instance_path():
     """Retrieve the `instance_path`. Raise an exception if not defined."""
@@ -27,4 +28,5 @@ def create_app(instance_dir):
     # setup blueprints
     app.register_blueprint(entrybp, url_prefix="/")
     app.register_blueprint(parsebp, url_prefix="/parse")
+    app.register_blueprint(dbinsertbp, url_prefix="/dbinsert")
     return app
diff --git a/qc_app/dbinsert.py b/qc_app/dbinsert.py
new file mode 100644
index 0000000..0733e5f
--- /dev/null
+++ b/qc_app/dbinsert.py
@@ -0,0 +1,63 @@
+"Handle inserting data into the database"
+import os
+import json
+from functools import reduce
+
+import requests
+from redis import Redis
+from flask import request, Blueprint, render_template, current_app as app
+
+from . import jobs
+
+dbinsertbp = Blueprint("dbinsert", __name__)
+
+def render_error(error_msg):
+    "Render the generic error page"
+    return render_template("dbupdate_error.html", error_message=error_msg), 400
+
+def make_menu_items_grouper(grouping_fn=lambda item: item):
+    "Build function to be used to group menu items."
+    def __grouper__(acc, row):
+        grouping = grouping_fn(row[2])
+        row_values = (row[0].strip(), row[1].strip())
+        if acc.get(grouping) is None:
+            return {**acc, grouping: (row_values,)}
+        return {**acc, grouping: (acc[grouping] + (row_values,))}
+    return __grouper__
+
+@dbinsertbp.route("/select-dataset", methods=["POST"])
+def select_dataset():
+    "Select the dataset to add the file contents against"
+    job_id = request.form["job_id"]
+    with Redis.from_url(app.config["REDIS_URL"], decode_responses=True) as rconn:
+        job = jobs.job(rconn, job_id)
+        if job:
+            filename = job["filename"]
+            filepath = f"{app.config['UPLOAD_FOLDER']}/{filename}"
+            if os.path.exists(filepath):
+                req = requests.get(
+                    "https://genenetwork.org/api3/api/menu/generate/json")
+                menu_contents = req.json()
+                default_species = "mouse"
+                mouse_groups = reduce(
+                    make_menu_items_grouper(
+                        lambda item: item.strip()[7:].strip()),
+                    menu_contents["groups"][default_species], {})
+                default_group = "BXD"
+                group_types = reduce(
+                    make_menu_items_grouper(),
+                    menu_contents["types"][default_species][default_group], {})
+                default_type = group_types[tuple(group_types)[0]][0][0]
+                datasets = menu_contents[
+                    "datasets"][default_species][default_group][
+                        default_type]
+
+                return render_template(
+                    "select_dataset.html", job_id=job_id, job_name=filename,
+                    species=menu_contents["species"],
+                    default_species=default_species, groups=mouse_groups,
+                    types=group_types, datasets=datasets,
+                    menu_contents=json.dumps(menu_contents))
+            return render_error(f"File '{filename}' no longer exists.")
+        return render_error(f"Job '{job_id}' no longer exists.")
+    return render_error("Unknown error")
diff --git a/qc_app/parse.py b/qc_app/parse.py
index 5d75c37..2a33fd0 100644
--- a/qc_app/parse.py
+++ b/qc_app/parse.py
@@ -104,7 +104,8 @@ def results(job_id: str):
             "parse_results.html",
             errors=errors,
             job_name = f"Parsing '{filename}'",
-            user_aborted = job.get("user_aborted"))
+            user_aborted = job.get("user_aborted"),
+            job_id=job["job_id"])
 
     return render_template("no_such_job.html", job_id=job_id)
 
diff --git a/qc_app/static/css/styles.css b/qc_app/static/css/styles.css
index 9e5a4ec..4d0fa8c 100644
--- a/qc_app/static/css/styles.css
+++ b/qc_app/static/css/styles.css
@@ -101,3 +101,22 @@ table {
     border: 2px solid;
     border-radius: 1em;
 }
+
+form {
+    width: 30%;
+}
+
+fieldset {
+    border-style: none;
+    display: grid;
+    grid-template-columns: 5em 1fr;
+    column-gap: 5px;
+}
+
+label {
+    grid-column: 1 / 2;
+}
+
+input,select,button {
+    grid-column: 2 / 3;
+}
diff --git a/qc_app/static/js/dbinsert.js b/qc_app/static/js/dbinsert.js
new file mode 100644
index 0000000..3c0be54
--- /dev/null
+++ b/qc_app/static/js/dbinsert.js
@@ -0,0 +1,105 @@
+function remove_children(element) {
+    Array.from(element.children).forEach(child => {
+	element.removeChild(child);
+    });
+}
+
+function trigger_change_event(element) {
+    evt = new Event("change");
+    element.dispatchEvent(evt);
+}
+
+function setup_groups(group_data) {
+    elt = document.getElementById("group");
+    remove_children(elt);
+    the_groups = group_data.reduce(
+	function(acc, row) {
+	    grouping = row[2].slice(7).trim();
+	    if(acc[grouping] === undefined) {
+		acc[grouping] = [];
+	    }
+	    acc[grouping].push([row[0], row[1]]);
+	    return acc;
+	},
+	{});
+    for(grouping in the_groups) {
+	optgrp = document.createElement("optgroup");
+	optgrp.setAttribute("label", grouping);
+	the_groups[grouping].forEach(group => {
+	    opt = document.createElement("option");
+	    opt.setAttribute("value", group[0]);
+	    opt.appendChild(document.createTextNode(group[1]));
+	    optgrp.appendChild(opt);
+	});
+	elt.appendChild(optgrp);
+    }
+    trigger_change_event(elt);
+}
+
+function setup_types(type_data) {
+    elt = document.getElementById("type");
+    remove_children(elt);
+    the_types = type_data.reduce(function(acc, row) {
+	grp = row[2];
+	if(acc[grp] === undefined) {
+	    acc[grp] = [];
+	}
+	acc[grp].push([row[0], row[1]]);
+	return acc;
+    }, {});
+    for(type_group in the_types) {
+	optgrp = document.createElement("optgroup");
+	optgrp.setAttribute("label", type_group);
+	the_types[type_group].forEach(type => {
+	    opt = document.createElement("option");
+	    opt.setAttribute("value", type[0]);
+	    opt.appendChild(document.createTextNode(type[1]));
+	    optgrp.appendChild(opt);
+	});
+	elt.appendChild(optgrp);
+    }
+    trigger_change_event(elt);
+}
+
+function setup_datasets(dataset_data) {
+    console.info("DATASET DATA:", dataset_data);
+    elt = document.getElementById("dataset");
+    remove_children(elt);
+    dataset_data.forEach(dataset => {
+	opt = document.createElement("option");
+	opt.setAttribute("value", dataset[0]);
+	opt.appendChild(document.createTextNode(
+	    "[" + dataset[1] + "] " + dataset[2]));
+	elt.appendChild(opt);
+    });
+    trigger_change_event(elt);
+}
+
+function menu_contents() {
+    return JSON.parse(
+	document.getElementsByTagName("form")[0].getAttribute(
+	    "data-menu-content"));
+}
+
+function update_menu(event) {
+    menu = menu_contents();
+
+    species_elt = document.getElementById("species");
+    group_elt = document.getElementById("group");
+    type_elt = document.getElementById("type");
+    dataset_elt = document.getElementById("dataset");
+
+    if(event.target == species_elt) {
+	setup_groups(menu["groups"][species_elt.value]);
+    }
+
+    if(event.target == group_elt) {
+	setup_types(menu["types"][species_elt.value][group_elt.value]);
+    }
+
+    if(event.target == type_elt) {
+	setup_datasets(
+	    menu["datasets"][species_elt.value][group_elt.value][type_elt.value]
+	);
+    }
+}
diff --git a/qc_app/templates/dbupdate_error.html b/qc_app/templates/dbupdate_error.html
new file mode 100644
index 0000000..83e34fe
--- /dev/null
+++ b/qc_app/templates/dbupdate_error.html
@@ -0,0 +1,12 @@
+{%extends "base.html"%}
+
+{%block title%}DB Update Error{%endblock%}
+
+{%block contents%}
+<h1 class="heading">database update error</h2>
+
+<p class="alert-error">
+  <strong>Database Update Error</strong>: {{error_message}}
+</p>
+
+{%endblock%}
diff --git a/qc_app/templates/parse_results.html b/qc_app/templates/parse_results.html
index 358c5e8..1a224e8 100644
--- a/qc_app/templates/parse_results.html
+++ b/qc_app/templates/parse_results.html
@@ -12,4 +12,11 @@
 
 {{errors_display(errors, "No errors found in the file", "We found the following errors")}}
 
+{%if errors | length == 0 %}
+<form method="post" action="{{url_for('dbinsert.select_dataset')}}">
+  <input type="hidden" name="job_id" value="{{job_id}}" />
+  <input type="submit" value="update database" class="btn btn-main" />
+</form>
+{%endif%}
+
 {%endblock%}
diff --git a/qc_app/templates/select_dataset.html b/qc_app/templates/select_dataset.html
new file mode 100644
index 0000000..5fefccc
--- /dev/null
+++ b/qc_app/templates/select_dataset.html
@@ -0,0 +1,75 @@
+{%extends "base.html"%}
+
+{%block title%}Select Dataset{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}: select dataset</h2>
+
+<form method="POST" data-menu-content="{{menu_contents}}">
+  <input type="hidden" name="job_id" value="{{job_id}}" />
+
+  <fieldset>
+    <label for="species">species:</label>
+    <select id="species" name="species">
+      {%for row in species:%}
+      <option value="{{row[0]}}"
+	      {%if row[0] == default_species:%}
+	      selected="selected"
+	      {%endif%}>
+	{{row[1]}}
+      </option>
+      {%endfor%}
+    </select>
+  </fieldset>
+
+  <fieldset>
+    <label for="group">group:</label>
+    <select id="group" name="group">
+      {%for grouping, grps in groups.items():%}
+      <optgroup label="{{grouping}}">
+	{%for group in grps:%}
+	<option value="{{group[0]}}">{{group[1]}}</option>
+	{%endfor%}
+      </optgroup>
+      {%endfor%}
+    </select>
+  </fieldset>
+
+  <fieldset>
+    <label for="type">type:</label>
+    <select id="type" name="type">
+      {%for grouping, typs in types.items():%}
+      <optgroup label="{{grouping}}">
+	{%for type in typs:%}
+	<option value="{{type[0]}}">{{type[1]}}</option>
+	{%endfor%}
+      </optgroup>
+      {%endfor%}
+    </select>
+  </fieldset>
+
+  <fieldset>
+    <label for="dataset">dataset:</label>
+    <select id="dataset" name="dataset">
+      {%for dataset_id, name1, name2 in datasets:%}
+      <option value="{{dataset_id}}">[{{name1}}] {{name2}}</option>
+      {%endfor%}
+    </select>
+  </fieldset>
+
+  <fieldset>
+    <input type="submit" class="btn btn-main" value="update database" />
+  </fieldset>
+
+</form>
+
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/dbinsert.js"></script>
+<script type="text/javascript">
+  document.getElementById("species").addEventListener("change", update_menu);
+  document.getElementById("group").addEventListener("change", update_menu);
+  document.getElementById("type").addEventListener("change", update_menu);
+</script>
+{%endblock%}