Initialise the "Expression Data" section.

9 files changed, 379 insertions, 72 deletions

diff --git a/uploader/templates/expression-data/base.html b/uploader/templates/expression-data/base.html
new file mode 100644
index 0000000..d63fd7e
--- /dev/null
+++ b/uploader/templates/expression-data/base.html
@@ -0,0 +1,13 @@
+{%extends "populations/base.html"%}
+
+{%block lvl3_breadcrumbs%}
+<li {%if activelink=="expression-data"%}
+    class="breadcrumb-item active"
+    {%else%}
+    class="breadcrumb-item"
+    {%endif%}>
+  <a href="{{url_for('species.populations.expression-data.index')}}">
+    Expression Data</a>
+</li>
+{%block lvl4_breadcrumbs%}{%endblock%}
+{%endblock%}
diff --git a/uploader/templates/expression-data/data-review.html b/uploader/templates/expression-data/data-review.html
new file mode 100644
index 0000000..c985b03
--- /dev/null
+++ b/uploader/templates/expression-data/data-review.html
@@ -0,0 +1,85 @@
+{%extends "base.html"%}
+
+{%block title%}Data Review{%endblock%}
+
+{%block contents%}
+<h1 class="heading">data review</h1>
+
+<div class="row">
+  <h2 id="data-concerns">Data Concerns</h2>
+  <p>The following are some of the requirements that the data in your file
+    <strong>MUST</strong> fulfil before it is considered valid for this system:
+  </p>
+
+  <ol>
+    <li>File headings
+      <ul>
+	<li>The first row in the file should contains the headings. The number of
+	  headings in this first row determines the number of columns expected for
+	  all other lines in the file.</li>
+	<li>Each heading value in the first row MUST appear in the first row
+	  <strong>ONE AND ONLY ONE</strong> time</li>
+	<li>The sample/cases (previously 'strains') headers in your first row will be
+          against those in the <a href="https://genenetwork.org"
+                                  title="Link to the GeneNetwork service">
+            GeneNetwork</a> database.<br />
+          <small class="text-muted">
+            If you encounter an error saying your sample(s)/case(s) do not exist
+            in the GeneNetwork database, then you will have to use the
+            <a href="{{url_for('species.populations.samples.index')}}"
+               title="Upload samples/cases feature">Upload Samples/Cases</a>
+            option on this system to upload them.
+          </small>
+      </ul>
+    </li>
+
+    <li>Data
+      <ol>
+	<li><strong>NONE</strong> of the data cells/fields is allowed to be empty.
+	  All fields/cells <strong>MUST</strong> contain a value.</li>
+	<li>The first column of the data rows will be considered a textual field,
+	  holding the "identifier" for that row<li>
+	<li>Except for the first column/field for each data row,
+	  <strong>NONE</strong> of the data columns/cells/fields should contain
+	  spurious characters like `eeeee`, `5.555iloveguix`, etc...<br />
+	  All of them should be decimal values</li>
+	<li>decimal numbers must conform to the following criteria:
+	  <ul>
+	    <li>when checking an average file decimal numbers must have exactly three
+	      decimal places to the right of the decimal point.</li>
+	    <li>when checking a standard error file decimal numbers must have six or
+	      greater decimal places to the right of the decimal point.</li>
+	    <li>there must be a number to the left side of the decimal place
+	      (e.g. 0.55555 is allowed but .55555 is not).</li>
+	  </ul>
+	</li>
+      </ol>
+    </li>
+  </ol>
+</div>
+
+
+<div class="row">
+  <h2 id="file-types">Supported File Types</h2>
+  We support the following file types:
+
+  <ul>
+    <li>Tab-Separated value files (.tsv)
+      <ul>
+	<li>The <strong>TAB</strong> character is used to separate the fields of each
+	  column</li>
+	<li>The values of each field <strong>ARE NOT</strong> quoted.</li>
+	<li>Here is an
+	  <a href="https://gitlab.com/fredmanglis/gnqc_py/-/blob/main/tests/test_data/no_data_errors.tsv"
+             target="_blank">example file</a> with a single data row.</li>
+      </ul>
+    </li>
+    <li>.txt files: Content has the same format as .tsv file above</li>
+    <li>.zip files: each zip file should contain
+      <strong>ONE AND ONLY ONE</strong> file of the .tsv or .txt type above.
+      <br />Any zip file with more than one file is invalid, and so is an empty
+      zip file.</li>
+  </ul>
+
+</div>
+{%endblock%}
diff --git a/uploader/templates/expression-data/index.html b/uploader/templates/expression-data/index.html
index ed5d8dd..9ba3582 100644
--- a/uploader/templates/expression-data/index.html
+++ b/uploader/templates/expression-data/index.html
@@ -1,5 +1,6 @@
-{%extends "base.html"%}
+{%extends "expression-data/base.html"%}
 {%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-select-species.html" import select_species_form%}
 
 {%block title%}Expression Data{%endblock%}
 
@@ -10,86 +11,23 @@
   <a href="{{url_for('base.index')}}">Home</a>
 </li>
 <li class="breadcrumb-item active">
-  <a href="{{url_for('expression-data.index.index')}}">Expression Data</a>
+  <a href="{{url_for('species.populations.expression-data.index')}}"
+     title="Upload expression data.">
+    Expression Data</a>
 </li>
 {%endblock%}
 
 {%block contents%}
 <div class="row">
-  {{flash_all_messages()}}
-
-  <h1 class="heading">data upload</h1>
-
-  <div class="explainer">
-    <p>Each of the sections below gives you a different option for data expression-data.
-      Please read the documentation for each section carefully to understand what
-      each section is about.</p>
-  </div>
-</div>
-
-<div class="row">
-  <h2 class="heading">R/qtl2 Bundles</h2>
-
-  <div class="explainer">
-    <p>This feature combines and extends the two upload methods below. Instead of
-      uploading one item at a time, the R/qtl2 bundle you upload can contain both
-      the genotypes data (samples/individuals/cases and their data) and the
-      expression data.</p>
-    <p>The R/qtl2 bundle, additionally, can contain extra metadata, that neither
-      of the methods below can handle.</p>
-
-    <a href="{{url_for('expression-data.rqtl2.select_species')}}"
-       title="Upload a zip bundle of R/qtl2 files">
-      <button class="btn btn-primary">upload R/qtl2 bundle</button></a>
-  </div>
-</div>
-
-
-<div class="row">
   <h2 class="heading">Expression Data</h2>
+  {{flash_all_messages()}}
 
-  <div class="explainer">
-    <p>This feature enables you to upload expression data. It expects the data to
-      be in <strong>tab-separated values (TSV)</strong> files. The data should be
-      a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a
-      list of the <em>phenotypes</em> and the first row is a list of
-      <em>samples/cases</em>.</p>
-
-    <p>If you haven't done so please go to this page to learn the requirements for
-      file formats and helpful suggestions to enter your data in a fast and easy
-      way.</p>
-
-    <ol>
-      <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies
-        with our system requirements. (
-        <a href="{{url_for('expression-data.index.data_review')}}#data-concerns"
-	   title="Details for the data expectations.">Help</a>
-        )</li>
-      <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept
-        <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong>
-        files (<a href="{{url_for('expression-data.index.data_review')}}#file-types"
-	          title="Details for the data expectations.">Help</a>)</li>
-    </ol>
-  </div>
-
-  <a href="{{url_for('expression-data.index.upload_file')}}"
-     title="Upload your expression data"
-     class="btn btn-primary">upload expression data</a>
+  <p>This section allows you to enter the expression data for your experiment.
+    You will need to select the species that your data concerns below.</p>
 </div>
 
 <div class="row">
-  <h2 class="heading">samples/cases</h2>
-
-  <div class="explainer">
-    <p>For the expression data above, you need the samples/cases in your file to
-      already exist in the GeneNetwork database. If there are any samples that do
-      not already exist the upload of the expression data will fail.</p>
-    <p>This section gives you the opportunity to upload any missing samples</p>
-  </div>
-
-  <a href="{{url_for('expression-data.samples.select_species')}}"
-     title="Upload samples/cases/individuals for your data"
-     class="btn btn-primary">upload Samples/Cases</a>
+  {{select_species_form(url_for("species.populations.expression-data.index"),
+  species)}}
 </div>
-
 {%endblock%}
diff --git a/uploader/templates/expression-data/job-progress.html b/uploader/templates/expression-data/job-progress.html
new file mode 100644
index 0000000..ef264e1
--- /dev/null
+++ b/uploader/templates/expression-data/job-progress.html
@@ -0,0 +1,47 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block extrameta%}
+<meta http-equiv="refresh" content="5">
+{%endblock%}
+
+{%block title%}Job Status{%endblock%}
+
+{%block contents%}
+<h1 class="heading">{{job_name}}</h2>
+
+<div class="row">
+  <form action="{{url_for('species.populations.expression-data.abort',
+                species_id=species.SpeciesId,
+                population_id=population.Id)}}" method="POST">
+    <legend class="heading">Status</legend>
+    <div class="form-group">
+      <label for="job_status" class="form-label">status:</label>
+      <span class="form-text">{{job_status}}: {{message}}</span><br />
+    </div>
+
+    <div class="form-group">
+      <label for="job_{{job_id}}" class="form-label">parsing: </label>
+      <progress id="job_{{job_id}}"
+                value="{{progress/100}}"
+                class="form-control">
+        {{progress}}</progress>
+      <span class="form-text text-muted">{{"%.2f" | format(progress)}}%</span>
+    </div>
+
+    <input type="hidden" name="job_id" value="{{job_id}}" />
+
+    <button type="submit" class="btn btn-danger">Abort</button>
+  </form>
+</div>
+
+<div class="row">
+  {{errors_display(errors, "No errors found so far", "We have found the following errors so far", False)}}
+</div>
+
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/expression-data/no-such-job.html b/uploader/templates/expression-data/no-such-job.html
new file mode 100644
index 0000000..d22c429
--- /dev/null
+++ b/uploader/templates/expression-data/no-such-job.html
@@ -0,0 +1,15 @@
+{%extends "base.html"%}
+
+{%block extrameta%}
+<meta http-equiv="refresh"
+      content="5;url={{url_for('species.populations.expression-data.index.upload_file')}}">
+{%endblock%}
+
+{%block title%}No Such Job{%endblock%}
+
+{%block contents%}
+<h1 class="heading">No Such Job: {{job_id}}</h2>
+
+<p>No job, with the id '<em>{{job_id}}</em>' was found!</p>
+
+{%endblock%}
diff --git a/uploader/templates/expression-data/parse-failure.html b/uploader/templates/expression-data/parse-failure.html
new file mode 100644
index 0000000..31f6be8
--- /dev/null
+++ b/uploader/templates/expression-data/parse-failure.html
@@ -0,0 +1,26 @@
+{%extends "base.html"%}
+
+{%block title%}Worker Failure{%endblock%}
+
+{%block contents%}
+<h1 class="heading">Worker Failure</h1>
+
+<p>
+  There was an error while parsing your file.
+</p>
+
+<p>
+  Please notify the developers of this issue when you encounter it,
+  providing the information below.
+</p>
+
+<h4>Debugging Information</h4>
+
+<ul>
+  <li><strong>job id</strong>: {{job["job_id"]}}</li>
+  <li><strong>filename</strong>: {{job["filename"]}}</li>
+  <li><strong>line number</strong>: {{job["line_number"]}}</li>
+  <li><strong>Progress</strong>: {{job["percent"]}} %</li>
+</ul>
+
+{%endblock%}
diff --git a/uploader/templates/expression-data/parse-results.html b/uploader/templates/expression-data/parse-results.html
new file mode 100644
index 0000000..03a23e2
--- /dev/null
+++ b/uploader/templates/expression-data/parse-results.html
@@ -0,0 +1,39 @@
+{%extends "base.html"%}
+{%from "errors_display.html" import errors_display%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Parse Results{%endblock%}
+
+{%block contents%}
+
+<div class="row">
+  <h2 class="heading">{{job_name}}: parse results</h2>
+
+  {%if user_aborted%}
+  <span class="alert-warning">Job aborted by the user</span>
+  {%endif%}
+
+  {{errors_display(errors, "No errors found in the file", "We found the following errors", True)}}
+
+  {%if errors | length == 0 and not user_aborted %}
+  <form method="post" action="{{url_for('dbinsert.select_platform')}}">
+    <input type="hidden" name="job_id" value="{{job_id}}" />
+    <input type="submit" value="update database" class="btn btn-primary" />
+  </form>
+  {%endif%}
+
+  {%if errors | length > 0 or user_aborted %}
+  <br />
+  <a href="{{url_for('species.populations.expression-data.upload_file',
+           species_id=species.SpeciesId,
+           population_id=population.Id)}}"
+     title="Back to index page."
+     class="btn btn-primary">Go back</a>
+
+  {%endif%}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
diff --git a/uploader/templates/expression-data/select-file.html b/uploader/templates/expression-data/select-file.html
new file mode 100644
index 0000000..4ca461e
--- /dev/null
+++ b/uploader/templates/expression-data/select-file.html
@@ -0,0 +1,115 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_messages%}
+{%from "upload_progress_indicator.html" import upload_progress_indicator%}
+{%from "populations/macro-display-population-card.html" import display_population_card%}
+
+{%block title%}Expression Data &mdash; Upload Data{%endblock%}
+
+{%block pagetitle%}Expression Data &mdash; Upload Data{%endblock%}
+
+{%block contents%}
+{{upload_progress_indicator()}}
+
+<div class="row">
+  <h2 class="heading">Upload Expression Data</h2>
+
+  <p>This feature enables you to upload expression data. It expects the data to
+    be in <strong>tab-separated values (TSV)</strong> files. The data should be
+    a simple matrix of <em>phenotype × sample</em>, i.e. The first column is a
+    list of the <em>phenotypes</em> and the first row is a list of
+    <em>samples/cases</em>.</p>
+
+  <p>If you haven't done so please go to this page to learn the requirements for
+    file formats and helpful suggestions to enter your data in a fast and easy
+    way.</p>
+
+  <ol>
+    <li><strong>PLEASE REVIEW YOUR DATA.</strong>Make sure your data complies
+      with our system requirements. (
+      <a href="{{url_for('species.populations.expression-data.data_review')}}#data-concerns"
+	 title="Details for the data expectations.">Help</a>
+      )</li>
+    <li><strong>UPLOAD YOUR DATA FOR DATA VERIFICATION.</strong> We accept
+      <strong>.csv</strong>, <strong>.txt</strong> and <strong>.zip</strong>
+      files (<a href="{{url_for('species.populations.expression-data.data_review')}}#file-types"
+	        title="Details for the data expectations.">Help</a>)</li>
+  </ol>
+</div>
+
+<div class="row">
+  <form action="{{url_for(
+                'species.populations.expression-data.upload_file',
+                species_id=species.SpeciesId,
+                population_id=population.Id)}}"
+        method="POST"
+        enctype="multipart/form-data"
+        id="frm-upload-expression-data">
+    {{flash_messages("error-expr-data")}}
+
+    <div class="form-group">
+      <legend class="heading">File Type</legend>
+
+      <div class="radio">
+        <label for="filetype_average" class="form-check-label">
+          <input type="radio" name="filetype" value="average" id="filetype_average"
+	         required="required" class="form-check-input" />
+          Average</label>
+        <p class="form-text text-muted">
+          <small>The averages data …</small></p>
+      </div>
+
+      <div class="radio">
+        <label for="filetype_standard_error" class="form-check-label">
+          <input type="radio" name="filetype" value="standard-error"
+	         id="filetype_standard_error" required="required"
+	         class="form-check-input" />
+          Standard Error
+        </label>
+        <p class="form-text text-muted">
+          <small>The standard errors computed from the averages …</small></p>
+      </div>
+    </div>
+
+    <div class="form-group">
+      <span id="no-file-error" class="alert-danger" style="display: none;">
+        No file selected
+      </span>
+      <label for="file_upload" class="form-label">Select File</label>
+      <input type="file" name="qc_text_file" id="file_upload"
+	     accept="text/plain, text/tab-separated-values, application/zip"
+	     class="form-control"/>
+      <p class="form-text text-muted">
+        <small>Select the file to upload.</small></p>
+    </div>
+
+    <button type="submit"
+            class="btn btn-primary"
+            data-toggle="modal"
+            data-target="#upload-progress-indicator">upload file</button>
+  </form>
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_population_card(species, population)}}
+{%endblock%}
+
+{%block javascript%}
+<script type="text/javascript" src="/static/js/upload_progress.js"></script>
+<script type="text/javascript">
+  function setup_formdata(form) {
+      var formdata = new FormData();
+      formdata.append(
+	  "qc_text_file",
+	  form.querySelector("input[type='file']").files[0]);
+      formdata.append(
+	  "filetype",
+	  selected_filetype(
+	      Array.from(form.querySelectorAll("input[type='radio']"))));
+      return formdata;
+  }
+
+  setup_upload_handlers(
+      "frm-upload-expression-data", make_data_uploader(setup_formdata));
+</script>
+{%endblock%}
diff --git a/uploader/templates/expression-data/select-population.html b/uploader/templates/expression-data/select-population.html
new file mode 100644
index 0000000..8555e27
--- /dev/null
+++ b/uploader/templates/expression-data/select-population.html
@@ -0,0 +1,29 @@
+{%extends "expression-data/base.html"%}
+{%from "flash_messages.html" import flash_all_messages%}
+{%from "species/macro-display-species-card.html" import display_species_card%}
+{%from "populations/macro-select-population.html" import select_population_form%}
+
+{%block title%}Expression Data{%endblock%}
+
+{%block pagetitle%}Expression Data{%endblock%}
+
+
+{%block contents%}
+{{flash_all_messages()}}
+
+<div class="row">
+  <p>You have selected the species. Now you need to select the population that
+  the expression data belongs to.</p>
+</div>
+
+<div class="row">
+  {{select_population_form(url_for(
+  "species.populations.expression-data.select_population",
+  species_id=species.SpeciesId),
+  populations)}}
+</div>
+{%endblock%}
+
+{%block sidebarcontents%}
+{{display_species_card(species)}}
+{%endblock%}