path: root/uploader/templates/phenotypes/add-phenotypes-raw-files.html

{%extends "phenotypes/add-phenotypes-base.html"%}
{%from "flash_messages.html" import flash_all_messages%}
{%from "macro-table-pagination.html" import table_pagination%}
{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%}
{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%}

{%block title%}Phenotypes{%endblock%}

{%block pagetitle%}Phenotypes{%endblock%}

{%block lvl4_breadcrumbs%}
<li {%if activelink=="add-phenotypes"%}
    class="breadcrumb-item active"
    {%else%}
    class="breadcrumb-item"
    {%endif%}>
  <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
           species_id=species.SpeciesId,
           population_id=population.Id,
           dataset_id=dataset.Id)}}">Add Phenotypes</a>
</li>
{%endblock%}

{%block frm_add_phenotypes_documentation%}
<p>This page will allow you to upload all the separate files that make up your
  phenotypes. Here, you will have to upload each separate file individually. If
  you want instead to upload all your files as a single ZIP file,
  <a href="{{url_for('species.populations.phenotypes.add_phenotypes',
           species_id=species.SpeciesId,
           population_id=population.Id,
           dataset_id=dataset.Id,
           use_bundle=true)}}"
     title="">click here</a>.</p>
{%endblock%}

{%block frm_add_phenotypes_elements%}
<fieldset id="fldset-file-metadata">
  <legend>File(s) Metadata</legend>
  <div class="form-group">
    <label for="txt-file-separator" class="form-label">File Separator</label>
    <div class="input-group">
      <input id="txt-file-separator"
             name="file-separator"
             type="text"
             value="&#9;"
             class="form-control"
             maxlength="1" />
      <span class="input-group-btn">
        <button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button>
      </span>
    </div>
    <span class="form-text text-muted">
      Provide the character that separates the fields in your file(s). It should
      be the same character for all files (if more than one is provided).<br />
      A tab character will be assumed if you leave this field blank. See
      <a href="#docs-file-separator"
         title="Documentation for file-separator characters">
        documentation for more information</a>.
    </span>
  </div>

  <div class="form-group">
    <label for="txt-file-comment-character" class="form-label">File Comment-Character</label>
    <div class="input-group">
      <input id="txt-file-comment-character"
             name="file-comment-character"
             type="text"
             value="#"
             class="form-control"
             maxlength="1" />
      <span class="input-group-btn">
        <button id="btn-reset-file-comment-character" class="btn btn-info">
          Reset Default</button>
      </span>
    </div>
    <span class="form-text text-muted">
      This specifies that lines that begin with the character provided will be
      considered comment lines and ignored in their entirety. See
      <a href="#docs-file-comment-character"
         title="Documentation for comment characters">
        documentation for more information</a>.
    </span>
  </div>

  <div class="form-group">
    <label for="txt-file-na" class="form-label">File "No-Value" Indicators</label>
    <div class="input-group">
      <input id="txt-file-na"
             name="file-na"
             type="text"
             value="- NA N/A"
             class="form-control" />
      <span class="input-group-btn">
        <button id="btn-reset-file-na" class="btn btn-info">Reset Default</button>
      </span>
    </div>
    <span class="form-text text-muted">
      This specifies strings in your file indicate that there is no value for a
      particular cell (a cell is where a column and row intersect). Provide a
      space-separated list of strings if you have more than one way of
      indicating no values. See
      <a href="#docs-file-na" title="Documentation for no-value fields">
        documentation for more information</a>.</span>
  </div>
</fieldset>

<fieldset id="fldset-data-files">
  <legend>Data File(s)</legend>

  <div class="form-group non-resumable-elements">
    <label for="finput-phenotype-descriptions" class="form-label">
      Phenotype Descriptions</label>
    <input id="finput-phenotype-descriptions"
           name="phenotype-descriptions"
           class="form-control"
           type="file"
           data-preview-table="tbl-preview-pheno-desc"
           required="required"  />
    <span class="form-text text-muted">
      Provide a file that contains only the phenotype descriptions,
      <a href="#docs-file-phenotype-description"
         title="Documentation of the phenotype data file format.">
        the documentation for the expected format of the file</a>.</span>
  </div>

  {{display_resumable_elements(
  "resumable-phenotype-descriptions",
  "phenotype descriptions",
  '<p>You can drop a CSV file that contains the phenotype descriptions here,
    or you can click the "Browse" button to select it from your computer.</p>
  <p>The CSV file must conform to some standards, as documented in the
    <a href="#docs-file-phenotype-description"
       title="Documentation of the phenotype data file format.">
      "Phenotypes Descriptions" documentation</a> section below.</p>')}}

  <div class="form-group">
    <label for="finput-phenotype-data" class="form-label">Phenotype Data</label>
    <input id="finput-phenotype-data"
           name="phenotype-data"
           class="form-control"
           type="file"
           data-preview-table="tbl-preview-pheno-data"
           required="required"  />
    <span class="form-text text-muted">
      Provide a file that contains only the phenotype data. See
      <a href="#docs-phenotype-data"
         title="Documentation of the phenotype data file format.">
        the documentation for the expected format of the file</a>.</span>
  </div>

  {%if population.Family in families_with_se_and_n%}
  <div class="form-group">
    <label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label>
    <input id="finput-phenotype-se"
           name="phenotype-se"
           class="form-control"
           type="file"
           data-preview-table="tbl-preview-pheno-se"
           required="required"  />
    <span class="form-text text-muted">
      Provide a file that contains only the standard errors for the phenotypes,
      computed from the data above.</span>
  </div>

  <div class="form-group">
    <label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label>
    <input id="finput-phenotype-n"
           name="phenotype-n"
           class="form-control"
           type="file"
           data-preview-table="tbl-preview-pheno-n"
           required="required"  />
    <span class="form-text text-muted">
      Provide a file that contains only the number of samples/individuals used in
      the computation of the standard errors above.</span>
  </div>
</fieldset>
{%endif%}
{%endblock%}


{%block page_documentation%}
<div class="row">
  <h2 class="heading" id="docs-help">Help</h2>
  <h3 class="subheading">Common Features</h3>
  <p>The following are the common expectations for <strong>ALL</strong> the
    files provided in the form above:
    <ul>
      <li>The file <strong>MUST</strong> be character-separated values (CSV)
        text file</li>
      <li>The first row in the file <strong>MUST</strong> be a heading row, and
        will be composed of the list identifiers for all of
        samples/individuals/cases involved in your study.</li>
      <li>The first column of data in the file <strong>MUST</strong> be the
        identifiers for all of the phenotypes you wish to upload.</li>
    </ul>
  </p>

  <p>If you do not specify the separator character, then we will assume a
    <strong>TAB</strong> character was used as your separator.</p>

  <p>We also assume you might include comments lines in your files. In that
    case, if you do not specify what character denotes that a line in your files
    is a comment line, we will assume the <strong>#</strong> character.<br />
    A comment <strong>MUST ALWAYS</strong> begin at the start of the line marked
    with the comment character specified.</p>

  <h3 class="subheading" id="docs-file-metadata">File Metadata</h3>
  <p>We request some details about your files to help us parse and process the
    files correctly. The details we collect are:</p>
  <dl>
    <dt id="docs-file-separator">File separator</dt>
    <dd>The files you provide should be character-separated value (CSV) files.
      We need to know what character you used to separate the values in your
      file. Some common ones are the Tab character, the comma, etc.<br />
      Providing that information makes it possible for the system to parse and
      process your files correctly.<br>
      <strong>NOTE:</strong> All the files you upload MUST use the same
      separator.</dd>

    <dt id="docs-file-comment-character">Comment character</dt>
    <dd>We support use of comment lines in your files. We only support one type
      of comment style, the <em>line comment</em>.<br />
      This mean the comment begins at the start of the line, and the end of that
      line indicates the end of that comment. If you have a really long comment,
      then you need to break it across multiple lines, marking each line a
      comment line.<br />
      The "comment character" is the character at the start of the line that
      indicates that the line is a line comment.</dd>

    <dt id="docs-file-na">No-Value indicator(s)</dt>
    <dd>Data in the real world is messy, and in some cases, entirely absent. You
      need to indicate, in your files, that a particular field did not have a
      value, and once you do that, you then need to let the system know how you
      mark such fields. Common ways of indicating "empty values" are, leaving
      the field blank, using a character such as '-', or using strings like
      "NA", "N/A", "NULL", etc.<br />
      Providing this information will help with parsing and processing such
      no-value fields the correct way.</dd>
  </dl>

  <h3 class="subheading" id="docs-file-phenotype-description">
    file: Phenotypes Descriptions</h3>
  <p>The data in this file is a matrix of <em>phenotypes × metadata-fields</em>.
    Please note we use the term "metadata-fields" above loosely, due to lack of
    a good word for this.</p>
  <p>The file <strong>MUST</strong> have columns in this order:
    <dl>
      <dt>Phenotype Identifiers</dt>
      <dd>These are the names/identifiers for your phenotypes. These
        names/identifiers are the same ones you will have in all the other files you are
        uploading.</dd>

      <dt>Descriptions</dt>
      <dd>Each phenotype will need a description. Good description are necessary
        to inform other people of what the data is about. Good description are
        hard to construct, so we provide
        <a href="https://info.genenetwork.org/faq.php#q-22"
           title="How to write phenotype descriptions">
          advice on describing your phenotypes.</a></dd>

      <dt>Units</dt>
      <dd>Each phenotype will need units for the measurements taken. If there are
        none, then indicate the field is a no-value field.</dd>
  </dl></p>
  <p>You can add more columns after those three if you want to, but these 3
    <strong>MUST</strong> be present.</p>
  <p>The file would, for example, look like the following:</p>
  <code>id,description,units,…<br />
    pheno10001|Central nervous system, behavior, cognition; …|mg|…<br />
    pheno10002|Aging, metabolism, central nervous system: …|mg|…<br />
    ⋮<br /></code>

  <p><strong>Note 01</strong>: The first usable row is the heading row.</p>
  <p><strong>Note 02: </strong>This example demonstrates a subtle issue that
    could make your CSV file invalid &mdash; the choice of your field separator
    character.<br >
    In the example above, we use the pipe character (<code>|</code>) as our
    field separator. This is because, if we follow the advice on how to write
    good descriptions, then we cannot use the comma as our separator &ndash; if
    we did, then our CSV file would be invalid because the system would have no
    way to tell the difference between the comma as a field separator, and the
    comma as a way to separate the "general category and ontology terms".</p>

  <h3 class="subheading">file: Phenotype Data, Standard Errors and/or Sample Counts</h3>
  <span id="docs-phenotype-data"></span>
  <span id="docs-phenotype-se"></span>
  <span id="docs-phenotype-n"></span>
  <p>The data is a matrix of <em>phenotypes × individuals</em>, e.g.</p>
  <code>
    # num-cases: 2549
    # num-phenos: 13
    id,IND001,IND002,IND003,IND004,…<br />
    pheno10001,61.400002,54.099998,483,49.799999,…<br />
    pheno10002,49,50.099998,403,45.5,…<br />
    pheno10003,62.5,53.299999,501,62.900002,…<br />
    pheno10004,53.099998,55.099998,403,NA,…<br />
    ⋮<br /></code>

  <p>where <code>IND001,IND002,IND003,IND004,…</code> are the
    samples/individuals/cases in your study, and
    <code>pheno10001,pheno10002,pheno10004,pheno10004,…</code> are the
    identifiers for your phenotypes.</p>
  <p>The lines beginning with the "<em>#</em>" symbol (i.e.
    <code># num-cases: 2549</code> and <code># num-phenos: 13</code> are comment
    lines and will be ignored</p>
  <p>In this example, the comma (,) is used as the file separator.</p>
</div>

{%endblock%}

{%block sidebarcontents%}
{{display_preview_table("tbl-preview-pheno-desc", "descriptions")}}
{{display_preview_table("tbl-preview-pheno-data", "data")}}
{%if population.Family in families_with_se_and_n%}
{{display_preview_table("tbl-preview-pheno-se", "standard errors")}}
{{display_preview_table("tbl-preview-pheno-n", "number of samples")}}
{%endif%}
{{display_pheno_dataset_card(species, population, dataset)}}
{%endblock%}


{%block more_javascript%}
<script src="{{url_for('base.node_modules',
             filename='resumablejs/resumable.js')}}"></script>
<script type="text/javascript" src="/static/js/files.js"></script>

<script type="text/javascript">
  $("#btn-reset-file-separator").on("click", (event) => {
      event.preventDefault();
      $("#txt-file-separator").val("\t");
      $("#txt-file-separator").trigger("change");
  });
  $("#btn-reset-file-comment-character").on("click", (event) => {
      event.preventDefault();
      $("#txt-file-comment-character").val("#");
      $("#txt-file-comment-character").trigger("change");
  });
  $("#btn-reset-file-na").on("click", (event) => {
      event.preventDefault();
      $("#txt-file-na").val("- NA N/A");
      $("#txt-file-na").trigger("change");
  });

  var update_preview = (table, filedata, formdata, numrows) => {
      table.find("thead tr").remove()
      table.find(".data-row").remove();
      var linenum = 0;
      var tableheader = table.find("thead");
      var tablebody = table.find("tbody");
      var numheadings = 0;
      var navalues = formdata
          .na_strings
          .split(" ")
          .map((v) => {return v.trim();})
          .filter((v) => {return Boolean(v);});
      filedata.forEach((line) => {
          if(line.startsWith(formdata.comment_char) || linenum >= numrows) {
              return false;
          }
          var row = $("<tr></tr>");
          line.split(formdata.separator)
              .map((field) => {
                  var value = field.trim();
                  if(navalues.includes(value)) {
                      return "⋘NUL⋙";
                  }
                  return value;
              })
              .filter((field) => {
                  return (field !== "" && field != undefined && field != null);
              })
              .forEach((field) => {
                  if(linenum == 0) {
                      numheadings += 1;
                      var tablefield = $("<th></th>");
                      tablefield.text(field);
                      row.append(tablefield);
                  } else {
                      add_class(row, "data-row");
                      var tablefield = $("<td></td>");
                      tablefield.text(field);
                      row.append(tablefield);
                  }
              });

          if(linenum == 0) {
              tableheader.append(row);
          } else {
              tablebody.append(row);
          }
          linenum += 1;
      });

      if(table.find("tbody tr.data-row").length > 0) {
          add_class(table.find(".data-row-template"), "hidden");
      } else {
          remove_class(table.find(".data-row-template"), "hidden");
      }
  };

  var preview_tables_to_elements_map = {
      "#tbl-preview-pheno-desc": "#finput-phenotype-descriptions",
      "#tbl-preview-pheno-data": "#finput-phenotype-data",
      "#tbl-preview-pheno-se": "#finput-phenotype-se",
      "#tbl-preview-pheno-n": "#finput-phenotype-n"
  };

  var files_metadata = () => {
      return {
          "separator": $("#txt-file-separator").val(),
          "comment_char": $(
              "#txt-file-comment-character").val(),
          "na_strings": $("#txt-file-na").val()
      }
  };

  var PREVIEW_ROWS = 5;

  var handler_update_previews = (event) => {
      Object.entries(preview_tables_to_elements_map).forEach((mapentry) => {
          var element = $(mapentry[1]);
          if(element.length === 1) {
              read_first_n_lines(
                  element[0],
                  10,
                  [(data) => {
                      update_preview(
                          $(mapentry[0]),
                          data,
                          files_metadata(),
                          PREVIEW_ROWS);}]);
          }
      });
  };

  [
      "#txt-file-separator",
      "#txt-file-comment-character",
      "#txt-file-na"
  ].forEach((elementid) => {
      $(elementid).on("change", handler_update_previews);
  });

  [
      "#finput-phenotype-descriptions",
      "#finput-phenotype-data",
      "#finput-phenotype-se",
      "#finput-phenotype-n"
  ].forEach((elementid) => {
      $(elementid).on("change", (event) => {
          read_first_n_lines(
              event.target,
              10,
              [(data) => {
                  update_preview(
                      $("#" + event.target.getAttribute("data-preview-table")),
                      data,
                      files_metadata(),
                      PREVIEW_ROWS);
              }]);
      });
  });


  var resumableDisplayFiles = (display_area, files) => {
      files.forEach((file) => {
          var display_element = display_area
              .find(".file-display-template")
              .clone();
          remove_class(display_element, "hidden");
          remove_class(display_element, "file-display-template");
          add_class(display_element, "file-display");
          display_element.find(".filename").text(file.name
                                                || file.fileName
                                                || file.relativePath
                                                || file.webkitRelativePath);
          display_element.find(".filesize").text(
              (file.size / (1024*1024)).toFixed(2) + "MB");
          display_element.find(".fileuniqueid").text(file.uniqueIdentifier);
          display_element.find(".filemimetype").text(file.file.type);
          display_area.append(display_element);
      });
  };


  var indicateProgress = (resumable, progress_bar) => {
      return (event) => {
          var progress = (resumable.progress() * 100).toFixed(2);
          var pbar = progress_bar.find(".progress-bar");
          remove_class(progress_bar, "hidden");
          pbar.css("width", progress+"%");
          pbar.attr("aria-valuenow", progress);
          pbar.text("Uploading: " + progress + "%");
      };
  };

  var retryUpload = (retry_button, cancel_button) => {
      retry_button.on("click", (event) => {
          resumable.files.forEach((file) => {file.retry();});
          add_class(retry_button, "hidden");
          remove_class(cancel_button, "hidden");
          add_class(browse_button, "hidden");
      });
  };

  var cancelUpload = (cancel_button, retry_button) => {
      cancel_button.on("click", (event) => {
          resumable.files.forEach((file) => {
              if(file.isUploading()) {
                  file.abort();
              }
          });
          add_class(cancel_button, "hidden");
          remove_class(retry_button, "hidden");
          remove_class(browse_button, "hidden");
      });
  };


  var startUpload = (browse_button, retry_button, cancel_button) => {
      return (event) => {
          remove_class(cancel_button, "hidden");
          add_class(retry_button, "hidden");
          add_class(browse_button, "hidden");
      };
  };


  var uploadSuccess = () => {
      return (file, message) => {
          console.log("THE FILE:", file);
          console.log("THE SUCCESS MESSAGE:", message);
          // TODOS:
          // * Save filename/filepath somewhere
          // * Trigger some function that will run when all files have succeeded
      };
  };


  var uploadError = () => {
      return (message, file) => {
          console.log("THE FILE:", file);
          console.log("THE ERROR MESSAGE:", message);
      };
  };


  var r = errorHandler(
      fileSuccessHandler(
          uploadStartHandler(
              filesAddedHandler(
                  markResumableDragAndDropElement(
                      makeResumableElement(
                          $("#frm-add-phenotypes").attr("data-resumable-target"),
                          $("#finput-phenotype-descriptions").parent(),
                          $("#resumable-phenotype-descriptions"),
                          $("#frm-add-phenotypes input[type=submit]"),
                          ["csv", "tsv"]),
                      $("#finput-phenotype-descriptions").parent(),
                      $("#resumable-phenotype-descriptions"),
                      $("#resumable-phenotype-descriptions-browse-button")),
                  (files) => {
                      // TODO: Also trigger preview!
                      resumableDisplayFiles(
                          $("#resumable-phenotype-descriptions-selected-files"), files);
                  }),
              startUpload($("#resumable-phenotype-descriptions-browse-button"),
                          $("#resumable-phenotype-descriptions-retry-button"),
                          $("#resumable-phenotype-descriptions-cancel-button"))),
          uploadSuccess()),
      uploadError());

  progressHandler(
      r,
      indicateProgress(r, $("#resumable-phenotype-descriptions-progress-bar")));


  $("#frm-add-phenotypes input[type=submit]").on("click", (event) => {
      event.preventDefault();
      // TODO: Check all the relevant files exist
      // TODO: Check all fields
      // Start the uploads.
      r.upload();
  });
</script>
{%endblock%}