From b8cba2b0585604935d26bec6916aaadd114a2fcf Mon Sep 17 00:00:00 2001 From: Frederick Muriuki Muriithi Date: Tue, 16 Dec 2025 12:24:36 -0600 Subject: Implement streamlined UI for phenotypes upload page. --- uploader/phenotypes/views.py | 6 +- .../macro-display-pheno-dataset-card.html | 28 +- .../phenotypes/sui-add-phenotypes-base.html | 155 ++++ .../phenotypes/sui-add-phenotypes-raw-files.html | 829 +++++++++++++++++++++ .../sui-add-phenotypes-with-rqtl2-bundle.html | 189 +++++ uploader/templates/phenotypes/sui-base.html | 25 + uploader/templates/populations/sui-base.html | 2 +- 7 files changed, 1230 insertions(+), 4 deletions(-) create mode 100644 uploader/templates/phenotypes/sui-add-phenotypes-base.html create mode 100644 uploader/templates/phenotypes/sui-add-phenotypes-raw-files.html create mode 100644 uploader/templates/phenotypes/sui-add-phenotypes-with-rqtl2-bundle.html create mode 100644 uploader/templates/phenotypes/sui-base.html diff --git a/uploader/phenotypes/views.py b/uploader/phenotypes/views.py index 5b6f965..c4fd7fd 100644 --- a/uploader/phenotypes/views.py +++ b/uploader/phenotypes/views.py @@ -423,9 +423,11 @@ def add_phenotypes(species: dict, population: dict, dataset: dict, **kwargs):# p with Redis.from_url(_redisuri, decode_responses=True) as rconn: if request.method == "GET": today = datetime.date.today() + _sui="sui-" if request.args.get("streamlined_ui") else "" return render_template( - ("phenotypes/add-phenotypes-with-rqtl2-bundle.html" - if use_bundle else "phenotypes/add-phenotypes-raw-files.html"), + (f"phenotypes/{_sui}add-phenotypes-with-rqtl2-bundle.html" + if use_bundle + else f"phenotypes/{_sui}add-phenotypes-raw-files.html"), species=species, population=population, dataset=dataset, diff --git a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html index 11b108b..641421d 100644 --- a/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html +++ b/uploader/templates/phenotypes/macro-display-pheno-dataset-card.html @@ -1,4 +1,4 @@ -{%from "populations/macro-display-population-card.html" import display_population_card%} +{%from "populations/macro-display-population-card.html" import display_population_card, display_sui_population_card%} {%macro display_pheno_dataset_card(species, population, dataset)%} {{display_population_card(species, population)}} @@ -29,3 +29,29 @@ {%endmacro%} + +{%macro display_sui_pheno_dataset_card(species, population, dataset)%} +{{display_sui_population_card(species, population)}} + +
+ + + + + + + + + + + + + + + + + + +
Current dataset
Name{{dataset.Name}}
Full Name{{dataset.FullName}}
Short Name{{dataset.ShortName}}
+
+{%endmacro%} diff --git a/uploader/templates/phenotypes/sui-add-phenotypes-base.html b/uploader/templates/phenotypes/sui-add-phenotypes-base.html new file mode 100644 index 0000000..1e71267 --- /dev/null +++ b/uploader/templates/phenotypes/sui-add-phenotypes-base.html @@ -0,0 +1,155 @@ +{%extends "phenotypes/sui-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block contents%} +{{super()}} +{{flash_all_messages()}} + +
+
+ Add New Phenotypes + +
+ {%block frm_add_phenotypes_documentation%}{%endblock%} +

This will not update any existing phenotypes!

+
+ + {%block frm_add_phenotypes_elements%}{%endblock%} + +
+ Publication Information + + + Select a publication for your data.
+ Can't find a publication you can use? Go ahead and + create a new publication.
+ + + + + + + + + + + +
#PubMed IDTitleAuthors
+
+ +
+ +
+
+
+ +
+ {%block page_documentation%}{%endblock%} +
+ +{%endblock%} + + + + +{%block javascript%} + + +{%block more_javascript%}{%endblock%} +{%endblock%} diff --git a/uploader/templates/phenotypes/sui-add-phenotypes-raw-files.html b/uploader/templates/phenotypes/sui-add-phenotypes-raw-files.html new file mode 100644 index 0000000..6038617 --- /dev/null +++ b/uploader/templates/phenotypes/sui-add-phenotypes-raw-files.html @@ -0,0 +1,829 @@ +{%extends "phenotypes/sui-add-phenotypes-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} +{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%} +{%from "phenotypes/macro-display-resumable-elements.html" import display_resumable_elements%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block frm_add_phenotypes_documentation%} +

This page will allow you to upload all the separate files that make up your + phenotypes. Here, you will have to upload each separate file individually. If + you want instead to upload all your files as a single ZIP file, + click here.

+{%endblock%} + +{%block frm_add_phenotypes_elements%} +
+ File(s) Metadata +
+ +
+ + + + +
+ + Provide the character that separates the fields in your file(s). It should + be the same character for all files (if more than one is provided).
+ A tab character will be assumed if you leave this field blank. See + + documentation for more information. +
+
+ +
+ +
+ + + + +
+ + This specifies that lines that begin with the character provided will be + considered comment lines and ignored in their entirety. See + + documentation for more information. + +
+ +
+ +
+ + + + +
+ + This specifies strings in your file indicate that there is no value for a + particular cell (a cell is where a column and row intersect). Provide a + space-separated list of strings if you have more than one way of + indicating no values. See + + documentation for more information. +
+
+ +
+ Data File(s) + +
+
+
+ + +
+ +
+ + + + Provide a file that contains only the phenotype descriptions, + + the documentation for the expected format of the file. +
+ {{display_resumable_elements( + "resumable-phenotype-descriptions", + "phenotype descriptions", + '

Drag and drop the CSV file that contains the descriptions of your + phenotypes here.

+ +

The CSV file should be a matrix of + phenotypes × descriptions i.e. The first column + contains the phenotype names/identifiers whereas the first row is a list + of metadata fields like, "description", "units", etc.

+ +

If the format is transposed (i.e. + descriptions × phenotypes) select the checkbox above. +

+ +

Please see the + + "Phenotypes Descriptions" documentation section below for more + information on the expected format of the file provided here.

')}} + {{display_preview_table( + "tbl-preview-pheno-desc", "phenotype descriptions")}} +
+
+ + +
+
+
+ + +
+ +
+ + + + Provide a file that contains only the phenotype data. See + + the documentation for the expected format of the file. +
+ + {{display_resumable_elements( + "resumable-phenotype-data", + "phenotype data", + '

Drag and drop a CSV file that contains the phenotypes numerical data + here. You can click the "Browse" button (below and to the right) to + select the file from your computer.

+ +

The CSV should be a matrix of samples × phenotypes, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.

+ +

If the format is transposed (i.e phenotypes × samples) + select the checkbox above.

+

Please see the + + "Phenotypes Data" documentation section below for more information + on the expected format for the file provided here.

')}} + {{display_preview_table("tbl-preview-pheno-data", "phenotype data")}} +
+
+ + + {%if population.Family in families_with_se_and_n%} +
+
+
+ + +
+
+ + + + Provide a file that contains only the standard errors for the phenotypes, + computed from the data above. +
+ + {{display_resumable_elements( + "resumable-phenotype-se", + "standard errors", + '

Drag and drop a CSV file that contains the phenotypes standard-errors + data here. You can click the "Browse" button (below and to the right) to + select the file from your computer.

+ +

The CSV should be a matrix of samples × phenotypes, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.

+ +

If the format is transposed (i.e phenotypes × samples) + select the checkbox above.

+ +

Please see the + + "Phenotypes Data" documentation section below for more information + on the expected format of the file provided here.

')}} + + {{display_preview_table("tbl-preview-pheno-se", "standard errors")}} +
+
+ + +
+
+
+ + +
+
+ + + + Provide a file that contains only the number of samples/individuals used in + the computation of the standard errors above. +
+ + {{display_resumable_elements( + "resumable-phenotype-n", + "number of samples/individuals", + '

Drag and drop a CSV file that contains the samples\' phenotypes counts + data here. You can click the "Browse" button (below and to the right) to + select the file from your computer.

+ +

The CSV should be a matrix of samples × phenotypes, + i.e. The first column contains the samples identifiers while the first + row is the list of phenotypes identifiers occurring in the phenotypes + descriptions file.

+ +

If the format is transposed (i.e phenotypes × samples) + select the checkbox above.

+ +

Please see the + + "Phenotypes Data" documentation section below for more information + on the expected format of the file provided here.

')}} + + {{display_preview_table("tbl-preview-pheno-n", "number of samples/individuals")}} +
+
+
+{%endif%} +{%endblock%} + + +{%block page_documentation%} +
+

Help

+

Common Features

+

The following are the common expectations for ALL the + files provided in the form above: +

+

+ +

If you do not specify the separator character, then we will assume a + TAB character was used as your separator.

+ +

We also assume you might include comments lines in your files. In that + case, if you do not specify what character denotes that a line in your files + is a comment line, we will assume the # character.
+ A comment MUST ALWAYS begin at the start of the line marked + with the comment character specified.

+ +

File Metadata

+

We request some details about your files to help us parse and process the + files correctly. The details we collect are:

+
+
File separator
+
The files you provide should be character-separated value (CSV) files. + We need to know what character you used to separate the values in your + file. Some common ones are the Tab character, the comma, etc.
+ Providing that information makes it possible for the system to parse and + process your files correctly.
+ NOTE: All the files you upload MUST use the same + separator.
+ +
Comment character
+
We support use of comment lines in your files. We only support one type + of comment style, the line comment.
+ This mean the comment begins at the start of the line, and the end of that + line indicates the end of that comment. If you have a really long comment, + then you need to break it across multiple lines, marking each line a + comment line.
+ The "comment character" is the character at the start of the line that + indicates that the line is a line comment.
+ +
No-Value indicator(s)
+
Data in the real world is messy, and in some cases, entirely absent. You + need to indicate, in your files, that a particular field did not have a + value, and once you do that, you then need to let the system know how you + mark such fields. Common ways of indicating "empty values" are, leaving + the field blank, using a character such as '-', or using strings like + "NA", "N/A", "NULL", etc.
+ Providing this information will help with parsing and processing such + no-value fields the correct way.
+
+ +

+ file: Phenotypes Descriptions

+

The data in this file is a matrix of phenotypes × metadata-fields. + Please note we use the term "metadata-fields" above loosely, due to lack of + a good word for this.

+

The file MUST have columns in this order: +

+
Phenotype Identifiers
+
These are the names/identifiers for your phenotypes. These + names/identifiers are the same ones you will have in all the other files you are + uploading.
+ +
Descriptions
+
Each phenotype will need a description. Good description are necessary + to inform other people of what the data is about. Good description are + hard to construct, so we provide + + advice on describing your phenotypes.
+ +
Units
+
Each phenotype will need units for the measurements taken. If there are + none, then indicate the field is a no-value field.
+

+

You can add more columns after those three if you want to, but these 3 + MUST be present.

+

The file would, for example, look like the following:

+ id,description,units,…
+ pheno10001|Central nervous system, behavior, cognition; …|mg|…
+ pheno10002|Aging, metabolism, central nervous system: …|mg|…
+ ⋮
+ +

Note 01: The first usable row is the heading row.

+

Note 02: This example demonstrates a subtle issue that + could make your CSV file invalid — the choice of your field separator + character.
+ In the example above, we use the pipe character (|) as our + field separator. This is because, if we follow the advice on how to write + good descriptions, then we cannot use the comma as our separator – if + we did, then our CSV file would be invalid because the system would have no + way to tell the difference between the comma as a field separator, and the + comma as a way to separate the "general category and ontology terms".

+ +

file: Phenotype Data, Standard Errors and/or Sample Counts

+ + + +

The data is a matrix of samples(or individuals) × phenotypes, e.g.

+ + # num-cases: 2549 + # num-phenos: 13 + id,pheno10001,pheno10002,pheno10003,pheno10004,53.099998,…
+ IND001,61.400002,49,62.5,55.099998,…
+ IND002,54.099998,50.099998,53.299999,55.099998,…
+ IND003,483,403,501,403,…
+ IND004,49.799999,45.5,62.900002,NA,…
+ ⋮
+ +

where IND001,IND002,IND003,IND004,… are the + samples/individuals/cases in your study, and + pheno10001,pheno10002,pheno10004,pheno10004,… are the + identifiers for your phenotypes.

+

The lines beginning with the "#" symbol (i.e. + # num-cases: 2549 and # num-phenos: 13 are comment + lines and will be ignored

+

In this example, the comma (,) is used as the file separator.

+
+ +{%endblock%} + + +{%block more_javascript%} + + + + +{%endblock%} diff --git a/uploader/templates/phenotypes/sui-add-phenotypes-with-rqtl2-bundle.html b/uploader/templates/phenotypes/sui-add-phenotypes-with-rqtl2-bundle.html new file mode 100644 index 0000000..29a8dea --- /dev/null +++ b/uploader/templates/phenotypes/sui-add-phenotypes-with-rqtl2-bundle.html @@ -0,0 +1,189 @@ +{%extends "phenotypes/sui-add-phenotypes-base.html"%} +{%from "flash_messages.html" import flash_all_messages%} +{%from "macro-table-pagination.html" import table_pagination%} + +{%block title%}Phenotypes{%endblock%} + +{%block pagetitle%}Phenotypes{%endblock%} + +{%block frm_add_phenotypes_documentation%} +

Select the zip file bundle containing information on the phenotypes you + wish to upload, then click the "Upload Phenotypes" button below to + upload the data.

+

If you wish to upload the files individually instead, + click here.

+

See the File Formats section below + to get an understanding of what is expected of the bundle files you + upload.

+{%endblock%} + +{%block frm_add_phenotypes_elements%} +
+ + +
+{%endblock%} + +{%block page_documentation%} +
+

File Formats

+

We accept an extended form of the + + input files' format used with the R/qtl2 software as a single ZIP + file

+

The files that are used for this feature are: +

+

+

Other files within the bundle will be ignored, for this feature.

+

The following section will detail the expectations for each of the + different file types within the uploaded ZIP file bundle for phenotypes:

+ +

Control File

+

There MUST be one, and only one file that acts + as the control file. This file can be: +

+

+ +

The control file is useful for defining things about the bundle such as:

+ + +

pheno File(s)

+

These files are the main data files. You must have at least one of these + files in your bundle for it to be valid for this step.

+

The data is a matrix of individuals × phenotypes by default, as + below:
+ + id,10001,10002,10003,10004,…
+ BXD1,61.400002,54.099998,483,49.799999,…
+ BXD2,49,50.099998,403,45.5,…
+ BXD5,62.5,53.299999,501,62.900002,…
+ BXD6,53.099998,55.099998,403,NA,…
+ ⋮
+

+

If the pheno_transposed value is set to True, + then the data will be a phenotypes × individuals matrix as in the + example below:
+ + id,BXD1,BXD2,BXD5,BXD6,…
+ 10001,61.400002,49,62.5,53.099998,…
+ 10002,54.099998,50.099998,53.299999,55.099998,…
+ 10003,483,403,501,403,…
+ 10004,49.799999,45.5,62.900002,NA,…
+ ⋮ +
+

+ + +

phenocovar File(s)

+

At least one phenotypes metadata file with the metadata values such as + descriptions, PubMed Identifier, publication titles (if present), etc.

+

The data in this/these file(s) is a matrix of + phenotypes × phenotypes-covariates. The first column is always the + phenotype names/identifiers — same as in the R/qtl2 format.

+

phenocovar files should never be transposed!

+

This file MUST be present in the bundle, and have data for + the bundle to be considered valid by our system for this step.
+ In addition to that, the following are the fields that must be + present, and + have values, in the file before the file is considered valid: +

+ +

The following optional fields can also be provided: +

+

+

These files will be marked up in the control file with the + phenocovar key, as in the examples below: +

    +
  1. JSON: single file
    + {
    +   ⋮,
    +   "phenocovar": "your_covariates_file.csv",
    +   ⋮
    + } +
    +
  2. +
  3. JSON: multiple files
    + {
    +   ⋮,
    +   "phenocovar": [
    +     "covariates_file_01.csv",
    +     "covariates_file_01.csv",
    +     ⋮
    +   ],
    +   ⋮
    + } +
    +
  4. +
  5. YAML: single file or
    + + ⋮
    + phenocovar: your_covariates_file.csv
    + ⋮ +
    +
  6. +
  7. YAML: multiple files
    + + ⋮
    + phenocovar:
    + - covariates_file_01.csv
    + - covariates_file_02.csv
    + - covariates_file_03.csv
    + …
    + ⋮ +
    +
  8. +
+

+ +

phenose and phenonum File(s)

+

These are extensions to the R/qtl2 standard, i.e. these types ofs file are + not supported by the original R/qtl2 file format

+

We use these files to upload the standard errors (phenose) when + the data file (pheno) is average data. In that case, the + phenonum file(s) contains the number of individuals that were + involved when computing the averages.

+

Both types of files are matrices of individuals × phenotypes by + default. Like the related pheno files, if + pheno_transposed: True, then the file will be a matrix of + phenotypes × individuals.

+
+{%endblock%} diff --git a/uploader/templates/phenotypes/sui-base.html b/uploader/templates/phenotypes/sui-base.html new file mode 100644 index 0000000..d7d980f --- /dev/null +++ b/uploader/templates/phenotypes/sui-base.html @@ -0,0 +1,25 @@ +{%extends "populations/sui-base.html"%} +{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_sui_pheno_dataset_card%} + +{%block breadcrumbs%} +{{super()}} + +{%endblock%} + +{%block contents%} +
+

{{dataset.FullName}} ({{dataset.Name}})

+
+{%endblock%} + + +{%block sidebarcontents%} +{{display_sui_pheno_dataset_card(species, population, dataset)}} +{%endblock%} diff --git a/uploader/templates/populations/sui-base.html b/uploader/templates/populations/sui-base.html index cc01c9e..0ca5c59 100644 --- a/uploader/templates/populations/sui-base.html +++ b/uploader/templates/populations/sui-base.html @@ -6,7 +6,7 @@ - {{population["FullName"]}} + {{population["Name"]}} {%endblock%} -- cgit 1.4.1