1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
|
{%extends "phenotypes/add-phenotypes-base.html"%}
{%from "flash_messages.html" import flash_all_messages%}
{%from "macro-table-pagination.html" import table_pagination%}
{%from "phenotypes/macro-display-pheno-dataset-card.html" import display_pheno_dataset_card%}
{%from "phenotypes/macro-display-preview-table.html" import display_preview_table%}
{%block title%}Phenotypes{%endblock%}
{%block pagetitle%}Phenotypes{%endblock%}
{%block lvl4_breadcrumbs%}
<li {%if activelink=="add-phenotypes"%}
class="breadcrumb-item active"
{%else%}
class="breadcrumb-item"
{%endif%}>
<a href="{{url_for('species.populations.phenotypes.add_phenotypes',
species_id=species.SpeciesId,
population_id=population.Id,
dataset_id=dataset.Id)}}">Add Phenotypes</a>
</li>
{%endblock%}
{%block frm_add_phenotypes_documentation%}
<p>This page will allow you to upload all the separate files that make up your
phenotypes. Here, you will have to upload each separate file individually. If
you want instead to upload all your files as a single ZIP file,
<a href="{{url_for('species.populations.phenotypes.add_phenotypes',
species_id=species.SpeciesId,
population_id=population.Id,
dataset_id=dataset.Id,
use_bundle=true)}}"
title="">click here</a>.</p>
{%endblock%}
{%block frm_add_phenotypes_elements%}
<fieldset id="fldset-file-metadata">
<legend>File(s) Metadata</legend>
<div class="form-group">
<label for="txt-file-separator" class="form-label">File Separator</label>
<div class="input-group">
<input id="txt-file-separator"
name="file-separator"
type="text"
value="	"
class="form-control"
maxlength="1" />
<span class="input-group-btn">
<button id="btn-reset-file-separator" class="btn btn-info">Reset Default</button>
</span>
</div>
<span class="form-text text-muted">
Provide the character that separates the fields in your file(s). It should
be the same character for all files (if more than one is provided).<br />
A tab character will be assumed if you leave this field blank. See
<a href="#docs-file-separator"
title="Documentation for file-separator characters">
documentation for more information</a>.
</span>
</div>
<div class="form-group">
<label for="txt-file-comment-character" class="form-label">File Comment-Character</label>
<div class="input-group">
<input id="txt-file-comment-character"
name="file-comment-character"
type="text"
value="#"
class="form-control"
maxlength="1" />
<span class="input-group-btn">
<button id="btn-reset-file-comment-character" class="btn btn-info">
Reset Default</button>
</span>
</div>
<span class="form-text text-muted">
This specifies that lines that begin with the character provided will be
considered comment lines and ignored in their entirety. See
<a href="#docs-file-comment-character"
title="Documentation for comment characters">
documentation for more information</a>.
</span>
</div>
<div class="form-group">
<label for="txt-file-na" class="form-label">File "No-Value" Indicators</label>
<div class="input-group">
<input id="txt-file-na"
name="file-na"
type="text"
value="- NA N/A"
class="form-control" />
<span class="input-group-btn">
<button id="btn-reset-file-na" class="btn btn-info">Reset Default</button>
</span>
</div>
<span class="form-text text-muted">
This specifies strings in your file indicate that there is no value for a
particular cell (a cell is where a column and row intersect). Provide a
space-separated list of strings if you have more than one way of
indicating no values. See
<a href="#docs-file-na" title="Documentation for no-value fields">
documentation for more information</a>.</span>
</div>
</fieldset>
<fieldset id="fldset-data-files">
<legend>Data File(s)</legend>
<div class="form-group">
<label for="finput-phenotype-descriptions" class="form-label">
Phenotype Descriptions</label>
<input id="finput-phenotype-descriptions"
name="phenotype-descriptions"
class="form-control"
type="file"
required="required" />
<span class="form-text text-muted">
Provide a file that contains only the phenotype descriptions,
<a href="#docs-file-phenotype-description"
title="Documentation of the phenotype data file format.">
the documentation for the expected format of the file</a>.</span>
</div>
<div class="form-group">
<label for="finput-phenotype-data" class="form-label">Phenotype Data</label>
<input id="finput-phenotype-data"
name="phenotype-data"
class="form-control"
type="file"
required="required" />
<span class="form-text text-muted">
Provide a file that contains only the phenotype data. See
<a href="#docs-phenotype-data"
title="Documentation of the phenotype data file format.">
the documentation for the expected format of the file</a>.</span>
</div>
{%if population.Family in families_with_se_and_n%}
<div class="form-group">
<label for="finput-phenotype-se" class="form-label">Phenotype: Standard Errors</label>
<input id="finput-phenotype-se"
name="phenotype-se"
class="form-control"
type="file"
required="required" />
<span class="form-text text-muted">
Provide a file that contains only the standard errors for the phenotypes,
computed from the data above.</span>
</div>
<div class="form-group">
<label for="finput-phenotype-n" class="form-label">Phenotype: Number of Samples/Individuals</label>
<input id="finput-phenotype-n"
name="phenotype-n"
class="form-control"
type="file"
required="required" />
<span class="form-text text-muted">
Provide a file that contains only the number of samples/individuals used in
the computation of the standard errors above.</span>
</div>
</fieldset>
{%endif%}
{%endblock%}
{%block page_documentation%}
<div class="row">
<h2 class="heading" id="docs-help">Help</h2>
<h3 class="subheading">Common Features</h3>
<p>The following are the common expectations for <strong>ALL</strong> the
files provided in the form above:
<ul>
<li>The file <strong>MUST</strong> be character-separated values (CSV)
text file</li>
<li>The first row in the file <strong>MUST</strong> be a heading row, and
will be composed of the list identifiers for all of
samples/individuals/cases involved in your study.</li>
<li>The first column of data in the file <strong>MUST</strong> be the
identifiers for all of the phenotypes you wish to upload.</li>
</ul>
</p>
<p>If you do not specify the separator character, then we will assume a
<strong>TAB</strong> character was used as your separator.</p>
<p>We also assume you might include comments lines in your files. In that
case, if you do not specify what character denotes that a line in your files
is a comment line, we will assume the <strong>#</strong> character.<br />
A comment <strong>MUST ALWAYS</strong> begin at the start of the line marked
with the comment character specified.</p>
<h3 class="subheading" id="docs-file-metadata">File Metadata</h3>
<p>We request some details about your files to help us parse and process the
files correctly. The details we collect are:</p>
<dl>
<dt id="docs-file-separator">File separator</dt>
<dd>The files you provide should be character-separated value (CSV) files.
We need to know what character you used to separate the values in your
file. Some common ones are the Tab character, the comma, etc.<br />
Providing that information makes it possible for the system to parse and
process your files correctly.<br>
<strong>NOTE:</strong> All the files you upload MUST use the same
separator.</dd>
<dt id="docs-file-comment-character">Comment character</dt>
<dd>We support use of comment lines in your files. We only support one type
of comment style, the <em>line comment</em>.<br />
This mean the comment begins at the start of the line, and the end of that
line indicates the end of that comment. If you have a really long comment,
then you need to break it across multiple lines, marking each line a
comment line.<br />
The "comment character" is the character at the start of the line that
indicates that the line is a line comment.</dd>
<dt id="docs-file-na">No-Value indicator(s)</dt>
<dd>Data in the real world is messy, and in some cases, entirely absent. You
need to indicate, in your files, that a particular field did not have a
value, and once you do that, you then need to let the system know how you
mark such fields. Common ways of indicating "empty values" are, leaving
the field blank, using a character such as '-', or using strings like
"NA", "N/A", "NULL", etc.<br />
Providing this information will help with parsing and processing such
no-value fields the correct way.</dd>
</dl>
<h3 class="subheading" id="docs-file-phenotype-description">
file: Phenotypes Descriptions</h3>
<p>The data in this file is a matrix of <em>phenotypes × metadata-fields</em>.
Please note we use the term "metadata-fields" above loosely, due to lack of
a good word for this.</p>
<p>The file <strong>MUST</strong> have columns in this order:
<dl>
<dt>Phenotype Identifiers</dt>
<dd>These are the names/identifiers for your phenotypes. These
names/identifiers are the same ones you will have in all the other files you are
uploading.</dd>
<dt>Descriptions</dt>
<dd>Each phenotype will need a description. Good description are necessary
to inform other people of what the data is about. Good description are
hard to construct, so we provide
<a href="https://info.genenetwork.org/faq.php#q-22"
title="How to write phenotype descriptions">
advice on describing your phenotypes.</a></dd>
<dt>Units</dt>
<dd>Each phenotype will need units for the measurements taken. If there are
none, then indicate the field is a no-value field.</dd>
</dl></p>
<p>You can add more columns after those three if you want to, but these 3
<strong>MUST</strong> be present.</p>
<p>The file would, for example, look like the following:</p>
<code>id,description,units,…<br />
pheno10001|Central nervous system, behavior, cognition; …|mg|…<br />
pheno10002|Aging, metabolism, central nervous system: …|mg|…<br />
⋮<br /></code>
<p><strong>Note 01</strong>: The first usable row is the heading row.</p>
<p><strong>Note 02: </strong>This example demonstrates a subtle issue that
could make your CSV file invalid — the choice of your field separator
character.<br >
In the example above, we use the pipe character (<code>|</code>) as our
field separator. This is because, if we follow the advice on how to write
good descriptions, then we cannot use the comma as our separator – if
we did, then our CSV file would be invalid because the system would have no
way to tell the difference between the comma as a field separator, and the
comma as a way to separate the "general category and ontology terms".</p>
<h3 class="subheading">file: Phenotype Data, Standard Errors and/or Sample Counts</h3>
<span id="docs-phenotype-data"></span>
<span id="docs-phenotype-se"></span>
<span id="docs-phenotype-n"></span>
<p>The data is a matrix of <em>phenotypes × individuals</em>, e.g.</p>
<code>
# num-cases: 2549
# num-phenos: 13
id,IND001,IND002,IND003,IND004,…<br />
pheno10001,61.400002,54.099998,483,49.799999,…<br />
pheno10002,49,50.099998,403,45.5,…<br />
pheno10003,62.5,53.299999,501,62.900002,…<br />
pheno10004,53.099998,55.099998,403,NA,…<br />
⋮<br /></code>
<p>where <code>IND001,IND002,IND003,IND004,…</code> are the
samples/individuals/cases in your study, and
<code>pheno10001,pheno10002,pheno10004,pheno10004,…</code> are the
identifiers for your phenotypes.</p>
<p>The lines beginning with the "<em>#</em>" symbol (i.e.
<code># num-cases: 2549</code> and <code># num-phenos: 13</code> are comment
lines and will be ignored</p>
<p>In this example, the comma (,) is used as the file separator.</p>
</div>
{%endblock%}
{%block sidebarcontents%}
{{display_preview_table("tbl-preview-pheno-desc", "descriptions")}}
{{display_preview_table("tbl-preview-pheno-data", "data")}}
{%if population.Family in families_with_se_and_n%}
{{display_preview_table("tbl-preview-pheno-se", "standard errors")}}
{{display_preview_table("tbl-preview-pheno-n", "number of samples")}}
{%endif%}
{{display_pheno_dataset_card(species, population, dataset)}}
{%endblock%}
{%block more_javascript%}
<script type="text/javascript" src="/static/js/files.js"></script>
<script type="text/javascript">
$("#btn-reset-file-separator").on("click", (event) => {
event.preventDefault();
$("#txt-file-separator").val("\t");
$("#txt-file-separator").trigger("change");
});
$("#btn-reset-file-comment-character").on("click", (event) => {
event.preventDefault();
$("#txt-file-comment-character").val("#");
$("#txt-file-comment-character").trigger("change");
});
$("#btn-reset-file-na").on("click", (event) => {
event.preventDefault();
$("#txt-file-na").val("- NA N/A");
$("#txt-file-na").trigger("change");
});
var update_preview = (table, filedata, formdata) => {
table.find("thead tr").remove()
table.find(".data-row").remove();
var linenum = 0;
var tableheader = table.find("thead");
var tablebody = table.find("tbody");
var numheadings = 0;
filedata.forEach((line) => {
if(line.startsWith(formdata.comment_char)) {
return false;
}
var row = $("<tr></tr>");
line.split(formdata.separator)
.map((field) => {return field.trim();})
.filter((field) => {
return (field !== "" && field != undefined && field != null);
})
.forEach((field) => {
if(linenum == 0) {
numheadings += 1;
var tablefield = $("<th></th>");
tablefield.text(field);
row.append(tablefield);
} else {
add_class(row, "data-row");
var tablefield = $("<td></td>");
tablefield.text(field);
row.append(tablefield);
}
});
if(linenum == 0) {
tableheader.append(row);
} else {
tablebody.append(row);
}
linenum += 1;
});
if(table.find("tbody tr.data-row").length > 0) {
add_class(table.find(".data-row-template"), "hidden");
} else {
remove_class(table.find(".data-row-template"), "hidden");
}
};
</script>
{%endblock%}
|