1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
|
//! Bayesian network learning with BNW, called out to its own driver.
//!
//! BNW is the Bayesian Network Webserver: it searches the space of directed
//! acyclic graphs over a set of variables and reports, for every candidate
//! edge, the share of the k-best networks that contain it. In GeneNetwork its
//! use is not the picture: it is the *structure* -- which traits move together
//! under a causal ordering -- and, downstream, module scores that can be mapped
//! like any other trait.
//!
//! This module knows nothing about how the search works. BNW's knowledge lives
//! in the `gn-bnw` repository, which ships a headless driver
//! (`bin/bnw learn`, see its `info_files/gn4-integration-plan.md`); GN4
//! prepares a variables × samples matrix, runs that driver, and parses what it
//! prints. That boundary is the point: the pipeline (three C programs, a
//! private work directory, a type line in the input file) can change without a
//! GN4 release, and GN4 cannot accidentally grow a second copy of BNW's
//! semantics.
//!
//! The driver's contract, which this module codes against:
//!
//! ```text
//! bnw learn --input matrix.tsv [--maxparents N] [--k N] [--tiers N]
//! [--threshold X] [--format tsv|json] [--workdir DIR]
//! stdout: "# bnw <version> variables=N samples=N maxparents=… …"
//! "from\tto\tposterior\tin_structure"
//! rows
//! exit: 0 ok · 2 bad input · 3 computation failed
//! 77 the BNW binaries are not installed (the "skip" convention)
//! ```
use std::path::{Path, PathBuf};
use serde_json::Value;
/// Columns of a finished network, in order.
pub const COLUMNS: [&str; 4] = ["from", "to", "posterior", "in_structure"];
/// BNW's own defaults, from `sourcecodes/create_tiers_gom_part1.php`: the web
/// application writes these into `<key>parent.txt`, `<key>k.txt` and
/// `<key>thr.txt` unless the user changes them. Since the endpoint is a single
/// call, it has to pass them explicitly.
pub const DEFAULT_MAXPARENTS: u32 = 4;
pub const DEFAULT_K: u32 = 1;
pub const DEFAULT_TIERS: u32 = 3;
pub const DEFAULT_THRESHOLD: f64 = 0.5;
/// The most variables a request may carry. The structure search is
/// exponential in in-degree, so a wide matrix is hours rather than seconds --
/// refused here rather than left to the caller to discover.
pub const MAX_VARIABLES: usize = 40;
/// Why a run failed.
#[derive(Debug)]
pub enum BnwError {
/// Reading or writing the matrix this module prepares.
Io(String),
/// The driver was not found, or is not runnable.
NotInstalled(String),
/// The driver rejected the input (its exit 2).
BadInput(String),
/// The driver's computation failed (its exit 3), in its own words.
CommandFailed(String),
/// The reply could not be read.
UnexpectedOutput(String),
}
impl std::fmt::Display for BnwError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
BnwError::Io(what)
| BnwError::NotInstalled(what)
| BnwError::BadInput(what)
| BnwError::CommandFailed(what)
| BnwError::UnexpectedOutput(what) => write!(f, "{what}"),
}
}
}
/// Which variables enter the network, and their values by sample.
#[derive(Debug, Clone)]
pub struct Matrix {
/// Variable names, in column order.
pub variables: Vec<String>,
/// One row per sample: the sample's name and one value per variable, in
/// the same order. A sample with a missing value in any variable is
/// dropped (BNW's continuous model has no missing data).
pub rows: Vec<(String, Vec<f64>)>,
}
impl Matrix {
/// Build a matrix from per-variable value lists, keeping only the samples
/// that have a value for *every* variable.
///
/// Dropping incomplete samples is deliberate: BNW models a complete
/// multivariate normal, so one missing cell makes the whole row unusable,
/// and a caller who loses half their samples deserves to be told how many
/// survived (the reply's notes do that).
pub fn from_values(
variables: Vec<(String, Vec<(String, Option<f64>)>)>,
) -> Result<Matrix, BnwError> {
if variables.len() < 2 {
return Err(BnwError::BadInput(
"a network needs at least 2 variables".to_string(),
));
}
if variables.len() > MAX_VARIABLES {
return Err(BnwError::BadInput(format!(
"{} variables given; at most {MAX_VARIABLES} can be searched (the number of \
candidate structures grows exponentially in the number of parents, so a wider \
network is hours rather than seconds)",
variables.len()
)));
}
// Samples in the order of the first variable; a sample missing from a
// later variable is incomplete, so the intersection is what remains.
let order: Vec<String> = variables[0]
.1
.iter()
.map(|(name, _)| name.clone())
.collect();
let mut rows: Vec<(String, Vec<f64>)> = Vec::new();
for sample in &order {
let mut row = Vec::with_capacity(variables.len());
let mut complete = true;
for (_, values) in &variables {
match values.iter().find(|(name, _)| name == sample) {
Some((_, Some(value))) if value.is_finite() => row.push(*value),
_ => {
complete = false;
break;
}
}
}
if complete {
rows.push((sample.clone(), row));
}
}
if rows.len() < 3 {
return Err(BnwError::BadInput(format!(
"only {} sample(s) have a value for every variable; a network needs at least 3",
rows.len()
)));
}
Ok(Matrix {
variables: variables.into_iter().map(|(name, _)| name).collect(),
rows,
})
}
pub fn samples(&self) -> usize {
self.rows.len()
}
pub fn variables(&self) -> usize {
self.variables.len()
}
/// The matrix as the driver reads it: a header line, then one row per
/// sample. (The driver adds BNW's type line itself; see its README.)
pub fn to_tsv(&self) -> String {
let mut out = String::new();
out.push_str(&self.variables.join("\t"));
out.push('\n');
for (_, values) in &self.rows {
let cells: Vec<String> = values.iter().map(|value| format!("{value}")).collect();
out.push_str(&cells.join("\t"));
out.push('\n');
}
out
}
}
/// What the driver printed, once parsed.
#[derive(Debug, Clone)]
pub struct Learned {
/// One per candidate edge, in the order the driver printed them (strongest
/// posterior first).
pub rows: Vec<Value>,
/// The parameter line the driver echoed, e.g. `variables=6 samples=200 …`.
pub summary: String,
}
/// One network request's parameters.
#[derive(Debug, Clone, Copy)]
pub struct Parameters {
pub maxparents: u32,
pub k: u32,
pub tiers: u32,
pub threshold: f64,
}
impl Default for Parameters {
fn default() -> Self {
Parameters {
maxparents: DEFAULT_MAXPARENTS,
k: DEFAULT_K,
tiers: DEFAULT_TIERS,
threshold: DEFAULT_THRESHOLD,
}
}
}
/// Run the driver on `matrix` and parse its reply.
pub async fn learn(
bnw_cmd: &str,
bin_dir: Option<&str>,
cache_dir: &str,
matrix: &Matrix,
parameters: Parameters,
) -> Result<Learned, BnwError> {
std::fs::create_dir_all(cache_dir).map_err(|error| BnwError::Io(format!("{cache_dir}: {error}")))?;
let unique = format!("{}-{}", std::process::id(), nanos());
let matrix_path = Path::new(cache_dir).join(format!("{unique}-matrix.tsv"));
std::fs::write(&matrix_path, matrix.to_tsv())
.map_err(|error| BnwError::Io(format!("{}: {error}", matrix_path.display())))?;
let matrix_arg = matrix_path.to_string_lossy().into_owned();
let maxparents = parameters.maxparents.to_string();
let k = parameters.k.to_string();
let tiers = parameters.tiers.to_string();
let threshold = parameters.threshold.to_string();
let mut args: Vec<&str> = vec![
"learn",
"--input",
&matrix_arg,
"--maxparents",
&maxparents,
"--k",
&k,
"--tiers",
&tiers,
"--threshold",
&threshold,
];
if let Some(bin_dir) = bin_dir.filter(|dir| !dir.is_empty()) {
args.push("--bin-dir");
args.push(bin_dir);
}
let output = crate::streaming::run_streaming(bnw_cmd, &args, None)
.await
.map_err(|error| match error.kind() {
std::io::ErrorKind::NotFound => BnwError::NotInstalled(format!(
"the BNW driver was not found: {bnw_cmd} (set BNW_CMD, and BNW_BIN_DIR if the \
compiled C programs live elsewhere)"
)),
_ => BnwError::Io(format!("{bnw_cmd}: {error}")),
})?;
let _ = std::fs::remove_file(&matrix_path); // best effort
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
if !output.status.success() {
let code = output.status.code().unwrap_or(-1);
let complaint = driver_complaint(&stderr, &stdout);
return match code {
77 => Err(BnwError::NotInstalled(complaint)),
2 => Err(BnwError::BadInput(complaint)),
_ => Err(BnwError::CommandFailed(complaint)),
};
}
parse(&stdout)
}
/// The driver's own sentence about what went wrong: it writes `bnw: <reason>`
/// on stderr, so that line is the whole message and the last line is the
/// fallback only when it printed nothing.
fn driver_complaint(stderr: &str, stdout: &str) -> String {
let lines: Vec<&str> = stderr
.lines()
.chain(stdout.lines())
.map(str::trim)
.filter(|line| !line.is_empty())
.collect();
if let Some(line) = lines.iter().find(|line| line.starts_with("bnw:")) {
return line.trim_start_matches("bnw:").trim().to_string();
}
lines
.last()
.map(|line| (*line).to_string())
.unwrap_or_else(|| "the BNW driver failed without saying why".to_string())
}
/// Parse the driver's TSV: a `#` header line naming the parameters, a column
/// header, then rows.
fn parse(stdout: &str) -> Result<Learned, BnwError> {
let mut summary = String::new();
let mut rows = Vec::new();
let mut columns: Option<Vec<&str>> = None;
for line in stdout.lines() {
let line = line.trim_end_matches('\r');
if line.trim().is_empty() {
continue;
}
if let Some(rest) = line.strip_prefix("# ") {
summary = rest.trim().to_string();
continue;
}
let cells: Vec<&str> = line.split('\t').collect();
match &columns {
None => {
if cells != COLUMNS {
return Err(BnwError::UnexpectedOutput(format!(
"expected columns {:?}, got {cells:?}",
COLUMNS
)));
}
columns = Some(cells);
}
Some(_) => {
if cells.len() != COLUMNS.len() {
return Err(BnwError::UnexpectedOutput(format!(
"row has {} field(s), expected {}: {line:?}",
cells.len(),
COLUMNS.len()
)));
}
let posterior: f64 = cells[2].parse().map_err(|_| {
BnwError::UnexpectedOutput(format!(
"posterior {:?} is not a number",
cells[2]
))
})?;
let in_structure = match cells[3] {
"1" => true,
"0" => false,
other => {
return Err(BnwError::UnexpectedOutput(format!(
"in_structure {other:?} is not 0 or 1"
)))
}
};
rows.push(serde_json::json!([
cells[0],
cells[1],
posterior,
in_structure
]));
}
}
}
if columns.is_none() {
return Err(BnwError::UnexpectedOutput(
"the driver printed no edge table".to_string(),
));
}
Ok(Learned { rows, summary })
}
fn nanos() -> u128 {
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|elapsed| elapsed.as_nanos())
.unwrap_or(0)
}
/// Where the matrix file lives for one request (exposed for tests).
pub fn matrix_path(cache_dir: &str, unique: &str) -> PathBuf {
Path::new(cache_dir).join(format!("{unique}-matrix.tsv"))
}
/// One module of a learned network: a node and its parents, scored per sample.
#[derive(Debug, Clone)]
pub struct Module {
/// The node whose parents define the module (the network's "child").
pub name: String,
/// The node and its parents, in that order.
pub members: Vec<String>,
/// The module score per sample.
pub scores: Vec<(String, f64)>,
/// Share of the module's variance the score explains (the first principal
/// component's eigenvalue over the number of members) -- how much of a
/// summary the score really is.
pub variance_explained: f64,
}
/// Score each node's module -- the node plus its parents -- per sample.
///
/// The score is the **first principal component** of the module's standardised
/// members, sign-corrected so that it correlates positively with the node
/// itself (a principal component's sign is arbitrary; a score that means "more
/// of the child" must not come out negated). That is the definition the reply's
/// notes state, because a module score is only citable if its construction is
/// named.
///
/// Only nodes with at least one parent form a module: a node the search left
/// unconnected is not a summary of anything, and inventing a score for it would
/// be exactly the kind of number that gets mapped and then over-interpreted.
pub fn module_scores(matrix: &Matrix, edges: &[Value], threshold: f64) -> Vec<Module> {
// node -> its parents, in the network at or above the threshold
let mut parents: Vec<(String, Vec<String>)> = Vec::new();
for edge in edges {
let Some(from) = edge.get(0).and_then(Value::as_str) else { continue };
let Some(to) = edge.get(1).and_then(Value::as_str) else { continue };
let posterior = edge.get(2).and_then(Value::as_f64).unwrap_or(0.0);
if posterior < threshold {
continue;
}
match parents.iter_mut().find(|(node, _)| node == to) {
Some((_, list)) => {
if !list.iter().any(|name| name == from) {
list.push(from.to_string());
}
}
None => parents.push((to.to_string(), vec![from.to_string()])),
}
}
parents.sort_by(|a, b| a.0.cmp(&b.0));
let mut modules = Vec::new();
for (node, node_parents) in parents {
let members: Vec<String> = std::iter::once(node.clone()).chain(node_parents).collect();
let columns: Vec<Vec<f64>> = members
.iter()
.map(|member| {
let index = matrix
.variables
.iter()
.position(|name| name == member)
.unwrap_or(usize::MAX);
matrix
.rows
.iter()
.map(|(_, values)| values.get(index).copied().unwrap_or(f64::NAN))
.collect()
})
.collect();
let standardised: Option<Vec<Vec<f64>>> = columns.iter().map(|column| standardise(column)).collect();
let Some(standardised) = standardised else { continue };
let d = standardised.len();
let n = matrix.rows.len();
if d < 2 || n < 3 {
continue;
}
// Correlation matrix of the members.
let mut correlation = vec![vec![0.0; d]; d];
for i in 0..d {
for j in 0..d {
let dot: f64 = (0..n).map(|s| standardised[i][s] * standardised[j][s]).sum();
correlation[i][j] = dot / (n as f64 - 1.0);
}
}
let (loadings, eigenvalue) = first_component(&correlation);
// Sign: the score must rise with the child.
let mut score: Vec<f64> = (0..n)
.map(|s| (0..d).map(|k| loadings[k] * standardised[k][s]).sum())
.collect();
let child = &standardised[0];
let covariance: f64 = (0..n).map(|s| score[s] * child[s]).sum();
if covariance < 0.0 {
for value in &mut score {
*value = -*value;
}
}
modules.push(Module {
name: node,
members,
scores: matrix
.rows
.iter()
.enumerate()
.map(|(index, (sample, _))| (sample.clone(), score[index]))
.collect(),
variance_explained: if d > 0 { eigenvalue / d as f64 } else { 0.0 },
});
}
modules
}
/// Standardise a column: mean 0, standard deviation 1. `None` when it does not
/// vary (a constant variable has no component to contribute, and dividing by
/// its zero deviation would produce NaNs that travel into the score).
fn standardise(column: &[f64]) -> Option<Vec<f64>> {
let n = column.len() as f64;
if column.is_empty() {
return None;
}
let mean: f64 = column.iter().sum::<f64>() / n;
let variance: f64 = column.iter().map(|value| (value - mean).powi(2)).sum::<f64>() / (n - 1.0).max(1.0);
if !variance.is_finite() || variance <= f64::EPSILON {
return None;
}
let deviation = variance.sqrt();
Some(column.iter().map(|value| (value - mean) / deviation).collect())
}
/// The dominant eigenvector of a symmetric matrix, by power iteration.
///
/// Deterministic -- it starts from the vector of ones -- which matters because
/// a score that changes run to run is not a trait anyone can map. Returns the
/// eigenvector (normalised) and its eigenvalue.
fn first_component(matrix: &[Vec<f64>]) -> (Vec<f64>, f64) {
let d = matrix.len();
let mut vector = vec![1.0 / (d as f64).sqrt(); d];
for _ in 0..200 {
let mut next: Vec<f64> = (0..d)
.map(|i| (0..d).map(|j| matrix[i][j] * vector[j]).sum())
.collect();
let norm: f64 = next.iter().map(|value| value * value).sum::<f64>().sqrt();
if norm <= f64::EPSILON {
break;
}
for value in &mut next {
*value /= norm;
}
let change: f64 = next
.iter()
.zip(vector.iter())
.map(|(a, b)| (a - b).abs())
.fold(0.0, f64::max);
vector = next;
if change < 1e-12 {
break;
}
}
let eigenvalue: f64 = (0..d)
.map(|i| (0..d).map(|j| vector[i] * matrix[i][j] * vector[j]).sum::<f64>())
.sum();
// Canonical orientation: the largest-magnitude loading positive, so the
// vector does not flip between runs on a sign-symmetric matrix.
if let Some((index, _)) = vector
.iter()
.enumerate()
.max_by(|a, b| a.1.abs().partial_cmp(&b.1.abs()).unwrap_or(std::cmp::Ordering::Equal))
{
if vector[index] < 0.0 {
for value in &mut vector {
*value = -*value;
}
}
}
(vector, eigenvalue)
}
/// One worked example shipped with BNW, as the endpoint offers it.
#[derive(Debug, Clone)]
pub struct Example {
/// File stem, e.g. "example_data_8nodes".
pub name: String,
/// BNW's own description of it, when we know it.
pub title: String,
pub variables: usize,
pub samples: usize,
/// True when a column looks like a genotype code (whole numbers 1..4):
/// this driver is continuous-only, so such a column enters the network as
/// a number, which is what the web application does too on the GN2 path.
pub has_genotype_column: bool,
}
/// A small, curated table: the datasets in the gn-bnw checkout that BNW's own
/// workflow page presents, with the titles it gives them. Anything else in the
/// directory is listed by file stem, without a title.
const KNOWN_EXAMPLES: [(&str, &str); 5] = [
(
"example_data_8nodes",
"Synthetic genetic network: 2 genotypes and 6 traits (BNW tutorial 1 / example 3)",
),
(
"sci_5node_input_data",
"Scientific 5-node network: a genotype, 3 genes and a phenotype (BNW tutorial 2)",
),
(
"sci_5node_input_data2",
"The same 5-node design with 500 samples instead of 100",
),
(
"spleen_chr2_example_data",
"Immune-related genes in the spleens of BXD mice, chromosome 2 (BNW example 1)",
),
(
"time_series_example",
"Synthetic time-series network: a genotype and three traits at three time points (BNW example 4)",
),
];
/// The worked examples available in `dir`.
pub fn examples(dir: &Path) -> Result<Vec<Example>, BnwError> {
let entries = std::fs::read_dir(dir).map_err(|error| {
BnwError::Io(format!(
"{}: {error} (set BNW_EXAMPLES_DIR to the gn-bnw checkout's \
sourcecodes/example_datasets)",
dir.display()
))
})?;
let mut found = Vec::new();
for entry in entries.filter_map(Result::ok) {
let path = entry.path();
if path.extension().and_then(|ext| ext.to_str()) != Some("txt") {
continue;
}
let Some(stem) = path.file_stem().and_then(|stem| stem.to_str()) else { continue };
if stem.ends_with('~') {
continue;
}
let Ok(matrix) = read_example(&path, None) else { continue };
let has_genotype = (0..matrix.variables.len()).any(|column| {
matrix
.rows
.iter()
.all(|(_, values)| (1.0..=4.0).contains(&values[column]) && values[column].fract() == 0.0)
});
found.push(Example {
name: stem.to_string(),
title: KNOWN_EXAMPLES
.iter()
.find(|(name, _)| *name == stem)
.map(|(_, title)| (*title).to_string())
.unwrap_or_else(|| format!("{stem} (BNW example data)")),
variables: matrix.variables(),
samples: matrix.samples(),
has_genotype_column: has_genotype,
});
}
found.sort_by(|a, b| a.name.cmp(&b.name));
Ok(found)
}
/// One example's matrix, optionally restricted to named columns.
///
/// Restricting matters: BNW's spleen example has more variables than the
/// endpoint will search, so a caller has to choose a subset.
pub fn example_matrix(dir: &Path, name: &str, columns: Option<&[String]>) -> Result<Matrix, BnwError> {
if name.is_empty()
|| !name
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-' || c == '.')
{
return Err(BnwError::BadInput(format!("{name:?} is not an example name")));
}
let path = dir.join(format!("{name}.txt"));
read_example(&path, columns)
}
/// Read one BNW example file: a header of variable names, then data rows.
///
/// These files carry no type line (that is written by the web application's
/// preparation step), so every column is taken as continuous -- and the
/// endpoint says so when a column is a genotype code.
fn read_example(path: &Path, columns: Option<&[String]>) -> Result<Matrix, BnwError> {
let text = std::fs::read_to_string(path)
.map_err(|error| BnwError::Io(format!("{}: {error}", path.display())))?;
let mut lines = text.lines().filter(|line| !line.trim().is_empty());
let Some(header) = lines.next() else {
return Err(BnwError::BadInput(format!("{} is empty", path.display())));
};
let names: Vec<String> = header.split('\t').map(|cell| cell.trim().to_string()).collect();
let keep: Vec<usize> = match columns {
None => (0..names.len()).collect(),
Some(wanted) => {
let mut keep = Vec::new();
for name in wanted {
match names.iter().position(|candidate| candidate == name) {
Some(index) => keep.push(index),
None => {
return Err(BnwError::BadInput(format!(
"no column {name:?} in {}; it has: {}",
path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("the example"),
names.join(", ")
)))
}
}
}
keep
}
};
let mut rows: Vec<(String, Vec<f64>)> = Vec::new();
for (index, line) in lines.enumerate() {
let cells: Vec<&str> = line.split('\t').collect();
if cells.len() != names.len() {
return Err(BnwError::BadInput(format!(
"{} line {}: {} field(s), expected {}",
path.display(),
index + 2,
cells.len(),
names.len()
)));
}
let mut row = Vec::with_capacity(keep.len());
for &column in &keep {
match cells[column].trim().parse::<f64>() {
Ok(value) if value.is_finite() => row.push(value),
_ => return Err(BnwError::BadInput(format!(
"{} line {}: {:?} is not a number (BNW's example {} may be discrete data, \
which this driver does not model)",
path.display(),
index + 2,
cells[column],
path.file_stem().and_then(|stem| stem.to_str()).unwrap_or("set")
))),
}
}
rows.push((format!("sample{}", index + 1), row));
}
let variables: Vec<String> = keep.iter().map(|&index| names[index].clone()).collect();
if rows.len() < 3 {
return Err(BnwError::BadInput(format!(
"{} has {} data row(s); a network needs at least 3",
path.display(),
rows.len()
)));
}
Ok(Matrix { variables, rows })
}
#[cfg(test)]
mod tests {
use super::*;
fn values(name: &str, pairs: &[(&str, Option<f64>)]) -> (String, Vec<(String, Option<f64>)>) {
(
name.to_string(),
pairs
.iter()
.map(|(sample, value)| ((*sample).to_string(), *value))
.collect(),
)
}
#[test]
fn a_matrix_keeps_only_samples_complete_across_every_variable() {
let matrix = Matrix::from_values(vec![
values("a", &[("s1", Some(1.0)), ("s2", Some(2.0)), ("s3", Some(3.0)), ("s4", Some(0.5))]),
values("b", &[("s1", Some(4.0)), ("s2", None), ("s3", Some(6.0)), ("s4", Some(1.0))]),
// s5 is missing from the variables above, so it is not in the matrix
values("c", &[("s1", Some(7.0)), ("s2", Some(2.0)), ("s3", Some(9.0)), ("s4", Some(3.0))]),
])
.unwrap();
assert_eq!(matrix.variables, vec!["a", "b", "c"]);
assert_eq!(matrix.rows.len(), 3, "s1, s3 and s4 are complete");
assert_eq!(matrix.rows[0].0, "s1");
assert_eq!(matrix.rows[0].1, vec![1.0, 4.0, 7.0]);
assert_eq!(matrix.rows[2].0, "s4");
}
#[test]
fn a_single_variable_is_refused() {
let error = Matrix::from_values(vec![values("a", &[("s1", Some(1.0))])]).unwrap_err();
assert!(error.to_string().contains("at least 2 variables"), "{error}");
}
#[test]
fn too_few_complete_samples_is_refused_and_says_how_many() {
let error = Matrix::from_values(vec![
values("a", &[("s1", Some(1.0)), ("s2", Some(2.0))]),
values("b", &[("s1", Some(3.0)), ("s2", Some(4.0))]),
])
.unwrap_err();
assert!(error.to_string().contains("only 2 sample(s)"), "{error}");
}
#[test]
fn the_matrix_is_written_as_a_header_then_one_row_per_sample() {
let matrix = Matrix::from_values(vec![
values("a", &[("s1", Some(1.5)), ("s2", Some(2.5)), ("s3", Some(3.5))]),
values("b", &[("s1", Some(4.0)), ("s2", Some(5.0)), ("s3", Some(6.0))]),
])
.unwrap();
assert_eq!(matrix.to_tsv(), "a\tb\n1.5\t4\n2.5\t5\n3.5\t6\n");
}
#[test]
fn the_drivers_table_parses_into_rows() {
let stdout = "# bnw 0.1.0 variables=2 samples=3 maxparents=4 k=1 tiers=3 threshold=0.5\n\
from\tto\tposterior\tin_structure\n\
Trait2\tTrait1\t1\t1\n\
Trait1\tTrait2\t0.25\t0\n";
let learned = parse(stdout).unwrap();
assert_eq!(learned.summary, "bnw 0.1.0 variables=2 samples=3 maxparents=4 k=1 tiers=3 threshold=0.5");
assert_eq!(learned.rows.len(), 2);
assert_eq!(learned.rows[0], serde_json::json!(["Trait2", "Trait1", 1.0, true]));
assert_eq!(learned.rows[1], serde_json::json!(["Trait1", "Trait2", 0.25, false]));
}
#[test]
fn an_unexpected_table_is_refused_rather_than_guessed() {
assert!(matches!(
parse("# header\nwrong\tcolumns\n"),
Err(BnwError::UnexpectedOutput(_))
));
assert!(matches!(
parse("# header\nfrom\tto\tposterior\tin_structure\nA\tB\tnot-a-number\t1\n"),
Err(BnwError::UnexpectedOutput(_))
));
assert!(matches!(
parse("# header only\n"),
Err(BnwError::UnexpectedOutput(_))
));
}
fn matrix_of(variables: &[&str], rows: &[&[f64]]) -> Matrix {
Matrix {
variables: variables.iter().map(|name| (*name).to_string()).collect(),
rows: rows
.iter()
.enumerate()
.map(|(index, row)| (format!("s{index}"), row.to_vec()))
.collect(),
}
}
#[test]
fn a_module_is_the_child_and_its_parents_at_or_above_the_threshold() {
let matrix = matrix_of(
&["a", "b", "c"],
&[&[1.0, 2.0, 9.0], &[2.0, 4.1, 9.0], &[3.0, 5.9, 9.0], &[4.0, 8.2, 9.0]],
);
let edges = vec![
serde_json::json!(["a", "b", 1.0, true]),
serde_json::json!(["c", "b", 0.2, false]), // below 0.5: not a parent
];
let modules = module_scores(&matrix, &edges, 0.5);
assert_eq!(modules.len(), 1, "only b has a parent above the threshold");
assert_eq!(modules[0].name, "b");
assert_eq!(modules[0].members, vec!["b", "a"]);
assert_eq!(modules[0].scores.len(), 4);
}
#[test]
fn a_score_rises_with_its_child_and_is_signed_that_way() {
// b tracks a closely; the principal component could point either way,
// so the sign correction is what makes "more of b" mean "higher score".
let matrix = matrix_of(
&["a", "b", "c"],
&[&[1.0, 1.1, 5.0], &[2.0, 1.9, 4.0], &[3.0, 3.2, 3.0], &[4.0, 3.8, 2.0]],
);
let edges = vec![serde_json::json!(["a", "b", 1.0, true])];
let modules = module_scores(&matrix, &edges, 0.5);
let scores: Vec<f64> = modules[0].scores.iter().map(|(_, value)| *value).collect();
assert!(scores[0] < scores[3], "the score must rise with b: {scores:?}");
assert!(
(0.5..=1.0).contains(&modules[0].variance_explained),
"two correlated variables should be summarised well: {}",
modules[0].variance_explained
);
}
#[test]
fn an_unconnected_node_gets_no_module() {
let matrix = matrix_of(
&["a", "b"],
&[&[1.0, 2.0], &[2.0, 3.0], &[3.0, 4.0]],
);
assert!(module_scores(&matrix, &[], 0.5).is_empty());
}
#[test]
fn a_constant_member_is_dropped_rather_than_dividing_by_zero() {
let matrix = matrix_of(
&["a", "b"],
&[&[1.0, 2.0], &[1.0, 3.0], &[1.0, 4.0]],
);
let edges = vec![serde_json::json!(["a", "b", 1.0, true])];
assert!(
module_scores(&matrix, &edges, 0.5).is_empty(),
"a constant column has no component to contribute"
);
}
#[test]
fn the_drivers_own_words_become_the_complaint() {
assert_eq!(
driver_complaint("bnw: variable 'a' appears twice\n", ""),
"variable 'a' appears twice"
);
// nothing on stderr: the last line is better than nothing
assert_eq!(driver_complaint("", "some noise\n"), "some noise");
assert_eq!(
driver_complaint("", ""),
"the BNW driver failed without saying why"
);
}
}
|