diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/bnw.rs | 460 | ||||
| -rw-r--r-- | src/compute_gate.rs | 12 | ||||
| -rw-r--r-- | src/config.rs | 25 | ||||
| -rw-r--r-- | src/lib.rs | 2 | ||||
| -rw-r--r-- | src/routes/bnw.rs | 349 | ||||
| -rw-r--r-- | src/routes/mod.rs | 1 |
6 files changed, 849 insertions, 0 deletions
diff --git a/src/bnw.rs b/src/bnw.rs new file mode 100644 index 0000000..cc73efd --- /dev/null +++ b/src/bnw.rs @@ -0,0 +1,460 @@ +//! Bayesian network learning with BNW, called out to its own driver. +//! +//! BNW is the Bayesian Network Webserver: it searches the space of directed +//! acyclic graphs over a set of variables and reports, for every candidate +//! edge, the share of the k-best networks that contain it. In GeneNetwork its +//! use is not the picture: it is the *structure* -- which traits move together +//! under a causal ordering -- and, downstream, module scores that can be mapped +//! like any other trait. +//! +//! This module knows nothing about how the search works. BNW's knowledge lives +//! in the `gn-bnw` repository, which ships a headless driver +//! (`bin/bnw learn`, see its `info_files/gn4-integration-plan.md`); GN4 +//! prepares a variables × samples matrix, runs that driver, and parses what it +//! prints. That boundary is the point: the pipeline (three C programs, a +//! private work directory, a type line in the input file) can change without a +//! GN4 release, and GN4 cannot accidentally grow a second copy of BNW's +//! semantics. +//! +//! The driver's contract, which this module codes against: +//! +//! ```text +//! bnw learn --input matrix.tsv [--maxparents N] [--k N] [--tiers N] +//! [--threshold X] [--format tsv|json] [--workdir DIR] +//! stdout: "# bnw <version> variables=N samples=N maxparents=… …" +//! "from\tto\tposterior\tin_structure" +//! rows +//! exit: 0 ok · 2 bad input · 3 computation failed +//! 77 the BNW binaries are not installed (the "skip" convention) +//! ``` + +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +/// Columns of a finished network, in order. +pub const COLUMNS: [&str; 4] = ["from", "to", "posterior", "in_structure"]; + +/// BNW's own defaults, from `sourcecodes/create_tiers_gom_part1.php`: the web +/// application writes these into `<key>parent.txt`, `<key>k.txt` and +/// `<key>thr.txt` unless the user changes them. Since the endpoint is a single +/// call, it has to pass them explicitly. +pub const DEFAULT_MAXPARENTS: u32 = 4; +pub const DEFAULT_K: u32 = 1; +pub const DEFAULT_TIERS: u32 = 3; +pub const DEFAULT_THRESHOLD: f64 = 0.5; + +/// The most variables a request may carry. The structure search is +/// exponential in in-degree, so a wide matrix is hours rather than seconds -- +/// refused here rather than left to the caller to discover. +pub const MAX_VARIABLES: usize = 40; + +/// Why a run failed. +#[derive(Debug)] +pub enum BnwError { + /// Reading or writing the matrix this module prepares. + Io(String), + /// The driver was not found, or is not runnable. + NotInstalled(String), + /// The driver rejected the input (its exit 2). + BadInput(String), + /// The driver's computation failed (its exit 3), in its own words. + CommandFailed(String), + /// The reply could not be read. + UnexpectedOutput(String), +} + +impl std::fmt::Display for BnwError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BnwError::Io(what) + | BnwError::NotInstalled(what) + | BnwError::BadInput(what) + | BnwError::CommandFailed(what) + | BnwError::UnexpectedOutput(what) => write!(f, "{what}"), + } + } +} + +/// Which variables enter the network, and their values by sample. +#[derive(Debug, Clone)] +pub struct Matrix { + /// Variable names, in column order. + pub variables: Vec<String>, + /// One row per sample: the sample's name and one value per variable, in + /// the same order. A sample with a missing value in any variable is + /// dropped (BNW's continuous model has no missing data). + pub rows: Vec<(String, Vec<f64>)>, +} + +impl Matrix { + /// Build a matrix from per-variable value lists, keeping only the samples + /// that have a value for *every* variable. + /// + /// Dropping incomplete samples is deliberate: BNW models a complete + /// multivariate normal, so one missing cell makes the whole row unusable, + /// and a caller who loses half their samples deserves to be told how many + /// survived (the reply's notes do that). + pub fn from_values( + variables: Vec<(String, Vec<(String, Option<f64>)>)>, + ) -> Result<Matrix, BnwError> { + if variables.len() < 2 { + return Err(BnwError::BadInput( + "a network needs at least 2 variables".to_string(), + )); + } + if variables.len() > MAX_VARIABLES { + return Err(BnwError::BadInput(format!( + "{} variables given; at most {MAX_VARIABLES} can be searched (the number of \ + candidate structures grows exponentially in the number of parents, so a wider \ + network is hours rather than seconds)", + variables.len() + ))); + } + // Samples in the order of the first variable; a sample missing from a + // later variable is incomplete, so the intersection is what remains. + let order: Vec<String> = variables[0] + .1 + .iter() + .map(|(name, _)| name.clone()) + .collect(); + let mut rows: Vec<(String, Vec<f64>)> = Vec::new(); + for sample in &order { + let mut row = Vec::with_capacity(variables.len()); + let mut complete = true; + for (_, values) in &variables { + match values.iter().find(|(name, _)| name == sample) { + Some((_, Some(value))) if value.is_finite() => row.push(*value), + _ => { + complete = false; + break; + } + } + } + if complete { + rows.push((sample.clone(), row)); + } + } + if rows.len() < 3 { + return Err(BnwError::BadInput(format!( + "only {} sample(s) have a value for every variable; a network needs at least 3", + rows.len() + ))); + } + Ok(Matrix { + variables: variables.into_iter().map(|(name, _)| name).collect(), + rows, + }) + } + + pub fn samples(&self) -> usize { + self.rows.len() + } + + pub fn variables(&self) -> usize { + self.variables.len() + } + + /// The matrix as the driver reads it: a header line, then one row per + /// sample. (The driver adds BNW's type line itself; see its README.) + pub fn to_tsv(&self) -> String { + let mut out = String::new(); + out.push_str(&self.variables.join("\t")); + out.push('\n'); + for (_, values) in &self.rows { + let cells: Vec<String> = values.iter().map(|value| format!("{value}")).collect(); + out.push_str(&cells.join("\t")); + out.push('\n'); + } + out + } +} + +/// What the driver printed, once parsed. +#[derive(Debug, Clone)] +pub struct Learned { + /// One per candidate edge, in the order the driver printed them (strongest + /// posterior first). + pub rows: Vec<Value>, + /// The parameter line the driver echoed, e.g. `variables=6 samples=200 …`. + pub summary: String, +} + +/// One network request's parameters. +#[derive(Debug, Clone, Copy)] +pub struct Parameters { + pub maxparents: u32, + pub k: u32, + pub tiers: u32, + pub threshold: f64, +} + +impl Default for Parameters { + fn default() -> Self { + Parameters { + maxparents: DEFAULT_MAXPARENTS, + k: DEFAULT_K, + tiers: DEFAULT_TIERS, + threshold: DEFAULT_THRESHOLD, + } + } +} + +/// Run the driver on `matrix` and parse its reply. +pub async fn learn( + bnw_cmd: &str, + bin_dir: Option<&str>, + cache_dir: &str, + matrix: &Matrix, + parameters: Parameters, +) -> Result<Learned, BnwError> { + std::fs::create_dir_all(cache_dir).map_err(|error| BnwError::Io(format!("{cache_dir}: {error}")))?; + let unique = format!("{}-{}", std::process::id(), nanos()); + let matrix_path = Path::new(cache_dir).join(format!("{unique}-matrix.tsv")); + std::fs::write(&matrix_path, matrix.to_tsv()) + .map_err(|error| BnwError::Io(format!("{}: {error}", matrix_path.display())))?; + + let matrix_arg = matrix_path.to_string_lossy().into_owned(); + let maxparents = parameters.maxparents.to_string(); + let k = parameters.k.to_string(); + let tiers = parameters.tiers.to_string(); + let threshold = parameters.threshold.to_string(); + let mut args: Vec<&str> = vec![ + "learn", + "--input", + &matrix_arg, + "--maxparents", + &maxparents, + "--k", + &k, + "--tiers", + &tiers, + "--threshold", + &threshold, + ]; + if let Some(bin_dir) = bin_dir.filter(|dir| !dir.is_empty()) { + args.push("--bin-dir"); + args.push(bin_dir); + } + + let output = crate::streaming::run_streaming(bnw_cmd, &args, None) + .await + .map_err(|error| match error.kind() { + std::io::ErrorKind::NotFound => BnwError::NotInstalled(format!( + "the BNW driver was not found: {bnw_cmd} (set BNW_CMD, and BNW_BIN_DIR if the \ + compiled C programs live elsewhere)" + )), + _ => BnwError::Io(format!("{bnw_cmd}: {error}")), + })?; + let _ = std::fs::remove_file(&matrix_path); // best effort + + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + if !output.status.success() { + let code = output.status.code().unwrap_or(-1); + let complaint = driver_complaint(&stderr, &stdout); + return match code { + 77 => Err(BnwError::NotInstalled(complaint)), + 2 => Err(BnwError::BadInput(complaint)), + _ => Err(BnwError::CommandFailed(complaint)), + }; + } + parse(&stdout) +} + +/// The driver's own sentence about what went wrong: it writes `bnw: <reason>` +/// on stderr, so that line is the whole message and the last line is the +/// fallback only when it printed nothing. +fn driver_complaint(stderr: &str, stdout: &str) -> String { + let lines: Vec<&str> = stderr + .lines() + .chain(stdout.lines()) + .map(str::trim) + .filter(|line| !line.is_empty()) + .collect(); + if let Some(line) = lines.iter().find(|line| line.starts_with("bnw:")) { + return line.trim_start_matches("bnw:").trim().to_string(); + } + lines + .last() + .map(|line| (*line).to_string()) + .unwrap_or_else(|| "the BNW driver failed without saying why".to_string()) +} + +/// Parse the driver's TSV: a `#` header line naming the parameters, a column +/// header, then rows. +fn parse(stdout: &str) -> Result<Learned, BnwError> { + let mut summary = String::new(); + let mut rows = Vec::new(); + let mut columns: Option<Vec<&str>> = None; + for line in stdout.lines() { + let line = line.trim_end_matches('\r'); + if line.trim().is_empty() { + continue; + } + if let Some(rest) = line.strip_prefix("# ") { + summary = rest.trim().to_string(); + continue; + } + let cells: Vec<&str> = line.split('\t').collect(); + match &columns { + None => { + if cells != COLUMNS { + return Err(BnwError::UnexpectedOutput(format!( + "expected columns {:?}, got {cells:?}", + COLUMNS + ))); + } + columns = Some(cells); + } + Some(_) => { + if cells.len() != COLUMNS.len() { + return Err(BnwError::UnexpectedOutput(format!( + "row has {} field(s), expected {}: {line:?}", + cells.len(), + COLUMNS.len() + ))); + } + let posterior: f64 = cells[2].parse().map_err(|_| { + BnwError::UnexpectedOutput(format!( + "posterior {:?} is not a number", + cells[2] + )) + })?; + let in_structure = match cells[3] { + "1" => true, + "0" => false, + other => { + return Err(BnwError::UnexpectedOutput(format!( + "in_structure {other:?} is not 0 or 1" + ))) + } + }; + rows.push(serde_json::json!([ + cells[0], + cells[1], + posterior, + in_structure + ])); + } + } + } + if columns.is_none() { + return Err(BnwError::UnexpectedOutput( + "the driver printed no edge table".to_string(), + )); + } + Ok(Learned { rows, summary }) +} + +fn nanos() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|elapsed| elapsed.as_nanos()) + .unwrap_or(0) +} + +/// Where the matrix file lives for one request (exposed for tests). +pub fn matrix_path(cache_dir: &str, unique: &str) -> PathBuf { + Path::new(cache_dir).join(format!("{unique}-matrix.tsv")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn values(name: &str, pairs: &[(&str, Option<f64>)]) -> (String, Vec<(String, Option<f64>)>) { + ( + name.to_string(), + pairs + .iter() + .map(|(sample, value)| ((*sample).to_string(), *value)) + .collect(), + ) + } + + #[test] + fn a_matrix_keeps_only_samples_complete_across_every_variable() { + let matrix = Matrix::from_values(vec![ + values("a", &[("s1", Some(1.0)), ("s2", Some(2.0)), ("s3", Some(3.0)), ("s4", Some(0.5))]), + values("b", &[("s1", Some(4.0)), ("s2", None), ("s3", Some(6.0)), ("s4", Some(1.0))]), + // s5 is missing from the variables above, so it is not in the matrix + values("c", &[("s1", Some(7.0)), ("s2", Some(2.0)), ("s3", Some(9.0)), ("s4", Some(3.0))]), + ]) + .unwrap(); + assert_eq!(matrix.variables, vec!["a", "b", "c"]); + assert_eq!(matrix.rows.len(), 3, "s1, s3 and s4 are complete"); + assert_eq!(matrix.rows[0].0, "s1"); + assert_eq!(matrix.rows[0].1, vec![1.0, 4.0, 7.0]); + assert_eq!(matrix.rows[2].0, "s4"); + } + + #[test] + fn a_single_variable_is_refused() { + let error = Matrix::from_values(vec![values("a", &[("s1", Some(1.0))])]).unwrap_err(); + assert!(error.to_string().contains("at least 2 variables"), "{error}"); + } + + #[test] + fn too_few_complete_samples_is_refused_and_says_how_many() { + let error = Matrix::from_values(vec![ + values("a", &[("s1", Some(1.0)), ("s2", Some(2.0))]), + values("b", &[("s1", Some(3.0)), ("s2", Some(4.0))]), + ]) + .unwrap_err(); + assert!(error.to_string().contains("only 2 sample(s)"), "{error}"); + } + + #[test] + fn the_matrix_is_written_as_a_header_then_one_row_per_sample() { + let matrix = Matrix::from_values(vec![ + values("a", &[("s1", Some(1.5)), ("s2", Some(2.5)), ("s3", Some(3.5))]), + values("b", &[("s1", Some(4.0)), ("s2", Some(5.0)), ("s3", Some(6.0))]), + ]) + .unwrap(); + assert_eq!(matrix.to_tsv(), "a\tb\n1.5\t4\n2.5\t5\n3.5\t6\n"); + } + + #[test] + fn the_drivers_table_parses_into_rows() { + let stdout = "# bnw 0.1.0 variables=2 samples=3 maxparents=4 k=1 tiers=3 threshold=0.5\n\ + from\tto\tposterior\tin_structure\n\ + Trait2\tTrait1\t1\t1\n\ + Trait1\tTrait2\t0.25\t0\n"; + let learned = parse(stdout).unwrap(); + assert_eq!(learned.summary, "bnw 0.1.0 variables=2 samples=3 maxparents=4 k=1 tiers=3 threshold=0.5"); + assert_eq!(learned.rows.len(), 2); + assert_eq!(learned.rows[0], serde_json::json!(["Trait2", "Trait1", 1.0, true])); + assert_eq!(learned.rows[1], serde_json::json!(["Trait1", "Trait2", 0.25, false])); + } + + #[test] + fn an_unexpected_table_is_refused_rather_than_guessed() { + assert!(matches!( + parse("# header\nwrong\tcolumns\n"), + Err(BnwError::UnexpectedOutput(_)) + )); + assert!(matches!( + parse("# header\nfrom\tto\tposterior\tin_structure\nA\tB\tnot-a-number\t1\n"), + Err(BnwError::UnexpectedOutput(_)) + )); + assert!(matches!( + parse("# header only\n"), + Err(BnwError::UnexpectedOutput(_)) + )); + } + + #[test] + fn the_drivers_own_words_become_the_complaint() { + assert_eq!( + driver_complaint("bnw: variable 'a' appears twice\n", ""), + "variable 'a' appears twice" + ); + // nothing on stderr: the last line is better than nothing + assert_eq!(driver_complaint("", "some noise\n"), "some noise"); + assert_eq!( + driver_complaint("", ""), + "the BNW driver failed without saying why" + ); + } +} diff --git a/src/compute_gate.rs b/src/compute_gate.rs index fe5f5d5..9ee4f2c 100644 --- a/src/compute_gate.rs +++ b/src/compute_gate.rs @@ -464,6 +464,18 @@ pub fn qtl2() -> &'static Gate { /// BulkLMM scans: two at a time. A permutation is cheap (it rotates the /// phenotypes rather than repeating the scan), so the gate treats it as a /// GEMMA-sized job rather than an R/qtl nperm-sized one. +/// The BNW gate: one network search at a time, always gated. +/// +/// The threshold is 0 rather than a panel size because there is no such thing +/// as a small BNW run: the cost is the structure search, which is exponential +/// in the number of parents, not the number of individuals. A request that +/// reached this gate has already passed the variable ceiling, so any single +/// run deserves the one slot. +pub fn bnw() -> &'static Gate { + static GATE: std::sync::OnceLock<Gate> = std::sync::OnceLock::new(); + GATE.get_or_init(|| Gate::new(1, 0)) +} + pub fn bulklmm() -> &'static Gate { static GATE: std::sync::OnceLock<Gate> = std::sync::OnceLock::new(); GATE.get_or_init(|| Gate::new(2, GEMMA_LARGE_PANEL)) diff --git a/src/config.rs b/src/config.rs index cf5476e..0ab3fca 100644 --- a/src/config.rs +++ b/src/config.rs @@ -148,6 +148,16 @@ pub struct Config { pub basket_dir: String, /// The script that draws a trait by genotype class (`GENOTYPE_PLOTTER`). pub genotype_plotter: String, + /// The BNW driver for `/compute/bnw` (`BNW_CMD`): the headless + /// `bin/bnw` from the gn-bnw repository, which runs BNW's C programs + /// `BNW_BIN_DIR` overrides where those compiled programs live, for a + /// deployment that installs the driver and the binaries separately. + pub bnw_cmd: String, + /// Directory holding the compiled BNW programs (`BNW_BIN_DIR`); empty + /// means "beside the driver", which is how `build.sh` lays them out. + pub bnw_bin_dir: String, + /// Where the per-request matrices are written (`BNW_CACHE_DIR`). + pub bnw_cache_dir: String, /// The `julia` binary for BulkLMM (`BULKLMM_CMD`), the project its /// packages are instantiated into (`BULKLMM_ENV`), the depot they live in /// (`BULKLMM_DEPOT`), the driver script (`BULKLMM_SCRIPT`), the threads it @@ -331,6 +341,15 @@ impl Config { // BulkLMM.jl (src/bulklmm.rs): the Julia binary, the project its // packages are instantiated into, and the depot those packages live // in. `guix.scm` supplies the binary; the depot is built once. + values.insert("BNW_CMD".to_string(), "bnw".to_string()); + values.insert("BNW_BIN_DIR".to_string(), String::new()); + values.insert( + "BNW_CACHE_DIR".to_string(), + std::env::temp_dir() + .join("gn4-bnw") + .to_string_lossy() + .into_owned(), + ); values.insert( "BULKLMM_CMD".to_string(), "/gnu/store/4w6ivx7b27djzg16js6mkks1n4dvwq0f-julia-1.8.5/bin/julia".to_string(), @@ -428,6 +447,9 @@ impl Config { "WIKIDATA_TTL_SECONDS", "BASKET_DIR", "GENOTYPE_PLOTTER", + "BNW_CMD", + "BNW_BIN_DIR", + "BNW_CACHE_DIR", "BULKLMM_CMD", "BULKLMM_ENV", "BULKLMM_DEPOT", @@ -482,6 +504,9 @@ impl Config { wikidata_ttl: values["WIKIDATA_TTL_SECONDS"].parse().unwrap_or(604800), basket_dir: values["BASKET_DIR"].clone(), genotype_plotter: values["GENOTYPE_PLOTTER"].clone(), + bnw_cmd: values["BNW_CMD"].clone(), + bnw_bin_dir: values["BNW_BIN_DIR"].clone(), + bnw_cache_dir: values["BNW_CACHE_DIR"].clone(), bulklmm_cmd: values["BULKLMM_CMD"].clone(), bulklmm_env: values["BULKLMM_ENV"].clone(), bulklmm_depot: values["BULKLMM_DEPOT"].clone(), diff --git a/src/lib.rs b/src/lib.rs index 1df4151..2675c6b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -8,6 +8,7 @@ use utoipa_axum::routes; pub mod auth; pub mod baskets; +pub mod bnw; pub mod bulklmm; pub mod compute_gate; pub mod covariates; @@ -162,6 +163,7 @@ pub fn app() -> Router { .routes(routes!(routes::genotypes::group_marker_location)) .routes(routes!(routes::genotypes::unlisted_genotype_markers)) .routes(routes!(routes::genotypes::unlisted_marker_location)) + .routes(routes!(routes::bnw::compute_bnw)) .routes(routes!(routes::bulklmm::compute_bulklmm)) .routes(routes!(routes::gemma::compute_gemma)) .routes(routes!(routes::gemma::compute_gemma_ravanan)) diff --git a/src/routes/bnw.rs b/src/routes/bnw.rs new file mode 100644 index 0000000..eb8e17b --- /dev/null +++ b/src/routes/bnw.rs @@ -0,0 +1,349 @@ +//! `POST /compute/bnw`: learn a Bayesian network over a set of variables. +//! +//! BNW (the Bayesian Network Webserver) searches directed acyclic graphs and +//! reports, per candidate edge, the share of the k-best networks containing it. +//! In GeneNetwork the useful part is the *structure* — which traits belong +//! together under a causal ordering — and the module scores that follow from +//! it, not the drawing. +//! +//! The computation is done by BNW's own headless driver (`bnw learn`, in the +//! `gn-bnw` repository), called out to exactly the way `gemma-wrapper` and the +//! Julia/R drivers are: GN4 builds a variables × samples matrix, runs it, and +//! parses the reply. Nothing about BNW's pipeline lives here — see +//! [`crate::bnw`] for the contract and `gn-bnw/info_files/gn4-integration-plan.md` +//! for why. +//! +//! An edge posterior is **not a p-value**: it says how often the search put +//! that arrow in the k-best networks, which is a statement about the sample's +//! correlation structure, not about surprise. The reply says so in its notes, +//! because it is the easiest thing for a reader to over-interpret. + +use axum::Json; +use axum::http::StatusCode; +use serde::Deserialize; + +use gnauth_macros::compute_gate; + +use crate::auth::AuthToken; +use crate::bnw::{self, BnwError, Matrix, Parameters}; +use crate::compute_gate as gate; +use crate::config::Config; +use crate::db; +use crate::error::ApiError; +use crate::response::{Ctx, Envelope, RequestInfo}; + +/// The request: either published traits of a group, or an explicit matrix. +#[derive(Deserialize, utoipa::ToSchema)] +pub struct BnwRequest { + /// Group whose published traits form the variables (an InbredSet name, + /// e.g. "BXD"). Give this with `traits`, or `values` instead. + pub group: Option<String>, + /// `PublishXRef.Id` values of that group, one per variable. At most 40 — + /// the structure search grows exponentially with the number of nodes. + #[serde(default)] + pub traits: Vec<u32>, + /// An explicit matrix instead of `group` + `traits`: one entry per + /// variable, each with its per-sample values. Useful for data that is not + /// in the database (a basket's columns, a handful of probe sets). + pub values: Option<Vec<BnwVariable>>, + + /// Maximum parents per node (BNW's default 4; 0 means "no restriction", + /// which BNW reads as variables − 1). + pub maxparents: Option<u32>, + /// Networks to average over (BNW's default 1: no averaging). + pub k: Option<u32>, + /// Variable-ordering tiers (BNW's default 3). + pub tiers: Option<u32>, + /// Edge-posterior threshold for the reported structure (default 0.5). + pub threshold: Option<f64>, + + /// How to label this network in the reply, e.g. "BXD::hippocampus". + pub trait_name: Option<String>, +} + +/// One variable of an explicit matrix. +#[derive(Deserialize, utoipa::ToSchema)] +pub struct BnwVariable { + /// The variable's name, as it should appear in the edges. + pub name: String, + /// Its value per sample; samples with a missing value anywhere are dropped. + pub samples: Vec<BnwSample>, +} + +#[derive(Deserialize, utoipa::ToSchema)] +pub struct BnwSample { + pub name: String, + pub value: Option<f64>, +} + +#[utoipa::path(post, path = "/compute/bnw", + summary = "Learn a Bayesian network over a set of traits (BNW)", + description = "Searches the space of directed acyclic graphs over the given \ +variables and returns one row per candidate edge: `from`, `to`, the edge's \ +posterior (the share of the k-best networks containing it) and whether it \ +survives the threshold. BNW's own defaults are used unless the request \ +overrides them (maxparents 4, k 1, tiers 3, threshold 0.5). The computation \ +runs in BNW's headless driver (see the gn-bnw repository); a posterior is not a \ +p-value.", + request_body = BnwRequest, + responses( + (status = 200, description = "One row per candidate edge, positional (column names in meta.columns)", body = Envelope<serde_json::Value>), + crate::error::Unauthorized, + (status = 409, description = "A BNW run is already in progress for this caller", body = ApiError), + (status = 422, description = "Too few or too many variables, or too few complete samples", body = ApiError), + (status = 502, description = "The BNW driver failed", body = ApiError) + ) +)] +#[compute_gate(bnw, label = "BNW")] +pub async fn compute_bnw( + ctx: Ctx, + token: AuthToken, + Json(request): Json<BnwRequest>, +) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { + let rid = ctx.request_info(); + let config = Config::load(); + + let parameters = Parameters { + maxparents: request.maxparents.unwrap_or(bnw::DEFAULT_MAXPARENTS), + k: request.k.unwrap_or(bnw::DEFAULT_K), + tiers: request.tiers.unwrap_or(bnw::DEFAULT_TIERS), + threshold: request.threshold.unwrap_or(bnw::DEFAULT_THRESHOLD), + }; + if parameters.maxparents > 0 && parameters.maxparents > 15 { + return Err(unprocessable( + "maxparents_too_large", + "Too many parents", + format!( + "maxparents {} is beyond what the structure search can do in a request; at most \ + 15 is accepted (BNW's own default is {}).", + parameters.maxparents, + bnw::DEFAULT_MAXPARENTS + ), + rid, + )); + } + if parameters.k == 0 { + return Err(unprocessable( + "invalid_k", + "Invalid k", + "k must be at least 1 (1 means no model averaging).".to_string(), + rid, + )); + } + if !(0.0..=1.0).contains(¶meters.threshold) { + return Err(unprocessable( + "invalid_threshold", + "Invalid threshold", + format!( + "threshold {} is not an edge posterior: give a value in [0, 1] (BNW's default is \ + {}).", + parameters.threshold, + bnw::DEFAULT_THRESHOLD + ), + rid, + )); + } + + // The variables: published traits of a group, or an explicit matrix. + let (variables, label) = match (&request.group, &request.values) { + (Some(group), None) => { + if request.traits.len() < 2 { + return Err(unprocessable( + "too_few_variables", + "Too few variables", + format!( + "{} trait(s) given; a network needs at least 2 (and at most {}).", + request.traits.len(), + bnw::MAX_VARIABLES + ), + rid, + )); + } + let group_name = group.clone(); + let traits = request.traits.clone(); + let fetched = tokio::task::spawn_blocking(move || { + let row = db::get_group_by_name(&group_name) + .map_err(|error| error.to_string())? + .ok_or_else(|| format!("no group named '{group_name}'"))?; + let mut variables = Vec::with_capacity(traits.len()); + for trait_id in traits { + let values = db::get_group_trait_values(row.id, trait_id) + .map_err(|error| error.to_string())?; + if values.is_empty() { + return Err(format!( + "group '{group_name}' has no values for trait {trait_id}" + )); + } + variables.push(( + format!("{group_name}::{trait_id}"), + values + .into_iter() + .map(|value| (value.name, value.value)) + .collect::<Vec<_>>(), + )); + } + Ok::<_, String>(variables) + }) + .await + .map_err(|error| internal(error.to_string(), rid.clone()))? + .map_err(|detail| unprocessable("invalid_variables", "Unusable variables", detail, rid.clone()))?; + let label = request + .trait_name + .clone() + .unwrap_or_else(|| format!("{}::network", group)); + (fetched, label) + } + (None, Some(values)) => { + let variables = values + .iter() + .map(|variable| { + ( + variable.name.clone(), + variable + .samples + .iter() + .map(|sample| (sample.name.clone(), sample.value)) + .collect::<Vec<_>>(), + ) + }) + .collect::<Vec<_>>(); + let label = request + .trait_name + .clone() + .unwrap_or_else(|| "network".to_string()); + (variables, label) + } + _ => { + return Err(unprocessable( + "ambiguous_input", + "Ambiguous network input", + "give `group` + `traits`, or `values` -- one of the two.".to_string(), + rid, + )) + } + }; + + let matrix = Matrix::from_values(variables) + .map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; + let variables_count = matrix.variables(); + let samples = matrix.samples(); + + let learned = bnw::learn( + &config.bnw_cmd, + Some(&config.bnw_bin_dir), + &config.bnw_cache_dir, + &matrix, + parameters, + ) + .await + .map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; + + let in_structure = learned + .rows + .iter() + .filter(|row| row.get(3).and_then(serde_json::Value::as_bool).unwrap_or(false)) + .count(); + let peak = learned.rows.first().and_then(|row| { + let from = row.get(0)?.as_str()?; + let to = row.get(1)?.as_str()?; + let posterior = row.get(2)?.as_f64()?; + Some((from.to_string(), to.to_string(), posterior)) + }); + + let candidate_edges = learned.rows.len(); + let mut envelope = ctx + .envelope(serde_json::Value::Array(learned.rows)) + .with_columns( + bnw::COLUMNS.iter().map(|name| (*name).to_string()).collect(), + candidate_edges, + ) + .with_note(&format!( + "Network: {label}, {variables_count} variables over {samples} samples that have a \ + value for every one of them (samples missing any value are dropped: BNW models a \ + complete multivariate normal)." + )) + .with_note(&format!( + "Parameters: maxparents {}, k {}, tiers {}, threshold {} -- BNW's own defaults unless \ + the request overrode them. The parameters are echoed because a network learned at \ + threshold 0.5 and one at 0.9 are different answers to the same question.", + parameters.maxparents, parameters.k, parameters.tiers, parameters.threshold + )) + .with_note( + &"A posterior is not a p-value: it is the share of the k-best networks that contain \ + the edge, i.e. how often the search put that arrow there in this sample -- not how \ + surprising it is. `in_structure` says whether it survived the threshold. A network \ + summarises correlation under a causal ordering; it does not show a QTL, and an edge \ + is not an effect size.", + ) + .with_note(&format!( + "Driver: {} ({}).", + config.bnw_cmd, + if learned.summary.is_empty() { + "no version line".to_string() + } else { + learned.summary.clone() + } + )); + if let Some((from, to, posterior)) = peak { + envelope = envelope.with_note(&format!( + "Strongest edge: {from} -> {to} (posterior {posterior:.4}); {in_structure} of {} \ + candidate edges survive the threshold.", + candidate_edges + )); + } + Ok(Json(envelope)) +} + +fn unprocessable( + code: &'static str, + title: &'static str, + detail: String, + rid: RequestInfo, +) -> ApiError { + ApiError::new( + StatusCode::UNPROCESSABLE_ENTITY, + code, + title, + detail, + "compute_bnw", + ) + .with_request_id(rid) +} + +fn internal(detail: String, rid: RequestInfo) -> ApiError { + ApiError::new( + StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + "Internal error", + detail, + "compute_bnw", + ) + .with_request_id(rid) +} + +/// Map the driver's failures onto the API's: bad input is the caller's (422), +/// a missing or failing driver is ours (502) -- and a driver that is not +/// installed says so, rather than looking like a computation bug. +fn bnw_error_to_api_error(error: BnwError, rid: RequestInfo) -> ApiError { + match error { + BnwError::BadInput(detail) => unprocessable("invalid_network_input", "Unusable network input", detail, rid), + BnwError::NotInstalled(detail) => ApiError::new( + StatusCode::BAD_GATEWAY, + "bnw_not_installed", + "BNW is not available", + detail, + "compute_bnw", + ) + .with_request_id(rid), + BnwError::Io(detail) | BnwError::CommandFailed(detail) | BnwError::UnexpectedOutput(detail) => { + ApiError::new( + StatusCode::BAD_GATEWAY, + "bnw_failed", + "BNW failed", + detail, + "compute_bnw", + ) + .with_request_id(rid) + } + } +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index adda88b..55c8ce2 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -5,6 +5,7 @@ pub mod accessions; pub mod baskets; +pub mod bnw; pub mod bulklmm; pub mod correlation; pub mod datasets; |
