//! `POST /compute/bnw`: learn a Bayesian network over a set of variables. //! //! BNW (the Bayesian Network Webserver) searches directed acyclic graphs and //! reports, per candidate edge, the share of the k-best networks containing it. //! In GeneNetwork the useful part is the *structure* — which traits belong //! together under a causal ordering — and the module scores that follow from //! it, not the drawing. //! //! The computation is done by BNW's own headless driver (`bnw learn`, in the //! `gn-bnw` repository), called out to exactly the way `gemma-wrapper` and the //! Julia/R drivers are: GN4 builds a variables × samples matrix, runs it, and //! parses the reply. Nothing about BNW's pipeline lives here — see //! [`crate::bnw`] for the contract and `gn-bnw/info_files/gn4-integration-plan.md` //! for why. //! //! An edge posterior is **not a p-value**: it says how often the search put //! that arrow in the k-best networks, which is a statement about the sample's //! correlation structure, not about surprise. The reply says so in its notes, //! because it is the easiest thing for a reader to over-interpret. use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::{compute_gate, gnauth}; use crate::auth::AuthToken; use crate::bnw::{self, BnwError, Matrix, Parameters}; use crate::compute_gate as gate; use crate::config::Config; use crate::db; use crate::error::ApiError; use crate::response::{Ctx, Envelope, RequestInfo}; /// The request: either published traits of a group, or an explicit matrix. #[derive(Deserialize, utoipa::ToSchema)] pub struct BnwRequest { /// Group whose published traits form the variables (an InbredSet name, /// e.g. "BXD"). Give this with `traits`, or `values` instead. pub group: Option, /// `PublishXRef.Id` values of that group, one per variable. At most 40 — /// the structure search grows exponentially with the number of nodes. #[serde(default)] pub traits: Vec, /// An explicit matrix instead of `group` + `traits`: one entry per /// variable, each with its per-sample values. Useful for data that is not /// in the database (a basket's columns, a handful of probe sets). pub values: Option>, /// Maximum parents per node (BNW's default 4; 0 means "no restriction", /// which BNW reads as variables − 1). pub maxparents: Option, /// Networks to average over (BNW's default 1: no averaging). pub k: Option, /// Variable-ordering tiers (BNW's default 3). pub tiers: Option, /// Edge-posterior threshold for the reported structure (default 0.5). pub threshold: Option, /// How to label this network in the reply, e.g. "BXD::hippocampus". pub trait_name: Option, } /// One variable of an explicit matrix. #[derive(Deserialize, utoipa::ToSchema)] pub struct BnwVariable { /// The variable's name, as it should appear in the edges. pub name: String, /// Its value per sample; samples with a missing value anywhere are dropped. pub samples: Vec, } #[derive(Deserialize, utoipa::ToSchema)] pub struct BnwSample { pub name: String, pub value: Option, } #[utoipa::path(post, path = "/compute/bnw", summary = "Learn a Bayesian network over a set of traits (BNW)", description = "Searches the space of directed acyclic graphs over the given \ variables and returns one row per candidate edge: `from`, `to`, the edge's \ posterior (the share of the k-best networks containing it) and whether it \ survives the threshold. BNW's own defaults are used unless the request \ overrides them (maxparents 4, k 1, tiers 3, threshold 0.5). The computation \ runs in BNW's headless driver (see the gn-bnw repository); a posterior is not a \ p-value.", request_body = BnwRequest, responses( (status = 200, description = "One row per candidate edge, positional (column names in meta.columns)", body = Envelope), crate::error::Unauthorized, (status = 409, description = "A BNW run is already in progress for this caller", body = ApiError), (status = 422, description = "Too few or too many variables, or too few complete samples", body = ApiError), (status = 502, description = "The BNW driver failed", body = ApiError) ) )] #[compute_gate(bnw, label = "BNW")] pub async fn compute_bnw( ctx: Ctx, token: AuthToken, Json(request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); let parameters = Parameters { maxparents: request.maxparents.unwrap_or(bnw::DEFAULT_MAXPARENTS), k: request.k.unwrap_or(bnw::DEFAULT_K), tiers: request.tiers.unwrap_or(bnw::DEFAULT_TIERS), threshold: request.threshold.unwrap_or(bnw::DEFAULT_THRESHOLD), }; if parameters.maxparents > 0 && parameters.maxparents > 15 { return Err(unprocessable( "maxparents_too_large", "Too many parents", format!( "maxparents {} is beyond what the structure search can do in a request; at most \ 15 is accepted (BNW's own default is {}).", parameters.maxparents, bnw::DEFAULT_MAXPARENTS ), rid, )); } if parameters.k == 0 { return Err(unprocessable( "invalid_k", "Invalid k", "k must be at least 1 (1 means no model averaging).".to_string(), rid, )); } if !(0.0..=1.0).contains(¶meters.threshold) { return Err(unprocessable( "invalid_threshold", "Invalid threshold", format!( "threshold {} is not an edge posterior: give a value in [0, 1] (BNW's default is \ {}).", parameters.threshold, bnw::DEFAULT_THRESHOLD ), rid, )); } // The variables: published traits of a group, or an explicit matrix. let (variables, label) = match (&request.group, &request.values) { (Some(group), None) => { if request.traits.len() < 2 { return Err(unprocessable( "too_few_variables", "Too few variables", format!( "{} trait(s) given; a network needs at least 2 (and at most {}).", request.traits.len(), bnw::MAX_VARIABLES ), rid, )); } let group_name = group.clone(); let traits = request.traits.clone(); let fetched = tokio::task::spawn_blocking(move || { let row = db::get_group_by_name(&group_name) .map_err(|error| error.to_string())? .ok_or_else(|| format!("no group named '{group_name}'"))?; let mut variables = Vec::with_capacity(traits.len()); for trait_id in traits { let values = db::get_group_trait_values(row.id, trait_id) .map_err(|error| error.to_string())?; if values.is_empty() { return Err(format!( "group '{group_name}' has no values for trait {trait_id}" )); } variables.push(( format!("{group_name}::{trait_id}"), values .into_iter() .map(|value| (value.name, value.value)) .collect::>(), )); } Ok::<_, String>(variables) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|detail| unprocessable("invalid_variables", "Unusable variables", detail, rid.clone()))?; let label = request .trait_name .clone() .unwrap_or_else(|| format!("{}::network", group)); (fetched, label) } (None, Some(values)) => { let variables = values .iter() .map(|variable| { ( variable.name.clone(), variable .samples .iter() .map(|sample| (sample.name.clone(), sample.value)) .collect::>(), ) }) .collect::>(); let label = request .trait_name .clone() .unwrap_or_else(|| "network".to_string()); (variables, label) } _ => { return Err(unprocessable( "ambiguous_input", "Ambiguous network input", "give `group` + `traits`, or `values` -- one of the two.".to_string(), rid, )) } }; let matrix = Matrix::from_values(variables) .map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; let variables_count = matrix.variables(); let samples = matrix.samples(); let learned = bnw::learn( &config.bnw_cmd, Some(&config.bnw_bin_dir), &config.bnw_cache_dir, &matrix, parameters, ) .await .map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; let in_structure = learned .rows .iter() .filter(|row| row.get(3).and_then(serde_json::Value::as_bool).unwrap_or(false)) .count(); let peak = learned.rows.first().and_then(|row| { let from = row.get(0)?.as_str()?; let to = row.get(1)?.as_str()?; let posterior = row.get(2)?.as_f64()?; Some((from.to_string(), to.to_string(), posterior)) }); let candidate_edges = learned.rows.len(); let mut envelope = ctx .envelope(serde_json::Value::Array(learned.rows)) .with_columns( bnw::COLUMNS.iter().map(|name| (*name).to_string()).collect(), candidate_edges, ) .with_note(&format!( "Network: {label}, {variables_count} variables over {samples} samples that have a \ value for every one of them (samples missing any value are dropped: BNW models a \ complete multivariate normal)." )) .with_note(&format!( "Parameters: maxparents {}, k {}, tiers {}, threshold {} -- BNW's own defaults unless \ the request overrode them. The parameters are echoed because a network learned at \ threshold 0.5 and one at 0.9 are different answers to the same question.", parameters.maxparents, parameters.k, parameters.tiers, parameters.threshold )) .with_note( &"A posterior is not a p-value: it is the share of the k-best networks that contain \ the edge, i.e. how often the search put that arrow there in this sample -- not how \ surprising it is. `in_structure` says whether it survived the threshold. A network \ summarises correlation under a causal ordering; it does not show a QTL, and an edge \ is not an effect size.", ) .with_note(&format!( "Driver: {} ({}).", config.bnw_cmd, if learned.summary.is_empty() { "no version line".to_string() } else { learned.summary.clone() } )); if let Some((from, to, posterior)) = peak { envelope = envelope.with_note(&format!( "Strongest edge: {from} -> {to} (posterior {posterior:.4}); {in_structure} of {} \ candidate edges survive the threshold.", candidate_edges )); } Ok(Json(envelope)) } fn unprocessable( code: &'static str, title: &'static str, detail: String, rid: RequestInfo, ) -> ApiError { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, code, title, detail, "compute_bnw", ) .with_request_id(rid) } fn internal(detail: String, rid: RequestInfo) -> ApiError { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", detail, "compute_bnw", ) .with_request_id(rid) } /// Map the driver's failures onto the API's: bad input is the caller's (422), /// a missing or failing driver is ours (502) -- and a driver that is not /// installed says so, rather than looking like a computation bug. fn bnw_error_to_api_error(error: BnwError, rid: RequestInfo) -> ApiError { match error { BnwError::BadInput(detail) => unprocessable("invalid_network_input", "Unusable network input", detail, rid), BnwError::NotInstalled(detail) => ApiError::new( StatusCode::BAD_GATEWAY, "bnw_not_installed", "BNW is not available", detail, "compute_bnw", ) .with_request_id(rid), BnwError::Io(detail) | BnwError::CommandFailed(detail) | BnwError::UnexpectedOutput(detail) => { ApiError::new( StatusCode::BAD_GATEWAY, "bnw_failed", "BNW failed", detail, "compute_bnw", ) .with_request_id(rid) } } } /// What `/compute/bnw/features` takes: the same inputs as `/compute/bnw`, /// because a module score comes from the network those inputs learn. #[derive(Deserialize, utoipa::ToSchema)] pub struct BnwFeaturesRequest { /// Group whose published traits form the variables; give with `traits`. pub group: Option, #[serde(default)] pub traits: Vec, /// An explicit matrix instead of `group` + `traits`. pub values: Option>, pub maxparents: Option, pub k: Option, pub tiers: Option, pub threshold: Option, pub trait_name: Option, } #[derive(Deserialize, utoipa::IntoParams)] pub struct BnwFeaturesParams { /// Return this one module's scores as `{name, value}` per sample -- the /// shape every mapping endpoint takes -- instead of the module list. pub module: Option, } #[utoipa::path(post, path = "/compute/bnw/features", summary = "Score each module of a learned network, as trait values", description = "Learns the network (or reuses the same computation as \ POST /compute/bnw), then scores every node that has parents: the node plus its \ parents form a module, and its score is the first principal component of the \ module's standardised members, sign-corrected to rise with the node. Without \ `?module=` the reply is one row per module -- name, members, size, and the \ share of the module's variance the score explains. With `?module=NAME` the \ reply is that module's per-sample scores in the `{name, value}` shape every \ mapping endpoint (`/compute/rqtl`, `/compute/gemma`, `/compute/qtl2`) already \ takes, so a module can be mapped without any translation.", params(BnwFeaturesParams), request_body = BnwFeaturesRequest, responses( (status = 200, description = "One row per module, or one value per sample with ?module=", body = Envelope), crate::error::Unauthorized, (status = 409, description = "A BNW run is already in progress for this caller", body = ApiError), (status = 422, description = "Unusable inputs, or no module with that name", body = ApiError), (status = 502, description = "The BNW driver failed", body = ApiError) ) )] #[compute_gate(bnw, label = "BNW features")] pub async fn compute_bnw_features( ctx: Ctx, token: AuthToken, Query(params): Query, Json(request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); let parameters = Parameters { maxparents: request.maxparents.unwrap_or(bnw::DEFAULT_MAXPARENTS), k: request.k.unwrap_or(bnw::DEFAULT_K), tiers: request.tiers.unwrap_or(bnw::DEFAULT_TIERS), threshold: request.threshold.unwrap_or(bnw::DEFAULT_THRESHOLD), }; let (variables, label) = resolve_variables(&request.group, &request.traits, &request.values, request.trait_name.as_deref(), rid.clone()).await?; let matrix = Matrix::from_values(variables).map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; let learned = bnw::learn( &config.bnw_cmd, Some(&config.bnw_bin_dir), &config.bnw_cache_dir, &matrix, parameters, ) .await .map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; let modules = bnw::module_scores(&matrix, &learned.rows, parameters.threshold); if modules.is_empty() { return Err(unprocessable( "no_modules", "No modules to score", format!( "the network has no node with a parent at or above threshold {}; nothing to \ summarise. Lower the threshold, or widen the variables.", parameters.threshold ), rid, )); } if let Some(wanted) = params.module.as_deref() { let Some(module) = modules.iter().find(|module| module.name == wanted) else { let names: Vec<&str> = modules.iter().map(|module| module.name.as_str()).collect(); return Err(unprocessable( "unknown_module", "No such module", format!("no module {wanted:?} in this network; it has: {}", names.join(", ")), rid, )); }; let values: Vec = module .scores .iter() .map(|(name, value)| serde_json::json!({"name": name, "value": value})) .collect(); return Ok(Json( ctx.envelope(serde_json::Value::Array(values)) .with_note(&format!( "Module {label}::{}: the node and its parents ({}), scored by the first \ principal component of their standardised values, sign-corrected to rise with \ the node. These are trait values in the shape /compute/rqtl, /compute/gemma \ and /compute/qtl2 take: post them as `samples` to map the module.", module.name, module.members.join(" + ") )) .with_note(&format!( "The score explains {:.1}% of the module's variance over {} samples; a low \ share means the module is not one thing and the score is a poor summary.", module.variance_explained * 100.0, module.scores.len() )), )); } let rows: Vec = modules .iter() .map(|module| { serde_json::json!([ module.name, module.members.join(","), module.members.len(), (module.variance_explained * 10000.0).round() / 10000.0 ]) }) .collect(); let mut envelope = ctx .envelope(serde_json::Value::Array(rows)) .with_columns( vec![ "module".to_string(), "members".to_string(), "size".to_string(), "variance_explained".to_string(), ], modules.len(), ) .with_note(&format!( "Modules of the network {label}: every node with at least one parent at or above \ threshold {}, scored as the first principal component of the node and its parents' \ standardised values, sign-corrected to rise with the node. `variance_explained` is \ the share of the module's variance that component explains.", parameters.threshold )) .with_note( "Post `?module=` to get one module's per-sample scores in the `{name, value}` \ shape the mapping endpoints take, then map them like any trait. A module is a \ correlation structure, not a QTL: a mapped module says where its members' shared \ variation comes from, not which member causes which.", ); for module in &modules { envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: format!("/compute/bnw/features?module={}", module.name), description: format!( "The per-sample scores of module {} ({} members), ready to post as `samples` to a \ mapping endpoint", module.name, module.members.len() ), body: None, }); } Ok(Json(envelope)) } /// The variables a BNW request names, from the database or the request body. async fn resolve_variables( group: &Option, traits: &[u32], values: &Option>, trait_name: Option<&str>, rid: RequestInfo, ) -> Result<(Vec<(String, Vec<(String, Option)>)>, String), ApiError> { match (group, values) { (Some(group), None) => { if traits.len() < 2 { return Err(unprocessable( "too_few_variables", "Too few variables", format!( "{} trait(s) given; a network needs at least 2 (and at most {}).", traits.len(), bnw::MAX_VARIABLES ), rid, )); } let group_name = group.clone(); let wanted = traits.to_vec(); let fetched = tokio::task::spawn_blocking(move || { let row = db::get_group_by_name(&group_name) .map_err(|error| error.to_string())? .ok_or_else(|| format!("no group named '{group_name}'"))?; let mut variables = Vec::with_capacity(wanted.len()); for trait_id in wanted { let values = db::get_group_trait_values(row.id, trait_id) .map_err(|error| error.to_string())?; if values.is_empty() { return Err(format!("group '{group_name}' has no values for trait {trait_id}")); } variables.push(( format!("{group_name}::{trait_id}"), values.into_iter().map(|value| (value.name, value.value)).collect::>(), )); } Ok::<_, String>(variables) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|detail| unprocessable("invalid_variables", "Unusable variables", detail, rid.clone()))?; let label = trait_name .map(str::to_string) .unwrap_or_else(|| format!("{group}::network")); Ok((fetched, label)) } (None, Some(values)) => { let variables = values .iter() .map(|variable| { ( variable.name.clone(), variable.samples.iter().map(|sample| (sample.name.clone(), sample.value)).collect::>(), ) }) .collect(); Ok((variables, trait_name.unwrap_or("network").to_string())) } _ => Err(unprocessable( "ambiguous_input", "Ambiguous network input", "give `group` + `traits`, or `values` -- one of the two.".to_string(), rid, )), } } #[derive(Deserialize, utoipa::IntoParams)] pub struct ExampleParams { /// Restrict the matrix to these columns, comma separated. BNW's spleen /// example has more variables than the endpoint will search, so a caller /// chooses a subset. pub columns: Option, } #[utoipa::path(get, path = "/compute/bnw/examples", summary = "The worked examples BNW ships, ready to post to /compute/bnw", description = "BNW's own example datasets -- the ones its tutorial and examples \ pages present -- listed with the variables and samples each has, and an action \ carrying a ready-to-post body for POST /compute/bnw. GET \ /compute/bnw/examples/{name} returns one example's matrix in the `values` shape \ /compute/bnw takes, so an agent can explore a real network without inventing \ data. The files are read from BNW_EXAMPLES_DIR (the gn-bnw checkout's \ sourcecodes/example_datasets).", responses( (status = 200, description = "One row per example", body = Envelope), crate::error::Unauthorized, (status = 404, description = "No examples are configured on this deployment", body = ApiError) ) )] #[gnauth(token_only)] pub async fn bnw_examples( ctx: Ctx, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); if config.bnw_examples_dir.trim().is_empty() { return Err(ApiError::new( StatusCode::NOT_FOUND, "no_examples", "No examples configured", "this deployment has no BNW_EXAMPLES_DIR; point it at a gn-bnw checkout's \ sourcecodes/example_datasets to offer BNW's worked examples." .to_string(), "bnw_examples", ) .with_request_id(rid)); } let dir = std::path::PathBuf::from(&config.bnw_examples_dir); let examples = tokio::task::spawn_blocking(move || bnw::examples(&dir)) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| bnw_error_to_api_error(error, rid.clone()))?; if examples.is_empty() { return Err(ApiError::new( StatusCode::NOT_FOUND, "no_examples", "No examples found", format!("no *.txt example data in {}", config.bnw_examples_dir), "bnw_examples", ) .with_request_id(rid)); } let rows: Vec = examples .iter() .map(|example| { serde_json::json!([ example.name, example.title, example.variables, example.samples, example.has_genotype_column ]) }) .collect(); let mut envelope = ctx .envelope(serde_json::Value::Array(rows)) .with_columns( vec![ "name".to_string(), "title".to_string(), "variables".to_string(), "samples".to_string(), "genotype_column".to_string(), ], examples.len(), ) .with_note( "These are BNW's own example datasets. GET /compute/bnw/examples/{name} returns one \ as a matrix in the `values` shape POST /compute/bnw takes -- post that body straight \ back to learn the network the web application's tutorial shows.", ) .with_note( "`genotype_column` warns that a column is a genotype code (whole numbers). This \ driver is continuous-only, so such a column enters the network as a number, exactly \ as the web application's GeneNetwork path treats it -- not as a discrete variable \ with its own multinomial model. Anything the examples page marks as discrete \ (the car-diagnosis example) is not offered here.", ); for example in &examples { envelope = envelope.with_next( &format!("/compute/bnw/examples/{}", example.name), example.title.as_str(), ); } Ok(Json(envelope.with_links(crate::suggest::root_links()))) } #[utoipa::path(get, path = "/compute/bnw/examples/{name}", summary = "One of BNW's worked examples, as a matrix /compute/bnw takes", description = "The example's data in the `values` shape POST /compute/bnw expects: \ one entry per variable, each with its per-sample values. `?columns=a,b,c` restricts \ the matrix -- the spleen example has more variables than the endpoint searches.", params(ExampleParams, ("name" = String, Path, description = "An example name from GET /compute/bnw/examples")), responses( (status = 200, description = "The matrix, as `values`", body = Envelope), crate::error::Unauthorized, (status = 404, description = "No such example", body = ApiError), (status = 422, description = "A named column does not exist, or the file is not numeric", body = ApiError) ) )] #[gnauth(token_only)] pub async fn bnw_example( ctx: Ctx, _token: AuthToken, Path(name): Path, Query(params): Query, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); if config.bnw_examples_dir.trim().is_empty() { return Err(ApiError::new( StatusCode::NOT_FOUND, "no_examples", "No examples configured", "this deployment has no BNW_EXAMPLES_DIR.".to_string(), "bnw_example", ) .with_request_id(rid)); } let columns: Option> = params.columns.as_deref().map(|list| { list.split(',') .map(|column| column.trim().to_string()) .filter(|column| !column.is_empty()) .collect() }); let dir = std::path::PathBuf::from(&config.bnw_examples_dir); let wanted = name.clone(); let matrix = tokio::task::spawn_blocking(move || { bnw::example_matrix(&dir, &wanted, columns.as_deref()) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| match error { BnwError::Io(detail) if detail.contains("No such file") => ApiError::new( StatusCode::NOT_FOUND, "no_such_example", "No such example", format!("no example named {name:?}; see GET /compute/bnw/examples"), "bnw_example", ) .with_request_id(rid.clone()), other => bnw_error_to_api_error(other, rid.clone()), })?; let values: Vec = matrix .variables .iter() .enumerate() .map(|(column, variable)| { let samples: Vec = matrix .rows .iter() .map(|(sample, row)| serde_json::json!({"name": sample, "value": row[column]})) .collect(); serde_json::json!({"name": variable, "samples": samples}) }) .collect(); Ok(Json( ctx.envelope(serde_json::json!({"values": serde_json::Value::Array(values.clone())})) .with_note(&format!( "BNW example {name}: {} variables over {} samples.\nPost this body (or its \ `values`) to /compute/bnw to learn the network the web application's tutorial \ shows; add `maxparents`, `k` or `threshold` to explore how the structure moves.", matrix.variables.len(), matrix.rows.len() )) .with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: "/compute/bnw".to_string(), description: format!("Learn the network for example {name}"), body: Some(serde_json::json!({"values": serde_json::Value::Array(values.clone())})), }), )) }