//! Correlation endpoint: one trait against every trait of a target dataset, //! observations being individuals (the sample axis). //! //! The arithmetic and the policy live in `src/correlation.rs`; what this module //! does is resolve the request, fetch the values, and shape the reply. It is //! synchronous by decision: one POST computes and answers, no redis, no job //! (see docs/correlations.md). use axum::Json; use serde::Deserialize; use gnauth_macros::gnauth; use crate::auth::AuthToken; use crate::correlation::{self, Method, DEFAULT_TOP_N, MIN_OVERLAP}; use crate::db; use crate::error::{ApiError, Unauthorized}; use crate::response::{Ctx, Envelope}; /// The reply's columns, in this order, one row per correlated trait. pub const COLUMNS: [&str; 4] = ["trait", "r", "p", "n"]; /// A sample-axis correlation request. /// /// `dataset` is the reference trait's own dataset, `target_dataset` the one to /// correlate it against -- GN2's `corr_dataset`. Missing `method` means /// pearson, missing `top_n` the GN2 default of 500. #[derive(Deserialize, Clone, Default, utoipa::ToSchema)] pub struct CorrelationRequest { /// Dataset the reference trait lives in, e.g. "HC_M2_0606_P". pub dataset: String, /// The reference trait, e.g. "1427571_at". pub trait_name: String, /// Dataset to correlate against, e.g. "BXDPublish" or another ProbeSet set. pub target_dataset: String, /// What is correlated, GN2's `corr_type`: "sample" (default) pairs the two /// traits over individuals, "tissue" pairs them over tissue systems, and /// "literature" reads a precomputed gene-by-gene value. #[serde(default)] pub r#type: Option, /// "pearson" (default) or "spearman". #[serde(default)] pub method: Option, /// How many hits to return, strongest by |r| first. Defaults to 500. #[serde(default)] pub top_n: Option, /// Traits held constant (`type: "partial"`): each is a `{dataset, /// trait_name}` reference, as GN3's `control_traits`. #[serde(default)] pub control_traits: Vec, /// Which individuals the correlation runs over, GN2's `corr_samples_group`: /// "samples_primary" (default) restricts to the trait's own group, /// "samples_all" uses every individual the trait has a value for. /// "samples_other" is not supported yet (it needs the other sample lists). #[serde(default)] pub corr_samples_group: Option, } #[utoipa::path(post, path = "/compute/correlation", summary = "Correlate one trait against a dataset (sample axis)", description = "One trait against every trait of a target dataset, \ observations being individuals. Synchronous: the reply is the result. \ Correlations are computed by the correlation_rust engine (pearson/spearman), \ need at least six paired observations, and are ordered by |r| descending.", request_body = CorrelationRequest, responses( (status = 200, description = "Correlated traits, strongest first", body = Envelope), (status = 404, description = "No such dataset", body = ApiError), (status = 422, description = "Unknown method, or a trait with no values", body = ApiError), Unauthorized, (status = 500, description = "Internal error", body = ApiError) ))] #[gnauth(token_only)] pub async fn correlation( ctx: Ctx, _token: AuthToken, Json(request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let fail = |status: axum::http::StatusCode, code: &str, title: &str, detail: String| { ApiError::new(status, code, title, detail, "compute_correlation") .with_request_id(rid.clone()) }; let method = Method::parse(request.method.as_deref().unwrap_or("pearson")) .map_err(|detail| { fail( axum::http::StatusCode::UNPROCESSABLE_ENTITY, "invalid_method", "Invalid correlation method", detail, ) })?; let top_n = request.top_n.unwrap_or(DEFAULT_TOP_N); if top_n == 0 || top_n > correlation::MAX_TOP_N { return Err(fail( axum::http::StatusCode::UNPROCESSABLE_ENTITY, "invalid_top_n", "Invalid top_n", format!( "top_n must be between 1 and {}; got {top_n}.", correlation::MAX_TOP_N ), )); } let axis = Axis::parse(request.r#type.as_deref()).map_err(|failure| { failure.into_api(rid.clone()) })?; let wanted = request.clone(); let computed = tokio::task::spawn_blocking(move || compute(&request, axis, method, top_n)) .await .map_err(|error| { fail( axum::http::StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), ) })?; let computed = computed.map_err(|failure| failure.into_api(rid.clone()))?; let (reference_name, rows, columns, num_rows, partial_note) = match computed { Computed::Standard(reference_name, hits) => { let rows = serde_json::Value::Array( hits.iter() .map(|hit| serde_json::json!([hit.name, hit.r, hit.p, hit.n])) .collect(), ); ( reference_name, rows, COLUMNS.to_vec(), hits.len(), String::new(), ) } Computed::Partial(reference_name, hits, skipped) => { let rows = serde_json::Value::Array( hits.iter() .map(|hit| { serde_json::json!([ hit.name, hit.r, hit.p, hit.n, hit.zero_order_r, hit.zero_order_p ]) }) .collect(), ); let note = format!( "Partial correlations: r and p hold the control trait(s) constant (t on \ n - controls - 2 degrees of freedom); zero_order_r and zero_order_p are the \ same pairs with nothing held constant.{}{}", if skipped > 0 { format!(" {skipped} trait(s) had too few complete cases.") } else { String::new() }, if hits.is_empty() { " No trait had enough complete observations across the reference, the \ target and every control." .to_string() } else { String::new() } ); ( reference_name, rows, vec![ "trait", "r", "p", "n", "zero_order_r", "zero_order_p", ], hits.len(), note, ) } }; let mut envelope = ctx .envelope(rows) .with_columns(columns.iter().map(|name| name.to_string()).collect(), num_rows) .with_note(&format!( "{} against {} trait(s) of {}.", reference_name, num_rows, wanted.target_dataset )) .with_note(&match axis { Axis::Sample | Axis::Tissue | Axis::Partial => format!( "Method {}; at least {MIN_OVERLAP} paired {} are required, ordered by |r| \ descending with undefined values last, top {top_n}.", method.as_str(), if axis == Axis::Tissue { "tissues" } else { "observations" } ), Axis::Literature => format!( "Rows are precomputed LCorrRamin3 values, not a statistic: p and n are \ absent, and the order is by |value| descending, top {top_n}." ), }) .with_note(match axis { Axis::Sample | Axis::Tissue => { "Rows are positional: trait, r, p, n. p is the two-sided p-value of the \ engine's t statistic; n is how many pairs that correlation used." } Axis::Literature => { "Rows are positional: trait, r, p, n -- with p and n null for this axis." } Axis::Partial => partial_note.as_str(), }) .with_next("/mapping-methods", "The mapping tools GeneNetwork has run"); if axis == Axis::Partial { envelope = envelope.with_next( "/compute/correlation", "The sample axis for the same trait, without controls", ); } Ok(Json(envelope)) } /// A trait named by the dataset that holds it, GN3's `{dataset, trait_name}`. #[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)] pub struct TraitRef { /// Dataset the trait lives in. pub dataset: String, /// Trait name within that dataset (a probeset name, or a numeric id for a /// phenotype dataset). pub trait_name: String, } /// What an axis produced: the axes that answer with r/p/n, and the partial one /// that also carries the uncontrolled correlation beside it. enum Computed { Standard(String, Vec), Partial(String, Vec, usize), } /// GN2's `corr_samples_group`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum SampleGroup { /// The trait's own group -- GN2's default. Primary, /// Every individual the reference trait has a value for. All, } impl SampleGroup { pub(crate) fn parse(name: Option<&str>) -> Result { match name.unwrap_or("samples_primary").trim().to_ascii_lowercase().as_str() { "" | "samples_primary" => Ok(SampleGroup::Primary), "samples_all" => Ok(SampleGroup::All), "samples_other" => Err(Failure::NoValues( "corr_samples_group=samples_other is not supported yet; use \ samples_primary or samples_all" .to_string(), )), other => Err(Failure::NoValues(format!( "unknown corr_samples_group '{other}'; use samples_primary or samples_all." ))), } } fn as_str(&self) -> &'static str { match self { SampleGroup::Primary => "samples_primary", SampleGroup::All => "samples_all", } } } /// Why a correlation request could not be answered. pub(crate) enum Failure { NotFound(String), NoValues(String), Database(String), } impl Failure { pub(crate) fn into_api(self, rid: crate::response::RequestInfo) -> ApiError { let (status, code, title, detail) = match self { Failure::NotFound(detail) => ( axum::http::StatusCode::NOT_FOUND, "not_found", "No such dataset", detail, ), Failure::NoValues(detail) => ( axum::http::StatusCode::UNPROCESSABLE_ENTITY, "no_values", "No values to correlate", detail, ), Failure::Database(detail) => ( axum::http::StatusCode::INTERNAL_SERVER_ERROR, "database_error", "Database error", detail, ), }; ApiError::new(status, code, title, detail, "compute_correlation").with_request_id(rid) } } /// The individuals a `samples_primary` correlation runs over: the group's /// genotype panel, which is where GN2's `group.samplelist` comes from. /// /// Measured on the live site: trait 1427571_at of HC_M2_0606_P carries 99 /// values, the BXD panel holds 67 of those individuals, and GN2 correlates /// over exactly those 67 -- not cosmetic, it reorders the whole reply. /// /// A group with no genotype file on this machine falls back to its /// used-for-mapping strains; for BXD those are the same 67 individuals, so the /// axis survives either way. pub(crate) fn sample_axis(dataset: &str) -> Result>, Failure> { let Some(group) = db::get_dataset_group_name(dataset).map_err(db_failure)? else { return Ok(None); }; let root = crate::config::Config::load().genotype_files_dir.clone(); match crate::genotypes::resolve(std::path::Path::new(&root), &group, None) { Ok(resolved) => Ok(Some(resolved.panel)), Err(_) => Ok(Some( db::get_group_mapping_samples(&group).map_err(db_failure)?, )), } } /// The blocking half: resolve both datasets, fetch the values, correlate. /// /// Either side may be a ProbeSet (expression) dataset or a Publish (phenotype) /// one: GN2's `corr_dataset` menu mixes them, so a pheno trait correlated /// against an expression dataset -- the common case -- has to work. fn compute( request: &CorrelationRequest, axis: Axis, method: Method, top_n: usize, ) -> Result { match axis { Axis::Sample => compute_sample(request, method, top_n).map(|(name, hits)| Computed::Standard(name, hits)), Axis::Tissue => compute_tissue(request, method, top_n).map(|(name, hits)| Computed::Standard(name, hits)), Axis::Literature => compute_literature(request, top_n).map(|(name, hits)| Computed::Standard(name, hits)), Axis::Partial => compute_partial(request, method, top_n), } } /// GN2's `corr_type`. #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum Axis { Sample, Tissue, Literature, Partial, } impl Axis { fn parse(name: Option<&str>) -> Result { match name.unwrap_or("sample").trim().to_ascii_lowercase().as_str() { "" | "sample" => Ok(Axis::Sample), "tissue" => Ok(Axis::Tissue), "literature" | "lit" => Ok(Axis::Literature), "partial" => Ok(Axis::Partial), other => Err(Failure::NoValues(format!( "unknown correlation type '{other}'; use sample, tissue or literature." ))), } } fn as_str(&self) -> &'static str { match self { Axis::Sample => "sample", Axis::Tissue => "tissue", Axis::Literature => "literature", Axis::Partial => "partial", } } } /// The sample axis: the two traits paired over individuals. fn compute_sample( request: &CorrelationRequest, method: Method, top_n: usize, ) -> Result<(String, Vec), Failure> { let group = SampleGroup::parse(request.corr_samples_group.as_deref())?; let mut reference = resolve_reference(request)?; let kept = match group { SampleGroup::Primary => { let axis = sample_axis(&request.dataset)?; match axis { Some(samples) => reference.retain_within(&samples), // No group behind the dataset: nothing to restrict to. None => reference.len(), } } SampleGroup::All => reference.len(), }; if kept == 0 { return Err(Failure::NoValues(format!( "'{}' has no values among the individuals this request selects", request.trait_name ))); } if reference.values.is_empty() { return Err(Failure::NoValues(format!( "'{}' has no values in '{}', so there is nothing to correlate", request.trait_name, request.dataset ))); } if reference.len() < MIN_OVERLAP { return Err(Failure::NoValues(format!( "'{}' has {} value(s) in '{}'; a correlation needs at least {MIN_OVERLAP}", request.trait_name, reference.len(), request.dataset ))); } let rows = resolve_target_rows(&request.target_dataset, &reference.names)?; let hits = correlation::correlate_many(&reference, method, rows, top_n); Ok(( format!( "{} ({}, {} observations, {})", request.trait_name, request.dataset, reference.len(), group.as_str() ), hits, )) } /// The tissue axis: the two traits paired over tissue systems. /// /// The values are keyed by `TissueID` and paired *by tissue*, which is where /// this departs from GN2 on purpose. GN2 keeps each gene's tissue values as a /// bare list and pairs two lists by position, on the assumption that every /// probe covers the same tissues. On this database it does not: every probe /// carries exactly 26 values (liver, kidney, spleen, ... -- peripheral tissues) /// or exactly 32 (striatum, hippocampus, neocortex, ... -- brain regions), and /// the two sets are **disjoint**. Pairing such lists by position correlates /// liver expression with striatum expression because they happen to be entry /// one of their respective lists, which is not a correlation of anything. So a /// gene from one panel has *no* tissue correlation against a gene from the /// other, and pairing by tissue makes that visible instead of inventing a /// number. fn compute_tissue( request: &CorrelationRequest, method: Method, top_n: usize, ) -> Result<(String, Vec), Failure> { let dataset = db::get_dataset_by_name(&request.dataset) .map_err(db_failure)? .ok_or_else(|| { Failure::NoValues(format!( "the tissue axis needs a probeset dataset; '{}' is not one", request.dataset )) })?; let symbols = db::get_dataset_symbols(dataset.id).map_err(db_failure)?; let reference_symbol = symbols .iter() .find(|(name, _)| name == &request.trait_name) .map(|(_, symbol)| symbol.clone()) .ok_or_else(|| { Failure::NoValues(format!( "'{}' in '{}' has no gene symbol, so it has no tissue values", request.trait_name, request.dataset )) })?; let target_dataset = db::get_dataset_by_name(&request.target_dataset) .map_err(db_failure)? .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.target_dataset)))?; let target_symbols = db::get_dataset_symbols(target_dataset.id).map_err(db_failure)?; let mut wanted: Vec = target_symbols.iter().map(|(_, s)| s.clone()).collect(); wanted.push(reference_symbol.clone()); wanted.sort(); wanted.dedup(); let tissue = db::get_tissue_values(&wanted).map_err(db_failure)?; let reference_values = tissue.get(&reference_symbol.to_lowercase()).ok_or_else(|| { Failure::NoValues(format!( "'{reference_symbol}' has no tissue values in TissueProbeSetData" )) })?; // The tissue axis names its observations by tissue id, so the engine's // alignment by name becomes an alignment by tissue for free. let reference = correlation::Reference::from_pairs( reference_values .iter() .map(|(tissue_id, value)| (format!("tissue:{tissue_id}"), Some(*value))), ); let rows: Vec<(String, String, Option)> = target_symbols .iter() .flat_map(|(probeset, symbol)| { tissue .get(&symbol.to_lowercase()) .map(|values| (probeset.clone(), values)) .into_iter() }) .flat_map(|(probeset, values)| { values.iter().map(move |(tissue_id, value)| { ( probeset.clone(), format!("tissue:{tissue_id}"), Some(*value), ) }) }) .collect(); let hits = correlation::correlate_many(&reference, method, rows, top_n); Ok(( format!( "{} (symbol {}, {} tissues) by tissue", request.trait_name, reference_symbol, reference.len() ), hits, )) } /// The literature axis: a precomputed gene-by-gene value, not a statistic. /// /// GN2 looks the pair up in `LCorrRamin3`, both directions, after translating /// rat and human ids to mouse through `GeneIDXRef`. Two deliberate differences: /// a trait with no gene id or no row is **left out** rather than reported as /// 0.0 (GN2 fills the tail of its top-N with zeroes, which reads as "measured, /// no relationship"), and the reply says how many were left out. fn compute_literature( request: &CorrelationRequest, top_n: usize, ) -> Result<(String, Vec), Failure> { let species = db::get_dataset_species(&request.dataset) .map_err(db_failure)? .unwrap_or_else(|| "mouse".to_string()); let dataset = db::get_dataset_by_name(&request.dataset) .map_err(db_failure)? .ok_or_else(|| { Failure::NoValues(format!( "the literature axis needs a probeset dataset; '{}' is not one", request.dataset )) })?; let reference_gene = db::get_dataset_geneids(dataset.id) .map_err(db_failure)? .into_iter() .find(|(name, _)| name == &request.trait_name) .map(|(_, gene_id)| gene_id) .ok_or_else(|| { Failure::NoValues(format!( "'{}' in '{}' has no gene id, so it has no literature correlation", request.trait_name, request.dataset )) })?; let target_dataset = db::get_dataset_by_name(&request.target_dataset) .map_err(db_failure)? .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.target_dataset)))?; let target_genes = db::get_dataset_geneids(target_dataset.id).map_err(db_failure)?; // GN2 skips ids that name several genes ("1;2"). let single: Vec<(String, String)> = target_genes .into_iter() .filter(|(_, gene_id)| !gene_id.contains(';')) .collect(); let reference_mouse = db::mouse_gene_ids(&species, std::slice::from_ref(&reference_gene)) .map_err(db_failure)? .get(&reference_gene) .copied() .ok_or_else(|| { Failure::NoValues(format!( "gene id '{reference_gene}' has no mouse id in GeneIDXRef" )) })?; let raw_ids: Vec = single.iter().map(|(_, gene_id)| gene_id.clone()).collect(); let mouse = db::mouse_gene_ids(&species, &raw_ids).map_err(db_failure)?; let mut by_mouse: Vec<(i64, String)> = Vec::new(); for (probeset, gene_id) in &single { if let Some(mouse_id) = mouse.get(gene_id) { by_mouse.push((*mouse_id, probeset.clone())); } } let targets: Vec = by_mouse.iter().map(|(gene, _)| *gene).collect(); let values = db::get_literature_values(reference_mouse, &targets).map_err(db_failure)?; let mut hits: Vec = by_mouse .iter() .filter_map(|(gene, probeset)| { values.get(gene).map(|value| correlation::Hit { name: probeset.clone(), // No statistic was computed: serde_json writes NaN as null, // which is what "not a p-value" should look like here. r: *value, p: f64::NAN, n: 0, }) }) .collect(); let missing = single.len().saturating_sub(hits.len()); hits.sort_by(|a, b| { b.r.abs() .partial_cmp(&a.r.abs()) .unwrap_or(std::cmp::Ordering::Equal) }); hits.truncate(top_n); Ok(( format!( "{} (gene id {reference_gene}, mouse {reference_mouse}) against '{}' in LCorrRamin3; {missing} trait(s) had no gene id or no LCorrRamin3 row and are left out", request.trait_name, request.target_dataset ), hits, )) } /// The partial axis: the two traits with the control traits held constant. /// /// GN3 computes this with `pingouin.partial_corr` over a pandas frame of /// complete cases; the maths is in `crate::partial`, and the reply carries the /// uncontrolled correlation beside each partial one so a reader can see what /// the controls did. /// /// Two things worth knowing when comparing with GN3: /// /// * GN3 accepts a correlation from **4** complete observations. With one /// control that leaves a single degree of freedom, so the coefficient can be /// any value in [-1, 1] and its p-value is meaningless. We keep the six-pair /// floor the rest of the API uses and require `n >= controls + 3`. /// * GN3's `method` reaches pingouin, which rank-transforms for /// `method="spearman"` and then regresses. Ours does the same, with averaged /// ranks, so the two agree. fn compute_partial( request: &CorrelationRequest, method: Method, top_n: usize, ) -> Result { if request.control_traits.is_empty() { return Err(Failure::NoValues( "a partial correlation needs at least one control trait in 'control_traits'" .to_string(), )); } let reference = resolve_reference(request)?; let mut controls: Vec = Vec::with_capacity(request.control_traits.len()); for control in &request.control_traits { controls.push(values_of(&control.dataset, &control.trait_name)?); } // The controls have to overlap the reference, or there is nothing to hold // constant and the "partial" r would silently be the zero-order one. for control in &controls { let shared = control .names .iter() .filter(|name| reference.names.contains(name)) .count(); if shared < crate::correlation::MIN_OVERLAP { return Err(Failure::NoValues(format!( "a control trait shares only {shared} individual(s) with '{}'; a partial correlation needs at least {}", request.trait_name, crate::correlation::MIN_OVERLAP ))); } } let rows = resolve_target_rows(&request.target_dataset, &reference.names)?; let hits = crate::partial::correlate_partial(&reference, &controls, rows, method, top_n); let controls_named = request .control_traits .iter() .map(|control| format!("{}::{}", control.dataset, control.trait_name)) .collect::>() .join(", "); Ok(Computed::Partial( // No "against " here: the handler writes that sentence, and // naming it twice read as "against 'X' against 5 trait(s) of X". format!( "{} ({} observations, {} control(s): {})", request.trait_name, reference.len(), controls.len(), controls_named ), hits, 0, )) } /// The reference trait's values, from whichever kind of dataset holds it. fn resolve_reference(request: &CorrelationRequest) -> Result { values_of(&request.dataset, &request.trait_name) } /// One trait's values, from whichever kind of dataset holds it: a probeset by /// name, a phenotype trait by numeric id. fn values_of(dataset: &str, trait_name: &str) -> Result { let request = CorrelationRequest { dataset: dataset.to_string(), trait_name: trait_name.to_string(), ..CorrelationRequest::default() }; if let Some(dataset) = db::get_dataset_by_name(&request.dataset).map_err(db_failure)? { let values = db::get_dataset_probeset_values(dataset.id, &request.trait_name) .map_err(db_failure)?; return Ok(correlation::Reference::from_pairs( values.into_iter().map(|value| (value.name, value.value)), )); } if let Some(publish) = db::get_publish_freeze_by_name(&request.dataset).map_err(db_failure)? { // A publish trait is addressed by its numeric id within the set. let trait_id: u32 = request.trait_name.trim().parse().map_err(|_| { Failure::NoValues(format!( "'{}' is a phenotype dataset, so its traits are addressed by id (a number), \ not by name '{}'", request.dataset, request.trait_name )) })?; let values = db::get_group_trait_values(publish.inbred_set_id, trait_id) .map_err(db_failure)?; let values: Vec<(String, Option)> = values .into_iter() .map(|value| (value.name, value.value)) .collect(); return Ok(correlation::Reference::from_pairs(values)); } Err(Failure::NotFound(format!( "no dataset named '{}'", request.dataset ))) } /// Every candidate trait's values in the target dataset, restricted to the /// reference's individuals. fn resolve_target_rows( dataset: &str, individuals: &[String], ) -> Result)>, Failure> { if let Some(target) = db::get_dataset_by_name(dataset).map_err(db_failure)? { return db::get_dataset_values_for_individuals(target.id, individuals) .map_err(db_failure); } if let Some(publish) = db::get_publish_freeze_by_name(dataset).map_err(db_failure)? { return db::get_publish_values_for_individuals(publish.inbred_set_id, individuals) .map_err(db_failure); } Err(Failure::NotFound(format!("no dataset named '{dataset}'"))) } pub(crate) fn db_failure(error: db::DbError) -> Failure { Failure::Database(error.to_string()) }