diff options
| author | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-15 01:37:51 +0000 |
|---|---|---|
| committer | Pjotr Prins <pjotr.public01@thebird.nl> | 2026-11-15 01:37:51 +0000 |
| commit | c8ce261f2bf499ab3260ff5ce186c801491b9fea (patch) | |
| tree | a5d26ae5871df85fc13d7ec75e9aaccbdaebc171 /src | |
| parent | 41c70f042731e36fb91b8fb3f4f393a4aa654e9d (diff) | |
| download | genenetwork4-c8ce261f2bf499ab3260ff5ce186c801491b9fea.tar.gz | |
correlation: the sample-axis endpoint, synchronous
POST /compute/correlation now computes instead of returning canned data: one reference trait against every trait of a target dataset, observations being individuals, pearson or spearman, top-N by |r|. The reply is positional -- trait, r, p, n (column names in meta.columns) -- with the method, the minimum overlap and the top-N stated in the notes. `bicor` is refused with the methods that exist; a bad top_n is refused rather than clamped; a dataset or trait with nothing to correlate says so instead of returning an empty list. src/db.rs gains the bulk fetch the sample axis needs: every value of every trait of a dataset restricted to the reference trait's individuals, one record per (trait, individual, value), ordered by trait so the caller groups as it streams. It never loads a dataset whole -- ProbeSetData is 5.3e9 rows, so the query narrows by dataset and by individuals, which is also why the sweep, not the arithmetic, is the whole cost. Measured on the live db_webqtl: HC_M2_0606_P / 1427571_at against its own dataset, top 5, HTTP 200 in 16 s; the trait correlates with itself at 1.0 over 99 observations, then 0.802 (p 1.9e-23), 0.790, -0.789, -0.788. Synchronous is affordable at this size; GN2's job indirection exists for top-N 20000 across datasets, which is the next thing to time. Known and deliberate for now: the publish/genotype axes are not wired (the fetch is probeset-only), and a perfect correlation reports p as null -- GSL's t statistic is undefined at |r| = 1, whose r comes out as 1.0000000000000002, so 1 - r^2 is slightly negative and the engine returns NaN. Reproducing the engine exactly, rather than "fixing" the p-value here, is the decision recorded in docs/correlations.md.
Diffstat (limited to 'src')
| -rw-r--r-- | src/db.rs | 42 | ||||
| -rw-r--r-- | src/routes/correlation.rs | 239 |
2 files changed, 231 insertions, 50 deletions
diff --git a/src/db.rs b/src/db.rs index 9b2428f..7464125 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1552,6 +1552,48 @@ pub fn get_dataset_probeset_values( .map_err(DbError::from) } +/// Every value of every trait in a probeset dataset, restricted to a set of +/// individuals: one record per (trait, individual, value). +/// +/// This is the bulk half of the sample-axis correlation, and the whole cost of +/// it: `ProbeSetData` is 5.3e9 rows, so the query narrows by the dataset and by +/// the individuals the reference trait has values for, and never loads a +/// dataset whole. Ordering is by trait then individual so the caller can group +/// as it streams. +pub fn get_dataset_values_for_individuals( + dataset_id: u16, + individuals: &[String], +) -> Result<Vec<(String, String, Option<f64>)>, DbError> { + if individuals.is_empty() { + return Ok(Vec::new()); + } + let pool = pool()?; + let mut conn = pool.get_conn()?; + // The mysql crate sends a Vec as one value, so the IN list is written out + // as placeholders. + let placeholders = std::iter::repeat_n("?", individuals.len()) + .collect::<Vec<_>>() + .join(","); + let query = format!( + "SELECT ProbeSet.Name, Strain.Name, ProbeSetData.value \ + FROM ProbeSetData, Strain, ProbeSetXRef, ProbeSet \ + WHERE ProbeSetXRef.ProbeSetFreezeId = ? \ + AND ProbeSetXRef.ProbeSetId = ProbeSet.Id \ + AND ProbeSetXRef.DataId = ProbeSetData.Id \ + AND ProbeSetData.StrainId = Strain.Id \ + AND Strain.Name IN ({placeholders}) \ + ORDER BY ProbeSet.Name, Strain.Name" + ); + let mut params = mysql::Params::Positional(vec![mysql::Value::from(dataset_id)]); + if let mysql::Params::Positional(values) = &mut params { + values.extend(individuals.iter().map(|name| mysql::Value::from(name.as_str()))); + } + conn.exec_map(&query, params, |(trait_name, individual, value)| { + (trait_name, individual, value) + }) + .map_err(DbError::from) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/routes/correlation.rs b/src/routes/correlation.rs index b0b6d16..36faedb 100644 --- a/src/routes/correlation.rs +++ b/src/routes/correlation.rs @@ -1,47 +1,57 @@ -//! Correlation domain endpoint. +//! Correlation endpoint: one trait against every trait of a target dataset, +//! observations being individuals (the sample axis). //! -//! STUB: returns canned data in the descriptive envelope so the route -//! and the response shape can be exercised end to end. The engine -//! integration is specified in topics/gn4/gn4-correlation.gmi -//! (gn-gemtext-threads): correlation_rust in-process, rev-pinned. +//! The arithmetic and the policy live in `src/correlation.rs`; what this module +//! does is resolve the request, fetch the values, and shape the reply. It is +//! synchronous by decision: one POST computes and answers, no redis, no job +//! (see docs/correlations.md). use axum::Json; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use gnauth_macros::gnauth; use crate::auth::AuthToken; +use crate::correlation::{self, Method, DEFAULT_TOP_N, MIN_OVERLAP}; +use crate::db; use crate::error::{ApiError, Unauthorized}; use crate::response::{Ctx, Envelope}; -/// The request shape is part of the API contract (validation and -/// openapi); the stub computes nothing yet, so its fields stay unread -/// until the correlation engine integration reads them. -#[derive(Deserialize, utoipa::ToSchema)] -#[allow(dead_code)] +/// The reply's columns, in this order, one row per correlated trait. +pub const COLUMNS: [&str; 4] = ["trait", "r", "p", "n"]; + +/// A sample-axis correlation request. +/// +/// `dataset` is the reference trait's own dataset, `target_dataset` the one to +/// correlate it against -- GN2's `corr_dataset`. Missing `method` means +/// pearson, missing `top_n` the GN2 default of 500. +#[derive(Deserialize, Clone, utoipa::ToSchema)] pub struct CorrelationRequest { - /// Correlation method: pearson, spearman or bicor. - pub method: String, - /// Dataset to correlate against, e.g. "BXD liver". + /// Dataset the reference trait lives in, e.g. "HC_M2_0606_P". pub dataset: String, - /// The trait to correlate, e.g. "1435863_at". + /// The reference trait, e.g. "1427571_at". pub trait_name: String, -} - -#[derive(Serialize, utoipa::ToSchema)] -pub struct CorrelationHit { - pub trait_id: u32, - pub correlation: f64, + /// Dataset to correlate against, e.g. "BXDPublish" or another ProbeSet set. + pub target_dataset: String, + /// "pearson" (default) or "spearman". + #[serde(default)] + pub method: Option<String>, + /// How many hits to return, strongest by |r| first. Defaults to 500. + #[serde(default)] + pub top_n: Option<usize>, } #[utoipa::path(post, path = "/compute/correlation", - summary = "Compute a sample correlation (STUB: canned data)", - description = "Phase A target per topics/gn4/gn4-correlation.gmi. \ -Currently returns canned data wrapped in the descriptive envelope \ -({data, next}); the correlation_rust engine integration is pending.", + summary = "Correlate one trait against a dataset (sample axis)", + description = "One trait against every trait of a target dataset, \ +observations being individuals. Synchronous: the reply is the result. \ +Correlations are computed by the correlation_rust engine (pearson/spearman), \ +need at least six paired observations, and are ordered by |r| descending.", request_body = CorrelationRequest, responses( - (status = 200, description = "Correlated traits, strongest first", body = Envelope<Vec<CorrelationHit>>), + (status = 200, description = "Correlated traits, strongest first", body = Envelope<serde_json::Value>), + (status = 404, description = "No such dataset", body = ApiError), + (status = 422, description = "Unknown method, or a trait with no values", body = ApiError), Unauthorized, (status = 500, description = "Internal error", body = ApiError) ))] @@ -50,30 +60,159 @@ pub async fn correlation( ctx: Ctx, _token: AuthToken, Json(request): Json<CorrelationRequest>, -) -> Result<Json<Envelope<Vec<CorrelationHit>>>, ApiError> { - // The stub accepts the request but computes nothing yet. - let _ = &request.method; +) -> Result<Json<Envelope<serde_json::Value>>, ApiError> { + let rid = ctx.request_info(); + let fail = |status: axum::http::StatusCode, code: &str, title: &str, detail: String| { + ApiError::new(status, code, title, detail, "compute_correlation") + .with_request_id(rid.clone()) + }; + + let method = Method::parse(request.method.as_deref().unwrap_or("pearson")) + .map_err(|detail| { + fail( + axum::http::StatusCode::UNPROCESSABLE_ENTITY, + "invalid_method", + "Invalid correlation method", + detail, + ) + })?; + let top_n = request.top_n.unwrap_or(DEFAULT_TOP_N); + if top_n == 0 || top_n > correlation::MAX_TOP_N { + return Err(fail( + axum::http::StatusCode::UNPROCESSABLE_ENTITY, + "invalid_top_n", + "Invalid top_n", + format!( + "top_n must be between 1 and {}; got {top_n}.", + correlation::MAX_TOP_N + ), + )); + } + + let wanted = request.clone(); + let computed = tokio::task::spawn_blocking(move || compute(&request, method, top_n)) + .await + .map_err(|error| { + fail( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "internal_error", + "Internal error", + error.to_string(), + ) + })?; + let (reference_name, hits) = computed.map_err(|failure| failure.into_api(rid.clone()))?; + + let num_rows = hits.len(); + let rows = serde_json::Value::Array( + hits.iter() + .map(|hit| serde_json::json!([hit.name, hit.r, hit.p, hit.n])) + .collect(), + ); + Ok(Json( - ctx.envelope(vec![ - CorrelationHit { - trait_id: 1_435_863, - correlation: 0.91, - }, - CorrelationHit { - trait_id: 1_435_987, - correlation: 0.87, - }, - ]) - .with_note("STUB: canned data; nothing was computed.") - // `/compute/correlation` is a POST, so it belongs in `actions`, not in - // `next`: following a `next` link must be a GET that returns 200. - .with_action(crate::response::SuggestedCall { - method: "POST".to_string(), - href: "/compute/correlation".to_string(), - description: "Correlate another pair of traits".to_string(), - body: None, - }) - .with_next("/species", "Narrow by species") - .with_links(crate::suggest::root_links()), + ctx.envelope(rows) + .with_columns(COLUMNS.iter().map(|name| name.to_string()).collect(), num_rows) + .with_note(&format!( + "{} against {} trait(s) of {}, observations being individuals.", + reference_name, num_rows, wanted.target_dataset + )) + .with_note(&format!( + "Method {}; at least {MIN_OVERLAP} paired observations are required, \ + ordered by |r| descending with undefined values last, top {top_n}.", + method.as_str() + )) + .with_note( + "Rows are positional: trait, r, p, n. p is the two-sided p-value of the \ + engine's t statistic; n is how many pairs that correlation used.", + ) + .with_next("/mapping-methods", "The mapping tools GeneNetwork has run"), + )) +} + +/// Why a correlation request could not be answered. +enum Failure { + NotFound(String), + NoValues(String), + Database(String), +} + +impl Failure { + fn into_api(self, rid: crate::response::RequestInfo) -> ApiError { + let (status, code, title, detail) = match self { + Failure::NotFound(detail) => ( + axum::http::StatusCode::NOT_FOUND, + "not_found", + "No such dataset", + detail, + ), + Failure::NoValues(detail) => ( + axum::http::StatusCode::UNPROCESSABLE_ENTITY, + "no_values", + "No values to correlate", + detail, + ), + Failure::Database(detail) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + "database_error", + "Database error", + detail, + ), + }; + ApiError::new(status, code, title, detail, "compute_correlation").with_request_id(rid) + } +} + +/// The blocking half: resolve both datasets, fetch the values, correlate. +fn compute( + request: &CorrelationRequest, + method: Method, + top_n: usize, +) -> Result<(String, Vec<correlation::Hit>), Failure> { + let reference_dataset = db::get_dataset_by_name(&request.dataset) + .map_err(|error| Failure::Database(error.to_string()))? + .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.dataset)))?; + + let values = db::get_dataset_probeset_values(reference_dataset.id, &request.trait_name) + .map_err(|error| Failure::Database(error.to_string()))?; + let reference = correlation::Reference::from_pairs( + values + .into_iter() + .map(|value| (value.name, value.value)), + ); + if reference.is_empty() { + return Err(Failure::NoValues(format!( + "'{}' has no values in '{}', so there is nothing to correlate", + request.trait_name, request.dataset + ))); + } + if reference.len() < MIN_OVERLAP { + return Err(Failure::NoValues(format!( + "'{}' has {} value(s) in '{}'; a correlation needs at least {MIN_OVERLAP}", + request.trait_name, + reference.len(), + request.dataset + ))); + } + + let target = db::get_dataset_by_name(&request.target_dataset) + .map_err(|error| Failure::Database(error.to_string()))? + .ok_or_else(|| { + Failure::NotFound(format!( + "no dataset named '{}'", + request.target_dataset + )) + })?; + + let rows = db::get_dataset_values_for_individuals(target.id, &reference.names) + .map_err(|error| Failure::Database(error.to_string()))?; + let hits = correlation::correlate_many(&reference, method, rows, top_n); + Ok(( + format!( + "{} ({}, {} observations)", + request.trait_name, + request.dataset, + reference.len() + ), + hits, )) } |
