diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/db.rs | 192 | ||||
| -rw-r--r-- | src/routes/correlation.rs | 281 |
2 files changed, 462 insertions, 11 deletions
diff --git a/src/db.rs b/src/db.rs index 2f9caf0..505ef51 100644 --- a/src/db.rs +++ b/src/db.rs @@ -1590,6 +1590,198 @@ pub fn get_group_mapping_samples(group_name: &str) -> Result<Vec<String>, DbErro .collect()) } +/// The species of the group a dataset belongs to (`InbredSet.SpeciesId` -> +/// `Species.Name`), which is how GN2 decides whether a gene id needs +/// translating to mouse before it is looked up in `LCorrRamin3`. +pub fn get_dataset_species(dataset_name: &str) -> Result<Option<String>, DbError> { + let pool = pool()?; + let mut conn = pool.get_conn()?; + conn.exec_first( + "SELECT Species.Name FROM ProbeSetFreeze \ + JOIN ProbeFreeze ON ProbeFreeze.ProbeFreezeId = ProbeSetFreeze.ProbeFreezeId \ + JOIN InbredSet ON InbredSet.Id = ProbeFreeze.InbredSetId \ + JOIN Species ON Species.Id = InbredSet.SpeciesId \ + WHERE ProbeSetFreeze.Name = :name LIMIT 1", + mysql::params! { "name" => dataset_name }, + ) + .map_err(DbError::from) +} + +/// `ProbeSet.Name` -> `ProbeSet.Symbol` for one probeset dataset, the axis +/// the tissue correlation is keyed on. +pub fn get_dataset_symbols(dataset_id: u16) -> Result<Vec<(String, String)>, DbError> { + let pool = pool()?; + let mut conn = pool.get_conn()?; + conn.exec_map( + "SELECT ProbeSet.Name, ProbeSet.Symbol \ + FROM ProbeSet JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetId = ProbeSet.Id \ + WHERE ProbeSetXRef.ProbeSetFreezeId = :dataset_id \ + AND ProbeSet.Symbol IS NOT NULL AND ProbeSet.Symbol <> '' \ + ORDER BY ProbeSet.Id", + mysql::params! { "dataset_id" => dataset_id }, + |(name, symbol): (String, String)| (name, symbol), + ) + .map_err(DbError::from) +} + +/// `ProbeSet.Name` -> `ProbeSetXRef.GeneId` for one probeset dataset, the axis +/// the literature correlation is keyed on. +pub fn get_dataset_geneids(dataset_id: u16) -> Result<Vec<(String, String)>, DbError> { + let pool = pool()?; + let mut conn = pool.get_conn()?; + conn.exec_map( + // The gene id lives on ProbeSet, not on ProbeSetXRef: GN3's own mapping + // is `ProbeSet.GeneId`, and ProbeSetXRef has no GeneId column here. + "SELECT ProbeSet.Name, ProbeSet.GeneId \ + FROM ProbeSet JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetId = ProbeSet.Id \ + WHERE ProbeSetXRef.ProbeSetFreezeId = :dataset_id \ + AND ProbeSet.GeneId IS NOT NULL AND ProbeSet.GeneId <> '' \ + ORDER BY ProbeSet.Id", + mysql::params! { "dataset_id" => dataset_id }, + |(name, gene_id): (String, String)| (name, gene_id), + ) + .map_err(DbError::from) +} + +/// Tissue values keyed by *symbol*, as `(TissueID, value)` pairs. +/// +/// GN2 reduces a symbol to one probe -- the one with the largest `Mean` in +/// `TissueProbeSetXRef` -- and then reads that probe's values. It keeps them as +/// a bare list and pairs two lists *by position*, which is only meaningful if +/// both probes cover the same tissues: on this database they do not. The +/// values are keyed by `TissueID` here so a caller can pair them by tissue +/// rather than by position. +pub fn get_tissue_values( + symbols: &[String], +) -> Result<std::collections::HashMap<String, Vec<(u16, f64)>>, DbError> { + if symbols.is_empty() { + return Ok(std::collections::HashMap::new()); + } + let pool = pool()?; + let mut conn = pool.get_conn()?; + let placeholders = std::iter::repeat("?") + .take(symbols.len()) + .collect::<Vec<_>>() + .join(", "); + let rows: Vec<(String, u16, f64)> = conn.exec( + format!( + "SELECT xref.Symbol, data.TissueID, data.value \ + FROM TissueProbeSetData data \ + JOIN TissueProbeSetXRef xref ON xref.DataId = data.Id \ + JOIN (SELECT Symbol, MAX(Mean) AS best_mean FROM TissueProbeSetXRef \ + WHERE TissueProbeSetFreezeId = 1 AND Symbol IN ({placeholders}) \ + GROUP BY Symbol) best \ + ON best.Symbol = xref.Symbol AND xref.Mean = best.best_mean \ + WHERE xref.TissueProbeSetFreezeId = 1 \ + ORDER BY xref.Symbol, data.TissueID" + ), + symbols.iter().collect::<Vec<_>>(), + )?; + let mut out: std::collections::HashMap<String, Vec<(u16, f64)>> = + std::collections::HashMap::new(); + for (symbol, tissue_id, value) in rows { + out.entry(symbol.to_lowercase()).or_default().push((tissue_id, value)); + } + Ok(out) +} + +/// Gene ids translated to the mouse id `LCorrRamin3` is keyed on. +/// +/// GN2's `convert_to_mouse_gene_id`: a mouse id is used as it stands, a rat or +/// human id is looked up in `GeneIDXRef`. +pub fn mouse_gene_ids( + species: &str, + gene_ids: &[String], +) -> Result<std::collections::HashMap<String, i64>, DbError> { + let mut out = std::collections::HashMap::new(); + if gene_ids.is_empty() { + return Ok(out); + } + let numeric: Vec<(String, i64)> = gene_ids + .iter() + .filter_map(|g| g.trim().parse::<i64>().ok().map(|v| (g.clone(), v))) + .collect(); + if species.eq_ignore_ascii_case("mouse") { + for (raw, value) in numeric { + out.insert(raw, value); + } + return Ok(out); + } + let column = match species.to_ascii_lowercase().as_str() { + "rat" => "rat", + "human" => "human", + _ => return Ok(out), + }; + let pool = pool()?; + let mut conn = pool.get_conn()?; + let placeholders = std::iter::repeat("?") + .take(numeric.len()) + .collect::<Vec<_>>() + .join(", "); + let rows: Vec<(i64, i64)> = conn.exec( + format!("SELECT {column}, mouse FROM GeneIDXRef WHERE {column} IN ({placeholders})"), + numeric.iter().map(|(_, value)| *value).collect::<Vec<_>>(), + )?; + let by_source: std::collections::HashMap<i64, i64> = rows.into_iter().collect(); + for (raw, value) in numeric { + if let Some(mouse) = by_source.get(&value) { + out.insert(raw, *mouse); + } + } + Ok(out) +} + +/// `LCorrRamin3` values for one reference gene against many targets, in both +/// directions, preferring GN2's direction (`GeneId1` = target, `GeneId2` = +/// reference) where both rows exist. +/// +/// Chunked because a probeset dataset has tens of thousands of gene ids and +/// MySQL refuses a statement with more than 65 535 placeholders (measured: +/// "Prepared statement contains too many placeholders" on the first attempt). +pub fn get_literature_values( + reference: i64, + targets: &[i64], +) -> Result<std::collections::HashMap<i64, f64>, DbError> { + let mut forward: std::collections::HashMap<i64, f64> = std::collections::HashMap::new(); + let mut reverse: std::collections::HashMap<i64, f64> = std::collections::HashMap::new(); + for chunk in targets.chunks(LITERATURE_CHUNK) { + let pool = pool()?; + let mut conn = pool.get_conn()?; + let placeholders = std::iter::repeat("?") + .take(chunk.len()) + .collect::<Vec<_>>() + .join(", "); + let mut params: Vec<i64> = Vec::with_capacity(chunk.len() * 2 + 2); + params.push(reference); + params.extend(chunk.iter().copied()); + params.push(reference); + params.extend(chunk.iter().copied()); + let rows: Vec<(i64, i64, f64)> = conn.exec( + format!( + "SELECT GeneId1, GeneId2, value FROM LCorrRamin3 \ + WHERE (GeneId2 = ? AND GeneId1 IN ({placeholders})) \ + OR (GeneId1 = ? AND GeneId2 IN ({placeholders}))" + ), + params, + )?; + for (gene1, gene2, value) in rows { + if gene2 == reference { + forward.insert(gene1, value); + } else if gene1 == reference { + reverse.insert(gene2, value); + } + } + } + for (target, value) in reverse { + forward.entry(target).or_insert(value); + } + Ok(forward) +} + +/// How many genes go into one `IN (...)` clause: MySQL's placeholder ceiling is +/// 65 535 per statement, and this query uses two lists. +const LITERATURE_CHUNK: usize = 4_000; + /// Every value of every trait in a probeset dataset, restricted to a set of /// individuals: one record per (trait, individual, value). /// diff --git a/src/routes/correlation.rs b/src/routes/correlation.rs index 9b91735..01ccb7a 100644 --- a/src/routes/correlation.rs +++ b/src/routes/correlation.rs @@ -33,6 +33,11 @@ pub struct CorrelationRequest { pub trait_name: String, /// Dataset to correlate against, e.g. "BXDPublish" or another ProbeSet set. pub target_dataset: String, + /// What is correlated, GN2's `corr_type`: "sample" (default) pairs the two + /// traits over individuals, "tissue" pairs them over tissue systems, and + /// "literature" reads a precomputed gene-by-gene value. + #[serde(default)] + pub r#type: Option<String>, /// "pearson" (default) or "spearman". #[serde(default)] pub method: Option<String>, @@ -95,8 +100,11 @@ pub async fn correlation( )); } + let axis = Axis::parse(request.r#type.as_deref()).map_err(|failure| { + failure.into_api(rid.clone()) + })?; let wanted = request.clone(); - let computed = tokio::task::spawn_blocking(move || compute(&request, method, top_n)) + let computed = tokio::task::spawn_blocking(move || compute(&request, axis, method, top_n)) .await .map_err(|error| { fail( @@ -119,18 +127,34 @@ pub async fn correlation( ctx.envelope(rows) .with_columns(COLUMNS.iter().map(|name| name.to_string()).collect(), num_rows) .with_note(&format!( - "{} against {} trait(s) of {}, observations being individuals.", + "{} against {} trait(s) of {}.", reference_name, num_rows, wanted.target_dataset )) - .with_note(&format!( - "Method {}; at least {MIN_OVERLAP} paired observations are required, \ - ordered by |r| descending with undefined values last, top {top_n}.", - method.as_str() - )) - .with_note( - "Rows are positional: trait, r, p, n. p is the two-sided p-value of the \ - engine's t statistic; n is how many pairs that correlation used.", - ) + .with_note(&match axis { + Axis::Sample | Axis::Tissue => format!( + "Method {}; at least {MIN_OVERLAP} paired {} are required, ordered by |r| \ + descending with undefined values last, top {top_n}.", + method.as_str(), + if axis == Axis::Tissue { + "tissues" + } else { + "observations" + } + ), + Axis::Literature => format!( + "Rows are precomputed LCorrRamin3 values, not a statistic: p and n are \ + absent, and the order is by |value| descending, top {top_n}." + ), + }) + .with_note(match axis { + Axis::Sample | Axis::Tissue => { + "Rows are positional: trait, r, p, n. p is the two-sided p-value of the \ + engine's t statistic; n is how many pairs that correlation used." + } + Axis::Literature => { + "Rows are positional: trait, r, p, n -- with p and n null for this axis." + } + }) .with_next("/mapping-methods", "The mapping tools GeneNetwork has run"), )) } @@ -231,6 +255,53 @@ fn sample_axis(dataset: &str) -> Result<Option<Vec<String>>, Failure> { /// against an expression dataset -- the common case -- has to work. fn compute( request: &CorrelationRequest, + axis: Axis, + method: Method, + top_n: usize, +) -> Result<(String, Vec<correlation::Hit>), Failure> { + match axis { + Axis::Sample => compute_sample(request, method, top_n), + Axis::Tissue => compute_tissue(request, method, top_n), + Axis::Literature => compute_literature(request, top_n), + } +} + +/// GN2's `corr_type`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Axis { + Sample, + Tissue, + Literature, +} + +impl Axis { + fn parse(name: Option<&str>) -> Result<Self, Failure> { + match name.unwrap_or("sample").trim().to_ascii_lowercase().as_str() { + "" | "sample" => Ok(Axis::Sample), + "tissue" => Ok(Axis::Tissue), + "literature" | "lit" => Ok(Axis::Literature), + "partial" => Err(Failure::NoValues( + "partial correlation is not implemented yet; use sample, tissue or literature" + .to_string(), + )), + other => Err(Failure::NoValues(format!( + "unknown correlation type '{other}'; use sample, tissue or literature." + ))), + } + } + + fn as_str(&self) -> &'static str { + match self { + Axis::Sample => "sample", + Axis::Tissue => "tissue", + Axis::Literature => "literature", + } + } +} + +/// The sample axis: the two traits paired over individuals. +fn compute_sample( + request: &CorrelationRequest, method: Method, top_n: usize, ) -> Result<(String, Vec<correlation::Hit>), Failure> { @@ -282,6 +353,194 @@ fn compute( )) } +/// The tissue axis: the two traits paired over tissue systems. +/// +/// The values are keyed by `TissueID` and paired *by tissue*, which is where +/// this departs from GN2 on purpose. GN2 keeps each gene's tissue values as a +/// bare list and pairs two lists by position, on the assumption that every +/// probe covers the same tissues. On this database it does not: every probe +/// carries exactly 26 values (liver, kidney, spleen, ... -- peripheral tissues) +/// or exactly 32 (striatum, hippocampus, neocortex, ... -- brain regions), and +/// the two sets are **disjoint**. Pairing such lists by position correlates +/// liver expression with striatum expression because they happen to be entry +/// one of their respective lists, which is not a correlation of anything. So a +/// gene from one panel has *no* tissue correlation against a gene from the +/// other, and pairing by tissue makes that visible instead of inventing a +/// number. +fn compute_tissue( + request: &CorrelationRequest, + method: Method, + top_n: usize, +) -> Result<(String, Vec<correlation::Hit>), Failure> { + let dataset = db::get_dataset_by_name(&request.dataset) + .map_err(db_failure)? + .ok_or_else(|| { + Failure::NoValues(format!( + "the tissue axis needs a probeset dataset; '{}' is not one", + request.dataset + )) + })?; + let symbols = db::get_dataset_symbols(dataset.id).map_err(db_failure)?; + let reference_symbol = symbols + .iter() + .find(|(name, _)| name == &request.trait_name) + .map(|(_, symbol)| symbol.clone()) + .ok_or_else(|| { + Failure::NoValues(format!( + "'{}' in '{}' has no gene symbol, so it has no tissue values", + request.trait_name, request.dataset + )) + })?; + + let target_dataset = db::get_dataset_by_name(&request.target_dataset) + .map_err(db_failure)? + .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.target_dataset)))?; + let target_symbols = db::get_dataset_symbols(target_dataset.id).map_err(db_failure)?; + + let mut wanted: Vec<String> = target_symbols.iter().map(|(_, s)| s.clone()).collect(); + wanted.push(reference_symbol.clone()); + wanted.sort(); + wanted.dedup(); + let tissue = db::get_tissue_values(&wanted).map_err(db_failure)?; + + let reference_values = tissue.get(&reference_symbol.to_lowercase()).ok_or_else(|| { + Failure::NoValues(format!( + "'{reference_symbol}' has no tissue values in TissueProbeSetData" + )) + })?; + // The tissue axis names its observations by tissue id, so the engine's + // alignment by name becomes an alignment by tissue for free. + let reference = correlation::Reference::from_pairs( + reference_values + .iter() + .map(|(tissue_id, value)| (format!("tissue:{tissue_id}"), Some(*value))), + ); + + let rows: Vec<(String, String, Option<f64>)> = target_symbols + .iter() + .flat_map(|(probeset, symbol)| { + tissue + .get(&symbol.to_lowercase()) + .map(|values| (probeset.clone(), values)) + .into_iter() + }) + .flat_map(|(probeset, values)| { + values.iter().map(move |(tissue_id, value)| { + ( + probeset.clone(), + format!("tissue:{tissue_id}"), + Some(*value), + ) + }) + }) + .collect(); + + let hits = correlation::correlate_many(&reference, method, rows, top_n); + Ok(( + format!( + "{} (symbol {}, {} tissues) against '{}' by tissue", + request.trait_name, + reference_symbol, + reference.len(), + request.target_dataset + ), + hits, + )) +} + +/// The literature axis: a precomputed gene-by-gene value, not a statistic. +/// +/// GN2 looks the pair up in `LCorrRamin3`, both directions, after translating +/// rat and human ids to mouse through `GeneIDXRef`. Two deliberate differences: +/// a trait with no gene id or no row is **left out** rather than reported as +/// 0.0 (GN2 fills the tail of its top-N with zeroes, which reads as "measured, +/// no relationship"), and the reply says how many were left out. +fn compute_literature( + request: &CorrelationRequest, + top_n: usize, +) -> Result<(String, Vec<correlation::Hit>), Failure> { + let species = db::get_dataset_species(&request.dataset) + .map_err(db_failure)? + .unwrap_or_else(|| "mouse".to_string()); + let dataset = db::get_dataset_by_name(&request.dataset) + .map_err(db_failure)? + .ok_or_else(|| { + Failure::NoValues(format!( + "the literature axis needs a probeset dataset; '{}' is not one", + request.dataset + )) + })?; + let reference_gene = db::get_dataset_geneids(dataset.id) + .map_err(db_failure)? + .into_iter() + .find(|(name, _)| name == &request.trait_name) + .map(|(_, gene_id)| gene_id) + .ok_or_else(|| { + Failure::NoValues(format!( + "'{}' in '{}' has no gene id, so it has no literature correlation", + request.trait_name, request.dataset + )) + })?; + + let target_dataset = db::get_dataset_by_name(&request.target_dataset) + .map_err(db_failure)? + .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.target_dataset)))?; + let target_genes = db::get_dataset_geneids(target_dataset.id).map_err(db_failure)?; + + // GN2 skips ids that name several genes ("1;2"). + let single: Vec<(String, String)> = target_genes + .into_iter() + .filter(|(_, gene_id)| !gene_id.contains(';')) + .collect(); + let reference_mouse = db::mouse_gene_ids(&species, std::slice::from_ref(&reference_gene)) + .map_err(db_failure)? + .get(&reference_gene) + .copied() + .ok_or_else(|| { + Failure::NoValues(format!( + "gene id '{reference_gene}' has no mouse id in GeneIDXRef" + )) + })?; + let raw_ids: Vec<String> = single.iter().map(|(_, gene_id)| gene_id.clone()).collect(); + let mouse = db::mouse_gene_ids(&species, &raw_ids).map_err(db_failure)?; + let mut by_mouse: Vec<(i64, String)> = Vec::new(); + for (probeset, gene_id) in &single { + if let Some(mouse_id) = mouse.get(gene_id) { + by_mouse.push((*mouse_id, probeset.clone())); + } + } + let targets: Vec<i64> = by_mouse.iter().map(|(gene, _)| *gene).collect(); + let values = db::get_literature_values(reference_mouse, &targets).map_err(db_failure)?; + + let mut hits: Vec<correlation::Hit> = by_mouse + .iter() + .filter_map(|(gene, probeset)| { + values.get(gene).map(|value| correlation::Hit { + name: probeset.clone(), + // No statistic was computed: serde_json writes NaN as null, + // which is what "not a p-value" should look like here. + r: *value, + p: f64::NAN, + n: 0, + }) + }) + .collect(); + let missing = single.len().saturating_sub(hits.len()); + hits.sort_by(|a, b| { + b.r.abs() + .partial_cmp(&a.r.abs()) + .unwrap_or(std::cmp::Ordering::Equal) + }); + hits.truncate(top_n); + Ok(( + format!( + "{} (gene id {reference_gene}, mouse {reference_mouse}) against '{}' in LCorrRamin3; {missing} trait(s) had no gene id or no LCorrRamin3 row and are left out", + request.trait_name, request.target_dataset + ), + hits, + )) +} + /// The reference trait's values, from whichever kind of dataset holds it. fn resolve_reference(request: &CorrelationRequest) -> Result<correlation::Reference, Failure> { if let Some(dataset) = |
