about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/db.rs192
-rw-r--r--src/routes/correlation.rs281
2 files changed, 462 insertions, 11 deletions
diff --git a/src/db.rs b/src/db.rs
index 2f9caf0..505ef51 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -1590,6 +1590,198 @@ pub fn get_group_mapping_samples(group_name: &str) -> Result<Vec<String>, DbErro
         .collect())
 }
 
+/// The species of the group a dataset belongs to (`InbredSet.SpeciesId` ->
+/// `Species.Name`), which is how GN2 decides whether a gene id needs
+/// translating to mouse before it is looked up in `LCorrRamin3`.
+pub fn get_dataset_species(dataset_name: &str) -> Result<Option<String>, DbError> {
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    conn.exec_first(
+        "SELECT Species.Name FROM ProbeSetFreeze \
+         JOIN ProbeFreeze ON ProbeFreeze.ProbeFreezeId = ProbeSetFreeze.ProbeFreezeId \
+         JOIN InbredSet ON InbredSet.Id = ProbeFreeze.InbredSetId \
+         JOIN Species ON Species.Id = InbredSet.SpeciesId \
+         WHERE ProbeSetFreeze.Name = :name LIMIT 1",
+        mysql::params! { "name" => dataset_name },
+    )
+    .map_err(DbError::from)
+}
+
+/// `ProbeSet.Name` -> `ProbeSet.Symbol` for one probeset dataset, the axis
+/// the tissue correlation is keyed on.
+pub fn get_dataset_symbols(dataset_id: u16) -> Result<Vec<(String, String)>, DbError> {
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    conn.exec_map(
+        "SELECT ProbeSet.Name, ProbeSet.Symbol \
+         FROM ProbeSet JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetId = ProbeSet.Id \
+         WHERE ProbeSetXRef.ProbeSetFreezeId = :dataset_id \
+           AND ProbeSet.Symbol IS NOT NULL AND ProbeSet.Symbol <> '' \
+         ORDER BY ProbeSet.Id",
+        mysql::params! { "dataset_id" => dataset_id },
+        |(name, symbol): (String, String)| (name, symbol),
+    )
+    .map_err(DbError::from)
+}
+
+/// `ProbeSet.Name` -> `ProbeSetXRef.GeneId` for one probeset dataset, the axis
+/// the literature correlation is keyed on.
+pub fn get_dataset_geneids(dataset_id: u16) -> Result<Vec<(String, String)>, DbError> {
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    conn.exec_map(
+        // The gene id lives on ProbeSet, not on ProbeSetXRef: GN3's own mapping
+        // is `ProbeSet.GeneId`, and ProbeSetXRef has no GeneId column here.
+        "SELECT ProbeSet.Name, ProbeSet.GeneId \
+         FROM ProbeSet JOIN ProbeSetXRef ON ProbeSetXRef.ProbeSetId = ProbeSet.Id \
+         WHERE ProbeSetXRef.ProbeSetFreezeId = :dataset_id \
+           AND ProbeSet.GeneId IS NOT NULL AND ProbeSet.GeneId <> '' \
+         ORDER BY ProbeSet.Id",
+        mysql::params! { "dataset_id" => dataset_id },
+        |(name, gene_id): (String, String)| (name, gene_id),
+    )
+    .map_err(DbError::from)
+}
+
+/// Tissue values keyed by *symbol*, as `(TissueID, value)` pairs.
+///
+/// GN2 reduces a symbol to one probe -- the one with the largest `Mean` in
+/// `TissueProbeSetXRef` -- and then reads that probe's values.  It keeps them as
+/// a bare list and pairs two lists *by position*, which is only meaningful if
+/// both probes cover the same tissues: on this database they do not.  The
+/// values are keyed by `TissueID` here so a caller can pair them by tissue
+/// rather than by position.
+pub fn get_tissue_values(
+    symbols: &[String],
+) -> Result<std::collections::HashMap<String, Vec<(u16, f64)>>, DbError> {
+    if symbols.is_empty() {
+        return Ok(std::collections::HashMap::new());
+    }
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    let placeholders = std::iter::repeat("?")
+        .take(symbols.len())
+        .collect::<Vec<_>>()
+        .join(", ");
+    let rows: Vec<(String, u16, f64)> = conn.exec(
+        format!(
+            "SELECT xref.Symbol, data.TissueID, data.value \
+             FROM TissueProbeSetData data \
+             JOIN TissueProbeSetXRef xref ON xref.DataId = data.Id \
+             JOIN (SELECT Symbol, MAX(Mean) AS best_mean FROM TissueProbeSetXRef \
+                   WHERE TissueProbeSetFreezeId = 1 AND Symbol IN ({placeholders}) \
+                   GROUP BY Symbol) best \
+               ON best.Symbol = xref.Symbol AND xref.Mean = best.best_mean \
+             WHERE xref.TissueProbeSetFreezeId = 1 \
+             ORDER BY xref.Symbol, data.TissueID"
+        ),
+        symbols.iter().collect::<Vec<_>>(),
+    )?;
+    let mut out: std::collections::HashMap<String, Vec<(u16, f64)>> =
+        std::collections::HashMap::new();
+    for (symbol, tissue_id, value) in rows {
+        out.entry(symbol.to_lowercase()).or_default().push((tissue_id, value));
+    }
+    Ok(out)
+}
+
+/// Gene ids translated to the mouse id `LCorrRamin3` is keyed on.
+///
+/// GN2's `convert_to_mouse_gene_id`: a mouse id is used as it stands, a rat or
+/// human id is looked up in `GeneIDXRef`.
+pub fn mouse_gene_ids(
+    species: &str,
+    gene_ids: &[String],
+) -> Result<std::collections::HashMap<String, i64>, DbError> {
+    let mut out = std::collections::HashMap::new();
+    if gene_ids.is_empty() {
+        return Ok(out);
+    }
+    let numeric: Vec<(String, i64)> = gene_ids
+        .iter()
+        .filter_map(|g| g.trim().parse::<i64>().ok().map(|v| (g.clone(), v)))
+        .collect();
+    if species.eq_ignore_ascii_case("mouse") {
+        for (raw, value) in numeric {
+            out.insert(raw, value);
+        }
+        return Ok(out);
+    }
+    let column = match species.to_ascii_lowercase().as_str() {
+        "rat" => "rat",
+        "human" => "human",
+        _ => return Ok(out),
+    };
+    let pool = pool()?;
+    let mut conn = pool.get_conn()?;
+    let placeholders = std::iter::repeat("?")
+        .take(numeric.len())
+        .collect::<Vec<_>>()
+        .join(", ");
+    let rows: Vec<(i64, i64)> = conn.exec(
+        format!("SELECT {column}, mouse FROM GeneIDXRef WHERE {column} IN ({placeholders})"),
+        numeric.iter().map(|(_, value)| *value).collect::<Vec<_>>(),
+    )?;
+    let by_source: std::collections::HashMap<i64, i64> = rows.into_iter().collect();
+    for (raw, value) in numeric {
+        if let Some(mouse) = by_source.get(&value) {
+            out.insert(raw, *mouse);
+        }
+    }
+    Ok(out)
+}
+
+/// `LCorrRamin3` values for one reference gene against many targets, in both
+/// directions, preferring GN2's direction (`GeneId1` = target, `GeneId2` =
+/// reference) where both rows exist.
+///
+/// Chunked because a probeset dataset has tens of thousands of gene ids and
+/// MySQL refuses a statement with more than 65 535 placeholders (measured:
+/// "Prepared statement contains too many placeholders" on the first attempt).
+pub fn get_literature_values(
+    reference: i64,
+    targets: &[i64],
+) -> Result<std::collections::HashMap<i64, f64>, DbError> {
+    let mut forward: std::collections::HashMap<i64, f64> = std::collections::HashMap::new();
+    let mut reverse: std::collections::HashMap<i64, f64> = std::collections::HashMap::new();
+    for chunk in targets.chunks(LITERATURE_CHUNK) {
+        let pool = pool()?;
+        let mut conn = pool.get_conn()?;
+        let placeholders = std::iter::repeat("?")
+            .take(chunk.len())
+            .collect::<Vec<_>>()
+            .join(", ");
+        let mut params: Vec<i64> = Vec::with_capacity(chunk.len() * 2 + 2);
+        params.push(reference);
+        params.extend(chunk.iter().copied());
+        params.push(reference);
+        params.extend(chunk.iter().copied());
+        let rows: Vec<(i64, i64, f64)> = conn.exec(
+            format!(
+                "SELECT GeneId1, GeneId2, value FROM LCorrRamin3 \
+                 WHERE (GeneId2 = ? AND GeneId1 IN ({placeholders})) \
+                    OR (GeneId1 = ? AND GeneId2 IN ({placeholders}))"
+            ),
+            params,
+        )?;
+        for (gene1, gene2, value) in rows {
+            if gene2 == reference {
+                forward.insert(gene1, value);
+            } else if gene1 == reference {
+                reverse.insert(gene2, value);
+            }
+        }
+    }
+    for (target, value) in reverse {
+        forward.entry(target).or_insert(value);
+    }
+    Ok(forward)
+}
+
+/// How many genes go into one `IN (...)` clause: MySQL's placeholder ceiling is
+/// 65 535 per statement, and this query uses two lists.
+const LITERATURE_CHUNK: usize = 4_000;
+
 /// Every value of every trait in a probeset dataset, restricted to a set of
 /// individuals: one record per (trait, individual, value).
 ///
diff --git a/src/routes/correlation.rs b/src/routes/correlation.rs
index 9b91735..01ccb7a 100644
--- a/src/routes/correlation.rs
+++ b/src/routes/correlation.rs
@@ -33,6 +33,11 @@ pub struct CorrelationRequest {
     pub trait_name: String,
     /// Dataset to correlate against, e.g. "BXDPublish" or another ProbeSet set.
     pub target_dataset: String,
+    /// What is correlated, GN2's `corr_type`: "sample" (default) pairs the two
+    /// traits over individuals, "tissue" pairs them over tissue systems, and
+    /// "literature" reads a precomputed gene-by-gene value.
+    #[serde(default)]
+    pub r#type: Option<String>,
     /// "pearson" (default) or "spearman".
     #[serde(default)]
     pub method: Option<String>,
@@ -95,8 +100,11 @@ pub async fn correlation(
         ));
     }
 
+    let axis = Axis::parse(request.r#type.as_deref()).map_err(|failure| {
+        failure.into_api(rid.clone())
+    })?;
     let wanted = request.clone();
-    let computed = tokio::task::spawn_blocking(move || compute(&request, method, top_n))
+    let computed = tokio::task::spawn_blocking(move || compute(&request, axis, method, top_n))
         .await
         .map_err(|error| {
             fail(
@@ -119,18 +127,34 @@ pub async fn correlation(
         ctx.envelope(rows)
             .with_columns(COLUMNS.iter().map(|name| name.to_string()).collect(), num_rows)
             .with_note(&format!(
-                "{} against {} trait(s) of {}, observations being individuals.",
+                "{} against {} trait(s) of {}.",
                 reference_name, num_rows, wanted.target_dataset
             ))
-            .with_note(&format!(
-                "Method {}; at least {MIN_OVERLAP} paired observations are required, \
-                 ordered by |r| descending with undefined values last, top {top_n}.",
-                method.as_str()
-            ))
-            .with_note(
-                "Rows are positional: trait, r, p, n. p is the two-sided p-value of the \
-                 engine's t statistic; n is how many pairs that correlation used.",
-            )
+            .with_note(&match axis {
+                Axis::Sample | Axis::Tissue => format!(
+                    "Method {}; at least {MIN_OVERLAP} paired {} are required, ordered by |r| \
+                     descending with undefined values last, top {top_n}.",
+                    method.as_str(),
+                    if axis == Axis::Tissue {
+                        "tissues"
+                    } else {
+                        "observations"
+                    }
+                ),
+                Axis::Literature => format!(
+                    "Rows are precomputed LCorrRamin3 values, not a statistic: p and n are \
+                     absent, and the order is by |value| descending, top {top_n}."
+                ),
+            })
+            .with_note(match axis {
+                Axis::Sample | Axis::Tissue => {
+                    "Rows are positional: trait, r, p, n. p is the two-sided p-value of the \
+                     engine's t statistic; n is how many pairs that correlation used."
+                }
+                Axis::Literature => {
+                    "Rows are positional: trait, r, p, n -- with p and n null for this axis."
+                }
+            })
             .with_next("/mapping-methods", "The mapping tools GeneNetwork has run"),
     ))
 }
@@ -231,6 +255,53 @@ fn sample_axis(dataset: &str) -> Result<Option<Vec<String>>, Failure> {
 /// against an expression dataset -- the common case -- has to work.
 fn compute(
     request: &CorrelationRequest,
+    axis: Axis,
+    method: Method,
+    top_n: usize,
+) -> Result<(String, Vec<correlation::Hit>), Failure> {
+    match axis {
+        Axis::Sample => compute_sample(request, method, top_n),
+        Axis::Tissue => compute_tissue(request, method, top_n),
+        Axis::Literature => compute_literature(request, top_n),
+    }
+}
+
+/// GN2's `corr_type`.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum Axis {
+    Sample,
+    Tissue,
+    Literature,
+}
+
+impl Axis {
+    fn parse(name: Option<&str>) -> Result<Self, Failure> {
+        match name.unwrap_or("sample").trim().to_ascii_lowercase().as_str() {
+            "" | "sample" => Ok(Axis::Sample),
+            "tissue" => Ok(Axis::Tissue),
+            "literature" | "lit" => Ok(Axis::Literature),
+            "partial" => Err(Failure::NoValues(
+                "partial correlation is not implemented yet; use sample, tissue or literature"
+                    .to_string(),
+            )),
+            other => Err(Failure::NoValues(format!(
+                "unknown correlation type '{other}'; use sample, tissue or literature."
+            ))),
+        }
+    }
+
+    fn as_str(&self) -> &'static str {
+        match self {
+            Axis::Sample => "sample",
+            Axis::Tissue => "tissue",
+            Axis::Literature => "literature",
+        }
+    }
+}
+
+/// The sample axis: the two traits paired over individuals.
+fn compute_sample(
+    request: &CorrelationRequest,
     method: Method,
     top_n: usize,
 ) -> Result<(String, Vec<correlation::Hit>), Failure> {
@@ -282,6 +353,194 @@ fn compute(
     ))
 }
 
+/// The tissue axis: the two traits paired over tissue systems.
+///
+/// The values are keyed by `TissueID` and paired *by tissue*, which is where
+/// this departs from GN2 on purpose. GN2 keeps each gene's tissue values as a
+/// bare list and pairs two lists by position, on the assumption that every
+/// probe covers the same tissues. On this database it does not: every probe
+/// carries exactly 26 values (liver, kidney, spleen, ... -- peripheral tissues)
+/// or exactly 32 (striatum, hippocampus, neocortex, ... -- brain regions), and
+/// the two sets are **disjoint**. Pairing such lists by position correlates
+/// liver expression with striatum expression because they happen to be entry
+/// one of their respective lists, which is not a correlation of anything. So a
+/// gene from one panel has *no* tissue correlation against a gene from the
+/// other, and pairing by tissue makes that visible instead of inventing a
+/// number.
+fn compute_tissue(
+    request: &CorrelationRequest,
+    method: Method,
+    top_n: usize,
+) -> Result<(String, Vec<correlation::Hit>), Failure> {
+    let dataset = db::get_dataset_by_name(&request.dataset)
+        .map_err(db_failure)?
+        .ok_or_else(|| {
+            Failure::NoValues(format!(
+                "the tissue axis needs a probeset dataset; '{}' is not one",
+                request.dataset
+            ))
+        })?;
+    let symbols = db::get_dataset_symbols(dataset.id).map_err(db_failure)?;
+    let reference_symbol = symbols
+        .iter()
+        .find(|(name, _)| name == &request.trait_name)
+        .map(|(_, symbol)| symbol.clone())
+        .ok_or_else(|| {
+            Failure::NoValues(format!(
+                "'{}' in '{}' has no gene symbol, so it has no tissue values",
+                request.trait_name, request.dataset
+            ))
+        })?;
+
+    let target_dataset = db::get_dataset_by_name(&request.target_dataset)
+        .map_err(db_failure)?
+        .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.target_dataset)))?;
+    let target_symbols = db::get_dataset_symbols(target_dataset.id).map_err(db_failure)?;
+
+    let mut wanted: Vec<String> = target_symbols.iter().map(|(_, s)| s.clone()).collect();
+    wanted.push(reference_symbol.clone());
+    wanted.sort();
+    wanted.dedup();
+    let tissue = db::get_tissue_values(&wanted).map_err(db_failure)?;
+
+    let reference_values = tissue.get(&reference_symbol.to_lowercase()).ok_or_else(|| {
+        Failure::NoValues(format!(
+            "'{reference_symbol}' has no tissue values in TissueProbeSetData"
+        ))
+    })?;
+    // The tissue axis names its observations by tissue id, so the engine's
+    // alignment by name becomes an alignment by tissue for free.
+    let reference = correlation::Reference::from_pairs(
+        reference_values
+            .iter()
+            .map(|(tissue_id, value)| (format!("tissue:{tissue_id}"), Some(*value))),
+    );
+
+    let rows: Vec<(String, String, Option<f64>)> = target_symbols
+        .iter()
+        .flat_map(|(probeset, symbol)| {
+            tissue
+                .get(&symbol.to_lowercase())
+                .map(|values| (probeset.clone(), values))
+                .into_iter()
+        })
+        .flat_map(|(probeset, values)| {
+            values.iter().map(move |(tissue_id, value)| {
+                (
+                    probeset.clone(),
+                    format!("tissue:{tissue_id}"),
+                    Some(*value),
+                )
+            })
+        })
+        .collect();
+
+    let hits = correlation::correlate_many(&reference, method, rows, top_n);
+    Ok((
+        format!(
+            "{} (symbol {}, {} tissues) against '{}' by tissue",
+            request.trait_name,
+            reference_symbol,
+            reference.len(),
+            request.target_dataset
+        ),
+        hits,
+    ))
+}
+
+/// The literature axis: a precomputed gene-by-gene value, not a statistic.
+///
+/// GN2 looks the pair up in `LCorrRamin3`, both directions, after translating
+/// rat and human ids to mouse through `GeneIDXRef`. Two deliberate differences:
+/// a trait with no gene id or no row is **left out** rather than reported as
+/// 0.0 (GN2 fills the tail of its top-N with zeroes, which reads as "measured,
+/// no relationship"), and the reply says how many were left out.
+fn compute_literature(
+    request: &CorrelationRequest,
+    top_n: usize,
+) -> Result<(String, Vec<correlation::Hit>), Failure> {
+    let species = db::get_dataset_species(&request.dataset)
+        .map_err(db_failure)?
+        .unwrap_or_else(|| "mouse".to_string());
+    let dataset = db::get_dataset_by_name(&request.dataset)
+        .map_err(db_failure)?
+        .ok_or_else(|| {
+            Failure::NoValues(format!(
+                "the literature axis needs a probeset dataset; '{}' is not one",
+                request.dataset
+            ))
+        })?;
+    let reference_gene = db::get_dataset_geneids(dataset.id)
+        .map_err(db_failure)?
+        .into_iter()
+        .find(|(name, _)| name == &request.trait_name)
+        .map(|(_, gene_id)| gene_id)
+        .ok_or_else(|| {
+            Failure::NoValues(format!(
+                "'{}' in '{}' has no gene id, so it has no literature correlation",
+                request.trait_name, request.dataset
+            ))
+        })?;
+
+    let target_dataset = db::get_dataset_by_name(&request.target_dataset)
+        .map_err(db_failure)?
+        .ok_or_else(|| Failure::NotFound(format!("no dataset named '{}'", request.target_dataset)))?;
+    let target_genes = db::get_dataset_geneids(target_dataset.id).map_err(db_failure)?;
+
+    // GN2 skips ids that name several genes ("1;2").
+    let single: Vec<(String, String)> = target_genes
+        .into_iter()
+        .filter(|(_, gene_id)| !gene_id.contains(';'))
+        .collect();
+    let reference_mouse = db::mouse_gene_ids(&species, std::slice::from_ref(&reference_gene))
+        .map_err(db_failure)?
+        .get(&reference_gene)
+        .copied()
+        .ok_or_else(|| {
+            Failure::NoValues(format!(
+                "gene id '{reference_gene}' has no mouse id in GeneIDXRef"
+            ))
+        })?;
+    let raw_ids: Vec<String> = single.iter().map(|(_, gene_id)| gene_id.clone()).collect();
+    let mouse = db::mouse_gene_ids(&species, &raw_ids).map_err(db_failure)?;
+    let mut by_mouse: Vec<(i64, String)> = Vec::new();
+    for (probeset, gene_id) in &single {
+        if let Some(mouse_id) = mouse.get(gene_id) {
+            by_mouse.push((*mouse_id, probeset.clone()));
+        }
+    }
+    let targets: Vec<i64> = by_mouse.iter().map(|(gene, _)| *gene).collect();
+    let values = db::get_literature_values(reference_mouse, &targets).map_err(db_failure)?;
+
+    let mut hits: Vec<correlation::Hit> = by_mouse
+        .iter()
+        .filter_map(|(gene, probeset)| {
+            values.get(gene).map(|value| correlation::Hit {
+                name: probeset.clone(),
+                // No statistic was computed: serde_json writes NaN as null,
+                // which is what "not a p-value" should look like here.
+                r: *value,
+                p: f64::NAN,
+                n: 0,
+            })
+        })
+        .collect();
+    let missing = single.len().saturating_sub(hits.len());
+    hits.sort_by(|a, b| {
+        b.r.abs()
+            .partial_cmp(&a.r.abs())
+            .unwrap_or(std::cmp::Ordering::Equal)
+    });
+    hits.truncate(top_n);
+    Ok((
+        format!(
+            "{} (gene id {reference_gene}, mouse {reference_mouse}) against '{}' in LCorrRamin3;              {missing} trait(s) had no gene id or no LCorrRamin3 row and are left out",
+            request.trait_name, request.target_dataset
+        ),
+        hits,
+    ))
+}
+
 /// The reference trait's values, from whichever kind of dataset holds it.
 fn resolve_reference(request: &CorrelationRequest) -> Result<correlation::Reference, Failure> {
     if let Some(dataset) =