//! Genotype call frequencies. //! //! `/groups/{group}/genotypes/{version}/frequencies` is the canonical shape: //! a tally is a fact about one version of one group's panel, so it belongs //! beside `/markers` in the version's own subtree, where the group's //! permissions, the group-to-version check and the "available: …" message on //! a wrong version all already apply. //! //! A file no group lists -- a panel in preparation, a fixture, the DO's `do/` //! cross -- is served from `/groups/unlisted/genotypes/{name}/…`, a namespace //! rather than a group: one prefix for the whole genotype surface, so an //! agent that has learned where genotype facts live does not have to learn a //! second spelling for the same tally. `unlisted` is not a group and //! `/groups/unlisted` has no record; only its genotype subtree answers. //! //! "What is this panel made of?" is a question the listing cannot answer and //! the compute endpoints do not: they report how many markers and //! individuals a file has, never how they are called. A panel whose calls are //! not the codes its header declares, or a marker that is 99 % missing, is //! invisible until a scan comes back flat -- and then the search for the //! cause starts at the wrong end. //! //! None of these serve genotypes. A tally is counted and discarded, so what //! leaves the server is a distribution, not a copy of the matrix. use std::path::PathBuf; use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::gnauth; use crate::auth::AuthToken; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; use crate::frequencies::{self, Counted, FreqError}; use crate::genotypes; use crate::response::{Ctx, Envelope, RequestInfo}; use crate::suggest::{self, Link}; #[derive(Deserialize, utoipa::IntoParams)] pub struct FrequencyParams { /// Tally one marker instead of the whole file. The marker route is the /// same call with the marker in the path. pub marker: Option, /// Bin width for probability-valued calls (default 0.05: twenty bins /// over [0, 1]). Ignored for call-valued and low-cardinality files. pub bin: Option, } #[derive(Deserialize, utoipa::IntoParams)] pub struct MarkerFrequencyParams { /// Bin width for probability-valued calls (default 0.05). pub bin: Option, } #[derive(Deserialize, utoipa::IntoParams)] pub struct NamedFrequencyParams { /// Tally one marker instead of the whole file. pub marker: Option, /// Bin width for probability-valued calls (default 0.05). pub bin: Option, } #[utoipa::path(get, path = "/groups/{group}/genotypes/{version}/frequencies", summary = "Genotype call frequencies for one genotype version", description = "Counts the calls in the version's `.geno` and returns their frequencies, e.g. \ {\"B\": 0.62, \"D\": 0.36, \"H\": 0.02}: what the panel is actually called as, which the \ listing and the compute endpoints never say. The genotypes themselves are counted and \ discarded. A file whose cells are probabilities is binned at 0.05 (twenty bins over [0, 1]); \ pass ?bin= for another width. Use ?marker= for one marker.", params( ("group" = String, Path, description = "InbredSet name, e.g. BXD"), ("version" = String, Path, description = "Version from /groups/{group}/genotypes"), FrequencyParams ), responses( (status = 200, description = "A call -> frequency map (counts in notes)", body = Envelope), Unauthorized, (status = 404, description = "No such group, version, or marker", body = ApiError) ) )] #[gnauth(token_only)] pub async fn group_version_frequencies( ctx: Ctx, Path((group, version)): Path<(String, String)>, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let root = Config::load().genotype_files_dir.clone(); let label = format!("{group} {version}"); let path = resolve_version(root.clone(), group.clone(), version.clone(), rid.clone()).await?; let links = suggest::genotype_version_links(&group, &version); let current = format!( "/groups/{}/genotypes/{}/frequencies", crate::response::path_segment(&group), crate::response::path_segment(&version) ); let envelope = count(ctx, path, params.marker, params.bin, label, links, current, rid).await?; Ok(Json(envelope)) } #[utoipa::path(get, path = "/groups/{group}/genotypes/{version}/markers/{marker}/frequencies", summary = "Genotype call frequencies for one marker", description = "The same tally as the version's `/frequencies`, for a single marker: what the \ calls at one locus look like across the panel. This is how a caller sees whether a candidate \ peak's marker is nearly all missing -- a flat LOD from a silent marker and a flat LOD from real \ data look the same in a scan.", params( ("group" = String, Path, description = "InbredSet name, e.g. BXD"), ("version" = String, Path, description = "Version from /groups/{group}/genotypes"), ("marker" = String, Path, description = "Marker name as the genotype file spells it"), MarkerFrequencyParams ), responses( (status = 200, description = "A call -> frequency map for one marker", body = Envelope), Unauthorized, (status = 404, description = "No such group, version, or marker", body = ApiError) ) )] #[gnauth(token_only)] pub async fn group_marker_frequencies( ctx: Ctx, Path((group, version, marker)): Path<(String, String, String)>, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let root = Config::load().genotype_files_dir.clone(); let label = format!("{group} {version}"); let path = resolve_version(root, group.clone(), version.clone(), rid.clone()).await?; let links = suggest::marker_links(&group, &version, &marker); let current = format!( "/groups/{}/genotypes/{}/markers/{}/frequencies", crate::response::path_segment(&group), crate::response::path_segment(&version), crate::response::path_segment(&marker) ); let envelope = count(ctx, path, Some(marker), params.bin, label, links, current, rid).await?; Ok(Json(envelope)) } #[utoipa::path(get, path = "/groups/unlisted/genotypes/{name}/frequencies", summary = "Genotype call frequencies for a genotype file no group lists", description = "The same tally as `/groups/{group}/genotypes/{version}/frequencies`, for a \ file the listing does not know about: a panel in preparation, a fixture, the DO's `do/` cross. \ A bare name is looked up as `genotype/{name}.geno`; a `.geno` name or a path relative to the \ genotype files directory also resolves. Prefer the group-scoped route where a group exists -- it \ validates the pair and says what versions do.", params(("name" = String, Path, description = "Version (e.g. BXD.4), .geno name, or path relative to the genotype files directory"), NamedFrequencyParams), responses( (status = 200, description = "A call -> frequency map (counts in notes)", body = Envelope), Unauthorized, (status = 404, description = "No such genotype file, or no such marker in it", body = ApiError) ) )] #[gnauth(token_only)] pub async fn unlisted_frequencies( ctx: Ctx, Path(name): Path, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let root = Config::load().genotype_files_dir.clone(); let requested = name.clone(); let path: PathBuf = tokio::task::spawn_blocking(move || { genotypes::resolve_named(std::path::Path::new(&root), &requested) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| map_genotype_error(error, rid.clone()))?; let links = suggest::unlisted_genotype_links(&name); let current = format!( "/groups/unlisted/genotypes/{}/frequencies", crate::response::path_segment(&name) ); let envelope = count(ctx, path, params.marker, params.bin, name, links, current, rid).await?; Ok(Json(envelope)) } /// `/genotypes/marker/{marker}?genotype=…`: the marker route for a named file, /// kept because a file outside the listing has no group-scoped path. #[utoipa::path(get, path = "/groups/unlisted/genotypes/{name}/markers/{marker}/frequencies", summary = "One marker's call frequencies in a genotype file no group lists", description = "The same tally as `/groups/{group}/genotypes/{version}/markers/{marker}/frequencies`, \ for a file no group lists; `{name}` is the version, `.geno` name, or path relative to the \ genotype files directory.", params( ("name" = String, Path, description = "Version, .geno name, or path relative to the genotype files directory"), ("marker" = String, Path, description = "Marker name as the genotype file spells it"), MarkerFrequencyParams ), responses( (status = 200, description = "A call -> frequency map for one marker", body = Envelope), Unauthorized, (status = 404, description = "No such genotype file, or no such marker in it", body = ApiError) ) )] #[gnauth(token_only)] pub async fn unlisted_marker_frequencies( ctx: Ctx, Path((name, marker)): Path<(String, String)>, Query(params): Query, _token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let root = Config::load().genotype_files_dir.clone(); let requested = name.clone(); let path: PathBuf = tokio::task::spawn_blocking(move || { genotypes::resolve_named(std::path::Path::new(&root), &requested) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| map_genotype_error(error, rid.clone()))?; let links = suggest::unlisted_marker_links(&name, &marker); let current = format!( "/groups/unlisted/genotypes/{}/markers/{}/frequencies", crate::response::path_segment(&name), crate::response::path_segment(&marker) ); let envelope = count(ctx, path, Some(marker), params.bin, name, links, current, rid).await?; Ok(Json(envelope)) } /// Resolve `group`+`version` the way every other group endpoint does, so a /// wrong version is reported with the ones that exist. async fn resolve_version( root: String, group: String, version: String, rid: RequestInfo, ) -> Result { tokio::task::spawn_blocking(move || { genotypes::resolve(std::path::Path::new(&root), &group, Some(&version)) .map(|resolved| resolved.geno) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| map_genotype_error(error, rid)) } /// Count, off the async runtime: a large panel is seconds of CSV. async fn count( ctx: Ctx, path: PathBuf, marker: Option, bin: Option, label: String, links: Vec, current: String, rid: RequestInfo, ) -> Result, ApiError> { let counted = tokio::task::spawn_blocking({ let marker = marker.clone(); move || frequencies::tally(&path, marker.as_deref(), bin) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; Ok(envelope(ctx, counted, &label, marker.as_deref(), links, ¤t)) } /// The response: the frequency map as `data`, the counts that produced it in /// `notes`. fn envelope( ctx: Ctx, counted: Counted, file: &str, marker: Option<&str>, links: Vec, current: &str, ) -> Envelope { let tally = counted.tally; let cells = tally.called + tally.missing; let missing_share = if cells == 0 { 0.0 } else { ((tally.missing as f64 / cells as f64) * 1e4).round() / 100.0 }; let mut envelope = ctx.envelope(serde_json::json!(tally.frequencies)); envelope.meta.num_rows = Some(tally.frequencies.len()); envelope = match marker { Some(marker) => envelope.with_note(&format!( "Marker {marker}{} in {file}: the calls across the panel, as frequencies", match (counted.chr.as_deref(), counted.mb) { (Some(chr), Some(mb)) => format!(" (chr{chr}, {mb:.2} Mb)"), (Some(chr), None) => format!(" (chr{chr})"), _ => String::new(), } )), None => envelope.with_note(&format!( "{file}: {} marker(s), {cells} genotype cell(s)", tally.markers )), }; envelope = envelope.with_note(&format!( "{} call(s) counted, {} missing ({missing_share} %); frequencies are over the calls that \ are present, so they sum to 1", tally.called, tally.missing, )); if let Some(width) = tally.bin_width { envelope = envelope.with_note(&format!( "The calls are probabilities, so they are binned {width:.2} wide (keys are the bin's \ edges, e.g. \"0.05-0.10\"); pass ?bin= for another width" )); } else if marker.is_none() { envelope = envelope.with_note( "Calls are reported exactly, as the file spells them; missing codes (U, N, NA, an \ empty cell) are counted as missing, not as a call", ); } envelope .with_note( "The genotypes themselves are counted and discarded: this endpoint reports a \ distribution, not the matrix.", ) .with_links_except(links, current) } fn internal(message: String, rid: RequestInfo) -> ApiError { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", message, "genotype_frequencies", ) .with_request_id(rid) } fn map_error(error: FreqError, rid: RequestInfo) -> ApiError { match error { FreqError::NotFound(message) => ApiError::new( StatusCode::NOT_FOUND, "genotype_not_found", "Genotype file or marker not found", message, "genotype_frequencies", ) .with_request_id(rid), FreqError::TooLarge(message) => ApiError::new( StatusCode::PAYLOAD_TOO_LARGE, "genotype_too_large", "Panel too large to tally whole", message, "genotype_frequencies", ) .with_request_id(rid), FreqError::Io(message) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "genotype_unreadable", "Genotype file could not be read", message, "genotype_frequencies", ) .with_request_id(rid), } } /// `genotypes::resolve` failures (group scoping) keep their own codes. fn map_genotype_error(error: genotypes::GenotypeError, rid: RequestInfo) -> ApiError { match error { genotypes::GenotypeError::Invalid(message) => ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_individuals", "Unusable individual selection", message, "genotype_frequencies", ) .with_request_id(rid), genotypes::GenotypeError::NotFound(message) => ApiError::new( StatusCode::NOT_FOUND, "genotype_not_found", "Genotype file or marker not found", message, "genotype_frequencies", ) .with_request_id(rid), genotypes::GenotypeError::Io(message) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "genotype_unreadable", "Genotype file could not be read", message, "genotype_frequencies", ) .with_request_id(rid), } } // --------------------------------------------------------------------------- // `GET /groups/{group}/genotypes/{version}/markers/{marker}/classes` // // The question that follows "why do GEMMA and R/qtl2 peak at different // markers": what the trait looks like within each genotype class at one // marker. It cannot be answered from the endpoints above -- they report the // distribution of calls, not the values behind them -- and it must not be // answered by serving genotypes, which this API never does. // // So the calls stay inside the server: they are joined to a trait's per-strain // values (with that trait's own authorization) and reduced to n/mean/sd/min/max // per class. A caller learns "the two A animals are 5.9 and 6.0" as // `{"n": 2, "mean": 5.95, ...}` and never learns which animal carried which // allele. // --------------------------------------------------------------------------- #[derive(Deserialize, utoipa::IntoParams)] pub struct ClassParams { /// The trait whose values are summarised, as `/groups/{group}/traits` /// reports its id (a PublishXRef.Id). `trait` is a Rust keyword, hence /// the serde rename. #[serde(rename = "trait")] pub trait_id: u32, } #[utoipa::path(get, path = "/groups/{group}/genotypes/{version}/markers/{marker}/classes", summary = "A trait's values summarised per genotype class at one marker", description = "n, mean, sd, min and max of a published trait within each genotype class \ (A, H, B, or whatever codes the file uses) at one marker. This is how a caller checks whether a \ peak rests on a handful of animals -- or whether the class a scan cannot use, like a nearly \ monomorphic biallelic marker, is where another scan's signal lives. The calls are joined to the \ trait's values inside the server and only the summary is returned; individual genotypes are \ never served, and the trait's own authorization applies.", params( ("group" = String, Path, description = "InbredSet name, e.g. DOL"), ("version" = String, Path, description = "Version from /groups/{group}/genotypes"), ("marker" = String, Path, description = "Marker name as the genotype file spells it"), ClassParams ), responses( (status = 200, description = "class -> {n, mean, sd, min, max}", body = Envelope), Unauthorized, (status = 403, description = "The trait is not public and you are not authorized", body = ApiError), (status = 404, description = "No such group, version, marker or trait", body = ApiError) ) )] #[gnauth(secured)] pub async fn marker_classes( ctx: Ctx, Path((group, version, marker)): Path<(String, String, String)>, Query(params): Query, token: AuthToken, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); let root = config.genotype_files_dir.clone(); let auth_server_url = config.auth_server_url.clone(); let trait_id = params.trait_id; // The genotype side: which individual carries which call at this marker. let (wanted_group, wanted_version, wanted_marker) = (group.clone(), version.clone(), marker.clone()); let calls = tokio::task::spawn_blocking(move || { let resolved = genotypes::resolve( std::path::Path::new(&root), &wanted_group, Some(&wanted_version), ) .map_err(|error| FreqError::NotFound(error.to_string()))?; frequencies::marker_calls(&resolved.geno, &wanted_marker) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; // The trait side: the same lookup, authorization and 404s as // /groups/{group}/traits/{id}/values. let (wanted_group, wanted_version) = (group.clone(), version.clone()); let values = tokio::task::spawn_blocking(move || { crate::routes::groups::get_group_trait_values_result( &wanted_group, trait_id, &auth_server_url, &token, ) }) .await .map_err(|error| internal(error.to_string(), rid.clone()))?; let values = match values { Ok(crate::routes::groups::GroupTraitLookup::Found(authorized, values)) => { let _: crate::auth::Authorized = authorized; values } Ok(crate::routes::groups::GroupTraitLookup::Forbidden) => { return Err(ApiError::new( StatusCode::FORBIDDEN, "forbidden", "Not authorized", format!("trait {trait_id} is not public, and you are not authorized to view it."), "marker_classes", ) .with_request_id(rid)); } Ok(crate::routes::groups::GroupTraitLookup::GroupNotFound) => { return Err(ApiError::new( StatusCode::NOT_FOUND, "group_not_found", "Group not found", format!("No group matching '{group}'."), "marker_classes", ) .with_request_id(rid)); } Ok(crate::routes::groups::GroupTraitLookup::TraitNotFound) => { return Err(ApiError::new( StatusCode::NOT_FOUND, "trait_not_found", "Trait not found", format!("No trait {trait_id} in group '{group}'."), "marker_classes", ) .with_request_id(rid)); } Err(crate::routes::groups::GroupAuthError::Db(error)) => { return Err(ApiError::from_db(error, "marker_classes").with_request_id(rid)); } Err(crate::routes::groups::GroupAuthError::Auth(detail)) => { return Err(ApiError::new( StatusCode::BAD_GATEWAY, "auth_server_error", "Could not check authorization", detail, "marker_classes", ) .with_request_id(rid)); } }; // Join by individual name, then summarise per class. let by_name: std::collections::HashMap<&str, Option> = values .iter() .map(|value| (value.name.as_str(), value.value)) .collect(); let mut classes: std::collections::BTreeMap> = std::collections::BTreeMap::new(); let mut without_call = 0usize; let mut without_value = 0usize; for (name, call) in &calls.calls { match (call, by_name.get(name.as_str()).copied().flatten()) { (Some(call), Some(value)) => classes.entry(call.clone()).or_default().push(value), (None, Some(_)) => without_call += 1, (Some(_), None) => without_value += 1, (None, None) => {} } } let panel: std::collections::HashSet<&str> = calls.calls.iter().map(|(name, _)| name.as_str()).collect(); let outside_panel = values .iter() .filter(|value| !panel.contains(value.name.as_str())) .count(); let mut data = serde_json::Map::new(); for (class, mut numbers) in classes { let n = numbers.len(); numbers.sort_by(|left, right| left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal)); let mean = numbers.iter().sum::() / n as f64; let sd = if n > 1 { let variance = numbers .iter() .map(|value| (value - mean).powi(2)) .sum::() / (n - 1) as f64; variance.sqrt() } else { 0.0 }; data.insert( class, serde_json::json!({ "n": n, "mean": crate::routes::probesets::round(Some(mean), 4.0), "sd": crate::routes::probesets::round(Some(sd), 4.0), "min": crate::routes::probesets::round(numbers.first().copied(), 4.0), "max": crate::routes::probesets::round(numbers.last().copied(), 4.0), }), ); } let marker_label = format!( "{marker}{} in {group} {version}", match (calls.chr.as_deref(), calls.mb) { (Some(chr), Some(mb)) => format!(" (chr{chr}, {mb:.2} Mb)"), (Some(chr), None) => format!(" (chr{chr})"), _ => String::new(), } ); let mut envelope = ctx .envelope(serde_json::Value::Object(data)) .with_note(&format!( "{marker_label}: trait {trait_id}'s values summarised per genotype class. Counting \ only animals that have both a call and a value; {without_call} have a value but no \ call (missing at this marker) and {without_value} have a call but no value." )) .with_note( "Individual genotypes are not served: the calls are joined to the values here and \ reduced to these statistics. sd is the sample standard deviation.", ); if calls.calls.len() == 0 { envelope = envelope.with_note("This marker has no individuals in the file."); } else if let Some((class, smallest)) = envelope .data .as_object() .and_then(|classes| { classes .iter() .filter_map(|(class, stats)| Some((class.clone(), stats.get("n")?.as_u64()?))) .min_by_key(|(_, n)| *n) }) { if smallest < 10 { let warn = envelope.with_note(&format!( "The smallest class is '{class}' with {smallest} animal(s): a peak resting on that \ many is sensitive to one extreme value." )); envelope = warn; } } if outside_panel > 0 { envelope = envelope.with_note(&format!( "{outside_panel} animal(s) with a value for this trait are not in this genotype file \ and were left out." )); } envelope = envelope.with_next( &format!( "/groups/{}/genotypes/{}/markers/{}/frequencies", crate::response::path_segment(&group), crate::response::path_segment(&version), crate::response::path_segment(&marker) ), "This marker's calls across the panel, as frequencies", ); let links = suggest::genotype_version_links(&group, &version); let current = format!( "/groups/{}/genotypes/{}/markers/{}/classes", crate::response::path_segment(&group), crate::response::path_segment(&version), crate::response::path_segment(&marker) ); Ok(Json(envelope.with_links_except(links, ¤t))) }