//! `POST /compute/bulklmm`: a genome scan with BulkLMM.jl. //! //! The engine GEMMA cannot be asked for thresholds. GEMMA's LOCO scan is the //! reference for a biallelic panel, but a permutation repeats the whole scan, //! which is why `/compute/gemma` refuses `nperm` outright; the usual stand-in //! is R/qtl's `nperm`, whose null shuffles phenotypes as if the individuals //! were unrelated. BulkLMM fits an LMM with a global kinship matrix and //! permutes by rotating the data through the eigen decomposition of that //! matrix -- relatedness included -- and 200 permutations cost seconds. //! //! The differences from GEMMA are stated in the reply, not implied: this is a //! **global-kinship** LMM, not a leave-one-chromosome-out one, and `h2_null` //! says how much relatedness it found (when that is ~0 it degenerates to a //! fixed-effects fit). use axum::Json; use axum::extract::Query; use axum::http::StatusCode; use serde::Deserialize; use serde_json::Value; use gnauth_macros::{compute_gate, gnauth}; use crate::auth::AuthToken; use crate::bulklmm::{self, BulklmmError}; use crate::compute_gate as gate; use crate::config::Config; use crate::covariates; use crate::error::{ApiError, Unauthorized}; use crate::genotypes; use crate::normalize; use crate::response::{Ctx, Envelope, RequestInfo, path_segment}; use crate::rqtl::{Sample, write_pheno_file}; #[derive(Deserialize, utoipa::IntoParams)] pub struct ComputeBulklmmParams { /// Log progress for `GET /compute/rqtl/status/{id}` (shared with the other /// R-class engines, since this one also has to be watched). pub id: Option, } #[derive(Deserialize, utoipa::ToSchema)] pub struct BulklmmSample { pub name: String, pub value: Option, } #[derive(Deserialize, utoipa::ToSchema)] pub struct BulklmmRequest { /// Group to scan (an InbredSet name, e.g. "BXD"). pub group: Option, /// A version from `/groups/{group}/genotypes`. pub genotype_version: Option, /// A of a user's uploaded genotype (`/uploads/genotypes`, by name). pub genotype_upload: Option, /// Include parents and F1s; off by default, as elsewhere. pub include_parents_f1: Option, /// Individuals to keep or drop on top of the parent/F1 rule /// (`genotypes::Individuals`); `exclude` wins over `include`. #[serde(default)] pub individuals: genotypes::Individuals, /// The trait's values by individual name, or `trait_values` positionally. pub samples: Option>, /// A basket from `/baskets`, in place of `samples`. pub basket: Option, /// Label for the trait, e.g. "BXD::16627"; reported in `notes`. pub trait_name: Option, /// Covariates, as on `/compute/rqtl`. #[serde(default)] pub covariates: Vec, /// Permutations for the thresholds. 0 (default) scans without them. pub nperm: Option, /// Seed for the rotations, so a threshold set is reproducible. pub seed: Option, /// Estimate the variance components by REML rather than ML. pub reml: Option, /// `null` (default: variance components estimated once, under the null) or /// `alt` (per marker, slower and more accurate). pub assumption: Option, /// Julia threads (`--threads`), default `BULKLMM_THREADS` (4). The marker /// blocks scanned in parallel come from `BULKLMM_BLOCKS` (4). pub cores: Option, /// Normalisation of the trait values, as on `/compute/rqtl`. pub normalize: Option, pub winsor_quantile: Option, pub winsor_fences: Option, } #[utoipa::path(post, path = "/compute/bulklmm", summary = "Scan a panel with BulkLMM.jl: an LMM with permutation thresholds", description = "Fits one linear mixed model per marker with a global kinship matrix \ (BulkLMM.jl), and -- unlike /compute/gemma, which refuses permutations because each repeats \ the whole LOCO scan -- permutes by rotating the data through the kinship's eigen decomposition, \ so `nperm` costs seconds. Rows are positional: marker, chr, pos (Mb), lod, beta, h2. The reply \ notes the permutation thresholds (alpha 0.10, 0.05, 0.01), the seed, and the null heritability: \ this is a global-kinship model, not a LOCO one, and a null h2 near zero means it collapsed to a \ fixed-effects fit.", params(ComputeBulklmmParams), request_body = BulklmmRequest, responses( (status = 200, description = "One row per marker, positional (meta.columns names them)", body = Envelope), Unauthorized, (status = 409, description = "A scan is already running for this caller", body = ApiError), (status = 422, description = "No usable panel, trait values or covariate", body = ApiError), (status = 502, description = "Julia or BulkLMM failed", body = ApiError) ) )] #[gnauth(token_only)] #[compute_gate(bulklmm, label = "BulkLMM", run_id = params.id.as_deref())] pub async fn compute_bulklmm( ctx: Ctx, token: AuthToken, Query(params): Query, Json(mut request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); let trait_label = request .trait_name .clone() .filter(|name| !name.trim().is_empty()) .unwrap_or_else(|| format!("{}::bulklmm", request.group.clone().unwrap_or_default())); let pipeline = normalize::Pipeline::parse( request.normalize.as_deref(), request.winsor_quantile, request.winsor_fences, ) .map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_bulklmm", ) .with_request_id(rid.clone()) })?; let assumption = request.assumption.clone().unwrap_or_else(|| "null".to_string()); if !matches!(assumption.as_str(), "null" | "alt") { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_assumption", "Invalid variance-component assumption", format!("assumption '{assumption}' not recognized; use 'null' or 'alt'."), "compute_bulklmm", ) .with_request_id(rid)); } let nperm = request.nperm.unwrap_or(0).min(10_000); let seed = request.seed.unwrap_or(0); let reml = request.reml.unwrap_or(false); let threads = request.cores.unwrap_or(config.bulklmm_threads).clamp(1, 32); let blocks = config.bulklmm_blocks.clamp(1, 64); let mut notes: Vec = Vec::new(); // A basket is another source for `samples`. let mut basket_note: Option = None; if let Some(name) = request.basket.clone() { if request.samples.is_some() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_samples", "Ambiguous values", "give `basket` or `samples`, not both.".to_string(), "compute_bulklmm", ) .with_request_id(rid)); } let (resolved, note) = crate::routes::baskets::resolve_for_mapping( token.sub.clone(), config.basket_dir.clone(), name, "compute_bulklmm", rid.clone(), ) .await?; basket_note = Some(note); request.samples = Some( resolved .values .iter() .map(|(name, value)| BulklmmSample { name: name.clone(), value: *value, }) .collect(), ); } let samples: Vec = request .samples .take() .unwrap_or_default() .into_iter() .map(|sample| Sample { name: sample.name, value: sample.value }) .collect(); if samples.is_empty() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_samples", "No trait values", "give `samples`, a `basket`, or `trait_values`.".to_string(), "compute_bulklmm", ) .with_request_id(rid)); } // The panel: resolved and prepared exactly as GEMMA's BIMBAM input is, // because that pair is already the dosage matrix BulkLMM wants. let directory = config.genotype_files_dir.clone(); let cache_dir = config.bulklmm_cache_dir.clone(); let mut version = request.genotype_version.clone(); if request.genotype_upload.is_some() && request.group.is_some() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give genotype_upload or group -- one of the two.".to_string(), "compute_bulklmm", ) .with_request_id(rid)); } if let Some(upload) = request.genotype_upload.clone() { // The upload carries its own base version; resolving it yields the // version `genotypes::resolve` knows. let info = crate::genoupload::get( std::path::Path::new(&config.basket_dir), &token.sub, &upload, ) .map_err(|error| { ApiError::new( StatusCode::NOT_FOUND, "genotype_upload_not_found", "Uploaded genotype not found", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })?; version = Some(info.base_version); notes.push(format!( "Genotype: uploaded panel '{upload}' ({}, {} markers).", info.value_kind, info.markers )); } let Some(group) = request.group.clone().or_else(|| Some(String::new())).filter(|group| !group.is_empty() || version.is_some()) else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_genotype", "No genotype input", "give group (+ genotype_version), genotype_upload, or genotype_file.".to_string(), "compute_bulklmm", ) .with_request_id(rid)); }; let include_parents_f1 = request.include_parents_f1.unwrap_or(false); let individuals = request.individuals.clone(); let values: Vec<(String, Option)> = samples .iter() .map(|sample| (sample.name.clone(), sample.value)) .collect(); let group_for_scan = if group.is_empty() { // An upload resolves through its base version; the group is the one // that version belongs to. None } else { Some(group.clone()) }; let directory_for_scan = directory.clone(); let cache_for_scan = cache_dir.clone(); let version_for_scan = version.clone(); let prepared = tokio::task::spawn_blocking(move || { let resolved = match &group_for_scan { Some(group) => genotypes::resolve( std::path::Path::new(&directory_for_scan), group, version_for_scan.as_deref(), )?, None => { // A bare version (an upload's base): resolve it by name. let geno = genotypes::resolve_named( std::path::Path::new(&directory_for_scan), version_for_scan.as_deref().unwrap_or_default(), )?; let panel = genotypes::panel_of(&geno)?; genotypes::Resolved { version: version_for_scan.clone().unwrap_or_default(), title: None, geno, bimbam_geno: None, bimbam_snps: None, panel, } } }; let parents_f1s = genotypes::non_strains( std::path::Path::new(&directory_for_scan), &resolved.version, &resolved.panel, ); let selection = genotypes::select_panel( &resolved.panel, &parents_f1s, include_parents_f1, &individuals, )?; let keep: Vec = selection.keep.clone(); let (aligned, matched) = genotypes::align(&resolved.panel, &keep, &values); let prepared = genotypes::prepare( std::path::Path::new(&directory_for_scan), &resolved, genotypes::Engine::Gemma, &keep, std::path::Path::new(&cache_for_scan), )?; let kept_names: Vec = keep .iter() .map(|index| resolved.panel[*index].clone()) .collect(); let samples: Vec = kept_names .iter() .cloned() .zip(aligned) .map(|(name, value)| Sample { name, value }) .collect(); Ok::<_, genotypes::GenotypeError>((resolved.version.clone(), prepared, kept_names, samples, matched)) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })? .map_err(|error| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "genotype_error", "The genotype files could not be read", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })?; let (version_used, prepared, kept_names, aligned_samples, matched) = prepared; if matched == 0 { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "no_samples_aligned", "No samples aligned", "none of the supplied samples appear in this genotype panel".to_string(), "compute_bulklmm", ) .with_request_id(rid)); } notes.push(format!( "Genotype: {} (version {}); {} of {} supplied values aligned onto it by name.", prepared.note, version_used, matched, samples.len() )); let samples = aligned_samples; let samples = if pipeline.is_identity() { samples } else { let raw: Vec> = samples.iter().map(|s| s.value).collect(); let (normalized, applied) = pipeline.apply(&raw).map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_bulklmm", ) .with_request_id(rid.clone()) })?; notes.push(pipeline.describe(&applied)); samples .into_iter() .zip(normalized) .map(|(sample, value)| Sample { name: sample.name, value }) .collect() }; let (covariate_list, covariate_notes) = covariates::resolve( &request.covariates, &samples.iter().map(|s| s.name.clone()).collect::>(), request.group.as_deref(), Some(&samples.iter().map(|s| s.value.is_some()).collect::>()), rid.clone(), ) .await?; notes.extend(covariate_notes); // What Julia reads: the phenotype file, the dosage matrix, the marker map, // and the covariates when there are any. let work = bulklmm::work_dir(&cache_dir).map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })?; let unique = format!( "{}-{}", std::process::id(), std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_nanos()) .unwrap_or(0) ); let pheno_path = work.join(format!("{unique}-pheno.txt")); write_pheno_file(&pheno_path, &trait_label, &samples, &covariate_list).map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })?; let genotype_csv = work.join(format!("{unique}-geno.csv")); let markers_csv = work.join(format!("{unique}-markers.csv")); let Some(bimbam_geno) = Some(prepared.genotype_file.clone()) else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "no_dosage_input", "No dosage genotypes for this panel", "BulkLMM reads allele dosages (the `bimbam/` pair); this version has none. \ `/groups/{group}/genotypes` says which versions have GEMMA input." .to_string(), "compute_bulklmm", ) .with_request_id(rid)); }; let Some(bimbam_snps) = prepared.snps_file.clone() else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "no_dosage_input", "No SNP map for this panel", "the `bimbam/` pair is incomplete for this version".to_string(), "compute_bulklmm", ) .with_request_id(rid)); }; let individuals = kept_names.clone(); let (genotype_csv_for_job, markers_csv_for_job) = (genotype_csv.clone(), markers_csv.clone()); let markers = tokio::task::spawn_blocking(move || { bulklmm::write_inputs( &bimbam_geno, &bimbam_snps, &individuals, &genotype_csv_for_job, &markers_csv_for_job, ) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })? .map_err(|error| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "genotype_error", "The genotype matrix could not be written", error.to_string(), "compute_bulklmm", ) .with_request_id(rid.clone()) })?; if markers == 0 { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "empty_panel", "The panel has no markers", "the dosage matrix is empty".to_string(), "compute_bulklmm", ) .with_request_id(rid)); } let individuals_count = samples.len(); __gn4_claim .claim_slot_or_busy(individuals_count, rid.clone()) .await?; let rid_for_job = rid.clone(); let output = gate::bulklmm() .run(__gn4_claim, async move { bulklmm::compute( &config.bulklmm_cmd, &config.bulklmm_env, &config.bulklmm_depot, &config.bulklmm_script, &work, &genotype_csv, &markers_csv, &pheno_path, None, nperm, seed, reml, &assumption, threads, blocks, ) .await .map_err(|error| match error { BulklmmError::CommandFailed(detail) => ApiError::new( StatusCode::BAD_GATEWAY, "bulklmm_failed", "BulkLMM failed", detail, "compute_bulklmm", ) .with_request_id(rid_for_job.clone()), other => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", other.to_string(), "compute_bulklmm", ) .with_request_id(rid_for_job.clone()), }) }) .await?; if let Some(note) = basket_note { notes.push(note); } if let Some(h2) = output.h2_null { let boundary = if h2 < 0.01 { " At a boundary (h2 ~ 0 the model is ordinary regression, which is why its LODs can \ match R/qtl's to the digit)." } else if h2 > 0.99 { " At a boundary (h2 ~ 1 the fit is almost all kinship), so its LODs are not \ comparable with an engine that has no kinship term." } else { "" }; notes.push(format!( "Null heritability h2 = {h2:.3}: {}", if h2 < 0.01 { "relatedness explains almost nothing here, so this fit is effectively a fixed-\ effects model (which is why it can agree with R/qtl where GEMMA differs)." .to_string() } else { "the kinship term is doing work; GEMMA's LOCO scan weights each chromosome \ differently, so its p-values are not these." .to_string() } )); notes.push(format!( "Variance components: h2 = {h2:.6}, sigma2_e = {:.6}, estimated once under the null \ (intercept) model from y and K, not per marker. With a few dozen phenotyped \ individuals these estimates pile up at the boundaries 0 and 1 -- the likelihood is \ flat there -- so read h2 as \"kinship matters a lot / not at all\" rather than as a \ precise proportion.{boundary}", output.sigma2_e.unwrap_or(f64::NAN) )); } if !output.thresholds.is_empty() { let thresholds: Vec = output .thresholds .iter() .map(|(alpha, lod)| format!("alpha {alpha}: LOD {lod:.2}")) .collect(); notes.push(format!( "Permutation thresholds from {nperm} rotations (seed {seed}): {}. These account for \ relatedness -- R/qtl's nperm on the same values does not.", thresholds.join(", ") )); } else if nperm == 0 { notes.push( "No permutations requested, so no threshold: the LOD is a LOD. Ask for \ `nperm` to get thresholds from BulkLMM's rotations (cheap here)." .to_string(), ); } if !output.summary.is_empty() { notes.push(output.summary.clone()); } let num_rows = output.markers; let mut envelope = ctx .envelope(output.rows) .with_columns( bulklmm::COLUMNS.iter().map(|name| name.to_string()).collect(), num_rows, ) .with_note(&format!("Trait: {trait_label}")) .with_note( "Rows are positional: marker, chr, pos (Mb), lod, beta, h2. BulkLMM fits one LMM per \ marker with a global kinship matrix (not GEMMA's leave-one-chromosome-out), by ML \ or REML, and reports the effect size `beta` and the null heritability `h2`.", ) .with_note(&format!( "Parallelism: BulkLMM's single-trait scan() is serial in Julia, so the markers are \ split into {blocks} independent blocks and scanned on {threads} Julia thread(s) \ (BULKLMM_BLOCKS / BULKLMM_THREADS; blocks 1 is a plain serial scan and gives the \ same LODs). Where the cores really pay is many traits at once." )); if let Some(group) = request.group.clone() { envelope = envelope.with_next( &format!("/groups/{}", path_segment(&group)), "The group's record", ); envelope = envelope.with_action(crate::response::SuggestedCall { method: "GET".to_string(), href: format!( "/groups/{}/traits/{}/peak-genotypes?marker={{marker}}", path_segment(&group), trait_label.rsplit("::").next().unwrap_or_default() ), description: "Count the animals in each genotype class at a marker of the peak" .to_string(), body: None, }); } // And, when thresholds were computed, the GEMMA plot can carry them: same // values, two models, one picture -- labelled so the lines cannot be mixed up. if let Some(lod) = output.thresholds.get("0.05") { envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: format!( "/plot/gemma?thresholds={lod:.2}&threshold_labels=bulklmm%20alpha%200.05" ), description: "Or draw a GEMMA result for the same values with this threshold on it \ (the request body is the /compute/gemma reply)" .to_string(), body: None, }); } envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: "/plot/bulklmm".to_string(), description: "Draw this scan: post this reply back and get the LOD plot, with the \ thresholds above drawn on it." .to_string(), body: None, }); envelope = envelope.with_links(crate::suggest::root_links()); for note in ¬es { envelope = envelope.with_note(note); } Ok(Json(envelope)) }