//! Compute domain: /compute/gemma invokes gemma-wrapper (and the real //! GEMMA binary it wraps) to map a QTL from a genotype file, a SNP //! annotation file and a list of trait values. `GET //! /compute/gemma/status/{run_id}` tails that run's progress while it //! is (or was) in flight, the same shape GeneNetwork2/3 use for R/qtl //! (gn3/computations/streaming.py, gn3/api/streaming.py): the caller //! picks a `run_id`, passes it as `?id=` on the POST, and polls the //! status endpoint with `?peek=` (the byte offset already read) on a //! second connection while the first is still blocked on the POST. See //! `crate::gemma` for the invocation and log-writing itself. use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::{compute_gate, gnauth}; use crate::auth::AuthToken; use crate::config::Config; use crate::compute_gate; use crate::covariates; use crate::error::{ApiError, Unauthorized}; use crate::genotypes; use crate::gemma::{self, GemmaError}; use crate::response::{Ctx, Envelope, RequestInfo, path_segment}; use crate::streaming::{self, RunStatus, RunStatusParams, StatusError}; /// `?id=` on `POST /compute/gemma`: an optional, caller-chosen run id /// to log this run's progress under and poll back via `GET /// /compute/gemma/status/{id}`. Omitted entirely, no log is written. /// Must be unique: a collision with an existing log (another run still /// in progress, or one finished and never polled/cleaned up) errors out /// rather than overwriting it -- see `gemma::GemmaError::RunIdInUse`. #[derive(Deserialize, utoipa::IntoParams)] pub struct ComputeGemmaParams { pub id: Option, } #[derive(Deserialize, utoipa::ToSchema)] pub struct GemmaRequest { /// Path to a BIMBAM genotype file (plain text or gzipped), passed /// as GEMMA's `-g`. Give this and `snps_file`, or `group` -- not /// both. pub genotype_file: Option, /// An uploaded genotype from `/uploads/genotypes`, in place of /// `genotype_file`: the server writes it out and maps over it. pub genotype_upload: Option, /// Path to a SNP annotation file (rs id, position, chromosome), /// passed as GEMMA's `-a`. pub snps_file: Option, /// Group to map over (an InbredSet name, e.g. "BXD"). The server /// resolves the BIMBAM pair from the deployment's genotype-files /// directory; `/groups/{group}/genotypes` lists what is available. pub group: Option, /// A version from that listing, e.g. "BXD.8". Defaults to the /// current one. pub genotype_version: Option, /// Include parents and F1s. Off by default, as in GeneNetwork2/3. pub include_parents_f1: Option, /// Individuals to keep or drop on top of the parent/F1 rule /// (`genotypes::Individuals`). `exclude` wins over `include`, and an /// unknown name is a 422 rather than a silently dropped animal. #[serde(default)] pub individuals: genotypes::Individuals, /// Covariates to fit alongside the trait: a case attribute of the /// group by name, or explicit per-sample numbers. Same shape as /// `/compute/rqtl`'s; GEMMA needs `group`, since the individuals have /// to be named to align them. #[serde(default)] pub covariates: Vec, /// One phenotype value per individual, in the prepared genotype /// file's order; `null` for a missing value. Alternatively give /// `samples`, which is matched onto the panel by name. pub trait_values: Option>>, /// Trait values by individual name. With `group` given they are /// aligned onto the panel (unknown names dropped, missing /// individuals `null`), so a trait's own values are enough. pub samples: Option>, /// A basket from `/baskets`, in place of `samples` (see /// `crate::routes::baskets`). Its revision and digest are recorded in /// `notes` so a result on an edited basket stays distinguishable. pub basket: Option, /// How the trait values' scale is changed before mapping: `none` /// (default), `log2`, `log10`, `sqrt`, `negate` (GeneNetwork2 calls it /// invert), `zscore`, `robust_zscore`, or `rank` (GeneNetwork2's /// `qnorm`). Applied over the samples that actually enter the fit, so /// the effects `beta`/`additive` come out on that scale. pub normalize: Option, /// Winsorise: clip each tail at this fraction, e.g. 0.05. Mutually /// exclusive with `winsor_fences`. pub winsor_quantile: Option, /// Winsorise at Tukey fences: `Q1 - k*IQR` / `Q3 + k*IQR` (1.5 is /// GeneNetwork2's outlier rule). Mutually exclusive with /// `winsor_quantile`. pub winsor_fences: Option, /// Label for what is being mapped, e.g. "BXD::20010". Optional -- GEMMA /// itself has no notion of a trait name, and the values are positional -- /// but without it two responses from the same panel are indistinguishable /// in `notes`, and the plot of each comes out titled "gemma". pub trait_name: Option, /// GEMMA's `-lmm` model selector: stock values are 1 = Wald, /// 2 = LRT, 3 = score, 4 = all three. Defaults to 9 when omitted -- /// not a stock value, but the one GeneNetwork2 /// (wqflask/marker_regression/gemma_mapping.py) and gn-guile /// (gn/runner/gemma.scm) both use, and the only value this has been /// run against in the --loco shape (see `crate::gemma`). Confirmed /// by actually running it: `gemma-gn2` accepts it and the output /// carries `p_lrt`, not `p_wald` -- some other mapping than the /// stock 1-4 scale, not independently verified here. pub lmm: Option, /// Refused unless 0: GEMMA permutations are not supported here (see /// `permutations_not_supported`). Named so that asking for them is an /// error rather than a silently ignored key. pub nperm: Option, /// Refused unless 0, as `nperm`. pub permutations: Option, /// Refused unless 0, as `nperm`. pub permutate: Option, } /// One trait value, by individual name. #[derive(Deserialize, utoipa::ToSchema)] pub struct GemmaSample { pub name: String, pub value: Option, } /// What a mapping request resolves to before any engine runs: the genotype /// file the engine reads, the SNP map beside it, the trait values in that /// file's individual order, how many individuals that is, and the notes /// describing what was aligned, dropped or refused along the way. /// /// Both GEMMA runners -- the direct one and the ravanan one -- go through /// this, because the rules are fiddly and had drifted once already: an /// upload is materialised into the pair the engine reads, a group is /// prepared with the parent/F1 and empty-genotype rules applied, named /// samples are aligned by name onto whatever individuals the file has, and /// positional `trait_values` are checked against the panel size. Writing /// them twice means fixing them twice. pub(crate) struct ResolvedInputs { pub genotype_file: String, pub snps_file: String, pub trait_values: Vec>, /// Individuals the fit will see: the gate's size for this request. pub individuals: usize, /// Their names, in the same order, for a group-scoped request -- the /// only kind whose individuals are named (covariates align against /// these). pub names: Option>, pub group: String, pub notes: Vec, } /// Resolve a request's genotype input and trait values. /// /// `endpoint` names this endpoint in errors ("compute_gemma" or /// "compute_gemma_ravanan"), so a caller is never told about the other one. pub(crate) async fn resolve_inputs( config: &Config, request: &mut GemmaRequest, token: &AuthToken, endpoint: &'static str, rid: RequestInfo, ) -> Result { let mut notes: Vec = Vec::new(); let mut aligned_names: Option> = None; let mut aligned_group = String::new(); if let Some(upload) = request.genotype_upload.clone() { if request.genotype_file.is_some() || request.group.is_some() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give genotype_upload, genotype_file or group -- one of the three.".to_string(), endpoint, ) .with_request_id(rid)); } // GEMMA reads BIMBAM, not `.geno`, so the upload is written out as // the pair (`-g`/`-a`) that the deployment's own GTEx/BXD runs use. let upload_for_values = upload.clone(); let upload_for_note = upload.clone(); let (geno, snps) = crate::routes::genoupload::resolve_upload( token.sub.clone(), config.basket_dir.clone(), config.genotype_files_dir.clone(), upload, endpoint, rid.clone(), true, ) .await?; let named: Vec<(String, Option)> = request .samples .as_ref() .map(|samples| { samples .iter() .map(|sample| (sample.name.clone(), sample.value)) .collect() }) .unwrap_or_default(); let trait_values = if let Some(values) = request.trait_values.clone() { values } else { let dir = config.basket_dir.clone(); let owner = token.sub.clone(); let individuals = tokio::task::spawn_blocking(move || { crate::genoupload::individuals(std::path::Path::new(&dir), &owner, &upload_for_values) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), endpoint, ) .with_request_id(rid.clone()) })? .map_err(|error| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_genotype_upload", "Invalid genotype upload", error.to_string(), endpoint, ) .with_request_id(rid.clone()) })?; let by_name: std::collections::HashMap<&str, Option> = named .iter() .map(|(name, value)| (name.as_str(), *value)) .collect(); individuals .iter() .map(|name| by_name.get(name.as_str()).copied().flatten()) .collect() }; notes.push(format!( "Genotype: uploaded pick '{}' written out as a BIMBAM pair; {} values used.", upload_for_note, trait_values.iter().filter(|value| value.is_some()).count() )); return Ok(ResolvedInputs { genotype_file: geno, snps_file: snps.unwrap_or_default(), individuals: trait_values.len(), trait_values, names: None, group: String::new(), notes, }); } // A basket is another source for the caller's values, resolved before // anything below looks at them (see crate::routes::baskets). if let Some(name) = request.basket.clone() { if request.samples.is_some() { return Err(crate::routes::baskets::ambiguous(endpoint, rid)); } let (resolved, note) = crate::routes::baskets::resolve_for_mapping( token.sub.clone(), config.basket_dir.clone(), name, endpoint, rid.clone(), ) .await?; notes.push(note); request.samples = Some( resolved .values .iter() .map(|(name, value)| GemmaSample { name: name.clone(), value: *value, }) .collect(), ); } let group = request.group.clone(); let version = request.genotype_version.clone(); let include_parents_f1 = request.include_parents_f1.unwrap_or(false); let individuals = request.individuals.clone(); let named: Vec<(String, Option)> = request .samples .as_ref() .map(|samples| { samples .iter() .map(|sample| (sample.name.clone(), sample.value)) .collect() }) .unwrap_or_default(); let supplied_values = request.trait_values.clone(); match (request.genotype_file.clone(), request.snps_file.clone(), group) { (Some(genotype_file), Some(snps_file), None) => { let trait_values = if let Some(values) = supplied_values { values } else if named.is_empty() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_trait_values", "No trait values", "give trait_values (one per individual, genotype-file order) or \ samples (named)." .to_string(), endpoint, ) .with_request_id(rid)); } else { let path = std::path::PathBuf::from(&genotype_file); let aligned = tokio::task::spawn_blocking(move || { genotypes::align_to_file(&path, &named) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), endpoint, ) .with_request_id(rid.clone()) })? .map_err(|error| genotype_error_to_api_error(error, endpoint, rid.clone()))?; aligned }; notes.push(format!( "Genotype: given file and SNP map; {} values used.", trait_values.iter().filter(|value| value.is_some()).count() )); Ok(ResolvedInputs { individuals: trait_values.len(), trait_values, genotype_file, snps_file, names: None, group: String::new(), notes, }) } (None, None, Some(group)) => { let directory = std::path::PathBuf::from(&config.genotype_files_dir); let cache_dir = std::path::PathBuf::from(&config.gemma_cache_dir); let wanted = group.clone(); let had_samples = !named.is_empty(); let supplied_count = named.len(); let ( prepared, aligned, matched, resolved, kept_names, dropped_empty, parents_f1s, included_by_name, excluded_by_name, ) = tokio::task::spawn_blocking(move || { let resolved = genotypes::resolve(&directory, &wanted, version.as_deref())?; let parents_f1s = genotypes::non_strains(&directory, &wanted, &resolved.panel); let selection = genotypes::select_panel( &resolved.panel, &parents_f1s, include_parents_f1, &individuals, )?; let keep: Vec = selection.keep.clone(); // An individual with no genotypes at all makes the // kinship singular, so leave those out before anything // else is aligned: they contribute nothing to the fit. let empty = resolved .bimbam_geno .as_deref() .and_then(genotypes::individuals_with_genotypes); let parents_f1s = resolved.panel.len() - keep.len(); let keep: Vec = match empty { Some(mask) => keep .into_iter() .filter(|index| mask.get(*index).copied().unwrap_or(true)) .collect(), None => keep, }; let dropped_empty = (resolved.panel.len() - parents_f1s) - keep.len(); let (aligned, matched) = genotypes::align(&resolved.panel, &keep, &named); let kept_names: Vec = keep .iter() .filter_map(|index| resolved.panel.get(*index).cloned()) .collect(); let prepared = genotypes::prepare( &directory, &resolved, genotypes::Engine::Gemma, &keep, &cache_dir, )?; Ok::<_, genotypes::GenotypeError>(( prepared, aligned, matched, resolved, kept_names, dropped_empty, parents_f1s, selection.included_by_name, selection.excluded_by_name, )) }) .await .map_err(|e| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", e.to_string(), endpoint, ) .with_request_id(rid.clone()) })? .map_err(|e| genotype_error_to_api_error(e, endpoint, rid.clone()))?; let individuals = prepared.individuals; let trait_values = if had_samples { if matched == 0 { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "no_samples_aligned", "No samples aligned", format!( "none of the {supplied_count} named samples appear in genotype \ version '{}'.", resolved.version ), endpoint, ) .with_request_id(rid)); } aligned } else { let Some(supplied) = supplied_values else { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_trait_values", "No trait values", "give samples (named) or trait_values (one per individual).".to_string(), endpoint, ) .with_request_id(rid)); }; if supplied.len() != individuals { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "trait_values_length", "Trait values do not match the panel", format!( "genotype version '{}' has {} individuals after the parent/F1 rule; \ got {} trait_values.", resolved.version, individuals, supplied.len() ), endpoint, ) .with_request_id(rid)); } supplied }; let empty_clause = if dropped_empty > 0 { format!( ", {dropped_empty} left out with no genotypes at all (an empty kinship row \ makes the matrix singular)" ) } else { String::new() }; // What the selection did, beyond the parent/F1 count: a caller // who names individuals deserves to be told which way it went. let selection_clause = { let mut parts = Vec::new(); if included_by_name > 0 { parts.push(format!("{included_by_name} re-included by name against the parent/F1 rule")); } if excluded_by_name > 0 { parts.push(format!("{excluded_by_name} excluded by name")); } if parts.is_empty() { String::new() } else { format!("; {}", parts.join(", ")) } }; notes.push(format!( "Genotype: {} (version {}, {} individuals, {} parents/F1s dropped{empty_clause}{selection_clause}); \ the kinship uses all {} (GEMMA does not read phenotypes for it), the association \ uses the {} with a trait value.", prepared.note, resolved.version, individuals, parents_f1s, individuals, trait_values.iter().filter(|value| value.is_some()).count() )); aligned_names = Some(kept_names); aligned_group = group.clone(); Ok(ResolvedInputs { genotype_file: prepared.genotype_file.to_string_lossy().into_owned(), snps_file: prepared .snps_file .map(|p| p.to_string_lossy().into_owned()) .unwrap_or_default(), individuals, trait_values, names: aligned_names, group: aligned_group, notes, }) } _ => Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give genotype_file + snps_file, or group (see /groups/{group}/genotypes).".to_string(), endpoint, ) .with_request_id(rid)), } } #[utoipa::path(post, path = "/compute/gemma", summary = "Map a QTL from a genotype file and trait values using GEMMA", description = "Runs gemma-wrapper's K-matrix step, then its LMM \ association step, over the given genotype and SNP annotation files and \ trait values, and returns the resulting rows of GEMMA's own \ .assoc.txt output. Pass ?id= to log progress for GET \ /compute/gemma/status/{id} to poll concurrently.", params(ComputeGemmaParams), request_body = GemmaRequest, responses( (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as positional arrays (column names in meta.columns)", body = Envelope), Unauthorized, (status = 409, description = "id is already in use by another run", body = ApiError), (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), (status = 502, description = "gemma-wrapper or GEMMA failed", body = ApiError) ))] #[gnauth(token_only)] #[compute_gate(gemma, run_id = params.id.as_deref())] pub async fn compute_gemma( ctx: Ctx, token: AuthToken, Query(params): Query, Json(mut request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let lmm = request.lmm.unwrap_or(9); // Only the values actually run against this GEMMA build (see the // `lmm` field doc above) are allowed; 5-8 are stock-invalid and // untested here, so they're rejected rather than guessed at. if !matches!(lmm, 1 | 2 | 3 | 4 | 9) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_lmm", "Invalid lmm", format!( "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)." ), "compute_gemma", ) .with_request_id(rid)); } if let Some(id) = ¶ms.id { if !streaming::is_valid_run_id(id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", "Invalid id", "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), "compute_gemma", ) .with_request_id(rid)); } } // Permutations are refused rather than ignored: gemma-wrapper's // `--permutate` shuffles the phenotype file and runs the *whole* LOCO // GWA again for each permutation, so 1000 of them is thousands of // full scans -- hours to days on a real panel, with the single slot // this endpoint has. R/qtl has the permutation test (nperm), on the // same panel and the same values. let requested_permutations = [request.nperm, request.permutations, request.permutate] .into_iter() .flatten() .max() .unwrap_or(0); if requested_permutations > 0 { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "permutations_not_supported", "GEMMA permutations are not supported", format!( "{requested_permutations} permutations requested, but this endpoint runs none: \ each one repeats the entire LOCO association scan (gemma-wrapper --permutate), \ so a thousand is thousands of scans and hours of the one slot GEMMA has here. \ Two ways to get a threshold instead: (1) send the same trait values to \ POST /compute/bulklmm with \"nperm\": 500 -- it is the same LMM family as this \ scan and permutes by rotating the data through the kinship decomposition, so \ its thresholds respect relatedness (R/qtl's nperm does not, and is \ anti-conservative on a structured panel). Measured on BXD trait 10001 with 500 \ rotations: both LMMs pick the same peak marker, rsm10000005700 on chr 8 (GEMMA \ -log10 p 6.23, BulkLMM LOD 3.87), and BulkLMM's own 0.05 threshold is LOD 3.78, \ so the peak clears it. The numbers are not interchangeable: BulkLMM's kinship \ is one genome-wide matrix rather than LOCO, and on the same panel and trait the \ LOCO permutation null came out lower (R/qtl2's scan1perm, 500 permutations: \ 0.05 threshold LOD 2.62). (2) Run GEMMA's own permutation locally on the same \ inputs -- the files this endpoint used are named in the reply's notes (the \ bimbam pair and the phenotype file), so `gemma-wrapper --permutate` or \ `gemma -lmm 9 --permutate` reproduces exactly the null GEMMA would use, at \ hours rather than seconds." ), "compute_gemma", ) .with_request_id(rid) // The error contract carries one `next` href; the description in the // detail above says what to do with it. .with_next("/compute/bulklmm")); } let config = Config::load(); let mut extra_notes: Vec = Vec::new(); let normalization = crate::normalize::Pipeline::parse( request.normalize.as_deref(), request.winsor_quantile, request.winsor_fences, ) .map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_gemma", ) .with_request_id(rid.clone()) })?; let request_group = request.group.clone(); let request_genotype_version = request.genotype_version.clone(); let request_covariates = request.covariates.clone(); let request_covariates_present = !request_covariates.is_empty(); // What to call this mapping in notes and suggestions: the caller's own // label when it gave one, else the group's -- GEMMA's values are // positional, so there is nothing else to name it by. let request_trait_label = request .trait_name .clone() .filter(|name| !name.trim().is_empty()) .unwrap_or_else(|| format!("{}::gemma", request.group.clone().unwrap_or_default())); let original_samples: Option> = request.samples.as_ref().map(|samples| { samples .iter() .map(|sample| serde_json::json!({"name": sample.name, "value": sample.value})) .collect() }); let mut covariate_list: Vec = Vec::new(); let mut covariate_notes: Vec = Vec::new(); let mut aligned_names: Option> = None; let mut aligned_group = String::new(); let ResolvedInputs { genotype_file, snps_file, trait_values, individuals: _, names: aligned_names, group: aligned_group, notes: input_notes, } = resolve_inputs(&config, &mut request, &token, "compute_gemma", rid.clone()).await?; extra_notes.extend(input_notes); // Covariates are aligned by sample name, so only a group-scoped request // can have them: with explicit files there is nothing to align against. if request_covariates_present && request_group.is_none() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "covariate_unavailable", "Covariate unavailable", "covariates need a `group` on the request: individual names are what they are \ aligned against, and explicit genotype_file/snps_file requests carry none" .to_string(), "compute_gemma", ) .with_request_id(rid)); } // Normalize after alignment: over exactly the individuals that enter the // fit, so `beta`/`additive` come out on the scale the note names. let trait_values = if normalization.is_identity() { trait_values } else { let (normalized, applied) = normalization.apply(&trait_values).map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_gemma", ) .with_request_id(rid.clone()) })?; extra_notes.push(normalization.describe(&applied)); normalized }; // A large panel takes the one GEMMA slot here, where the size is known; // a small one runs alongside whatever else is running. See // `compute_gate::GEMMA_LARGE_PANEL`. let panel_individuals = aligned_names.as_ref().map(Vec::len).unwrap_or(0); __gn4_claim .claim_slot_or_busy(panel_individuals, rid.clone()) .await?; // Covariates are per individual, so they need the panel's names: the // same individuals, in the same order, as the phenotype file above. // Resolved here rather than beside the panel because the note it // reports counts the individuals that have *both* the trait and the // covariate -- the complete cases the fit will actually use. if let Some(names) = &aligned_names { let has_trait: Vec = trait_values.iter().map(Option::is_some).collect(); (covariate_list, covariate_notes) = covariates::resolve( &request_covariates, names, Some(&aligned_group), Some(&has_trait), rid.clone(), ) .await?; } let wrapper = config.gemma_wrapper_cmd.clone(); let cache_dir = config.gemma_cache_dir.clone(); let run_id = params.id.clone(); let rid_for_job = rid.clone(); let trait_values_for_job = trait_values.clone(); let covariates_for_job = covariate_list.clone(); compute_gate::gemma() .run(__gn4_claim, async move { gemma::compute( &wrapper, &cache_dir, &genotype_file, &snps_file, &trait_values_for_job, &covariates_for_job, lmm, run_id.as_deref(), ) .await .map_err(|e| match e { GemmaError::RunIdInUse(id) => ApiError::new( StatusCode::CONFLICT, "run_id_in_use", "Run id already in use", format!("id '{id}' is already in use; pick a different id."), "compute_gemma", ) .with_request_id(rid_for_job.clone()), other => gemma_error_to_api_error(other, rid_for_job.clone()), }) }) .await .map(|output| { let num_rows = output.rows.as_array().map(Vec::len).unwrap_or(0); // First, as on /compute/rqtl: it is what tells two responses from // the same panel apart, and `/plot/gemma` reads it for its title. let mut envelope = ctx .envelope(output.rows) .with_columns(output.columns, num_rows) .with_note(&format!("Trait: {}", request_trait_label)) .with_note( "Rows come from GEMMA's own .assoc.txt output, positional (see meta.columns for names); column names depend on the lmm value used.", ) .with_note( "Effect size: `beta` is GEMMA's effect per copy of the maternal allele on the \ trait's own scale and `se` its standard error (95% CI = beta +/- 1.96*se); \ `additive` is the same effect in GeneNetwork's convention, -beta/2 -- halved \ because half the difference between the two homozygotes is the usual additive \ effect, and negated because GEMMA counts maternal copies while GeneNetwork's \ published sign is anchored on the other allele. It sits just before the \ p-value, which stays the last column. These describe *this run*, on these \ samples and this genotype version -- not the `additive` trait info reports, \ which came from the published mapping.", ); for note in &output.notes { envelope = envelope.with_note(note); } for note in &covariate_notes { envelope = envelope.with_note(note); } for note in &extra_notes { envelope = envelope.with_note(note); } // The peak is the row with the smallest p_lrt (the last column of // GEMMA's own .assoc output); a PheWAS there is the next question. if let Some(group) = request_group.as_deref() { if let Some(peak) = gemma::peak_marker(&envelope.data) { envelope = envelope.with_next( &format!( "/groups/{}/phewas?locus={}&window=10000", path_segment(group), path_segment(&peak) ), "PheWAS: every published trait peaking within 10 kb of this GWAS's peak", ); let version = request .genotype_version .clone() .unwrap_or_else(|| "{version}".to_string()); let (href, description) = crate::suggest::class_hint(group, &version, &peak); envelope = envelope.with_next(&href, description); envelope = envelope.with_links(crate::suggest::group_links(group)); } } envelope = envelope.with_note( "To plot this: rows are positional, so chr (meta.columns[0]) against \ -log10 of p_lrt (the last column) is a Manhattan plot with no reshaping.", ); envelope = envelope.with_note( "GEMMA reports a p-value per marker, not a permutation threshold: judge a peak \ against the smallest p_lrt it reached, not against a genome-wide threshold. \ Permutations are deliberately not run here -- each one repeats the whole LOCO \ scan -- so a request asking for them is refused, not silently ignored. Use \ POST /compute/rqtl with nperm for suggestive and significant LOD thresholds on \ the same values.", ); // The picture: post this response back (no body duplicated here // -- it is the response the caller already has). envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: format!("/plot/gemma?trait={}", path_segment(&request_trait_label)), description: "Draw this GWAS: POST this response back to /plot/gemma (as-is, or its \ `data` member) and get an SVG Manhattan plot of -log10(p_lrt) per \ marker. Add ?bonferroni=true for GEMMA's own line, or \ ?thresholds= for one computed elsewhere (see /compute/bulklmm)." .to_string(), body: None, }); // GEMMA has no permutation test and this endpoint passes it no // permutation option (gemma-wrapper's own --permutate shuffles // the phenotype file; see docs/genotypes.md). R/qtl does, on the // same panel and the same values, so that is the runnable // suggestion rather than a flag that would do nothing. if let Some(group) = request_group.clone() { let mut body = serde_json::Map::new(); body.insert("group".to_string(), serde_json::json!(group.clone())); if let Some(version) = request_genotype_version.clone() { body.insert("genotype_version".to_string(), serde_json::json!(version)); } body.insert( "trait_name".to_string(), serde_json::json!(request_trait_label.clone()), ); if let Some(samples) = original_samples.clone() { body.insert("samples".to_string(), serde_json::Value::Array(samples)); } for (key, value) in normalization.to_json() { body.insert(key, value); } body.insert("nperm".to_string(), serde_json::json!(100)); // The LMM approximation first: BulkLMM is the same model family as // this scan, so a threshold from it means something for a // relatedness-structured panel, which R/qtl's haley-knott // permutation does not. Same peak marker on BXD 10001, 500 // rotations in about fourteen seconds. let mut lmm_body = body.clone(); lmm_body.insert("nperm".to_string(), serde_json::json!(500)); lmm_body.insert("seed".to_string(), serde_json::json!(1)); envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: "/compute/bulklmm".to_string(), description: "Significance thresholds from the same LMM family: post these values \ to /compute/bulklmm with 500 rotations (nperm=500). It permutes \ through the kinship decomposition, so the null respects relatedness, \ and 500 rotations cost about 14 s against GEMMA's hours. Its kinship \ is one genome-wide matrix rather than LOCO, so treat the threshold as \ an approximation of GEMMA's -- on BXD trait 10001 both picked the same \ peak marker." .to_string(), body: Some(serde_json::Value::Object(lmm_body)), }); envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: "/compute/rqtl".to_string(), description: "A non-LMM cross-check: map the same values over the same panel \ with R/qtl and 100 permutations (nperm=100). Its haley-knott \ permutation ignores relatedness, so the threshold is \ anti-conservative on a structured panel -- prefer the BulkLMM call \ above for a null that does." .to_string(), body: Some(serde_json::Value::Object(body)), }); } Json(envelope) }) } #[utoipa::path(post, path = "/compute/gemma/ravanan", summary = "Map a QTL with GEMMA, run via ravanan instead of gemma-wrapper directly", description = "Same computation and request/response shape as POST \ /compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \ step, both --loco), but driven by ravanan \ (https://github.com/vepiteski/ravanan) running a CWL workflow compiled \ from the ccwl source at cwl/gemma/gemma-loco.scm instead of this server's own direct subprocess \ calls -- an alternative runner for the same two gemma-wrapper steps, \ for comparing the two (correctness, timing). Exploratory: no caching of \ ravanan's own store between requests beyond its --store directory, and \ ?id=/status polling tails ravanan's own stderr rather than gemma-wrapper's.", params(ComputeGemmaParams), request_body = GemmaRequest, responses( (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as positional arrays (column names in meta.columns)", body = Envelope), Unauthorized, (status = 409, description = "id is already in use by another run", body = ApiError), (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError), (status = 502, description = "ravanan or gemma-wrapper failed", body = ApiError) ))] #[gnauth(token_only)] #[compute_gate( ravanan, label = "GEMMA (ravanan)", status = "/compute/gemma/status", run_id = params.id.as_deref() )] pub async fn compute_gemma_ravanan( ctx: Ctx, token: AuthToken, Query(params): Query, Json(mut request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); let lmm = request.lmm.unwrap_or(9); if !matches!(lmm, 1 | 2 | 3 | 4 | 9) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_lmm", "Invalid lmm", format!( "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)." ), "compute_gemma_ravanan", ) .with_request_id(rid)); } if let Some(id) = ¶ms.id { if !streaming::is_valid_run_id(id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", "Invalid id", "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), "compute_gemma_ravanan", ) .with_request_id(rid)); } } let config = Config::load(); // This endpoint takes the same request shape as /compute/gemma -- // group or file, samples or trait_values, basket or upload -- because // both runners are handed exactly the same three things. Resolving it // here (rather than passing "/compute/gemma"'s fields through) is what // makes a `group` + `samples` request work at all; before, those fields // were silently dropped and the run died reading an empty path. let ResolvedInputs { genotype_file, snps_file, trait_values, individuals: panel_individuals, names: _, group: _, notes: input_notes, } = resolve_inputs(&config, &mut request, &token, "compute_gemma_ravanan", rid.clone()).await?; if genotype_file.is_empty() || snps_file.is_empty() || trait_values.is_empty() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_inputs", "No mapping inputs", "ravanan needs a genotype file and a SNP map (a group whose version has a BIMBAM \ pair, an upload, or genotype_file + snps_file) and at least one trait value." .to_string(), "compute_gemma_ravanan", ) .with_request_id(rid)); } // Covariates and normalization are the direct runner's: the CWL workflow // has no covariate step, and normalization is applied to the values // before they are written out there. Refused rather than ignored -- // a caller who sent them would otherwise get a fit they did not ask for. if !request.covariates.is_empty() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "covariates_not_supported", "Covariates are not supported here", "the ravanan workflow runs gemma-wrapper per chromosome and has no covariate \ step, so `covariates` cannot be fitted on this path. Use POST /compute/gemma \ for a covariate fit, or drop the covariates." .to_string(), "compute_gemma_ravanan", ) .with_request_id(rid)); } if request.normalize.as_deref().map(|n| !n.trim().is_empty()) == Some(true) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "normalization_not_supported", "Normalization is not supported here", "send the values already normalized, or use POST /compute/gemma, which applies \ its normalization pipeline before the run." .to_string(), "compute_gemma_ravanan", ) .with_request_id(rid)); } // Gated like the direct GEMMA run and by the same size rule, now with the // real panel size: a samples-only request used to be sized 0 and took no // slot at all. __gn4_claim .claim_slot_or_busy(panel_individuals, rid.clone()) .await?; let run_id = params.id.clone(); let job_rid = rid.clone(); compute_gate::ravanan() .run(__gn4_claim, async move { gemma::compute_ravanan( &config.ravanan_cmd, &config.ravanan_batch, &config.ccwl_cmd, &config.gemma_ravanan_workflow, &config.gemma_ravanan_manifest, &config.ravanan_cache_dir, &config.ravanan_scratch, &genotype_file, &snps_file, &trait_values, lmm, run_id.as_deref(), ) .await .map_err(|e| match e { GemmaError::RunIdInUse(id) => ApiError::new( StatusCode::CONFLICT, "run_id_in_use", "Run id already in use", format!("id '{id}' is already in use; pick a different id."), "compute_gemma_ravanan", ) .with_request_id(job_rid.clone()), other => gemma_error_to_api_error(other, job_rid.clone()), }) }) .await .map(|output| { let num_rows = output.rows.as_array().map(Vec::len).unwrap_or(0); let mut envelope = ctx .envelope(output.rows) .with_columns(output.columns, num_rows) .with_note( "Rows come from GEMMA's own .assoc.txt output, positional (see meta.columns for names); column names depend on the lmm value used.", ); // What the request resolved to first (which genotype version, how many // values aligned, what was dropped), then the run's own notes. for note in &input_notes { envelope = envelope.with_note(note); } for note in &output.notes { envelope = envelope.with_note(note); } Json(envelope) }) } #[utoipa::path(get, path = "/compute/gemma/status/{run_id}", summary = "Poll a running (or finished) /compute/gemma call's progress", description = "Returns the lines gemma-wrapper has written to stderr \ for this run_id since byte offset `peek`, and a new `pointer` to pass as \ `peek` next time. Call this on a second connection while the POST is \ still in flight, the same tail-the-log-file shape GeneNetwork2/3 use for \ R/qtl.", params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), RunStatusParams), responses( (status = 200, description = "New log lines and the new pointer", body = Envelope), Unauthorized, (status = 404, description = "No run with this id (unknown, or not started yet)", body = ApiError), (status = 422, description = "run_id not alnum/-/_ (max 64 chars)", body = ApiError) ))] #[gnauth(token_only)] pub async fn gemma_status( ctx: Ctx, _token: AuthToken, Path(run_id): Path, Query(params): Query, ) -> Result>, ApiError> { let rid = ctx.request_info(); if !streaming::is_valid_run_id(&run_id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", "Invalid run_id", "run_id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), "gemma_status", ) .with_request_id(rid)); } let config = Config::load(); let peek = params.peek.unwrap_or(0); // A run_id may belong to either runner (POST /compute/gemma or // POST /compute/gemma/ravanan), which log to different cache // dirs; try gemma's first, then fall back to ravanan's. let result = match streaming::status(&config.gemma_cache_dir, &run_id, peek) { Err(StatusError::NotFound) => { streaming::status(&config.ravanan_cache_dir, &run_id, peek) } other => other, }; result .map(|(lines, pointer)| { Json(ctx.envelope(RunStatus { run_id: run_id.clone(), lines, pointer, })) }) .map_err(|e| match e { StatusError::NotFound => ApiError::new( StatusCode::NOT_FOUND, "run_not_found", "Run not found", format!("No run with id '{run_id}' (unknown, or not started yet)."), "gemma_status", ) .with_request_id(rid), StatusError::Io(err) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", err.to_string(), "gemma_status", ) .with_request_id(rid), }) } /// A genotype resolution failure, as a caller-facing error (same shape as /// compute_rqtl's). fn genotype_error_to_api_error( error: genotypes::GenotypeError, operation: &str, rid: crate::response::RequestInfo, ) -> ApiError { let (status, code, title) = match error { genotypes::GenotypeError::Invalid(_) => ( StatusCode::UNPROCESSABLE_ENTITY, "invalid_individuals", "Unusable individual selection", ), genotypes::GenotypeError::NotFound(_) => ( StatusCode::NOT_FOUND, "genotypes_not_found", "Genotype file not found", ), genotypes::GenotypeError::Io(_) => ( StatusCode::INTERNAL_SERVER_ERROR, "genotypes_unreadable", "Genotype files could not be read", ), }; ApiError::new(status, code, title, error.to_string(), operation).with_request_id(rid) } fn gemma_error_to_api_error(error: GemmaError, rid: RequestInfo) -> ApiError { ApiError::new( StatusCode::BAD_GATEWAY, "gemma_failed", "GEMMA computation failed", error.to_string(), "compute_gemma", ) .with_request_id(rid) }