//! `POST /compute/fitqtl`: effect sizes for named markers. //! //! `/compute/rqtl` says *where* a QTL is (a LOD per marker); this says *how //! much*, by fitting R/qtl's `fitqtl` at the markers the caller names -- //! usually the peaks a scan already reported. It is a separate endpoint //! because it is a separate question, asked about a handful of positions //! rather than the whole genome: `fitqtl` fits one model per marker, so a //! genome-wide version of this would be thousands of fits. use axum::Json; use axum::extract::Query; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::{compute_gate, gnauth}; use crate::auth::AuthToken; use crate::compute_gate as gate; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; use crate::fitqtl; use crate::genotypes; use crate::normalize; use crate::response::{Ctx, Envelope, RequestInfo, path_segment}; use crate::rqtl::Sample; /// The most markers one request may fit. Each is a full model fit, so this /// is a guard against a caller handing over a whole genome by accident. const MAX_MARKERS: usize = 100; #[derive(Deserialize, utoipa::IntoParams)] pub struct ComputeFitqtlParams { /// Log progress for `GET /compute/rqtl/status/{id}` (shared with /// `/compute/rqtl`). pub id: Option, } /// One trait value, by individual name. #[derive(Clone, Deserialize, utoipa::ToSchema)] pub struct FitqtlSample { pub name: String, pub value: Option, } #[derive(Deserialize, utoipa::ToSchema)] pub struct FitqtlRequest { /// Group to fit over (an InbredSet name, e.g. "BXD"); the server /// resolves its genotype file. Give this or `genotype_file`. pub group: Option, /// A version from `/groups/{group}/genotypes`, e.g. "BXD.4". pub genotype_version: Option, /// Explicit `.geno` path on the server, instead of `group`. pub genotype_file: Option, /// An uploaded genotype from `/uploads/genotypes`, in place of /// `genotype_file`: the server writes it out and maps over it. pub genotype_upload: Option, /// Include parents and F1s; off by default, as elsewhere. pub include_parents_f1: Option, /// Individuals to keep or drop on top of the parent/F1 rule /// (`genotypes::Individuals`); `exclude` wins over `include`. #[serde(default)] pub individuals: genotypes::Individuals, /// The trait's values by individual name (aligned onto the panel), or /// `trait_values` positionally in the genotype file's order. pub samples: Option>, /// A basket from `/baskets`, in place of `samples` (see /// `crate::routes::baskets`). Its revision and digest are recorded in /// `notes` so a result on an edited basket stays distinguishable. pub basket: Option, /// Positional values, for a `genotype_file` request. pub trait_values: Option>>, /// Label for the trait, e.g. "BXD::20010"; used in `notes`. pub trait_name: Option, /// The markers to fit (from `/groups/{g}/genotypes/{v}/markers`), at /// most 100. pub markers: Vec, /// R/qtl's `model`: "normal" (default) or "np". pub model: Option, /// R/qtl's `method` for the QTL fit: "hk" (default), "ehk", "mr", "em", /// "imp". pub method: Option, /// "mb" (default) or "cm". pub scale: Option, /// Covariates to fit alongside the QTL, exactly as on `/compute/rqtl`: /// a case attribute of the group by name, or inline values. They need a /// `group` (the individuals are aligned by name). #[serde(default)] pub covariates: Vec, /// Normalisation of the trait values, as on `/compute/rqtl`. pub normalize: Option, /// Winsorise at a tail fraction, as on `/compute/rqtl`. pub winsor_quantile: Option, /// Winsorise at Tukey fences, as on `/compute/rqtl`. pub winsor_fences: Option, } #[utoipa::path(post, path = "/compute/fitqtl", summary = "Effect sizes for named markers, via R/qtl's fitqtl", description = "Fits R/qtl's `fitqtl` at each named marker and returns, per \ marker, the model LOD, the percentage of variance explained, and the QTL's \ additive effect with its standard error and t. Use /compute/rqtl to find the \ peaks, then this to size them: a scan statistic is not a fitted model. The \ `additive` column is fitqtl's estimate for the QTL term -- in R/qtl's +-1 \ coding, half the difference between the two homozygote classes, the same \ convention GeneNetwork's own additive uses.", params(ComputeFitqtlParams), request_body = FitqtlRequest, responses( (status = 200, description = "One row per fitted marker, positional (column names in meta.columns)", body = Envelope), Unauthorized, (status = 409, description = "A compute job is already running for this caller", body = ApiError), (status = 422, description = "No markers, too many, or an unusable genotype/trait input", body = ApiError), (status = 502, description = "R/qtl failed", body = ApiError) ) )] #[gnauth(token_only)] #[compute_gate(rqtl, label = "R/qtl fitqtl", run_id = params.id.as_deref())] pub async fn compute_fitqtl( ctx: Ctx, token: AuthToken, Query(params): Query, Json(mut request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); if let Some(upload) = request.genotype_upload.clone() { if request.genotype_file.is_some() || request.group.is_some() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give genotype_upload, genotype_file or group -- one of the three.".to_string(), "compute_fitqtl", ) .with_request_id(rid)); } let (path, _) = crate::routes::genoupload::resolve_upload( token.sub.clone(), Config::load().basket_dir, Config::load().genotype_files_dir, upload, "compute_fitqtl", rid.clone(), false, ) .await?; request.genotype_file = Some(path); } // A basket is another source for the caller's values, resolved before // anything below looks at them (see crate::routes::baskets). let mut basket_note: Option = None; if let Some(name) = request.basket.clone() { if request.samples.is_some() { return Err(crate::routes::baskets::ambiguous("compute_fitqtl", rid)); } let (resolved, note) = crate::routes::baskets::resolve_for_mapping( token.sub.clone(), Config::load().basket_dir, name, "compute_fitqtl", rid.clone(), ) .await?; basket_note = Some(note); request.samples = Some( resolved .values .iter() .map(|(name, value)| FitqtlSample { name: name.clone(), value: *value, }) .collect(), ); } let config = Config::load(); let trait_label = request .trait_name .clone() .filter(|name| !name.trim().is_empty()) .unwrap_or_else(|| format!("{}::fitqtl", request.group.clone().unwrap_or_default())); if request.markers.is_empty() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_markers", "No markers", "give `markers`: the marker names to fit (a scan's peaks, usually). \ /compute/rqtl finds them; this sizes them." .to_string(), "compute_fitqtl", ) .with_request_id(rid)); } if request.markers.len() > MAX_MARKERS { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "too_many_markers", "Too many markers", format!( "{} markers given; at most {MAX_MARKERS} are fitted per request (each is a \ model fit). Use /compute/rqtl for a genome-wide scan.", request.markers.len() ), "compute_fitqtl", ) .with_request_id(rid)); } let pipeline = normalize::Pipeline::parse( request.normalize.as_deref(), request.winsor_quantile, request.winsor_fences, ) .map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_fitqtl", ) .with_request_id(rid.clone()) })?; let request_group = request.group.clone(); let include_parents_f1 = request.include_parents_f1.unwrap_or(false); let individuals = request.individuals.clone(); let mut extra_notes: Vec = Vec::new(); let (genotype_file, samples) = match ( request.genotype_file.clone(), request.group.clone(), ) { (None, Some(group)) => { let wanted = group.clone(); let version = request.genotype_version.clone(); let values = request .samples .clone() .map(|samples| { samples .into_iter() .map(|sample| (sample.name, sample.value)) .collect::>() }) .unwrap_or_default(); let directory = config.genotype_files_dir.clone(); let cache_dir = config.rqtl_cache_dir.clone(); let supplied = values.len(); let (prepared, aligned, matched, resolved, kept_names) = tokio::task::spawn_blocking(move || { let resolved = genotypes::resolve( std::path::Path::new(&directory), &wanted, version.as_deref(), )?; let parents_f1s = genotypes::non_strains( std::path::Path::new(&directory), &wanted, &resolved.panel, ); let selection = genotypes::select_panel( &resolved.panel, &parents_f1s, include_parents_f1, &individuals, )?; let keep: Vec = selection.keep.clone(); let (aligned, matched) = genotypes::align(&resolved.panel, &keep, &values); let prepared = genotypes::prepare( std::path::Path::new(&directory), &resolved, genotypes::Engine::Rqtl, &keep, std::path::Path::new(&cache_dir), )?; let kept_names: Vec = keep .iter() .filter_map(|index| resolved.panel.get(*index).cloned()) .collect(); Ok::<_, genotypes::GenotypeError>(( prepared, aligned, matched, resolved, kept_names, )) }) .await .map_err(|e| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", e.to_string(), "compute_fitqtl", ) .with_request_id(rid.clone()) })? .map_err(|e| genotypes_error(e, rid.clone()))?; if matched == 0 { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "no_samples_aligned", "No samples aligned", format!( "none of the {supplied} named samples appear in genotype version '{}'.", resolved.version ), "compute_fitqtl", ) .with_request_id(rid)); } extra_notes.push(format!( "Genotype: {} (version {}, {} individuals, {} parents/F1s dropped); {} of {} supplied values aligned onto it by name.", prepared.note, resolved.version, prepared.individuals, prepared.excluded, matched, supplied )); let samples: Vec = kept_names .into_iter() .zip(aligned) .map(|(name, value)| Sample { name, value }) .collect(); (prepared.genotype_file.to_string_lossy().into_owned(), samples) } (Some(file), None) => { let values = request.trait_values.clone().unwrap_or_default(); let named: Vec<(String, Option)> = request .samples .clone() .unwrap_or_default() .into_iter() .map(|sample| (sample.name, sample.value)) .collect(); if values.is_empty() && named.is_empty() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_trait_values", "No trait values", "with an explicit genotype_file, give `samples` (name and value), a `basket`, \ or `trait_values` (one per individual, in the file's order)." .to_string(), "compute_fitqtl", ) .with_request_id(rid)); } let samples: Vec = if !named.is_empty() { // Values are for animals, not for columns: align them onto the // file's own individuals. let path = std::path::PathBuf::from(&file); let names = tokio::task::spawn_blocking({ let path = path.clone(); move || genotypes::panel_of(&path) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_fitqtl", ) .with_request_id(rid.clone()) })? .map_err(|error| genotypes_error(error, rid.clone()))?; let aligned = tokio::task::spawn_blocking(move || { genotypes::align_to_file(&path, &named) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_fitqtl", ) .with_request_id(rid.clone()) })? .map_err(|error| { genotypes_error(error, rid.clone()) })?; names .into_iter() .zip(aligned) .map(|(name, value)| Sample { name, value }) .collect() } else { values .into_iter() .enumerate() .map(|(index, value)| Sample { name: format!("individual-{}", index + 1), value, }) .collect() }; (file, samples) } (Some(_), Some(_)) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give either genotype_file or group, not both.".to_string(), "compute_fitqtl", ) .with_request_id(rid)); } (None, None) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_genotype", "No genotype input", "give genotype_file, or group (see /groups/{group}/genotypes).".to_string(), "compute_fitqtl", ) .with_request_id(rid)); } }; // Normalize after alignment, over the individuals that enter the fits -- // the effects come out on that scale, as on /compute/rqtl. let samples = if pipeline.is_identity() { samples } else { let raw: Vec> = samples.iter().map(|sample| sample.value).collect(); let (normalized, applied) = pipeline.apply(&raw).map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_fitqtl", ) .with_request_id(rid.clone()) })?; extra_notes.push(pipeline.describe(&applied)); samples .into_iter() .zip(normalized) .map(|(sample, value)| Sample { name: sample.name, value, }) .collect() }; // Covariates, on the same sample scale as the trait values -- the same // resolver `/compute/rqtl` uses, so the coding and the complete-case // count are reported identically. let sample_names: Vec = samples.iter().map(|sample| sample.name.clone()).collect(); let has_trait: Vec = samples.iter().map(|sample| sample.value.is_some()).collect(); let group_for_covariates = request_group.clone(); let (covariate_list, covariate_notes) = crate::covariates::resolve( &request.covariates, &sample_names, group_for_covariates.as_deref(), Some(&has_trait), rid.clone(), ) .await?; extra_notes.extend(covariate_notes); let individuals = samples.len(); __gn4_claim .claim_slot_or_busy(individuals, rid.clone()) .await?; let script = config.fitqtl_script_path.clone(); let cache_dir = config.rqtl_cache_dir.clone(); let markers = request.markers.clone(); let model = request.model.clone().unwrap_or_else(|| "normal".to_string()); let method = request.method.clone().unwrap_or_else(|| "hk".to_string()); let scale = request.scale.clone().unwrap_or_else(|| "mb".to_string()); let covariates_for_job = covariate_list.clone(); let trait_label_for_job = trait_label.clone(); let rid_for_job = rid.clone(); let output = gate::rqtl() .run(__gn4_claim, async move { fitqtl::compute( &script, &cache_dir, &genotype_file, &trait_label_for_job, &samples, &markers, &covariates_for_job, &model, &method, &scale, ) .await .map_err(|error| { ApiError::new( StatusCode::BAD_GATEWAY, "fitqtl_failed", "R/qtl fitqtl failed", error.to_string(), "compute_fitqtl", ) .with_request_id(rid_for_job) }) }) .await?; let num_rows = output.rows.len(); let rows = serde_json::Value::Array(output.rows); let peak_marker = crate::suggest::peak_marker_of(&rows, 0, 3); let mut envelope = ctx .envelope(rows) .with_columns( fitqtl::COLUMNS.iter().map(|name| name.to_string()).collect(), num_rows, ) .with_note(&format!("Trait: {trait_label}")) .with_note( "Rows come from R/qtl's fitqtl, one per marker that could be fitted (positional; see \ meta.columns). `additive` is the QTL effect in R/qtl's +-1 coding -- half the \ difference between the two homozygote classes, the same convention GeneNetwork's \ own additive uses -- with `se` and `t`; `pct_var` is the model's percentage of \ variance explained and `p` its p-value. These describe this run on these samples; \ markers that are not in the genotype file, or that fitqtl could not fit, are \ absent rather than reported as zero.", ) .with_note(&output.command) .with_note(&format!("fitqtl: {}", output.summary)); for note in &extra_notes { envelope = envelope.with_note(note); } if let Some(note) = &basket_note { envelope = envelope.with_note(note); } if let Some(group) = request_group { if let Some(peak) = &peak_marker { envelope = envelope.with_next( &format!( "/groups/{}/phewas?locus={}", path_segment(&group), path_segment(peak) ), "PheWAS: every published trait peaking at this fit's strongest marker", ); } let version = request .genotype_version .clone() .unwrap_or_else(|| "{version}".to_string()); if let Some(peak) = &peak_marker { let (href, description) = crate::suggest::class_hint(&group, &version, peak); envelope = envelope.with_next(&href, description); } envelope = envelope.with_links(crate::suggest::group_links(&group)); } Ok(Json(envelope)) } fn genotypes_error(error: genotypes::GenotypeError, rid: RequestInfo) -> ApiError { let (status, code, title) = match error { genotypes::GenotypeError::Invalid(_) => ( StatusCode::UNPROCESSABLE_ENTITY, "invalid_individuals", "Unusable individual selection", ), genotypes::GenotypeError::NotFound(_) => ( StatusCode::NOT_FOUND, "genotypes_not_found", "Genotype file not found", ), genotypes::GenotypeError::Io(_) => ( StatusCode::INTERNAL_SERVER_ERROR, "genotypes_unreadable", "Genotype files could not be read", ), }; ApiError::new(status, code, title, error.to_string(), "compute_fitqtl") .with_request_id(rid) }