//! Compute domain: /compute/rqtl runs GeneNetwork3's own //! `scripts/rqtl_wrapper.R` (via `Rscript`) to map a QTL from a //! genotype file and trait values, the same shape as `/compute/gemma` //! including its progress-streaming `?id=`/`GET .../status/{run_id}` //! pair -- see `crate::streaming` for that and `crate::rqtl` for the //! invocation and its documented scope (single scan + permutations; //! no covariates, interval mapping or pair-scan). use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::{compute_gate, gnauth}; use crate::auth::AuthToken; use crate::config::Config; use crate::compute_gate; use crate::covariates; use crate::error::{ApiError, Unauthorized}; use crate::genotypes; use crate::response::{Ctx, Envelope, RequestInfo, path_segment}; use crate::rqtl::{self, RqtlError, Sample}; use crate::streaming::{self, RunStatus, RunStatusParams, StatusError}; /// `?id=` on `POST /compute/rqtl`: same as `/compute/gemma`'s -- an /// optional, caller-chosen, unique run id to log progress under and /// poll back via `GET /compute/rqtl/status/{id}`. #[derive(Deserialize, utoipa::IntoParams)] pub struct ComputeRqtlParams { pub id: Option, } #[derive(Deserialize, utoipa::ToSchema)] pub struct RqtlSample { /// Must match the genotype file's sample-column order exactly; see /// `crate::rqtl`'s module doc for why (R/qtl's cross format has no /// per-sample name matching of its own). pub name: String, pub value: Option, } #[derive(Deserialize, utoipa::ToSchema)] pub struct RqtlRequest { /// Path to an R/qtl-format `.geno` file (riset/F2/4-way; `@type:` /// in its header). Give this, or `group` -- not both. pub genotype_file: Option, /// An uploaded genotype from `/uploads/genotypes`, in place of /// `genotype_file`: the server writes it out and maps over it. pub genotype_upload: Option, /// Group to map over (an InbredSet name, e.g. "BXD"). The server /// resolves the genotype file from the deployment's genotype-files /// directory, which is what `/groups/{group}/genotypes` lists. pub group: Option, /// A version from that listing, e.g. "BXD.8" or "4". Defaults to /// the current one. pub genotype_version: Option, /// Include parents and F1s in the panel. Off by default, matching /// GeneNetwork2 (`incparentsf1`) and GeneNetwork3 /// (`get_sample_list(inc_par=False, inc_f1=False)`). pub include_parents_f1: Option, /// Individuals to keep or drop on top of the parent/F1 rule /// (`genotypes::Individuals`); `exclude` wins over `include`. #[serde(default)] pub individuals: genotypes::Individuals, pub trait_name: String, /// Covariates to fit alongside the trait, each either inline (named /// numeric values, which you converted yourself) or a case attribute /// of the group, whose text the server codes for you /// (`/groups/{group}/traits/{id}/case-attributes/{attribute}` shows /// the values it will code). #[serde(default)] pub covariates: Vec, /// One entry per individual being mapped. With `group` given, they are /// matched onto the panel **by name** (unknown names are dropped, and /// missing individuals get `null`), so a trait's own values are /// enough. pub samples: Option>, /// A basket from `/baskets`, in place of `samples` (see /// `crate::routes::baskets`). Its revision and digest are recorded in /// `notes` so a result on an edited basket stays distinguishable. pub basket: Option, /// R/qtl's `--model`: "normal" or "np" (non-parametric). Defaults /// to "normal". pub model: Option, /// R/qtl's `--method`: hk, ehk, mr, em or imp. Defaults to "hk" /// (Haley-Knott), the R script's own default. pub method: Option, /// Number of permutations; 0 (default) skips permutation testing /// entirely. pub nperm: Option, /// R/qtl's `--scale`: "mb" or "cm". Defaults to "mb". The script uses /// the column you ask for, and falls back to the other one when the /// file does not have it (a cM-only panel still maps) -- so `pos` in /// the reply is in the units the file carries. pub scale: Option, /// A marker name (from the genotype file) to use as a covariate. pub control: Option, /// How the trait values' scale is changed before mapping: `none` /// (default), `log2`, `log10`, `sqrt`, `negate` (GeneNetwork2 calls it /// invert), `zscore`, `robust_zscore`, or `rank` (GeneNetwork2's /// `qnorm`). Applied over the samples that actually enter the scan. pub normalize: Option, /// Winsorise: clip each tail at this fraction, e.g. 0.05 for the /// 5th/95th percentile. Mutually exclusive with `winsor_fences`. pub winsor_quantile: Option, /// Winsorise at Tukey fences: `Q1 - k*IQR` / `Q3 + k*IQR` (k = 1.5 is /// GeneNetwork2's outlier rule). Mutually exclusive with /// `winsor_quantile`. pub winsor_fences: Option, } /// How many covariates one request may fit. #[utoipa::path(post, path = "/compute/rqtl", summary = "Map a QTL from a genotype file and trait values using R/qtl", description = "Runs GeneNetwork3's scripts/rqtl_wrapper.R (scanone, \ plus permutations if nperm > 0) and returns the parsed marker results. \ Pass ?id= to log progress for GET /compute/rqtl/status/{id} to poll \ concurrently -- same shape as /compute/gemma.", params(ComputeRqtlParams), request_body = RqtlRequest, responses( (status = 200, description = "Mapped QTL markers, as positional arrays (column names in meta.columns), and, if requested, permutation thresholds", body = Envelope), Unauthorized, (status = 409, description = "id is already in use by another run", body = ApiError), (status = 422, description = "id not alnum/-/_ (max 64 chars)", body = ApiError), (status = 502, description = "Rscript or rqtl_wrapper.R failed", body = ApiError) ))] #[gnauth(token_only)] #[compute_gate(rqtl, label = "R/qtl", run_id = params.id.as_deref())] pub async fn compute_rqtl( ctx: Ctx, token: AuthToken, Query(params): Query, Json(mut request): Json, ) -> Result>, ApiError> { let rid = ctx.request_info(); if let Some(upload) = request.genotype_upload.clone() { if request.genotype_file.is_some() || request.group.is_some() { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give genotype_upload, genotype_file or group -- one of the three.".to_string(), "compute_rqtl", ) .with_request_id(rid)); } let (path, _) = crate::routes::genoupload::resolve_upload( token.sub.clone(), Config::load().basket_dir, Config::load().genotype_files_dir, upload, "compute_rqtl", rid.clone(), false, ) .await?; request.genotype_file = Some(path); } if let Some(id) = ¶ms.id { if !streaming::is_valid_run_id(id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", "Invalid id", "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), "compute_rqtl", ) .with_request_id(rid)); } } // `scale` selects the geno file's `Mb` or `cM` map column via the // wrapper's `--scale`; anything else would be silently ignored // there, so reject it here, loudly. let scale = match request .scale .as_deref() .unwrap_or("mb") .to_ascii_lowercase() { s if s == "mb" || s == "cm" => s, other => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_scale", "Invalid scale", format!("scale '{other}' not recognized; use 'mb' or 'cm'."), "compute_rqtl", ) .with_request_id(rid)); } }; let config = Config::load(); let mut extra_notes: Vec = Vec::new(); let normalization = crate::normalize::Pipeline::parse( request.normalize.as_deref(), request.winsor_quantile, request.winsor_fences, ) .map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_rqtl", ) .with_request_id(rid.clone()) })?; let request_group = request.group.clone(); // A basket is another source for the caller's values, resolved before // anything below looks at them (see crate::routes::baskets). let mut basket_note: Option = None; if let Some(name) = request.basket.clone() { if request.samples.is_some() { return Err(crate::routes::baskets::ambiguous("compute_rqtl", rid)); } let (resolved, note) = crate::routes::baskets::resolve_for_mapping( token.sub.clone(), config.basket_dir.clone(), name, "compute_rqtl", rid.clone(), ) .await?; basket_note = Some(note); request.samples = Some( resolved .values .iter() .map(|(name, value)| RqtlSample { name: name.clone(), value: *value, }) .collect(), ); } let samples: Vec = request.samples.take().unwrap_or_default(); let request_genotype_file = request.genotype_file.clone(); let request_genotype_version = request.genotype_version.clone(); let request_nperm = request.nperm.unwrap_or(0); let request_model = request.model.clone().unwrap_or_else(|| "normal".to_string()); let request_method = request.method.clone().unwrap_or_else(|| "hk".to_string()); // A request naming both a group and a file is answered as ambiguous even // when it also forgot its values: the input is the first thing to fix. let both_inputs = request.genotype_file.is_some() && request.group.is_some(); if samples.is_empty() && !both_inputs { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_samples", "No trait values", "give `samples` (name and value per individual) or a `basket` from /baskets." .to_string(), "compute_rqtl", ) .with_request_id(rid)); } // The caller's own named values, kept to rebuild a runnable body for // the permutation suggestion below (the aligned panel-length vector // would be thousands of entries on a real panel). let original_samples: Vec = samples .iter() .map(|sample| serde_json::json!({"name": sample.name, "value": sample.value})) .collect(); let (genotype_file, samples): (String, Vec) = match (request.genotype_file.clone(), request.group.clone()) { (Some(file), None) => { // The values are for animals, not for columns: read the file's // own individuals and put each value where it belongs. let path = std::path::PathBuf::from(&file); let named: Vec<(String, Option)> = samples .iter() .map(|sample| (sample.name.clone(), sample.value)) .collect(); let aligned = tokio::task::spawn_blocking(move || { genotypes::align_to_file(&path, &named) }) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_rqtl", ) .with_request_id(rid.clone()) })? .map_err(|error| genotype_error_to_api_error(error, "compute_rqtl", rid.clone()))?; let names = { let path = std::path::PathBuf::from(&file); tokio::task::spawn_blocking(move || genotypes::panel_of(&path)) .await .map_err(|error| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", error.to_string(), "compute_rqtl", ) .with_request_id(rid.clone()) })? .map_err(|error| { genotype_error_to_api_error(error, "compute_rqtl", rid.clone()) })? }; ( file, names .into_iter() .zip(aligned) .map(|(name, value)| Sample { name, value }) .collect(), ) } (None, Some(group)) => { let directory = std::path::PathBuf::from(&config.genotype_files_dir); let cache_dir = std::path::PathBuf::from(&config.rqtl_cache_dir); let version = request.genotype_version.clone(); let include_parents_f1 = request.include_parents_f1.unwrap_or(false); let individuals = request.individuals.clone(); let values: Vec<(String, Option)> = samples .iter() .map(|s| (s.name.clone(), s.value)) .collect(); let (prepared, aligned, matched, included_by_name, excluded_by_name) = tokio::task::spawn_blocking(move || { let resolved = genotypes::resolve(&directory, &group, version.as_deref())?; let parents_f1s = genotypes::non_strains(&directory, &group, &resolved.panel); let selection = genotypes::select_panel( &resolved.panel, &parents_f1s, include_parents_f1, &individuals, )?; let keep: Vec = selection.keep.clone(); let (aligned, matched) = genotypes::align(&resolved.panel, &keep, &values); let prepared = genotypes::prepare( &directory, &resolved, genotypes::Engine::Rqtl, &keep, &cache_dir, )?; Ok::<_, genotypes::GenotypeError>(( prepared, aligned, (matched, resolved), selection.included_by_name, selection.excluded_by_name, )) }) .await .map_err(|e| { ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", e.to_string(), "compute_rqtl", ) .with_request_id(rid.clone()) })? .map_err(|e| genotype_error_to_api_error(e, "compute_rqtl", rid.clone()))?; let (matched, resolved) = matched; if matched == 0 { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "no_samples_aligned", "No samples aligned", format!( "none of the {} named samples appear in genotype version '{}'; check the names against /groups/{}/strains", samples.len(), resolved.version, request.group.clone().unwrap_or_default() ), "compute_rqtl", ) .with_request_id(rid)); } extra_notes.push(format!( "Genotype: {} (version {}, {} individuals, {} parents/F1s dropped{}); {} of {} supplied values aligned onto it by name.", prepared.note, resolved.version, prepared.individuals, prepared.excluded, { let mut parts = Vec::new(); if included_by_name > 0 { parts.push(format!("{included_by_name} re-included by name")); } if excluded_by_name > 0 { parts.push(format!("{excluded_by_name} excluded by name")); } if parts.is_empty() { String::new() } else { format!("; {}", parts.join(", ")) } }, matched, samples.len() )); let panel = resolved.panel.clone(); let file = prepared.genotype_file.to_string_lossy().into_owned(); // The prepared file's individuals are the kept panel // entries, in panel order; name them from the resolved // panel so the pheno file carries the right labels. let excluded = if include_parents_f1 { std::collections::BTreeSet::new() } else { genotypes::non_strains( std::path::Path::new(&config.genotype_files_dir), &request.group.clone().unwrap_or_default(), &panel, ) }; let kept_names: Vec = panel .into_iter() .filter(|name| !excluded.contains(name)) .collect(); let samples = kept_names .into_iter() .zip(aligned) .map(|(name, value)| Sample { name, value }) .collect(); (file, samples) } (Some(_), Some(_)) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "ambiguous_genotype", "Ambiguous genotype input", "give either genotype_file or group, not both.".to_string(), "compute_rqtl", ) .with_request_id(rid)); } (None, None) => { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "missing_genotype", "No genotype input", "give genotype_file, or group (see /groups/{group}/genotypes).".to_string(), "compute_rqtl", ) .with_request_id(rid)); } }; // Covariates, on the same sample scale as `samples` (see // src/covariates.rs: the spec shape, alignment and coding are shared // with /compute/gemma). let sample_names: Vec = samples.iter().map(|s| s.name.clone()).collect(); let has_trait: Vec = samples.iter().map(|sample| sample.value.is_some()).collect(); let (covariate_list, covariate_notes) = covariates::resolve( &request.covariates, &sample_names, request_group.as_deref(), Some(&has_trait), rid.clone(), ) .await?; // Normalize now: after alignment, so z-scores and ranks are computed over // exactly the samples that enter the scan, and before anything else reads // the values (the covariates' complete-case count, the scan itself). let samples = if normalization.is_identity() { samples } else { let raw: Vec> = samples.iter().map(|sample| sample.value).collect(); let (normalized, applied) = normalization.apply(&raw).map_err(|detail| { ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_normalization", "Invalid normalization", detail, "compute_rqtl", ) .with_request_id(rid.clone()) })?; extra_notes.push(normalization.describe(&applied)); samples .into_iter() .zip(normalized) .map(|(sample, value)| rqtl::Sample { name: sample.name, value, }) .collect() }; // A large cross takes one of the R/qtl slots here, where the size is // known; a small one runs alongside whatever else is running. See // `compute_gate::RQTL_LARGE_PANEL`. __gn4_claim .claim_slot_or_busy(samples.len(), rid.clone()) .await?; let wrapper_path = config.rqtl_wrapper_path.clone(); let cache_dir = config.rqtl_cache_dir.clone(); let trait_name = request.trait_name.clone(); let samples_for_job = samples.clone(); let covariates_for_job = covariate_list.clone(); let model = request.model.clone().unwrap_or_else(|| "normal".to_string()); let method = request.method.clone().unwrap_or_else(|| "hk".to_string()); let nperm = request.nperm.unwrap_or(0); let control = request.control.clone(); let run_id = params.id.clone(); let rid_for_job = rid.clone(); compute_gate::rqtl() .run(__gn4_claim, async move { rqtl::compute( &wrapper_path, &cache_dir, &genotype_file, &trait_name, &samples_for_job, &covariates_for_job, &model, &method, nperm, &scale, control.as_deref(), run_id.as_deref(), ) .await .map_err(|e| match e { RqtlError::RunIdInUse(id) => ApiError::new( StatusCode::CONFLICT, "run_id_in_use", "Run id already in use", format!("id '{id}' is already in use; pick a different id."), "compute_rqtl", ) .with_request_id(rid_for_job.clone()), other => rqtl_error_to_api_error(other, rid_for_job.clone()), }) }) .await .map(|output| { let notes = output.notes.clone(); let num_rows = output.markers.len(); // What was mapped comes first: it is the one thing that tells two // responses from the same panel apart, and `/plot/rqtl` reads it // for the title. let mut envelope = ctx .envelope(rqtl::to_json(output)) .with_columns(rqtl::MARKER_COLUMNS.iter().map(|s| s.to_string()).collect(), num_rows) .with_note(&format!("Trait: {}", request.trait_name)) .with_note( "markers come from R/qtl's scanone output, positional (see meta.columns for names); permutation is null unless nperm > 0 was requested.", ); for note in ¬es { envelope = envelope.with_note(note); } for note in &covariate_notes { if !note.is_empty() { envelope = envelope.with_note(note); } } for note in &extra_notes { envelope = envelope.with_note(note); } if let Some(note) = &basket_note { envelope = envelope.with_note(note); } // The peak is the strongest LOD in `data` (column 4, LOD); a // PheWAS there is the obvious next question, and it is a GET that // returns 200, so it belongs in `next`. if let Some(group) = request_group.as_deref() { if let Some(peak) = rqtl::peak_marker(&envelope.data) { envelope = envelope.with_next( &format!( "/groups/{}/phewas?locus={}&window=10000", path_segment(group), path_segment(&peak) ), "PheWAS: every published trait peaking within 10 kb of this QTL's peak", ); let version = request .genotype_version .clone() .unwrap_or_else(|| "{version}".to_string()); let (href, description) = crate::suggest::class_hint(group, &version, &peak); envelope = envelope.with_next(&href, description); envelope = envelope.with_links(crate::suggest::group_links(group)); } } envelope = envelope.with_note( "To plot this: rows are positional, so chr (meta.columns[1]) against LOD \ (meta.columns[3]) is a Manhattan plot with no reshaping.", ); // Rebuild the request that ran, once: both suggestions below are // "the same call, somewhere else". let mut body = serde_json::Map::new(); match (request_genotype_file.clone(), request_group.clone()) { (Some(file), None) => { body.insert("genotype_file".to_string(), serde_json::json!(file)); } _ => { if let Some(group) = request_group.clone() { body.insert("group".to_string(), serde_json::json!(group)); } if let Some(version) = request_genotype_version.clone() { body.insert("genotype_version".to_string(), serde_json::json!(version)); } } } body.insert("trait_name".to_string(), serde_json::json!(request.trait_name)); body.insert( "samples".to_string(), serde_json::Value::Array(original_samples.clone()), ); body.insert("model".to_string(), serde_json::json!(request_model)); body.insert("method".to_string(), serde_json::json!(request_method)); // The picture: post this response back. The body is not // duplicated here -- it is the response the caller already has. // The trait goes in the URL as well as the response: `/plot/rqtl` // reads it from the posted `notes` now, but a caller that posts // only `data` still gets the right title this way. envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: format!("/plot/rqtl?trait={}", path_segment(&request.trait_name)), description: "Draw this scan: POST this response back to /plot/rqtl (as-is, or its \ `data` member) and get an SVG Manhattan plot of these rows, with the \ permutation thresholds as dashed lines when nperm > 0. Nothing is \ recomputed." .to_string(), body: None, }); // A scan without permutations gives LOD scores, not thresholds. if request_nperm == 0 { let mut permutation_body = body; permutation_body.insert("nperm".to_string(), serde_json::json!(100)); envelope = envelope.with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: "/compute/rqtl".to_string(), description: "Estimate significance thresholds: the same scan with 100 permutations (nperm=100). The response's `permutation` then carries the suggestive and significant LOD thresholds this scan's peaks can be judged against. Costs about 100 times this run, so pass ?id= and poll its status endpoint." .to_string(), body: Some(serde_json::Value::Object(permutation_body)), }); } Json(envelope) }) } #[utoipa::path(get, path = "/compute/rqtl/status/{run_id}", summary = "Poll a running (or finished) /compute/rqtl call's progress", description = "Same as GET /compute/gemma/status/{run_id}, against \ the run_id passed as ?id= on POST /compute/rqtl.", params(("run_id" = String, Path, description = "The id passed as ?id= on the POST"), RunStatusParams), responses( (status = 200, description = "New log lines and the new pointer", body = Envelope), Unauthorized, (status = 404, description = "No run with this id (unknown, or not started yet)", body = ApiError), (status = 422, description = "run_id not alnum/-/_ (max 64 chars)", body = ApiError) ))] #[gnauth(token_only)] pub async fn rqtl_status( ctx: Ctx, _token: AuthToken, Path(run_id): Path, Query(params): Query, ) -> Result>, ApiError> { let rid = ctx.request_info(); if !streaming::is_valid_run_id(&run_id) { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_run_id", "Invalid run_id", "run_id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(), "rqtl_status", ) .with_request_id(rid)); } let config = Config::load(); let peek = params.peek.unwrap_or(0); streaming::status(&config.rqtl_cache_dir, &run_id, peek) .map(|(lines, pointer)| { Json(ctx.envelope(RunStatus { run_id: run_id.clone(), lines, pointer, })) }) .map_err(|e| match e { StatusError::NotFound => ApiError::new( StatusCode::NOT_FOUND, "run_not_found", "Run not found", format!("No run with id '{run_id}' (unknown, or not started yet)."), "rqtl_status", ) .with_request_id(rid), StatusError::Io(err) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "internal_error", "Internal error", err.to_string(), "rqtl_status", ) .with_request_id(rid), }) } /// A genotype resolution failure, as a caller-facing error: 404 when the /// group/version/file is not there, 500 when the directory itself is /// unreadable. fn genotype_error_to_api_error( error: genotypes::GenotypeError, operation: &str, rid: crate::response::RequestInfo, ) -> ApiError { let (status, code, title) = match error { genotypes::GenotypeError::Invalid(_) => ( StatusCode::UNPROCESSABLE_ENTITY, "invalid_individuals", "Unusable individual selection", ), genotypes::GenotypeError::NotFound(_) => ( StatusCode::NOT_FOUND, "genotypes_not_found", "Genotype file not found", ), genotypes::GenotypeError::Io(_) => ( StatusCode::INTERNAL_SERVER_ERROR, "genotypes_unreadable", "Genotype files could not be read", ), }; ApiError::new(status, code, title, error.to_string(), operation).with_request_id(rid) } fn rqtl_error_to_api_error(error: RqtlError, rid: RequestInfo) -> ApiError { ApiError::new( StatusCode::BAD_GATEWAY, "rqtl_failed", "R/qtl computation failed", error.to_string(), "compute_rqtl", ) .with_request_id(rid) }