//! `POST /uploads/genotypes`: upload a genotype matrix that *updates* an //! existing population. //! //! Tab-delimited, header = individual names from column 4, column 1 the marker, //! 2 the chromosome, 3 the position in Mb. Values must be the same kind as the //! base population's (letters, whole numbers or probabilities) -- see //! `src/genoupload.rs` for what is enforced and why. //! //! This is an update path, not an ingest: at most 50 individuals the base does //! not have, at least one that it does, and at most 200 000 markers. Anything //! larger is the sysadmin's to organise. use axum::Json; use axum::extract::{Path, Query}; use axum::http::StatusCode; use serde::Deserialize; use gnauth_macros::gnauth; use crate::auth::AuthToken; use crate::baskets::BasketError; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; use crate::genotypes; use crate::genoupload::{self, GenotypeInfo}; use crate::response::{Ctx, Envelope, RequestInfo}; #[derive(Deserialize, utoipa::ToSchema)] pub struct UploadGenotype { /// `-`: the name this upload is known by. pub name: String, /// The population it updates, e.g. "BXD.4". pub base_version: String, /// What changed, for the listing. pub description: Option, /// The matrix: tab-delimited, one header row, then one row per marker. pub contents: String, } #[derive(Deserialize, utoipa::IntoParams)] pub struct MarkerPage { /// One-based page. Default 1. pub page: Option, /// Markers per page. Default 100, at most 10000. pub limit: Option, } fn map_error(error: BasketError, rid: RequestInfo, operation: &'static str) -> ApiError { match error { BasketError::NotFound(message) => ApiError::new( StatusCode::NOT_FOUND, "genotype_upload_not_found", "Uploaded genotype not found", message, operation, ) .with_request_id(rid), BasketError::Invalid(message) => ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_genotype_upload", "Invalid genotype upload", message, operation, ) .with_request_id(rid), BasketError::Store(message) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "genotype_upload_store_error", "The upload store failed", message, operation, ) .with_request_id(rid), } } #[utoipa::path(post, path = "/uploads/genotypes", summary = "Upload a genotype matrix that updates an existing population", description = "Tab-delimited: the header names the individuals from column 4 on, column 1 \ is the marker, 2 the chromosome, 3 the position in Mb. The base population must exist; at most 50 \ individuals may be new and at least one must overlap, and at most 200000 markers -- anything \ larger the sysadmin has to organise. Values must be the same kind as the base's: single letters \ (A/B/H), whole numbers (0/1/2) or probabilities (0.02).", request_body = UploadGenotype, responses( (status = 201, description = "The stored upload", body = Envelope), Unauthorized, (status = 404, description = "No such base population", body = ApiError), (status = 422, description = "The matrix is malformed or does not fit the base", body = ApiError) ) )] #[gnauth(token_only)] pub async fn upload_genotype( ctx: Ctx, token: AuthToken, Json(request): Json, ) -> Result<(StatusCode, Json>), ApiError> { let rid = ctx.request_info(); let config = Config::load(); let owner = token.sub.clone(); let dir = config.basket_dir.clone(); let root = config.genotype_files_dir.clone(); let base_version = request.base_version.clone(); let parsed = genoupload::parse(&request.contents) .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?; // The population being updated: its individuals and the kind of value its // matrix holds. let base_for_job = base_version.clone(); let (base_kind, base_individuals) = tokio::task::spawn_blocking(move || { let path = genotypes::resolve_named(std::path::Path::new(&root), &base_for_job) .map_err(|error| BasketError::NotFound(error.to_string()))?; genoupload::base_shape(&path) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "upload_genotype"))? .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?; let (overlap, new_individuals) = genoupload::check_base(&parsed, &base_individuals, base_kind) .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?; let name = request.name.clone(); let description = request.description.clone(); let contents = request.contents.clone(); // Kept for the note: the parse and the base version move into the closure. let base_for_note = base_version.clone(); let markers_count = parsed.markers.len(); let kind_label = parsed.kind.as_str(); let stored = tokio::task::spawn_blocking(move || { genoupload::create( std::path::Path::new(&dir), &owner, &name, &base_version, description.as_deref(), &parsed, overlap, new_individuals, &contents, ) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "upload_genotype"))? .map_err(|error| map_error(error, rid.clone(), "upload_genotype"))?; let mut envelope = ctx .envelope(stored) .with_note(&format!( "Updates '{base_for_note}': {overlap} individual(s) shared with the base, \ {new_individuals} new, {markers_count} marker(s), values as {kind_label}." )) .with_note( "Kept in your own SQLite file. A mapping reads it as a GN .geno: GET \ /uploads/genotypes/{name}/geno writes it out and gives the path for `genotype_file`.", ); envelope = envelope.with_links(crate::suggest::root_links()); Ok((StatusCode::CREATED, Json(envelope))) } #[utoipa::path(get, path = "/uploads/genotypes", summary = "This user's uploaded genotypes, newest first", responses( (status = 200, description = "Every upload of this user", body = Envelope>), Unauthorized ) )] #[gnauth(token_only)] pub async fn list_genotypes( ctx: Ctx, token: AuthToken, ) -> Result>>, ApiError> { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let uploads = tokio::task::spawn_blocking(move || { genoupload::list(std::path::Path::new(&dir), &owner) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "list_genotypes"))? .map_err(|error| map_error(error, rid.clone(), "list_genotypes"))?; let mut envelope = ctx .envelope(uploads) .with_note("Uploads are yours, kept in your own SQLite file; they update a population, \ they do not replace one.") .with_links(crate::suggest::root_links()); envelope = envelope.with_note("Upload one: POST /uploads/genotypes."); Ok(Json(envelope)) } #[utoipa::path(get, path = "/uploads/genotypes/{name}", summary = "One uploaded genotype's metadata", params(("name" = String, Path, description = "Upload name, e.g. ``user-GENONAME``")), responses( (status = 200, description = "The upload", body = Envelope), Unauthorized, (status = 404, description = "No such upload for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn get_genotype( ctx: Ctx, token: AuthToken, Path(name): Path, ) -> Result>, ApiError> { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let wanted = name.clone(); let upload = tokio::task::spawn_blocking(move || { genoupload::get(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "get_genotype"))? .map_err(|error| map_error(error, rid.clone(), "get_genotype"))?; let mut envelope = ctx .envelope(upload.clone()) .with_note("`overlap` and `new_individuals` are measured against the base population.") .with_next( &format!("/uploads/genotypes/{}/markers", crate::response::path_segment(&name)), "The markers in this upload, paged", ) .with_next( &format!("/uploads/genotypes/{}/geno", crate::response::path_segment(&name)), "Write it out as a GN .geno and get the path for `genotype_file`", ); envelope = envelope.with_links(crate::suggest::root_links()); Ok(Json(envelope)) } #[utoipa::path(get, path = "/uploads/genotypes/{name}/markers", summary = "The markers in an uploaded genotype, paged", params(("name" = String, Path, description = "Upload name"), MarkerPage), responses( (status = 200, description = "marker, chr, pos rows (meta.columns names them)", body = Envelope), Unauthorized, (status = 404, description = "No such upload for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn genotype_markers( ctx: Ctx, token: AuthToken, Path(name): Path, Query(params): Query, ) -> Result>, ApiError> { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let wanted = name.clone(); let contents = tokio::task::spawn_blocking(move || { genoupload::contents(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "genotype_markers"))? .map_err(|error| map_error(error, rid.clone(), "genotype_markers"))?; let parsed = genoupload::parse(&contents) .map_err(|error| map_error(error, rid.clone(), "genotype_markers"))?; let limit = params.limit.unwrap_or(100).clamp(1, 10_000) as usize; let page = params.page.unwrap_or(1).max(1) as usize; let rows: Vec = parsed .markers .iter() .skip((page - 1) * limit) .take(limit) .map(|(marker, chr, pos)| serde_json::json!([marker, chr, pos])) .collect(); let num_rows = rows.len(); let mut envelope = ctx .envelope(serde_json::Value::Array(rows)) .with_columns( ["marker", "chr", "pos"] .iter() .map(|name| name.to_string()) .collect(), num_rows, ) .with_note(&format!( "{} markers in this upload; pos is in Mb, as uploaded.", parsed.markers.len() )); let pages = parsed.markers.len().div_ceil(limit).max(1); if page < pages { envelope = envelope.with_next( &format!( "/uploads/genotypes/{}/markers?page={}&limit={limit}", crate::response::path_segment(&name), page + 1 ), "The next page of markers", ); } envelope = envelope.with_next( &format!("/uploads/genotypes/{}", crate::response::path_segment(&name)), "This upload's metadata", ); Ok(Json(envelope)) } #[utoipa::path(get, path = "/uploads/genotypes/{name}/geno", summary = "Write an uploaded genotype out as a GN .geno", description = "Materialises the upload as the layout every R script here reads (Chr, Locus, \ cM, Mb, individuals) with the base population's @-metadata, and returns its path for the \ `genotype_file` parameter of the compute endpoints.", params(("name" = String, Path, description = "Upload name")), responses( (status = 200, description = "path, file and marker/individual counts", body = Envelope), Unauthorized, (status = 404, description = "No such upload for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn genotype_geno( ctx: Ctx, token: AuthToken, Path(name): Path, ) -> Result>, ApiError> { let rid = ctx.request_info(); let config = Config::load(); let owner = token.sub.clone(); let dir = config.basket_dir.clone(); let root = config.genotype_files_dir.clone(); let wanted = name.clone(); let base_version = { let owner = owner.clone(); let dir = dir.clone(); let wanted = name.clone(); tokio::task::spawn_blocking(move || { genoupload::get(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "genotype_geno"))? .map_err(|error| map_error(error, rid.clone(), "genotype_geno"))? .base_version }; let stored = tokio::task::spawn_blocking({ let dir = dir.clone(); let owner = owner.clone(); let wanted = wanted.clone(); move || { // The base's labels travel with the file so R/qtl reads it as the // same kind of cross as the population it updates. let labels = genotypes::resolve_named(std::path::Path::new(&root), &base_version) .ok() .and_then(|path| genotypes::file_meta(&path).ok()) .map(|meta| meta.labels) .unwrap_or_default(); let path = genoupload::materialise( std::path::Path::new(&dir), &owner, &wanted, &labels, )?; let info = genoupload::get(std::path::Path::new(&dir), &owner, &wanted)?; Ok::<_, BasketError>((path, info)) } }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "genotype_geno"))? .map_err(|error| map_error(error, rid.clone(), "genotype_geno"))?; let (path, info) = stored; let mut envelope = ctx .envelope(serde_json::json!({ "name": info.name, "file": path.to_string_lossy(), "individuals": info.individuals, "markers": info.markers, "value_kind": info.value_kind, "digest": info.digest, })) .with_note("The file is written on demand, in the layout the R scripts here read; \ `file` is the path to pass as `genotype_file`.") .with_note("Nothing about the uploaded population is written to GeneNetwork's database.") .with_action(crate::response::SuggestedCall { method: "POST".to_string(), href: "/compute/qtl2".to_string(), description: "Map a trait over this genotype: pass `genotype_file` with the path \ above and the trait's values, or a `basket`." .to_string(), body: None, }); envelope = envelope.with_links(crate::suggest::root_links()); Ok(Json(envelope)) } #[utoipa::path(delete, path = "/uploads/genotypes/{name}", summary = "Delete an uploaded genotype", params(("name" = String, Path, description = "Upload name")), responses( (status = 204, description = "Deleted"), Unauthorized, (status = 404, description = "No such upload for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn delete_genotype( ctx: Ctx, token: AuthToken, Path(name): Path, ) -> Result { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let wanted = name.clone(); tokio::task::spawn_blocking(move || { genoupload::delete(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), "delete_genotype"))? .map_err(|error| map_error(error, rid.clone(), "delete_genotype"))?; Ok(StatusCode::NO_CONTENT) } /// Resolve `genotype_upload` for a compute endpoint: materialise the upload in /// the form that engine reads, and hand back the path(s). /// /// R/qtl and R/qtl2 read a GN `.geno`; GEMMA reads a BIMBAM pair. Either way /// the caller names the upload and the server writes the file, so no user path /// is ever passed around. pub(crate) async fn resolve_upload( owner: String, basket_dir: String, genotype_dir: String, name: String, endpoint: &'static str, rid: RequestInfo, bimbam: bool, ) -> Result<(String, Option), ApiError> { tokio::task::spawn_blocking(move || { let dir = std::path::Path::new(&basket_dir); let info = genoupload::get(dir, &owner, &name)?; // The base's @-metadata travels with the file so R/qtl reads the // upload as the same kind of cross as the population it updates. let labels = genotypes::resolve_named( std::path::Path::new(&genotype_dir), &info.base_version, ) .ok() .and_then(|path| genotypes::file_meta(&path).ok()) .map(|meta| meta.labels) .unwrap_or_default(); if bimbam { let (geno, snps) = genoupload::materialise_bimbam(dir, &owner, &name, &labels)?; Ok::<_, BasketError>(( geno.to_string_lossy().into_owned(), Some(snps.to_string_lossy().into_owned()), )) } else { let path = genoupload::materialise(dir, &owner, &name, &labels)?; Ok::<_, BasketError>((path.to_string_lossy().into_owned(), None)) } }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone(), endpoint))? .map_err(|error| map_error(error, rid, endpoint)) }