//! `POST /matrices` and friends: upload new traits and covariates as one named //! table, checked against an existing genotype file. //! //! A matrix is a CSV/TSV table whose first column names the individuals and //! whose other columns are traits or covariates. It is stored in the uploader's //! own SQLite file -- never MariaDB -- and each of its columns is a *basket* //! (`.`), so a mapping names it the way it names any basket and //! every engine already knows how: //! //! ```text //! POST /matrices {"name":"my-traits","genotype_version":"BXD.4", //! "contents":"id,bw,glucose\nBXD1,23.1,9.4\n..."} //! → columns: bw, glucose (basket: "my-traits/bw") //! POST /compute/qtl2 {"group":"BXD","basket":"my-traits/bw", ...} //! ``` //! //! The check that matters is against the genotype panel: the upload must name //! each individual of that version exactly once. A matrix that names 92 of 93, //! or a name the panel does not have, is refused with the difference spelled //! out -- a mapping that silently drops individuals is worse than an error. use std::collections::BTreeMap; use axum::Json; use axum::extract::Path; use axum::http::StatusCode; use serde::Deserialize; use serde_json::Value; use gnauth_macros::gnauth; use crate::auth::AuthToken; use crate::baskets::{self, BasketError, MatrixInfo}; use crate::config::Config; use crate::error::{ApiError, Unauthorized}; use crate::genotypes; use crate::response::{Ctx, Envelope, RequestInfo}; #[derive(Deserialize, utoipa::ToSchema)] pub struct UploadMatrix { /// URL-safe, unique per user: letters, digits, `-`, `_`, `.`. pub name: String, /// The genotype version (or `.geno` file) the individuals must match, /// e.g. "BXD.4". pub genotype_version: String, /// What this table is, for the listing. pub description: Option, /// The table itself: a header row, then one row per individual. Tab, /// comma or semicolon separated; guessed from the header unless /// `separator` says. pub contents: String, /// `"\t"`, `","` or `";"`. pub separator: Option, /// Columns that are covariates rather than traits, e.g. /// `{"batch": "covariate"}`. Everything else is a trait. #[serde(default)] pub kinds: BTreeMap, } fn map_error(error: BasketError, rid: RequestInfo) -> ApiError { match error { BasketError::NotFound(message) => ApiError::new( StatusCode::NOT_FOUND, "matrix_not_found", "Matrix not found", message, "matrices", ) .with_request_id(rid), BasketError::Invalid(message) => ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_matrix", "Invalid matrix", message, "matrices", ) .with_request_id(rid), BasketError::Store(message) => ApiError::new( StatusCode::INTERNAL_SERVER_ERROR, "matrix_store_error", "The matrix store failed", message, "matrices", ) .with_request_id(rid), } } #[utoipa::path(post, path = "/matrices", summary = "Upload traits and covariates as one named matrix", description = "A delimited table: the first column names the individuals, the rest are \ traits (or covariates, via `kinds`). Every individual of `genotype_version` must appear exactly \ once -- the upload is checked against that version's panel and refused with the difference if it \ does not match. The matrix is stored in the uploader's own SQLite file; each column becomes a \ basket named `.`, so a mapping names it as `{\"basket\": \".\"}` \ on any compute endpoint.", request_body = UploadMatrix, responses( (status = 201, description = "The stored matrix and its columns", body = Envelope), Unauthorized, (status = 404, description = "No such genotype version", body = ApiError), (status = 422, description = "The table does not match the genotype panel", body = ApiError) ) )] #[gnauth(token_only)] pub async fn upload_matrix( ctx: Ctx, token: AuthToken, Json(request): Json, ) -> Result<(StatusCode, Json>), ApiError> { let rid = ctx.request_info(); let config = Config::load(); let owner = token.sub.clone(); let dir = config.basket_dir.clone(); let genotype_version = request.genotype_version.clone(); let separator = request .separator .as_deref() .and_then(|separator| separator.chars().next()); if let Some(separator) = request.separator.as_deref() { if !matches!(separator, "\\t" | "," | ";" | "\t") { return Err(ApiError::new( StatusCode::UNPROCESSABLE_ENTITY, "invalid_separator", "Invalid separator", "use a tab, a comma or a semicolon".to_string(), "upload_matrix", ) .with_request_id(rid)); } } let parsed = baskets::parse_matrix(&request.contents, separator, &request.kinds) .map_err(|error| map_error(error, rid.clone()))?; let cell_count = parsed.individuals.len() * parsed.columns.len(); // The panel check: a mapping that silently drops individuals is worse than // an error, so the count and every name are verified first. let root = config.genotype_files_dir.clone(); let version_for_panel = genotype_version.clone(); let panel = tokio::task::spawn_blocking(move || { let geno = genotypes::resolve_named(std::path::Path::new(&root), &version_for_panel) .map_err(|error| BasketError::NotFound(error.to_string()))?; genotypes::panel_of(&geno).map_err(|error| BasketError::Store(error.to_string())) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; baskets::check_panel(&parsed.individuals, &panel).map_err(|error| map_error(error, rid.clone()))?; let name = request.name.clone(); let description = request.description.clone(); let columns_count = parsed.columns.len(); let parsed_for_job = parsed; let genetics = genotype_version.clone(); let stored = tokio::task::spawn_blocking(move || { baskets::create_matrix( std::path::Path::new(&dir), &owner, &name, &genetics, description.as_deref(), &parsed_for_job, ) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; let mut envelope = ctx .envelope(stored) .with_note(&format!( "{cell_count} value(s) over {} individual(s) and {} column(s), checked against the \ individuals of genotype version '{genotype_version}'.", panel.len(), columns_count )) .with_note("Each column is a basket: map one with \ {\"basket\": \".\"} on /compute/{rqtl,gemma,qtl2,fitqtl}."); envelope = envelope.with_links(crate::suggest::root_links()); Ok((StatusCode::CREATED, Json(envelope))) } #[utoipa::path(get, path = "/matrices", summary = "This user's uploaded matrices, newest first", description = "Each with its genotype version, description and the columns it holds.", responses( (status = 200, description = "Every matrix this user uploaded", body = Envelope>), Unauthorized ) )] #[gnauth(token_only)] pub async fn list_matrices( ctx: Ctx, token: AuthToken, ) -> Result>>, ApiError> { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let matrices = tokio::task::spawn_blocking(move || { baskets::list_matrices(std::path::Path::new(&dir), &owner) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; let mut envelope = ctx .envelope(matrices) .with_note("Matrices are yours: they live in your own SQLite file, and their columns are \ baskets.") .with_note("Map a column: POST /compute/qtl2 with \"basket\": \".\".") .with_note("Upload one: POST /matrices with the table as `contents`."); envelope = envelope.with_links(crate::suggest::root_links()); Ok(Json(envelope)) } #[utoipa::path(get, path = "/matrices/{name}", summary = "One matrix's columns", description = "The genotype version it was checked against, its description, and each \ column with the basket name a mapping uses.", params(("name" = String, Path, description = "Matrix name")), responses( (status = 200, description = "The matrix", body = Envelope), Unauthorized, (status = 404, description = "No such matrix for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn get_matrix( ctx: Ctx, token: AuthToken, Path(name): Path, ) -> Result>, ApiError> { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let wanted = name.clone(); let matrix = tokio::task::spawn_blocking(move || { baskets::get_matrix(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; let first = matrix.columns.first().map(|column| column.basket.clone()); let mut envelope = ctx .envelope(matrix.clone()) .with_note("Each column carries its own revision history and digest (see /baskets)."); if let Some(basket) = first { envelope = envelope.with_next( &format!("/baskets/{}", crate::response::path_segment(&basket)), "The first column as a basket (revision, digest, values)", ); } envelope = envelope.with_next( &format!("/matrices/{}/values", crate::response::path_segment(&name)), "The whole table back", ); envelope = envelope.with_links(crate::suggest::root_links()); Ok(Json(envelope)) } #[utoipa::path(get, path = "/matrices/{name}/values", summary = "The matrix as a table", description = "What was uploaded, read back: the individuals in the matrix's own order and \ each column with its parsed values (`null` where a value was missing). RAW text stays in \ /baskets/{matrix}/{column}/values.", params(("name" = String, Path, description = "Matrix name")), responses( (status = 200, description = "individuals and columns", body = Envelope), Unauthorized, (status = 404, description = "No such matrix for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn matrix_values( ctx: Ctx, token: AuthToken, Path(name): Path, ) -> Result>, ApiError> { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let wanted = name.clone(); let matrix = tokio::task::spawn_blocking({ let dir = dir.clone(); let owner = owner.clone(); move || baskets::get_matrix(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; let mut individuals: Vec = Vec::new(); let mut columns: Vec = Vec::new(); for column in &matrix.columns { let wanted = column.basket.clone(); let owner_for_job = owner.clone(); let dir_for_job = dir.clone(); let cells = tokio::task::spawn_blocking(move || { baskets::cells_of(std::path::Path::new(&dir_for_job), &owner_for_job, &wanted, None) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; if individuals.is_empty() { individuals = cells.iter().map(|cell| cell.sample.clone()).collect(); } columns.push(serde_json::json!({ "name": column.name, "kind": column.kind, "basket": column.basket, "values": cells.iter().map(|cell| cell.value).collect::>(), })); } let mut envelope = ctx .envelope(serde_json::json!({ "name": matrix.name, "genotype_version": matrix.genotype_version, "individuals": individuals, "columns": columns, })) .with_note("Parsed values, in the individuals order this table carries; `null` is a \ missing value. The raw text is in each column's basket."); envelope = envelope.with_next( &format!("/matrices/{}", crate::response::path_segment(&name)), "The matrix's columns and basket names", ); Ok(Json(envelope)) } #[utoipa::path(delete, path = "/matrices/{name}", summary = "Delete a matrix and its columns", description = "Removes the matrix and every basket it created. A mapping that named a column \ can no longer be reproduced, so export first if that matters.", params(("name" = String, Path, description = "Matrix name")), responses( (status = 204, description = "Deleted"), Unauthorized, (status = 404, description = "No such matrix for this user", body = ApiError) ) )] #[gnauth(token_only)] pub async fn delete_matrix( ctx: Ctx, token: AuthToken, Path(name): Path, ) -> Result { let rid = ctx.request_info(); let owner = token.sub.clone(); let dir = Config::load().basket_dir; let wanted = name.clone(); tokio::task::spawn_blocking(move || { baskets::delete_matrix(std::path::Path::new(&dir), &owner, &wanted) }) .await .map_err(|error| map_error(BasketError::Store(error.to_string()), rid.clone()))? .map_err(|error| map_error(error, rid.clone()))?; Ok(StatusCode::NO_CONTENT) }