//! Uploading a genotype matrix that updates an existing population. //! //! The rules are about *fit*: the base population must exist, at most 50 //! individuals may be new, at least one must overlap, at most 200 000 markers, //! and the values must be the same kind as the base's. These tests build an //! upload from the deployment's own BXD.4 file -- so it is a real update -- and //! then break each rule in turn. use axum::http::StatusCode; use http_body_util::BodyExt; use serde_json::Value; mod common; fn unique() -> String { format!( "upload-test-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|elapsed| elapsed.as_millis()) .unwrap_or(0) ) } fn genotype_dir() -> std::path::PathBuf { std::path::PathBuf::from(genenetwork4::config::Config::load().genotype_files_dir) } /// Read a few markers from the deployment's own `.geno` and write them in the /// upload format: marker, chr, pos, then individuals. fn upload_from_bxd4(markers: usize, drop: usize, new_names: &[&str]) -> Option { let path = genotype_dir().join("genotype/BXD.4.geno"); let text = std::fs::read_to_string(&path).ok()?; let mut individuals: Vec = Vec::new(); let mut rows: Vec<(String, String, String, Vec)> = Vec::new(); for line in text.lines() { if line.starts_with('#') || line.starts_with('@') || line.trim().is_empty() { continue; } let fields: Vec<&str> = line.split('\t').collect(); if individuals.is_empty() { let start = if fields.get(3).copied() == Some("Mb") { 4 } else { 3 }; individuals = fields[start..].iter().map(|name| name.trim().to_string()).collect(); continue; } if rows.len() >= markers { continue; } let pos = fields.get(3).copied().unwrap_or(""); rows.push(( fields[1].trim().to_string(), fields[0].trim().to_string(), pos.trim().to_string(), fields[4..].iter().map(|cell| cell.trim().to_string()).collect(), )); } if individuals.is_empty() || rows.is_empty() { return None; } let kept = individuals.len() - drop; let mut text = String::from("marker\tchr\tpos"); for name in &individuals[..kept] { text.push('\t'); text.push_str(name); } for name in new_names { text.push('\t'); text.push_str(name); } text.push('\n'); for (marker, chr, pos, cells) in &rows { text.push_str(&format!("{marker}\t{chr}\t{pos}")); for cell in &cells[..kept] { text.push('\t'); text.push_str(cell); } for _ in new_names { text.push('\t'); text.push_str(cells.first().map(String::as_str).unwrap_or("")); } text.push('\n'); } Some(text) } #[tokio::test] async fn an_upload_updates_a_population_and_can_be_written_out() { let Some(token) = common::auth_token().await else { eprintln!("skipping an_upload_updates_a_population_and_can_be_written_out: no token"); return; }; let Some(contents) = upload_from_bxd4(6, 2, &["NEWCOMER1"]) else { eprintln!("skipping an_upload_updates_a_population_and_can_be_written_out: no genotype dir"); return; }; let name = format!("{}-GENO1", unique()); let response = common::post_json_with_token( "POST /uploads/genotypes", "/uploads/genotypes", &token, &serde_json::json!({ "name": name, "base_version": "BXD.4", "description": "two individuals dropped, one added", "contents": contents, }), ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::CREATED, "{}", String::from_utf8_lossy(&bytes)); let upload: Value = common::data(&bytes); assert_eq!(upload["markers"], 6); assert_eq!(upload["new_individuals"], 1); assert_eq!(upload["value_kind"], "letters", "BXD.4 is called with letters"); assert!(upload["overlap"].as_u64().unwrap() > 80); // The marker list is the upload's own. let markers = common::get_with_token( "GET /uploads/genotypes/{name}/markers", &format!("/uploads/genotypes/{name}/markers?limit=3"), &token, ) .await; assert_eq!(markers.status(), StatusCode::OK); let bytes = markers.into_body().collect().await.unwrap().to_bytes(); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(envelope["meta"]["columns"], serde_json::json!(["marker", "chr", "pos"])); assert_eq!(envelope["data"].as_array().unwrap().len(), 3); // And it writes out as a .geno the R scripts can read. let written = common::get_with_token( "GET /uploads/genotypes/{name}/geno", &format!("/uploads/genotypes/{name}/geno"), &token, ) .await; assert_eq!(written.status(), StatusCode::OK); let bytes = written.into_body().collect().await.unwrap().to_bytes(); let envelope: Value = common::data(&bytes); let file = envelope["file"].as_str().unwrap(); let text = std::fs::read_to_string(file).unwrap(); assert!(text.starts_with('@'), "the base's @-metadata travels: {text:?}"); assert!(text.contains("Chr\tLocus\tcM\tMb\t"), "GN layout: {}", &text[..120]); assert!(text.contains("NEWCOMER1")); let deleted = common::delete_with_token( "DELETE /uploads/genotypes/{name}", &format!("/uploads/genotypes/{name}"), &token, ) .await; assert_eq!(deleted.status(), StatusCode::NO_CONTENT); let _ = std::fs::remove_file(file); } #[tokio::test] async fn an_upload_that_is_not_an_update_is_refused() { let Some(token) = common::auth_token().await else { eprintln!("skipping an_upload_that_is_not_an_update_is_refused: no token"); return; }; let Some(contents) = upload_from_bxd4(3, 0, &[]) else { eprintln!("skipping an_upload_that_is_not_an_update_is_refused: no genotype dir"); return; }; // No overlap at all: a different population, not an update. Every // individual name is replaced. let strangers: String = { let mut lines = contents.lines(); let header = lines.next().unwrap(); let mut fields: Vec = header.split('\t').map(str::to_string).collect(); for field in fields.iter_mut().skip(3) { *field = format!("STRANGER-{field}"); } let mut out = fields.join("\t"); out.push('\n'); for line in lines { let mut row: Vec = line.split('\t').map(str::to_string).collect(); for field in row.iter_mut().skip(3) { *field = "B".to_string(); } out.push_str(&row.join("\t")); out.push('\n'); } out }; let response = common::post_json_with_token( "POST /uploads/genotypes (no overlap)", "/uploads/genotypes", &token, &serde_json::json!({ "name": format!("{}-GENO2", unique()), "base_version": "BXD.4", "contents": strangers, }), ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(error["code"], "invalid_genotype_upload"); assert!( error["detail"].as_str().unwrap().contains("updates a population"), "{error}" ); // Numbers where the base has letters: a mapping engine would read them as // the wrong thing, so this is refused rather than converted. let numeric = contents.replacen("marker\tchr\tpos", "marker\tchr\tpos", 1).lines() .enumerate() .map(|(index, line)| { if index == 0 { line.to_string() } else { let mut fields: Vec = line.split('\t').map(str::to_string).collect(); for field in fields.iter_mut().skip(3) { *field = "1".to_string(); } fields.join("\t") } }) .collect::>() .join("\n"); let response = common::post_json_with_token( "POST /uploads/genotypes (mixed kinds)", "/uploads/genotypes", &token, &serde_json::json!({ "name": format!("{}-GENO3", unique()), "base_version": "BXD.4", "contents": numeric, }), ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); let detail = error["detail"].as_str().unwrap(); assert!( detail.contains("integers") && detail.contains("letters"), "the refusal names both kinds: {detail}" ); // A comma-separated file is not this format, and says so. let response = common::post_json_with_token( "POST /uploads/genotypes (commas)", "/uploads/genotypes", &token, &serde_json::json!({ "name": format!("{}-GENO4", unique()), "base_version": "BXD.4", "contents": "marker,chr,pos,BXD1\nrs1,1,3.4,B\n", }), ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert!( error["detail"].as_str().unwrap().contains("tab-delimited"), "{error}" ); } /// Every compute endpoint that takes genotypes accepts an upload by name. /// /// GEMMA is left out here: its LOCO kinship wants more markers per chromosome /// than a 20-marker fixture has, so the engine is exercised against a full /// panel by hand; these three are the fast ones. #[tokio::test] async fn every_engine_maps_an_uploaded_genotype() { let Some(token) = common::auth_token().await else { eprintln!("skipping every_engine_maps_an_uploaded_genotype: no token"); return; }; if !common::db_available() { eprintln!("skipping every_engine_maps_an_uploaded_genotype: no database"); return; } let Some(contents) = upload_from_bxd4(400, 1, &[]) else { eprintln!("skipping every_engine_maps_an_uploaded_genotype: no genotype dir"); return; }; let name = format!("{}-GENO5", unique()); let created = common::post_json_with_token( "POST /uploads/genotypes", "/uploads/genotypes", &token, &serde_json::json!({"name": name, "base_version": "BXD.4", "contents": contents}), ) .await; assert_eq!(created.status(), StatusCode::CREATED); let values = common::get_with_token( "GET /groups/BXD/traits/10001/values", "/groups/BXD/traits/10001/values", &token, ) .await; assert_eq!(values.status(), StatusCode::OK); let bytes = values.into_body().collect().await.unwrap().to_bytes(); let samples: Value = common::data(&bytes); let markers = common::get_with_token( "GET /uploads/genotypes/{name}/markers", &format!("/uploads/genotypes/{name}/markers?limit=2"), &token, ) .await; assert_eq!(markers.status(), StatusCode::OK); let bytes = markers.into_body().collect().await.unwrap().to_bytes(); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let picked: Vec = envelope["data"] .as_array() .unwrap() .iter() .map(|row| row[0].as_str().unwrap().to_string()) .collect(); for endpoint in ["rqtl", "qtl2"] { let response = common::post_json_with_token( &format!("POST /compute/{endpoint} (upload)"), &format!("/compute/{endpoint}"), &token, &serde_json::json!({ "genotype_upload": name, "trait_name": "upload::demo", "samples": samples, }), ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{endpoint}: {}", String::from_utf8_lossy(&bytes)); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); // /compute/rqtl wraps its rows in data.markers; /compute/qtl2 returns // them as the array itself. let rows = envelope["data"]["markers"] .as_array() .or_else(|| envelope["data"].as_array()) .unwrap_or_else(|| panic!("{endpoint}: no rows in {envelope}")); assert!(!rows.is_empty(), "{endpoint} mapped nothing"); let notes = envelope["notes"].to_string(); assert!( notes.contains(&name), "{endpoint}'s notes name the upload: {notes}" ); } let response = common::post_json_with_token( "POST /compute/fitqtl (upload)", "/compute/fitqtl", &token, &serde_json::json!({ "genotype_upload": name, "trait_name": "upload::demo", "samples": samples, "markers": picked, }), ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "fitqtl: {}", String::from_utf8_lossy(&bytes)); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(envelope["data"].as_array().unwrap().len(), picked.len()); common::delete_with_token( "DELETE /uploads/genotypes/{name}", &format!("/uploads/genotypes/{name}"), &token, ) .await; } /// An upload replaces one of the same name: the user's call, and the digest /// says which version produced a result. #[tokio::test] async fn uploading_the_same_name_replaces_it() { let Some(token) = common::auth_token().await else { eprintln!("skipping uploading_the_same_name_replaces_it: no token"); return; }; let Some(first) = upload_from_bxd4(5, 0, &[]) else { eprintln!("skipping uploading_the_same_name_replaces_it: no genotype dir"); return; }; let Some(second) = upload_from_bxd4(8, 1, &["EXTRA1"]) else { eprintln!("skipping uploading_the_same_name_replaces_it: no genotype dir"); return; }; let name = format!("{}-GENO6", unique()); let created = common::post_json_with_token( "POST /uploads/genotypes (first)", "/uploads/genotypes", &token, &serde_json::json!({"name": name, "base_version": "BXD.4", "contents": first}), ) .await; assert_eq!(created.status(), StatusCode::CREATED); let bytes = created.into_body().collect().await.unwrap().to_bytes(); let before: Value = common::data(&bytes); let replaced = common::post_json_with_token( "POST /uploads/genotypes (same name)", "/uploads/genotypes", &token, &serde_json::json!({ "name": name, "base_version": "BXD.4", "description": "the second version", "contents": second, }), ) .await; let status = replaced.status(); let bytes = replaced.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::CREATED, "{}", String::from_utf8_lossy(&bytes)); let after: Value = common::data(&bytes); assert_eq!(after["markers"], 8, "the second upload's markers"); assert_eq!(after["new_individuals"], 1); assert_ne!(after["digest"], before["digest"], "a new digest for new contents"); assert_eq!(after["description"], "the second version"); let listing = common::get_with_token( "GET /uploads/genotypes", "/uploads/genotypes", &token, ) .await; let bytes = listing.into_body().collect().await.unwrap().to_bytes(); let uploads: Value = common::data(&bytes); let matches = uploads .as_array() .unwrap() .iter() .filter(|upload| upload["name"] == serde_json::json!(name)) .count(); assert_eq!(matches, 1, "one row, not two"); common::delete_with_token( "DELETE /uploads/genotypes/{name}", &format!("/uploads/genotypes/{name}"), &token, ) .await; }