//! Integration test for POST /compute/gemma. Runs the real gemma-wrapper //! and GEMMA binaries against the BXD fixture files copied from //! genenetwork3/tests/test_data (tests/test_data here); skips rather //! than fails when either binary is not on PATH, same "skip, don't //! fail" convention as common::db_available()/auth_server_available(). use axum::http::StatusCode; use http_body_util::BodyExt; use serde_json::Value; mod common; /// True only when gemma-wrapper is actually runnable. `-v` is cheap and /// does not touch the cache dir or invoke the real GEMMA binary. /// The three tests that run a real GEMMA job take this lock: they share /// one cache directory *and* one token, so on a cold cache two of them at /// once would both be gated (GEMMA's threshold is 50 individuals) and the /// second would be refused with `409 compute_busy` -- a gate doing its job, /// which would fail the test that was only trying to map something. static COMPUTE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Take [`COMPUTE_LOCK`], ignoring poisoning (one failing test must not /// fail the rest). fn serialised() -> std::sync::MutexGuard<'static, ()> { COMPUTE_LOCK .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) } fn gemma_wrapper_available() -> bool { std::process::Command::new("gemma-wrapper") .arg("-v") .output() .is_ok() } /// True only when ravanan is actually runnable. fn ravanan_available() -> bool { std::process::Command::new("ravanan") .arg("--version") .output() .is_ok() } /// True only when ccwl is actually runnable. compute_ravanan compiles /// cwl/gemma/gemma-loco.scm to CWL with it before every ravanan run. /// ccwl has no top-level --version; "compile --help" is the cheapest /// subcommand invocation that doesn't need a real source file. fn ccwl_available() -> bool { std::process::Command::new("ccwl") .args(["compile", "--help"]) .output() .is_ok() } fn test_data_path(name: &str) -> String { format!("{}/tests/test_data/{name}", env!("CARGO_MANIFEST_DIR")) } fn bxd_trait_values() -> Vec> { std::fs::read_to_string(test_data_path("BXD_pheno.txt")) .unwrap() .lines() .map(|line| line.trim().parse::().ok()) .collect() } #[tokio::test] async fn compute_gemma_maps_a_qtl_from_the_bxd_fixtures() { let _serial = serialised(); if !gemma_wrapper_available() { eprintln!( "skipping compute_gemma_maps_a_qtl_from_the_bxd_fixtures: gemma-wrapper not on PATH" ); return; } let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_gemma_maps_a_qtl_from_the_bxd_fixtures: no token (see docs/gn-auth.md)" ); return; }; let run_id = format!( "test-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() ); let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), }); // Run the POST (which blocks for the whole ~19s computation) on its // own task, so this test can poll the status endpoint on a second // connection while it is still in flight -- the real shape a // browser would use, not just a read of the finished log. let post_token = token.clone(); let post_run_id = run_id.clone(); let post_handle = tokio::spawn(async move { common::post_json_with_token( "POST /compute/gemma", &format!("/compute/gemma?id={post_run_id}"), &post_token, &body, ) .await }); // Poll until something besides the empty initial log file shows up, // or the main request finishes first (a fast machine might not // catch it mid-run; either way, a fresh log with the kinship step's // output should exist very soon after the POST starts). let mut saw_progress_while_running = false; let mut peek = 0u64; for _ in 0..40 { if post_handle.is_finished() { break; } tokio::time::sleep(std::time::Duration::from_millis(500)).await; let response = common::get_with_token( "GET /compute/gemma/status", &format!("/compute/gemma/status/{run_id}?peek={peek}"), &token, ) .await; if response.status() != StatusCode::OK { continue; // log file not created yet } let bytes = response.into_body().collect().await.unwrap().to_bytes(); let status: Value = common::data(&bytes); peek = status["pointer"].as_u64().unwrap_or(peek); if !status["lines"].as_array().unwrap().is_empty() { saw_progress_while_running = true; } } assert!( saw_progress_while_running, "expected at least one non-empty status poll while the computation was still running" ); let response = post_handle.await.unwrap(); let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!( status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes) ); // After completion, the log carries the completion marker, and // real gemma-wrapper progress text (not just the one line written // at log creation). let final_response = common::get_with_token( "GET /compute/gemma/status (final)", &format!("/compute/gemma/status/{run_id}?peek=0"), &token, ) .await; assert_eq!(final_response.status(), StatusCode::OK); let final_bytes = final_response .into_body() .collect() .await .unwrap() .to_bytes(); let final_status: Value = common::data(&final_bytes); let lines: Vec<&str> = final_status["lines"] .as_array() .unwrap() .iter() .map(|l| l.as_str().unwrap()) .collect(); assert!(lines.contains(&"GN4: computation complete"), "{lines:?}"); assert!( lines.iter().any(|l| l.contains("GEMMA")), "expected real gemma-wrapper progress text in the log: {lines:?}" ); let rows: Value = common::data(&bytes); let rows = rows.as_array().expect("data is a JSON array of rows"); assert!(!rows.is_empty(), "expected at least one mapped marker"); // BXD_snps.txt has 7320 markers; every one should have a row. assert_eq!(rows.len(), 7320); // Rows are positional now; look up each column's index via // meta.columns rather than a fixed offset, since it varies with lmm. let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let columns: Vec<&str> = envelope["meta"]["columns"] .as_array() .expect("meta.columns") .iter() .map(|c| c.as_str().unwrap()) .collect(); assert_eq!(envelope["meta"]["num_cols"], columns.len()); assert_eq!(envelope["meta"]["num_rows"], 7320); let index_of = |name: &str| columns.iter().position(|c| *c == name); assert!(index_of("chr").is_some(), "{columns:?}"); assert!(index_of("rs").is_some(), "{columns:?}"); // lmm defaults to 9 (GeneNetwork's convention); observed to behave // like LRT, carrying p_lrt rather than p_wald. assert!(index_of("p_lrt").is_some(), "{columns:?}"); let first = &rows[0]; assert!(first[index_of("chr").unwrap()].is_number(), "{first}"); // The notes carry the exact gemma-wrapper invocations used, not // just a generic description. let notes = envelope["notes"].as_array().unwrap(); let kinship_note = notes .iter() .find(|n| n.as_str().unwrap().starts_with("Kinship:")); let gwa_note = notes .iter() .find(|n| n.as_str().unwrap().starts_with("GWA:")); assert!(kinship_note.is_some(), "{notes:?}"); assert!(gwa_note.is_some(), "{notes:?}"); let kinship_note = kinship_note.unwrap().as_str().unwrap(); let gwa_note = gwa_note.unwrap().as_str().unwrap(); assert!( kinship_note.contains("--loco") && kinship_note.contains("-gk"), "{kinship_note}" ); assert!(!kinship_note.contains("-lmm"), "{kinship_note}"); assert!( gwa_note.contains("--loco") && gwa_note.contains("-lmm 9"), "{gwa_note}" ); assert!(!gwa_note.contains("-gk"), "{gwa_note}"); } /// The ravanan workflow has no covariate step and no normalization pass, so /// both are refused rather than silently dropped -- and refused before any /// job is submitted, which is why this test needs no ravanan, ccwl or /// cluster. #[tokio::test] async fn compute_gemma_ravanan_refuses_what_its_workflow_cannot_do() { let _serial = serialised(); let Some(token) = common::auth_token().await else { eprintln!("skipping compute_gemma_ravanan_refuses_what_its_workflow_cannot_do: no token"); return; }; let cases = [ ( serde_json::json!({ "group": "BXD", "covariates": [{"case_attribute": "Status"}], "samples": [{"name": "BXD1", "value": 1.0}], }), "covariates_not_supported", ), ( serde_json::json!({ "group": "BXD", "normalize": "zscore", "samples": [{"name": "BXD1", "value": 1.0}], }), "normalization_not_supported", ), ]; for (body, expected) in cases { let response = common::post_json_with_token( "POST /compute/gemma/ravanan", "/compute/gemma/ravanan", &token, &body, ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let reply: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!( status, StatusCode::UNPROCESSABLE_ENTITY, "{expected}: {}", String::from_utf8_lossy(&bytes) ); assert_eq!(reply["code"], expected, "{reply}"); assert!( reply["detail"].as_str().unwrap_or_default().contains("/compute/gemma"), "the refusal should point at the endpoint that can do it: {reply}" ); } } #[tokio::test] async fn compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures() { let _serial = serialised(); if !ravanan_available() || !gemma_wrapper_available() || !ccwl_available() { eprintln!( "skipping compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures: ravanan, gemma-wrapper or ccwl not on PATH" ); return; } let Some(token) = common::auth_token().await else { eprintln!("skipping compute_gemma_ravanan_maps_a_qtl_from_the_bxd_fixtures: no token (see docs/gn-auth.md)"); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), }); let response = common::post_json_with_token( "POST /compute/gemma/ravanan", "/compute/gemma/ravanan", &token, &body, ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let rows: Value = common::data(&bytes); let rows = rows.as_array().expect("data is a JSON array of rows"); // BXD_snps.txt has 7320 markers across 20 chromosomes (1-19, X); // every one should have a row, same as /compute/gemma's own total. assert_eq!(rows.len(), 7320); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let columns: Vec<&str> = envelope["meta"]["columns"] .as_array() .expect("meta.columns") .iter() .map(|c| c.as_str().unwrap()) .collect(); let index_of = |name: &str| columns.iter().position(|c| *c == name).unwrap(); // rs31443144's values are deterministic for this fixture (same // values README.md documents for /compute/gemma's own run against // it); confirms the per-chromosome scatter produces the same // answer as gemma-wrapper's own internal GNU-parallel fan-out. let marker = rows .iter() .find(|row| row[index_of("rs")] == "rs31443144") .expect("rs31443144 should be one of the mapped markers"); assert_eq!(marker[index_of("chr")], serde_json::json!(1.0)); assert_eq!(marker[index_of("ps")], serde_json::json!(3010274.0)); assert!((marker[index_of("p_lrt")].as_f64().unwrap() - 0.2178407).abs() < 1e-6, "{marker}"); let notes = envelope["notes"].as_array().unwrap(); let ravanan_note = notes.iter().find(|n| n.as_str().unwrap().starts_with("ravanan:")); let chromosomes_note = notes.iter().find(|n| n.as_str().unwrap().starts_with("Scattered")); assert!(ravanan_note.is_some(), "{notes:?}"); assert!(chromosomes_note.is_some(), "{notes:?}"); // 20 chromosomes (1-19, X) in the BXD fixtures. assert!(chromosomes_note.unwrap().as_str().unwrap().contains("20 chromosome"), "{notes:?}"); } #[tokio::test] async fn compute_gemma_rejects_an_out_of_range_lmm() { let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_gemma_rejects_an_out_of_range_lmm: no token (see docs/gn-auth.md)" ); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": [1.0, 2.0], // 9 is now accepted (GeneNetwork's convention); 5-8 are stock- // invalid and untested, so they're still rejected. "lmm": 5, }); let response = common::post_json_with_token( "POST /compute/gemma (bad lmm)", "/compute/gemma", &token, &body, ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); } #[tokio::test] async fn compute_gemma_requires_a_token() { let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": [1.0, 2.0], }); // No token: build the request directly rather than through // post_json_with_token, which always attaches one. let request = axum::http::Request::builder() .method(axum::http::Method::POST) .uri("/compute/gemma") .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( [127, 0, 0, 1], 40000, )))) .header("content-type", "application/json") .body(axum::body::Body::from(serde_json::to_vec(&body).unwrap())) .unwrap(); let response = tower::ServiceExt::oneshot(genenetwork4::app(), request) .await .unwrap(); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } #[tokio::test] async fn compute_gemma_rejects_an_invalid_run_id() { let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_gemma_rejects_an_invalid_run_id: no token (see docs/gn-auth.md)" ); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": [1.0, 2.0], }); let response = common::post_json_with_token( "POST /compute/gemma (bad id)", "/compute/gemma?id=../etc/passwd", &token, &body, ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); } #[tokio::test] async fn gemma_status_unknown_run_id_returns_404() { let Some(token) = common::auth_token().await else { eprintln!( "skipping gemma_status_unknown_run_id_returns_404: no token (see docs/gn-auth.md)" ); return; }; let response = common::get_with_token( "GET /compute/gemma/status (unknown)", "/compute/gemma/status/no-such-gn4-gemma-run?peek=0", &token, ) .await; assert_eq!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test] async fn gemma_status_rejects_an_invalid_run_id() { let Some(token) = common::auth_token().await else { eprintln!( "skipping gemma_status_rejects_an_invalid_run_id: no token (see docs/gn-auth.md)" ); return; }; let response = common::get_with_token( "GET /compute/gemma/status (bad id)", "/compute/gemma/status/..%2Fetc%2Fpasswd?peek=0", &token, ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); } #[tokio::test] async fn gemma_status_requires_a_token() { let request = axum::http::Request::builder() .method(axum::http::Method::GET) .uri("/compute/gemma/status/some-run?peek=0") .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( [127, 0, 0, 1], 40000, )))) .body(axum::body::Body::empty()) .unwrap(); let response = tower::ServiceExt::oneshot(genenetwork4::app(), request) .await .unwrap(); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } #[tokio::test] async fn compute_gemma_rejects_a_run_id_already_in_use() { let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_gemma_rejects_a_run_id_already_in_use: no token (see docs/gn-auth.md)" ); return; }; // The uniqueness check happens before any subprocess runs, so this // doesn't need gemma-wrapper on PATH: just pre-seed a log file for // an id and confirm the POST refuses to reuse it. let cache_dir = genenetwork4::config::Config::load().gemma_cache_dir; std::fs::create_dir_all(&cache_dir).unwrap(); let run_id = format!( "dup-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() ); std::fs::write( std::path::Path::new(&cache_dir).join(format!("{run_id}.log")), "a previous (or still running) call already owns this id\n", ) .unwrap(); let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": [1.0, 2.0], }); let response = common::post_json_with_token( "POST /compute/gemma (duplicate id)", &format!("/compute/gemma?id={run_id}"), &token, &body, ) .await; assert_eq!(response.status(), StatusCode::CONFLICT); std::fs::remove_file(std::path::Path::new(&cache_dir).join(format!("{run_id}.log"))).unwrap(); } /// GEMMA with a covariate: a case attribute of the group, dummy-expanded /// into a covariate file of its own. Needs gemma-wrapper, a database and /// the genotype files. /// /// BXD, its classic `BXD.4` panel (3 811 markers, so this is a mechanism /// test rather than a mapping) and the `Status` case attribute, which has /// five levels among this trait's 34 strains -- so four dummy columns, and /// a check that a categorical covariate really is expanded rather than /// fitted as one ordinal number. #[tokio::test] async fn compute_gemma_fits_a_case_attribute_as_a_covariate() { let _serial = serialised(); if !gemma_wrapper_available() { eprintln!( "skipping compute_gemma_fits_a_case_attribute_as_a_covariate: gemma-wrapper not on PATH" ); return; } let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_gemma_fits_a_case_attribute_as_a_covariate: no token (see docs/gn-auth.md)" ); return; }; if !common::db_available() { eprintln!( "skipping compute_gemma_fits_a_case_attribute_as_a_covariate: no usable database" ); return; } let directory = genenetwork4::config::Config::load().genotype_files_dir; if !std::path::Path::new(&directory).is_dir() { eprintln!( "skipping compute_gemma_fits_a_case_attribute_as_a_covariate: no genotype files" ); return; } let response = common::get_with_token( "GET /groups/BXD/traits/10001/values", "/groups/BXD/traits/10001/values", &token, ) .await; assert_eq!(response.status(), StatusCode::OK); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let samples: Vec = common::data(&bytes); // Runs one request and hands back its status and body, so each step // below reads as "post this, look at that". let post = |samples: Vec, covariates: Value| { let body = serde_json::json!({ "group": "BXD", "genotype_version": "BXD.4", "trait_name": "BXD::10001", "samples": samples, "covariates": covariates, }); let token = token.clone(); async move { let response = common::post_json_with_token("POST /compute/gemma", "/compute/gemma", &token, &body) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); (status, bytes) } }; // The same trait without covariates, for comparison. let (status, bytes) = post(samples.clone(), serde_json::json!([])).await; assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let plain: Value = serde_json::from_slice(&bytes).unwrap(); let (status, bytes) = post( samples.clone(), serde_json::json!([{"case_attribute": "Status"}]), ) .await; assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let with_status: Value = serde_json::from_slice(&bytes).unwrap(); assert!(!with_status["data"].as_array().unwrap().is_empty()); // First note names what was mapped: two GEMMA runs over one panel are // otherwise indistinguishable, and the plot of each would be titled // "gemma". assert_eq!( with_status["notes"][0].as_str().unwrap(), "Trait: BXD::10001", "the response names its trait" ); const _: () = (); let notes: Vec<&str> = with_status["notes"] .as_array() .unwrap() .iter() .map(|note| note.as_str().unwrap()) .collect(); let covariate_note = notes .iter() .find(|note| note.starts_with("Covariate Status")) .expect("a note saying what the covariate was"); // The attribute's own description is quoted: `Status` alone does not // say what its levels mean. assert!( covariate_note.contains("Cryopreserved"), "the note says what the attribute means: {covariate_note}" ); assert!(covariate_note.contains("categorical"), "{covariate_note}"); assert!( covariate_note.contains("coded"), "the coding is reported: {covariate_note}" ); // A categorical covariate is expanded, never fitted as one ordinal // number: several levels, one reference, a dummy for each of the rest. let columns_note = notes .iter() .find(|note| note.starts_with("GEMMA fits ")) .expect("a note naming the covariate columns it was handed"); assert!( columns_note.contains("Status.") && columns_note.contains(", Status."), "a categorical covariate is dummy-expanded, not fitted as one number: {columns_note}" ); // A covariate only ever reaches the GWA step. let kinship = notes .iter() .find(|note| note.starts_with("Kinship:")) .expect("the kinship command line"); assert!(!kinship.contains(" -c "), "{kinship}"); let gwa = notes .iter() .find(|note| note.starts_with("GWA:")) .expect("the GWA command line"); assert!(gwa.contains(" -c "), "{gwa}"); assert!(gwa.contains("covariates.txt"), "{gwa}"); // The covariate changed the fit: the wrapper hashes its file // arguments, so this also proves the covariate file was part of the // key rather than a cached no-covariate run being reused. let peak_p = |envelope: &Value| { let columns: Vec<&str> = envelope["meta"]["columns"] .as_array() .unwrap() .iter() .map(|c| c.as_str().unwrap()) .collect(); let index = columns.iter().position(|c| *c == "p_lrt").expect("p_lrt"); envelope["data"] .as_array() .unwrap() .iter() .filter_map(|row| row[index].as_f64()) .fold(f64::INFINITY, f64::min) }; assert!( (peak_p(&plain) - peak_p(&with_status)).abs() > 0.0, "the covariate must change the fit, not reuse the plain run: {} vs {}", peak_p(&plain), peak_p(&with_status) ); // Covariates are aligned by name, so explicit files cannot have them. let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), "covariates": [{"case_attribute": "Epoch"}], }); let response = common::post_json_with_token("POST /compute/gemma (paths)", "/compute/gemma", &token, &body) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(error["code"], "covariate_unavailable"); // An attribute the group does not define is refused, listing its own. let (status, bytes) = post(samples, serde_json::json!([{"case_attribute": "Sex"}])).await; assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(error["code"], "covariate_unavailable"); assert!( error["detail"].as_str().unwrap().contains("Status"), "the error lists what the group has: {error}" ); } /// Permutations are refused outright, not ignored: gemma-wrapper's /// `--permutate` repeats the whole LOCO scan per permutation, which is /// hours on a real panel. R/qtl has the test, so the error says so. #[tokio::test] async fn compute_gemma_refuses_permutations() { // The gate answers before the handler does, so a job from another test // in this binary would come back 409 instead of the refusal under test. let Some(token) = common::auth_token().await else { eprintln!("skipping compute_gemma_refuses_permutations: no token (see docs/gn-auth.md)"); return; }; for field in ["nperm", "permutations", "permutate"] { let mut body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), }); body[field] = serde_json::json!(100); let response = common::post_json_with_token("POST /compute/gemma (permutations)", "/compute/gemma", &token, &body) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!( status, StatusCode::UNPROCESSABLE_ENTITY, "{field} must be refused: {}", String::from_utf8_lossy(&bytes) ); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(error["code"], "permutations_not_supported", "{error}"); let detail = error["detail"].as_str().unwrap(); assert!(detail.contains("100 permutations"), "{detail}"); // Two routes are named: the endpoint that can permute in seconds, and // running GEMMA itself for its exact null. assert!( detail.contains("/compute/bulklmm") && detail.contains("nperm"), "the error points at the endpoint that can do it: {detail}" ); assert!( detail.contains("run GEMMA's own permutation locally"), "and at GEMMA itself, for the null only GEMMA has: {detail}" ); } // Zero permutations is not a request for any: the field is accepted // and ignored, the way every other endpoint treats its own off-switch. let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), "nperm": 0, "lmm": 99, }); let response = common::post_json_with_token("POST /compute/gemma (nperm 0)", "/compute/gemma", &token, &body) .await; // Refused for the lmm value, which proves the request got past the // permutation check rather than being rejected for it. assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert!( error["code"] != "permutations_not_supported", "nperm 0 is not a permutation request: {error}" ); } /// The effect size comes back **next to the score**: `additive = −beta/2` /// immediately before `p_lrt`, with GEMMA's own `beta`/`se` untouched and /// `p_lrt` still last (peak detection and the plotter read it there). #[tokio::test] async fn compute_gemma_reports_the_effect_next_to_the_score() { let _serial = serialised(); if !gemma_wrapper_available() { eprintln!("skipping compute_gemma_reports_the_effect_next_to_the_score: gemma-wrapper not on PATH"); return; } let Some(token) = common::auth_token().await else { eprintln!("skipping compute_gemma_reports_the_effect_next_to_the_score: no token (see docs/gn-auth.md)"); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("BXD_geno.txt.gz"), "snps_file": test_data_path("BXD_snps.txt"), "trait_values": bxd_trait_values(), "trait_name": "BXD::fixture", }); let response = common::post_json_with_token("POST /compute/gemma (effect)", "/compute/gemma", &token, &body) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let columns: Vec<&str> = envelope["meta"]["columns"] .as_array() .unwrap() .iter() .map(|name| name.as_str().unwrap()) .collect(); let beta_at = columns.iter().position(|name| *name == "beta").expect("beta"); let se_at = columns.iter().position(|name| *name == "se").expect("se"); let additive_at = columns .iter() .position(|name| *name == "additive") .expect("the effect size column"); let rows = envelope["data"].as_array().unwrap(); assert_eq!( additive_at, columns.len() - 2, "additive sits immediately before the score: {columns:?}" ); assert_eq!( columns[additive_at + 1], "p_lrt", "and the p-value stays last: {columns:?}" ); for row in rows.iter().take(50) { let beta = row[beta_at].as_f64().expect("a numeric beta"); let additive = row[additive_at].as_f64().expect("a numeric additive"); assert!( (additive - (-beta / 2.0)).abs() < 1e-9, "additive = -beta/2: beta {beta}, additive {additive}" ); assert!(row[se_at].is_number(), "and se is still there: {row}"); } // The last column is still the p-value, not the effect: every value in // it is a probability, and the additive column is not. let p_at = columns.len() - 1; assert!( rows.iter().all(|row| row[p_at] .as_f64() .is_some_and(|p| (0.0..=1.0).contains(&p))), "p_lrt stays a probability in the last column" ); let effects: Vec = rows .iter() .filter_map(|row| row[additive_at].as_f64()) .collect(); assert!( !effects.is_empty() && effects.iter().any(|value| *value != 0.0), "the effect column carries numbers, not placeholders: {effects:?}" ); let notes: Vec<&str> = envelope["notes"] .as_array() .unwrap() .iter() .map(|note| note.as_str().unwrap()) .collect(); assert!( notes.iter().any(|note| note.starts_with("Effect size:")), "and the response says what the three columns mean: {notes:?}" ); } /// A mostly-numeric attribute (`SeqCvge`: depths beside two-library values /// like `36+39L`) must be refused with a message pointing at `map`, not /// coded as a factor and handed to GEMMA -- which used to come back as a /// `502 gemma_failed` about LU decomposition, saying nothing about the /// covariate. #[tokio::test] async fn a_mostly_numeric_covariate_is_refused_with_a_map_hint() { let _serial = serialised(); let Some(token) = common::auth_token().await else { eprintln!("skipping a_mostly_numeric_covariate_is_refused_with_a_map_hint: no token (see docs/gn-auth.md)"); return; }; if !common::db_available() { eprintln!("skipping a_mostly_numeric_covariate_is_refused_with_a_map_hint: no usable database"); return; } let body = serde_json::json!({ "group": "BXD", "genotype_version": "BXD.4", "trait_name": "BXD::10001", "samples": [ {"name": "BXD1", "value": 61.4}, {"name": "BXD11", "value": 53.1}, {"name": "BXD12", "value": 45.9}, {"name": "BXD13", "value": 49.2}, {"name": "BXD14", "value": 45.5}, {"name": "BXD15", "value": 60.1} ], "covariates": [{"case_attribute": "SeqCvge"}] }); let response = common::post_json_with_token( "POST /compute/gemma (SeqCvge)", "/compute/gemma", &token, &body, ) .await; assert_eq!( response.status(), StatusCode::UNPROCESSABLE_ENTITY, "refused before GEMMA runs" ); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(error["code"], "invalid_covariate", "{error}"); let detail = error["detail"].as_str().unwrap(); assert!(detail.contains("mixes numbers"), "{detail}"); assert!(detail.contains("`map`"), "it says what to send: {detail}"); } /// An individual with no genotypes anywhere empties a kinship row, and the /// matrix that results is singular: GEMMA dies inside LAPACK with /// "eigenvalues close to zero" and then `LUDecomp ... input domain error`, /// which is a missing-genotype problem wearing a linear-algebra costume. /// BXD-AE.4 is the case in point -- 136 of its 344 individuals have no /// genotypes at all, and the panel used to fail with `502 gemma_failed`. #[tokio::test] async fn individuals_with_no_genotypes_are_left_out() { let _serial = serialised(); if !gemma_wrapper_available() { eprintln!("skipping individuals_with_no_genotypes_are_left_out: gemma-wrapper not on PATH"); return; } let Some(token) = common::auth_token().await else { eprintln!("skipping individuals_with_no_genotypes_are_left_out: no token (see docs/gn-auth.md)"); return; }; if !common::db_available() { eprintln!("skipping individuals_with_no_genotypes_are_left_out: no usable database"); return; } let response = common::get_with_token( "GET /groups/BXD-AE/traits/10001/values", "/groups/BXD-AE/traits/10001/values", &token, ) .await; assert_eq!(response.status(), StatusCode::OK); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let samples: Value = common::data(&bytes); let body = serde_json::json!({ "group": "BXD-AE", "genotype_version": "BXD-AE.4", "trait_name": "BXD-AE::10001", "samples": samples, }); let response = common::post_json_with_token("POST /compute/gemma (empty genotypes)", "/compute/gemma", &token, &body) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!( status, StatusCode::OK, "the panel maps once the genotype-less individuals are left out: {}", String::from_utf8_lossy(&bytes) ); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert!(envelope["meta"]["num_rows"].as_u64().unwrap() > 0); let notes: Vec<&str> = envelope["notes"] .as_array() .unwrap() .iter() .map(|note| note.as_str().unwrap()) .collect(); let note = notes .iter() .find(|note| note.contains("no genotypes at all")) .unwrap_or_else(|| panic!("the reply says what it left out: {notes:?}")); assert!( note.contains("singular"), "and why it matters: {note}" ); // 136 of 344, so the genotype line must not still claim 344. let genotype = notes .iter() .find(|note| note.starts_with("Genotype:")) .expect("the genotype note"); assert!( !genotype.contains(" 344 individuals"), "the panel size reflects the drop: {genotype}" ); } /// Choosing individuals: the parent/F1 rule is a default, not a policy. #[tokio::test] async fn individuals_can_be_added_back_or_excluded_by_name() { let _serial = serialised(); let Some(token) = common::auth_token().await else { eprintln!("skipping individuals_can_be_added_back_or_excluded_by_name: set GN4_TEST_AUTH_TOKEN"); return; }; if !common::db_available() || !gemma_wrapper_available() { eprintln!("skipping individuals_can_be_added_back_or_excluded_by_name: no database or gemma-wrapper"); return; } let response = common::get_with_token( "GET /groups/BXD/traits/10001/values", "/groups/BXD/traits/10001/values", &token, ) .await; let bytes = response.into_body().collect().await.unwrap().to_bytes(); let samples: Vec = common::data(&bytes); if samples.is_empty() { eprintln!("skipping individuals_can_be_added_back_or_excluded_by_name: no trait values"); return; } // A name that is not in the panel is refused, and the message names the panel. let response = common::post_json_with_token( "POST /compute/gemma", "/compute/gemma", &token, &serde_json::json!({ "group": "BXD", "genotype_version": "BXD.4", "trait_name": "BXD::10001", "samples": samples, "individuals": {"exclude": ["NOT-A-STRAIN"]}, }), ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let reply: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{reply}"); assert!( reply["detail"].as_str().unwrap_or_default().contains("NOT-A-STRAIN"), "{reply}" ); // Excluding a real strain shrinks the panel and the note says so. let response = common::post_json_with_token( "POST /compute/gemma", "/compute/gemma", &token, &serde_json::json!({ "group": "BXD", "genotype_version": "BXD.4", "trait_name": "BXD::10001", "samples": samples, "individuals": {"exclude": ["BXD1"]}, }), ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let reply: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(status, StatusCode::OK, "{reply}"); let notes = reply["notes"].as_array().unwrap(); let genotype_note = notes .iter() .find(|note| note.as_str().unwrap_or_default().starts_with("Genotype:")) .expect("the reply says what it mapped"); assert!( genotype_note.as_str().unwrap().contains("1 excluded by name"), "{genotype_note}" ); }