//! Integration test for POST /compute/rqtl. Runs the real Rscript/R-qtl //! against a small, hand-written BXD riset fixture (tests/test_data, //! not copied from genenetwork3 verbatim: its own //! tests/unit/db/.../genotype_sample1.geno has a genotype code outside //! its own header's declared set and duplicate marker names across //! chromosomes, neither of which actually matters for the unrelated //! parsing unit it's written for, but both would break a real R/qtl //! run); skips rather than fails when Rscript is not on PATH, same //! "skip, don't fail" convention as tests/gemma.rs. use axum::http::StatusCode; use http_body_util::BodyExt; use serde_json::Value; mod common; /// These tests stay serialised for the machine's sake, not the gate's: /// small panels are exempt from gating (`compute_gate::LARGE_PANEL`), so /// nothing here is refused, but several Rscript scans at once would still /// compete for cores and the shared R/qtl scratch directory. Cargo runs a /// file's tests on several threads, so they take this lock first. static COMPUTE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); /// Take [`COMPUTE_LOCK`], ignoring poisoning (one failing test must not /// fail the rest). fn serialised() -> std::sync::MutexGuard<'static, ()> { COMPUTE_LOCK .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()) } fn rscript_available() -> bool { std::process::Command::new("Rscript") .arg("--version") .output() .is_ok() } fn test_data_path(name: &str) -> String { format!("{}/tests/test_data/{name}", env!("CARGO_MANIFEST_DIR")) } fn bxd_samples() -> Value { serde_json::json!([ {"name": "BXD1", "value": 9.5}, {"name": "BXD2", "value": 8.2}, {"name": "BXD5", "value": 7.1}, {"name": "BXD6", "value": 6.4}, {"name": "BXD8", "value": 5.9}, {"name": "BXD9", "value": 4.3}, ]) } #[tokio::test] async fn compute_rqtl_maps_a_qtl_from_the_bxd_riset_fixture() { let _serial = serialised(); if !rscript_available() { eprintln!( "skipping compute_rqtl_maps_a_qtl_from_the_bxd_riset_fixture: Rscript not on PATH" ); return; } let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_rqtl_maps_a_qtl_from_the_bxd_riset_fixture: no token (see docs/gn-auth.md)" ); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1.geno"), "trait_name": "mytrait", "samples": bxd_samples(), }); let response = common::post_json_with_token("POST /compute/rqtl", "/compute/rqtl", &token, &body).await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!( status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes) ); // A scan without permutations suggests the same run with nperm=100, // as an `actions` entry because it is a POST (see tests/phewas.rs for // the group-scoped case, which also gets a PheWAS `next`). let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let actions = envelope["actions"].as_array().expect("suggestions"); let action = actions .iter() .find(|action| action["href"] == "/compute/rqtl") .expect("the permutation suggestion"); assert_eq!(action["method"], "POST"); assert_eq!(action["body"]["nperm"], 100); // ... and the plot of the same run, which is the other suggestion. // Its href carries the trait, so the plot titles itself even when the // caller posts only `data`. let plot = actions .iter() .find(|action| { action["href"] .as_str() .is_some_and(|href| href.starts_with("/plot/rqtl")) }) .unwrap_or_else(|| panic!("a plot suggestion: {envelope}")); assert!( plot["href"].as_str().unwrap().contains("trait=mytrait"), "the plot suggestion names the trait: {plot}" ); assert_eq!( action["body"]["genotype_file"], body["genotype_file"], "the suggestion reuses the files that ran: {action}" ); let data: Value = common::data(&bytes); let markers = data["markers"].as_array().unwrap(); // The fixture has 6 markers (3 per chromosome, 2 chromosomes). assert_eq!(markers.len(), 6); // Rows are positional (name, chr, pos, lod); see meta.columns. assert_eq!(markers[0][0], "rs31443144"); assert_eq!(markers[0][1], "1"); assert!(markers[0][3].as_f64().unwrap() >= 0.0); assert!(data["permutation"].is_null(), "nperm defaults to 0: {data}"); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(envelope["meta"]["columns"], serde_json::json!(["name", "chr", "pos", "lod"])); assert_eq!(envelope["meta"]["num_cols"], 4); assert_eq!(envelope["meta"]["num_rows"], 6); let notes = envelope["notes"].as_array().unwrap(); let rscript_note = notes .iter() .find(|n| n.as_str().unwrap().starts_with("Rscript")); assert!(rscript_note.is_some(), "{notes:?}"); let rscript_note = rscript_note.unwrap().as_str().unwrap(); assert!( rscript_note.contains("--geno") && rscript_note.contains("--pheno"), "{rscript_note}" ); } #[tokio::test] async fn compute_rqtl_with_permutations_returns_thresholds() { let _serial = serialised(); if !rscript_available() { eprintln!( "skipping compute_rqtl_with_permutations_returns_thresholds: Rscript not on PATH" ); return; } let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_rqtl_with_permutations_returns_thresholds: no token (see docs/gn-auth.md)" ); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1.geno"), "trait_name": "mytrait", "samples": bxd_samples(), "nperm": 20, }); let response = common::post_json_with_token( "POST /compute/rqtl (with permutations)", "/compute/rqtl", &token, &body, ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!( status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes) ); let data: Value = common::data(&bytes); let perm = &data["permutation"]; assert!(!perm.is_null(), "{data}"); assert_eq!(perm["values"].as_array().unwrap().len(), 20); let suggestive = perm["suggestive"].as_f64().unwrap(); let significant = perm["significant"].as_f64().unwrap(); assert!(significant >= suggestive, "{perm}"); } #[tokio::test] async fn compute_rqtl_requires_a_token() { let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1.geno"), "trait_name": "mytrait", "samples": bxd_samples(), }); let request = axum::http::Request::builder() .method(axum::http::Method::POST) .uri("/compute/rqtl") .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( [127, 0, 0, 1], 40000, )))) .header("content-type", "application/json") .body(axum::body::Body::from(serde_json::to_vec(&body).unwrap())) .unwrap(); let response = tower::ServiceExt::oneshot(genenetwork4::app(), request) .await .unwrap(); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } #[tokio::test] async fn compute_rqtl_rejects_an_invalid_run_id() { let _serial = serialised(); let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_rqtl_rejects_an_invalid_run_id: no token (see docs/gn-auth.md)" ); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1.geno"), "trait_name": "mytrait", "samples": bxd_samples(), }); let response = common::post_json_with_token( "POST /compute/rqtl (bad id)", "/compute/rqtl?id=../etc/passwd", &token, &body, ) .await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); } #[tokio::test] async fn rqtl_status_unknown_run_id_returns_404() { let Some(token) = common::auth_token().await else { eprintln!( "skipping rqtl_status_unknown_run_id_returns_404: no token (see docs/gn-auth.md)" ); return; }; let response = common::get_with_token( "GET /compute/rqtl/status (unknown)", "/compute/rqtl/status/no-such-gn4-rqtl-run?peek=0", &token, ) .await; assert_eq!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test] async fn rqtl_status_requires_a_token() { let request = axum::http::Request::builder() .method(axum::http::Method::GET) .uri("/compute/rqtl/status/some-run?peek=0") .extension(axum::extract::ConnectInfo(std::net::SocketAddr::from(( [127, 0, 0, 1], 40000, )))) .body(axum::body::Body::empty()) .unwrap(); let response = tower::ServiceExt::oneshot(genenetwork4::app(), request) .await .unwrap(); assert_eq!(response.status(), StatusCode::UNAUTHORIZED); } #[tokio::test] async fn compute_rqtl_rejects_a_run_id_already_in_use() { let _serial = serialised(); let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_rqtl_rejects_a_run_id_already_in_use: no token (see docs/gn-auth.md)" ); return; }; // Same reasoning as gemma's equivalent test: the uniqueness check // runs before any subprocess call, so this doesn't need Rscript. let cache_dir = genenetwork4::config::Config::load().rqtl_cache_dir; std::fs::create_dir_all(&cache_dir).unwrap(); let run_id = format!( "dup-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() ); std::fs::write( std::path::Path::new(&cache_dir).join(format!("{run_id}.log")), "a previous (or still running) call already owns this id\n", ) .unwrap(); let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1.geno"), "trait_name": "mytrait", "samples": bxd_samples(), }); let response = common::post_json_with_token( "POST /compute/rqtl (duplicate id)", &format!("/compute/rqtl?id={run_id}"), &token, &body, ) .await; assert_eq!(response.status(), StatusCode::CONFLICT); std::fs::remove_file(std::path::Path::new(&cache_dir).join(format!("{run_id}.log"))).unwrap(); } #[tokio::test] async fn compute_rqtl_streams_progress_while_running() { let _serial = serialised(); if !rscript_available() { eprintln!("skipping compute_rqtl_streams_progress_while_running: Rscript not on PATH"); return; } let Some(token) = common::auth_token().await else { eprintln!( "skipping compute_rqtl_streams_progress_while_running: no token (see docs/gn-auth.md)" ); return; }; let run_id = format!( "rqtl-test-{}", std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos() ); let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1.geno"), "trait_name": "mytrait", "samples": bxd_samples(), }); let post_token = token.clone(); let post_run_id = run_id.clone(); let post_handle = tokio::spawn(async move { common::post_json_with_token( "POST /compute/rqtl (streamed)", &format!("/compute/rqtl?id={post_run_id}"), &post_token, &body, ) .await }); let response = post_handle.await.unwrap(); assert_eq!(response.status(), StatusCode::OK); let final_response = common::get_with_token( "GET /compute/rqtl/status (final)", &format!("/compute/rqtl/status/{run_id}?peek=0"), &token, ) .await; assert_eq!(final_response.status(), StatusCode::OK); let final_bytes = final_response .into_body() .collect() .await .unwrap() .to_bytes(); let final_status: Value = common::data(&final_bytes); let lines: Vec<&str> = final_status["lines"] .as_array() .unwrap() .iter() .map(|l| l.as_str().unwrap()) .collect(); assert!(lines.contains(&"GN4: computation complete"), "{lines:?}"); } /// R/qtl with a covariate: a case attribute by name, coded by the server. /// Needs Rscript, a database and the genotype files. /// /// Uses UM-HET3 (`HET3-ITP`), whose panel is 894 markers rather than the /// BXD WGS panel's 21 056: the covariate path is what is under test, and /// a 21k-marker scan would make this a mapping rather than a test. #[tokio::test] async fn compute_rqtl_fits_a_case_attribute_as_a_covariate() { let _serial = serialised(); if !rscript_available() { eprintln!("skipping compute_rqtl_fits_a_case_attribute_as_a_covariate: Rscript not on PATH"); return; } let Some(token) = common::auth_token().await else { eprintln!("skipping compute_rqtl_fits_a_case_attribute_as_a_covariate: set GN4_TEST_AUTH_TOKEN (see docs/gn-auth.md)"); return; }; if !common::db_available() { eprintln!("skipping compute_rqtl_fits_a_case_attribute_as_a_covariate: no usable database"); return; } let directory = genenetwork4::config::Config::load().genotype_files_dir; if !std::path::Path::new(&directory).is_dir() { eprintln!("skipping compute_rqtl_fits_a_case_attribute_as_a_covariate: no genotype files"); return; } let response = common::get_with_token( "GET /groups/HET3-ITP/traits/10000/values", "/groups/HET3-ITP/traits/10000/values", &token, ) .await; assert_eq!(response.status(), StatusCode::OK); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let all: Value = common::data(&bytes); let samples: Vec = all.as_array().unwrap().iter().take(400).cloned().collect(); let body = serde_json::json!({ "group": "HET3-ITP", "trait_name": "HET3-ITP::10000", "samples": samples, "covariates": [{"case_attribute": "Sex"}], }); let response = common::post_json_with_token("POST /compute/rqtl", "/compute/rqtl", &token, &body).await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!( status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes) ); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert!(!envelope["data"]["markers"].as_array().unwrap().is_empty()); let notes: Vec<&str> = envelope["notes"] .as_array() .unwrap() .iter() .map(|note| note.as_str().unwrap()) .collect(); let covariate_note = notes .iter() .find(|note| note.starts_with("Covariate Sex")) .expect("a note saying what the covariate was"); // The attribute's description comes with it: "F= female, M= male". assert!( covariate_note.contains("female"), "the note says what the attribute means: {covariate_note}" ); assert!(covariate_note.contains("categorical"), "{covariate_note}"); assert!( covariate_note.contains("coded"), "the coding is reported: {covariate_note}" ); let rscript = notes .iter() .find(|note| note.starts_with("Rscript ")) .expect("the command line"); assert!(rscript.contains("--addcovar"), "{rscript}"); assert!(rscript.contains("--covarstruct"), "{rscript}"); // An attribute the group does not define is refused, listing its own. let body = serde_json::json!({ "group": "HET3-ITP", "trait_name": "t", "samples": samples, "covariates": [{"case_attribute": "Nonsense"}], }); let response = common::post_json_with_token("POST /compute/rqtl", "/compute/rqtl", &token, &body).await; assert_eq!(response.status(), StatusCode::UNPROCESSABLE_ENTITY); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let error: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!(error["code"], "covariate_unavailable"); assert!( error["detail"].as_str().unwrap().contains("Sex"), "the error lists what the group has: {error}" ); } /// A `.geno` with `cM` and no `Mb` — 44 of this deployment's files look like /// that. GeneNetwork3's wrapper assumed four metadata columns /// (`Chr, Locus, cM, Mb`) and started the samples at column 5, so those /// panels lost their first individual and died with "number of columns of /// result is not a multiple of vector length". The wrapper now uses /// whichever map column exists and starts the samples right after the /// position columns, and the scan must cover every marker and individual. #[tokio::test] async fn a_cm_only_genotype_file_maps() { let _serial = serialised(); if !rscript_available() { eprintln!("skipping a_cm_only_genotype_file_maps: Rscript not on PATH"); return; } let Some(token) = common::auth_token().await else { eprintln!("skipping a_cm_only_genotype_file_maps: set GN4_TEST_AUTH_TOKEN (see docs/gn-auth.md)"); return; }; let body = serde_json::json!({ "genotype_file": test_data_path("rqtl_sample1_cm.geno"), "trait_name": "cm-only", "samples": [ {"name": "BXD1", "value": 9.5}, {"name": "BXD2", "value": 8.2}, {"name": "BXD5", "value": 7.1}, {"name": "BXD6", "value": 6.4}, {"name": "BXD8", "value": 5.9}, {"name": "BXD9", "value": 4.3} ] }); let response = common::post_json_with_token("POST /compute/rqtl (cm only)", "/compute/rqtl", &token, &body) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let markers = envelope["data"]["markers"].as_array().unwrap(); assert_eq!( markers.len(), 6, "every marker of the fixture is scanned, not five: {markers:?}" ); assert!(envelope["meta"]["num_rows"].as_u64().unwrap() == 6); }