//! `POST /compute/bulklmm`: an LMM scan with permutation thresholds. //! //! The reason the endpoint exists is the *thresholds*: `/compute/gemma` refuses //! `nperm` because a permutation repeats the whole LOCO scan, and R/qtl's //! permutation is the wrong null for related individuals. BulkLMM rotates by //! the kinship eigen decomposition, so a threshold costs seconds and belongs to //! the model that produced the LODs. use axum::http::StatusCode; use http_body_util::BodyExt; use serde_json::Value; mod common; /// BulkLMM needs a Julia with its packages instantiated; skip when the /// deployment has none (a `guix shell` without the Julia side). /// /// The project file has to *name the packages the driver loads*, not merely /// exist: an env left behind by a half-instantiated run has a Project.toml /// listing BulkLMM and nothing else, and the scan then dies on `using CSV` /// with a stack trace instead of skipping here. `BULKLMM_ENV` decides which /// project is asked, so a shell that wants these tests sets it. fn bulklmm_available() -> bool { let config = genenetwork4::config::Config::load(); if !std::path::Path::new(&config.bulklmm_cmd).is_file() { return false; } let project = std::path::Path::new(&config.bulklmm_env).join("Project.toml"); let Ok(contents) = std::fs::read_to_string(&project) else { return false; }; ["BulkLMM", "CSV", "DataFrames"] .iter() .all(|package| contents.contains(package)) } #[tokio::test] async fn a_scan_reports_lods_and_permutation_thresholds() { let Some(token) = common::auth_token().await else { eprintln!("skipping: set GN4_TEST_AUTH_TOKEN"); return; }; if !common::db_available() { eprintln!("skipping a_scan_reports_lods_and_permutation_thresholds: no database"); return; } if !bulklmm_available() { eprintln!("skipping a_scan_reports_lods_and_permutation_thresholds: no Julia/BulkLMM"); return; } let values = common::get_with_token( "GET /groups/BXD/traits/16627/values", "/groups/BXD/traits/16627/values", &token, ) .await; assert_eq!(values.status(), StatusCode::OK); let bytes = values.into_body().collect().await.unwrap().to_bytes(); let samples: Value = common::data(&bytes); let response = common::post_json_with_token( "POST /compute/bulklmm", "/compute/bulklmm", &token, &serde_json::json!({ "group": "BXD", "genotype_version": "BXD.8", "trait_name": "BXD::16627", "samples": samples, "nperm": 20, "seed": 7, "cores": 2, }), ) .await; let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); // Both tests in this file share one caller and the gate gives a caller one // scan at a time, so whichever starts second sees 409. That is the gate // working, not a failure. if status == StatusCode::CONFLICT { eprintln!("skipping: the other BulkLMM test holds this caller's slot"); return; } assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert_eq!( envelope["meta"]["columns"], serde_json::json!(["marker", "chr", "pos", "lod", "beta", "h2"]) ); let rows = envelope["data"].as_array().unwrap(); assert!(rows.len() > 1000, "the whole panel: {}", rows.len()); let peak = rows .iter() .max_by(|left, right| { left[3] .as_f64() .unwrap() .partial_cmp(&right[3].as_f64().unwrap()) .unwrap() }) .unwrap(); // The published peak of this trait is rsm10000000888 (chr2, LOD 9.59); an // LMM with the same values has to see it. assert_eq!(peak[1], "2", "the peak stays on chr2: {peak}"); assert!( peak[3].as_f64().unwrap() > 3.0, "a real peak, not noise: {peak}" ); let notes: Vec<&str> = envelope["notes"] .as_array() .unwrap() .iter() .map(|note| note.as_str().unwrap()) .collect(); assert!( notes .iter() .any(|note| note.contains("Permutation thresholds from 20 rotations")), "the thresholds are in the reply: {notes:?}" ); assert!( notes.iter().any(|note| note.contains("seed 7")), "and the seed that produced them: {notes:?}" ); assert!( notes.iter().any(|note| note.contains("Null heritability")), "and what the kinship term found: {notes:?}" ); // The parallelism is a configured number, not folklore. assert!( notes .iter() .any(|note| note.contains("independent blocks") && note.contains("BULKLMM_BLOCKS")), "the block count and where to set it: {notes:?}" ); } /// Without permutations there is no threshold, and the reply says so rather /// than leaving a bare LOD to be read as significant. #[tokio::test] async fn a_scan_without_permutations_says_there_is_no_threshold() { let Some(token) = common::auth_token().await else { eprintln!("skipping: set GN4_TEST_AUTH_TOKEN"); return; }; if !common::db_available() || !bulklmm_available() { eprintln!("skipping a_scan_without_permutations_says_there_is_no_threshold"); return; } let values = common::get_with_token( "GET /groups/BXD/traits/16627/values", "/groups/BXD/traits/16627/values", &token, ) .await; let bytes = values.into_body().collect().await.unwrap().to_bytes(); let samples: Value = common::data(&bytes); let response = common::post_json_with_token( "POST /compute/bulklmm (no permutations)", "/compute/bulklmm", &token, &serde_json::json!({ "group": "BXD", "genotype_version": "BXD.8", "trait_name": "BXD::16627", "samples": samples, "cores": 2, }), ) .await; // The two tests share one caller, and the gate allows one scan at a time // per caller: a 409 here means the other test is running, not a failure. if response.status() == StatusCode::CONFLICT { eprintln!("skipping: another BulkLMM scan holds this caller's slot"); return; } assert_eq!(response.status(), StatusCode::OK); let bytes = response.into_body().collect().await.unwrap().to_bytes(); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); let notes = envelope["notes"].to_string(); assert!(notes.contains("No permutations requested"), "{notes}"); } /// Posting a scan back draws it, thresholds included; and a GEMMA permutation /// refusal points at this engine, since it is the one that can answer. #[tokio::test] async fn a_scan_plots_and_gemmas_refusal_points_here() { let Some(token) = common::auth_token().await else { eprintln!("skipping: set GN4_TEST_AUTH_TOKEN"); return; }; if !common::db_available() || !bulklmm_available() { eprintln!("skipping a_scan_plots_and_gemmas_refusal_points_here"); return; } let values = common::get_with_token( "GET /groups/BXD/traits/16627/values", "/groups/BXD/traits/16627/values", &token, ) .await; let bytes = values.into_body().collect().await.unwrap().to_bytes(); let samples: Value = common::data(&bytes); let response = common::post_json_with_token( "POST /compute/bulklmm (for the plot)", "/compute/bulklmm", &token, &serde_json::json!({ "group": "BXD", "genotype_version": "BXD.8", "trait_name": "BXD::16627", "samples": samples, "nperm": 20, "seed": 3, "cores": 2, }), ) .await; if response.status() == StatusCode::CONFLICT { eprintln!("skipping: another BulkLMM scan holds this caller's slot"); return; } let status = response.status(); let bytes = response.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let envelope: Value = serde_json::from_slice(&bytes).unwrap(); assert!( envelope["actions"] .as_array() .unwrap() .iter() .any(|action| action["href"] == "/plot/bulklmm"), "the reply offers its own plot: {envelope}" ); // The plot takes the reply back, thresholds and all. let plot = common::post_json_with_token( "POST /plot/bulklmm", "/plot/bulklmm", &token, &envelope, ) .await; let status = plot.status(); let bytes = plot.into_body().collect().await.unwrap().to_bytes(); assert_eq!(status, StatusCode::OK, "{}", String::from_utf8_lossy(&bytes)); let svg = String::from_utf8_lossy(&bytes); assert!(svg.trim_start().starts_with("