diff options
| author | Pjotr <wrk@bosa.lan> | 2026-10-24 09:33:40 +0000 |
|---|---|---|
| committer | Pjotr <wrk@bosa.lan> | 2026-10-24 09:33:40 +0000 |
| commit | a7ffb1cfecbdb8ac3fcc6286d7a09f7285c95d67 (patch) | |
| tree | 1cb865784d901fad979aed32694b77474019468d /src | |
| parent | cda325b6334ef36e3f49501cfd77d68a7bd6a295 (diff) | |
| download | genenetwork4-a7ffb1cfecbdb8ac3fcc6286d7a09f7285c95d67.tar.gz | |
gemma: Make ravanan's batch system configurable (optional Slurm).
POST /compute/gemma/ravanan used to run every CWL step locally, because compute_ravanan never passed ravanan a batch system. Add a RavananBatch setting: - RAVANAN_BATCH_SYSTEM=single-machine (default) or slurm-api, failing closed on anything else, exactly like GN4_DB_MODE; - slurm-api adds --batch-system=slurm-api, --slurm-api-endpoint, --slurm-jwt (a *file*, which is what ravanan reads) and, when set, --slurm-partition/--slurm-nice; a slurm selection without an endpoint or a JWT file refuses to start rather than 502-ing every request; - --scratch defaults to ravanan_cache_dir for slurm, so the store and the job scratch space live on the same shared filesystem (with slurm that directory has to be visible to the compute nodes); - the response's ravanan note records which backend ran. The switch building is RavananBatch::ravanan_args so it is unit testable; compute_ravanan takes the backend and the scratch directory, and the route passes them from config. docs/ravanan-slurm.md documents the keys, the shared-directory requirement, and the slurm-side auth wiring (AuthType=auth/munge + AuthAltTypes=auth/jwt with SLURM_JWT in slurmrestd's environment) that makes REST-submitted jobs actually run.
Diffstat (limited to 'src')
| -rw-r--r-- | src/config.rs | 310 | ||||
| -rw-r--r-- | src/gemma.rs | 25 | ||||
| -rw-r--r-- | src/routes/gemma.rs | 2 |
3 files changed, 334 insertions, 3 deletions
diff --git a/src/config.rs b/src/config.rs index 47cb4bf..9d06c4f 100644 --- a/src/config.rs +++ b/src/config.rs @@ -100,8 +100,21 @@ pub struct Config { /// run logs, and ravanan's own --store and --output-directory. /// Separate from `gemma_cache_dir` so the two runners don't share /// state. Defaults to a "gn4-gemma-ravanan" directory under the - /// system temp dir. + /// system temp dir. With `ravanan_batch` set to slurm this directory + /// must be on a filesystem the cluster's compute nodes can see (see + /// `ravanan_scratch`). pub ravanan_cache_dir: String, + /// How ravanan runs each step: locally (`single-machine`, the + /// default) or as jobs on a Slurm cluster (`slurm-api`, submitting + /// through slurmrestd's REST API). Set via RAVANAN_BATCH_SYSTEM; + /// see `RavananBatch` and docs/ravanan-slurm.md. + pub ravanan_batch: RavananBatch, + /// Directory ravanan stages job scratch space in (its own --scratch + /// switch). Only meaningful for the slurm backend, where it must be + /// a path every compute node can write; defaults to + /// `ravanan_cache_dir` when the backend is slurm. Empty means "do + /// not pass --scratch" (single-machine's own default). + pub ravanan_scratch: String, /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3) /// for POST /compute/rqtl. Defaults to that path relative to the /// working directory the server is started from. @@ -182,6 +195,17 @@ impl Config { .into_owned(), ); values.insert( + "RAVANAN_BATCH_SYSTEM".to_string(), + "single-machine".to_string(), + ); + // Only read when RAVANAN_BATCH_SYSTEM=slurm-api; left blank so + // the single-machine default stays a zero-configuration path. + values.insert("RAVANAN_SLURM_API_ENDPOINT".to_string(), String::new()); + values.insert("RAVANAN_SLURM_JWT".to_string(), String::new()); + values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new()); + values.insert("RAVANAN_SLURM_NICE".to_string(), String::new()); + values.insert("RAVANAN_SCRATCH".to_string(), String::new()); + values.insert( "RQTL_WRAPPER_PATH".to_string(), "scripts/rqtl_wrapper.R".to_string(), ); @@ -225,6 +249,12 @@ impl Config { "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", + "RAVANAN_BATCH_SYSTEM", + "RAVANAN_SLURM_API_ENDPOINT", + "RAVANAN_SLURM_JWT", + "RAVANAN_SLURM_PARTITION", + "RAVANAN_SLURM_NICE", + "RAVANAN_SCRATCH", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", ] { @@ -261,6 +291,8 @@ impl Config { gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(), gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(), ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(), + ravanan_batch: parse_ravanan_batch(&values), + ravanan_scratch: values["RAVANAN_SCRATCH"].clone(), rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(), rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(), } @@ -304,6 +336,116 @@ fn parse_switch(value: &str) -> bool { ) } +/// How ravanan is asked to run each step of a workflow. +/// +/// `single-machine` (the default) keeps the previous behaviour: ravanan +/// forks each step locally. `slurm-api` submits every step to a Slurm +/// cluster through slurmrestd's REST API instead, so the CWL scatter +/// really does fan out over the cluster's nodes. +/// +/// Selected with RAVANAN_BATCH_SYSTEM; the slurm-specific keys are +/// `RAVANAN_SLURM_API_ENDPOINT` (e.g. http://127.0.0.1:6820), +/// `RAVANAN_SLURM_JWT` (path to a *file* holding the JWT -- ravanan's +/// `--slurm-jwt` takes a file, not the token), `RAVANAN_SLURM_PARTITION` +/// and `RAVANAN_SLURM_NICE`. See docs/ravanan-slurm.md. +#[derive(Clone, PartialEq, Eq, Debug)] +pub enum RavananBatch { + SingleMachine, + SlurmApi { + /// slurmrestd base URL, e.g. http://127.0.0.1:6820. + endpoint: String, + /// Path to a file containing the JWT ravanan authenticates with. + jwt_file: String, + partition: Option<String>, + nice: Option<i64>, + }, +} + +impl RavananBatch { + /// The ravanan command-line switches this backend adds. + /// + /// `ravanan` takes `--slurm-api-endpoint`, `--slurm-jwt`, + /// `--slurm-partition`, `--slurm-nice` and `--scratch`. The last one + /// only matters for slurm (the directory the compute nodes stage job + /// scratch in), so `scratch` is empty for single-machine. + pub fn ravanan_args(&self, scratch: &str) -> Vec<String> { + let mut args = Vec::new(); + match self { + RavananBatch::SingleMachine => { + if !scratch.is_empty() { + args.push(format!("--scratch={scratch}")); + } + } + RavananBatch::SlurmApi { + endpoint, + jwt_file, + partition, + nice, + } => { + args.push("--batch-system=slurm-api".to_string()); + args.push(format!("--slurm-api-endpoint={endpoint}")); + args.push(format!("--slurm-jwt={jwt_file}")); + if let Some(partition) = partition { + args.push(format!("--slurm-partition={partition}")); + } + if let Some(nice) = nice { + args.push(format!("--slurm-nice={nice}")); + } + if !scratch.is_empty() { + args.push(format!("--scratch={scratch}")); + } + } + } + args + } +} + +/// Reads RAVANAN_BATCH_SYSTEM, failing closed: only `single-machine` and +/// `slurm-api` (any case, trimmed) are accepted, so a typo cannot +/// silently leave jobs running locally. Selecting slurm-api without an +/// endpoint or a JWT file is likewise refused at startup -- ravanan +/// would fail every step, and a clear refusal here beats 500s later. +fn parse_ravanan_batch(values: &HashMap<String, String>) -> RavananBatch { + let system = values["RAVANAN_BATCH_SYSTEM"].trim().to_ascii_lowercase(); + match system.as_str() { + "" | "single-machine" => RavananBatch::SingleMachine, + "slurm-api" => { + let endpoint = values["RAVANAN_SLURM_API_ENDPOINT"].trim(); + if endpoint.is_empty() { + panic!( + "RAVANAN_BATCH_SYSTEM=slurm-api needs RAVANAN_SLURM_API_ENDPOINT \ + (e.g. http://127.0.0.1:6820)" + ); + } + let jwt_file = values["RAVANAN_SLURM_JWT"].trim(); + if jwt_file.is_empty() { + panic!( + "RAVANAN_BATCH_SYSTEM=slurm-api needs RAVANAN_SLURM_JWT \ + (path to a file containing the JWT)" + ); + } + let partition = Some(values["RAVANAN_SLURM_PARTITION"].trim().to_string()) + .filter(|p| !p.is_empty()); + let nice = Some(values["RAVANAN_SLURM_NICE"].trim().to_string()) + .filter(|n| !n.is_empty()) + .map(|n| { + n.parse().unwrap_or_else(|_| { + panic!("RAVANAN_SLURM_NICE must be an integer, not {n:?}") + }) + }); + RavananBatch::SlurmApi { + endpoint: endpoint.to_string(), + jwt_file: jwt_file.to_string(), + partition, + nice, + } + } + other => panic!( + "RAVANAN_BATCH_SYSTEM must be \"single-machine\" or \"slurm-api\", not {other:?}" + ), + } +} + /// The proxy appends "search/" to this base; without a trailing slash /// the result would be ".../apisearch/". Operators may set either form. fn normalize_base_url(mut url: String) -> String { @@ -366,6 +508,12 @@ mod tests { "GEMMA_RAVANAN_WORKFLOW", "GEMMA_RAVANAN_MANIFEST", "RAVANAN_CACHE_DIR", + "RAVANAN_BATCH_SYSTEM", + "RAVANAN_SLURM_API_ENDPOINT", + "RAVANAN_SLURM_JWT", + "RAVANAN_SLURM_PARTITION", + "RAVANAN_SLURM_NICE", + "RAVANAN_SCRATCH", "RQTL_WRAPPER_PATH", "RQTL_CACHE_DIR", ] { @@ -492,4 +640,164 @@ mod tests { restore("XAPIAN_DB_PATH", prior_path); restore("SEARCH_ENGINE", prior_engine); } + + /// A values map with every ravanan batch key present (empty means + /// unset), so the panic assertions below fail for the reason under + /// test rather than on a missing HashMap entry. + fn ravanan_values(system: &str, endpoint: &str, jwt: &str, nice: &str) -> HashMap<String, String> { + let mut values = HashMap::new(); + values.insert("RAVANAN_BATCH_SYSTEM".to_string(), system.to_string()); + values.insert( + "RAVANAN_SLURM_API_ENDPOINT".to_string(), + endpoint.to_string(), + ); + values.insert("RAVANAN_SLURM_JWT".to_string(), jwt.to_string()); + values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new()); + values.insert("RAVANAN_SLURM_NICE".to_string(), nice.to_string()); + values + } + + /// The ravanan backend is single-machine unless asked for slurm, and + /// single-machine adds no switches of its own beyond --scratch. + #[test] + fn ravanan_batch_defaults_to_single_machine() { + let mut values = HashMap::new(); + values.insert("RAVANAN_BATCH_SYSTEM".to_string(), String::new()); + assert_eq!(parse_ravanan_batch(&values), RavananBatch::SingleMachine); + assert_eq!( + RavananBatch::SingleMachine.ravanan_args(""), + Vec::<String>::new() + ); + assert_eq!( + RavananBatch::SingleMachine.ravanan_args("/scratch"), + vec!["--scratch=/scratch".to_string()] + ); + } + + /// Selecting slurm builds exactly the switches ravanan's slurm-api + /// backend needs, and nothing when they are optional and unset. + #[test] + fn ravanan_batch_slurm_args() { + let mut values = HashMap::new(); + values.insert("RAVANAN_BATCH_SYSTEM".to_string(), " Slurm-API ".to_string()); + values.insert( + "RAVANAN_SLURM_API_ENDPOINT".to_string(), + "http://127.0.0.1:6820".to_string(), + ); + values.insert( + "RAVANAN_SLURM_JWT".to_string(), + "/etc/gn4/slurm-jwt".to_string(), + ); + values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new()); + values.insert("RAVANAN_SLURM_NICE".to_string(), String::new()); + let batch = parse_ravanan_batch(&values); + assert_eq!( + batch, + RavananBatch::SlurmApi { + endpoint: "http://127.0.0.1:6820".to_string(), + jwt_file: "/etc/gn4/slurm-jwt".to_string(), + partition: None, + nice: None, + } + ); + assert_eq!( + batch.ravanan_args("/srv/gn4-ravanan"), + vec![ + "--batch-system=slurm-api".to_string(), + "--slurm-api-endpoint=http://127.0.0.1:6820".to_string(), + "--slurm-jwt=/etc/gn4/slurm-jwt".to_string(), + "--scratch=/srv/gn4-ravanan".to_string(), + ] + ); + + values.insert("RAVANAN_SLURM_PARTITION".to_string(), "debug".to_string()); + values.insert("RAVANAN_SLURM_NICE".to_string(), "100".to_string()); + assert_eq!( + parse_ravanan_batch(&values).ravanan_args(""), + vec![ + "--batch-system=slurm-api".to_string(), + "--slurm-api-endpoint=http://127.0.0.1:6820".to_string(), + "--slurm-jwt=/etc/gn4/slurm-jwt".to_string(), + "--slurm-partition=debug".to_string(), + "--slurm-nice=100".to_string(), + ] + ); + } + + /// A typo in the backend name, or slurm without an endpoint or a JWT + /// file, refuses to start rather than silently running jobs locally + /// or failing every step at request time. + #[test] + #[should_panic(expected = "RAVANAN_BATCH_SYSTEM must be")] + fn ravanan_batch_rejects_unknown_backend() { + parse_ravanan_batch(&ravanan_values("slrum-api", "http://h:6820", "/jwt", "")); + } + + #[test] + #[should_panic(expected = "RAVANAN_SLURM_API_ENDPOINT")] + fn ravanan_batch_slurm_requires_endpoint() { + parse_ravanan_batch(&ravanan_values("slurm-api", "", "/jwt", "")); + } + + #[test] + #[should_panic(expected = "RAVANAN_SLURM_JWT")] + fn ravanan_batch_slurm_requires_jwt_file() { + parse_ravanan_batch(&ravanan_values("slurm-api", "http://127.0.0.1:6820", "", "")); + } + + #[test] + #[should_panic(expected = "RAVANAN_SLURM_NICE must be an integer")] + fn ravanan_batch_rejects_non_integer_nice() { + parse_ravanan_batch(&ravanan_values("slurm-api", "http://127.0.0.1:6820", "/jwt", "high")); + } + + /// The new keys layer like the rest: environment beats the default. + #[test] + fn ravanan_batch_keys_follow_env_over_default_layering() { + let keys = [ + "RAVANAN_BATCH_SYSTEM", + "RAVANAN_SLURM_API_ENDPOINT", + "RAVANAN_SLURM_JWT", + "RAVANAN_SCRATCH", + ]; + let prior: Vec<Option<String>> = keys.iter().map(|k| std::env::var(k).ok()).collect(); + let restore = |key: &str, previous: Option<String>| unsafe { + match previous { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + }; + + unsafe { + std::env::remove_var("RAVANAN_BATCH_SYSTEM"); + std::env::remove_var("RAVANAN_SLURM_API_ENDPOINT"); + std::env::remove_var("RAVANAN_SLURM_JWT"); + std::env::remove_var("RAVANAN_SCRATCH"); + } + let config = Config::load(); + assert_eq!(config.ravanan_batch, RavananBatch::SingleMachine); + assert_eq!(config.ravanan_scratch, ""); + + unsafe { + std::env::set_var("RAVANAN_BATCH_SYSTEM", "slurm-api"); + std::env::set_var("RAVANAN_SLURM_API_ENDPOINT", "http://slurm:6820"); + std::env::set_var("RAVANAN_SLURM_JWT", "/etc/gn4/jwt"); + std::env::set_var("RAVANAN_SCRATCH", "/srv/scratch"); + } + let config = Config::load(); + assert_eq!( + config.ravanan_batch, + RavananBatch::SlurmApi { + endpoint: "http://slurm:6820".to_string(), + jwt_file: "/etc/gn4/jwt".to_string(), + partition: None, + nice: None, + } + ); + assert_eq!(config.ravanan_scratch, "/srv/scratch"); + + for (key, previous) in keys.iter().zip(prior) { + restore(key, previous); + } + } } diff --git a/src/gemma.rs b/src/gemma.rs index af5113a..42a7de9 100644 --- a/src/gemma.rs +++ b/src/gemma.rs @@ -532,10 +532,12 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> { /// Backs `POST /compute/gemma/ravanan`. pub async fn compute_ravanan( ravanan_cmd: &str, + batch: &crate::config::RavananBatch, ccwl_cmd: &str, workflow_scm_path: &str, manifest_path: &str, cache_dir: &str, + scratch_dir: &str, genotype_file: &str, snps_file: &str, trait_values: &TraitValues, @@ -634,13 +636,32 @@ pub async fn compute_ravanan( // on either side alone. Harmless to skip: it's optional, and this // code reads paths straight out of ravanan's own --store via its // stdout output record, never through --output-directory. - let args = vec![ + // Backend switches: nothing for the default local runner, and + // --batch-system=slurm-api plus endpoint/JWT/partition/scratch for a + // cluster (see docs/ravanan-slurm.md). `scratch` is the cache dir + // unless configured otherwise: with slurm, ravanan stages each job's + // scratch there, and the compute nodes have to see the same path. + let scratch = if scratch_dir.is_empty() { + cache_dir + } else { + scratch_dir + }; + let mut args = vec![ store_arg.as_str(), manifest_arg.as_str(), compiled_workflow_arg.as_str(), inputs_yaml_arg.as_str(), ]; - let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" ")); + let batch_args = batch.ravanan_args(scratch); + args.extend(batch_args.iter().map(String::as_str)); + let ravanan_note = format!( + "ravanan ({ravanan_cmd}, {} backend): {}", + match batch { + crate::config::RavananBatch::SingleMachine => "single-machine", + crate::config::RavananBatch::SlurmApi { .. } => "slurm-api", + }, + args.join(" ") + ); let chromosomes_note = format!( "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \ (--chromosomes/--no-parallel per job, so GNU parallel is never \ diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs index 32d1818..e096150 100644 --- a/src/routes/gemma.rs +++ b/src/routes/gemma.rs @@ -204,10 +204,12 @@ pub async fn compute_gemma_ravanan( let config = Config::load(); gemma::compute_ravanan( &config.ravanan_cmd, + &config.ravanan_batch, &config.ccwl_cmd, &config.gemma_ravanan_workflow, &config.gemma_ravanan_manifest, &config.ravanan_cache_dir, + &config.ravanan_scratch, &request.genotype_file, &request.snps_file, &request.trait_values, |
