about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr <wrk@bosa.lan>2026-10-24 09:33:40 +0000
committerPjotr <wrk@bosa.lan>2026-10-24 09:33:40 +0000
commita7ffb1cfecbdb8ac3fcc6286d7a09f7285c95d67 (patch)
tree1cb865784d901fad979aed32694b77474019468d /src
parentcda325b6334ef36e3f49501cfd77d68a7bd6a295 (diff)
downloadgenenetwork4-a7ffb1cfecbdb8ac3fcc6286d7a09f7285c95d67.tar.gz
gemma: Make ravanan's batch system configurable (optional Slurm).
POST /compute/gemma/ravanan used to run every CWL step locally, because
compute_ravanan never passed ravanan a batch system.  Add a
RavananBatch setting:

- RAVANAN_BATCH_SYSTEM=single-machine (default) or slurm-api, failing
  closed on anything else, exactly like GN4_DB_MODE;
- slurm-api adds --batch-system=slurm-api, --slurm-api-endpoint,
  --slurm-jwt (a *file*, which is what ravanan reads) and, when set,
  --slurm-partition/--slurm-nice; a slurm selection without an endpoint
  or a JWT file refuses to start rather than 502-ing every request;
- --scratch defaults to ravanan_cache_dir for slurm, so the store and
  the job scratch space live on the same shared filesystem (with slurm
  that directory has to be visible to the compute nodes);
- the response's ravanan note records which backend ran.

The switch building is RavananBatch::ravanan_args so it is unit
testable; compute_ravanan takes the backend and the scratch directory,
and the route passes them from config.  docs/ravanan-slurm.md documents
the keys, the shared-directory requirement, and the slurm-side auth
wiring (AuthType=auth/munge + AuthAltTypes=auth/jwt with SLURM_JWT in
slurmrestd's environment) that makes REST-submitted jobs actually run.
Diffstat (limited to 'src')
-rw-r--r--src/config.rs310
-rw-r--r--src/gemma.rs25
-rw-r--r--src/routes/gemma.rs2
3 files changed, 334 insertions, 3 deletions
diff --git a/src/config.rs b/src/config.rs
index 47cb4bf..9d06c4f 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -100,8 +100,21 @@ pub struct Config {
     /// run logs, and ravanan's own --store and --output-directory.
     /// Separate from `gemma_cache_dir` so the two runners don't share
     /// state. Defaults to a "gn4-gemma-ravanan" directory under the
-    /// system temp dir.
+    /// system temp dir. With `ravanan_batch` set to slurm this directory
+    /// must be on a filesystem the cluster's compute nodes can see (see
+    /// `ravanan_scratch`).
     pub ravanan_cache_dir: String,
+    /// How ravanan runs each step: locally (`single-machine`, the
+    /// default) or as jobs on a Slurm cluster (`slurm-api`, submitting
+    /// through slurmrestd's REST API). Set via RAVANAN_BATCH_SYSTEM;
+    /// see `RavananBatch` and docs/ravanan-slurm.md.
+    pub ravanan_batch: RavananBatch,
+    /// Directory ravanan stages job scratch space in (its own --scratch
+    /// switch). Only meaningful for the slurm backend, where it must be
+    /// a path every compute node can write; defaults to
+    /// `ravanan_cache_dir` when the backend is slurm. Empty means "do
+    /// not pass --scratch" (single-machine's own default).
+    pub ravanan_scratch: String,
     /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3)
     /// for POST /compute/rqtl. Defaults to that path relative to the
     /// working directory the server is started from.
@@ -182,6 +195,17 @@ impl Config {
                 .into_owned(),
         );
         values.insert(
+            "RAVANAN_BATCH_SYSTEM".to_string(),
+            "single-machine".to_string(),
+        );
+        // Only read when RAVANAN_BATCH_SYSTEM=slurm-api; left blank so
+        // the single-machine default stays a zero-configuration path.
+        values.insert("RAVANAN_SLURM_API_ENDPOINT".to_string(), String::new());
+        values.insert("RAVANAN_SLURM_JWT".to_string(), String::new());
+        values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new());
+        values.insert("RAVANAN_SLURM_NICE".to_string(), String::new());
+        values.insert("RAVANAN_SCRATCH".to_string(), String::new());
+        values.insert(
             "RQTL_WRAPPER_PATH".to_string(),
             "scripts/rqtl_wrapper.R".to_string(),
         );
@@ -225,6 +249,12 @@ impl Config {
             "GEMMA_RAVANAN_WORKFLOW",
             "GEMMA_RAVANAN_MANIFEST",
             "RAVANAN_CACHE_DIR",
+            "RAVANAN_BATCH_SYSTEM",
+            "RAVANAN_SLURM_API_ENDPOINT",
+            "RAVANAN_SLURM_JWT",
+            "RAVANAN_SLURM_PARTITION",
+            "RAVANAN_SLURM_NICE",
+            "RAVANAN_SCRATCH",
             "RQTL_WRAPPER_PATH",
             "RQTL_CACHE_DIR",
         ] {
@@ -261,6 +291,8 @@ impl Config {
             gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(),
             gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(),
             ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(),
+            ravanan_batch: parse_ravanan_batch(&values),
+            ravanan_scratch: values["RAVANAN_SCRATCH"].clone(),
             rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(),
             rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(),
         }
@@ -304,6 +336,116 @@ fn parse_switch(value: &str) -> bool {
     )
 }
 
+/// How ravanan is asked to run each step of a workflow.
+///
+/// `single-machine` (the default) keeps the previous behaviour: ravanan
+/// forks each step locally. `slurm-api` submits every step to a Slurm
+/// cluster through slurmrestd's REST API instead, so the CWL scatter
+/// really does fan out over the cluster's nodes.
+///
+/// Selected with RAVANAN_BATCH_SYSTEM; the slurm-specific keys are
+/// `RAVANAN_SLURM_API_ENDPOINT` (e.g. http://127.0.0.1:6820),
+/// `RAVANAN_SLURM_JWT` (path to a *file* holding the JWT -- ravanan's
+/// `--slurm-jwt` takes a file, not the token), `RAVANAN_SLURM_PARTITION`
+/// and `RAVANAN_SLURM_NICE`. See docs/ravanan-slurm.md.
+#[derive(Clone, PartialEq, Eq, Debug)]
+pub enum RavananBatch {
+    SingleMachine,
+    SlurmApi {
+        /// slurmrestd base URL, e.g. http://127.0.0.1:6820.
+        endpoint: String,
+        /// Path to a file containing the JWT ravanan authenticates with.
+        jwt_file: String,
+        partition: Option<String>,
+        nice: Option<i64>,
+    },
+}
+
+impl RavananBatch {
+    /// The ravanan command-line switches this backend adds.
+    ///
+    /// `ravanan` takes `--slurm-api-endpoint`, `--slurm-jwt`,
+    /// `--slurm-partition`, `--slurm-nice` and `--scratch`. The last one
+    /// only matters for slurm (the directory the compute nodes stage job
+    /// scratch in), so `scratch` is empty for single-machine.
+    pub fn ravanan_args(&self, scratch: &str) -> Vec<String> {
+        let mut args = Vec::new();
+        match self {
+            RavananBatch::SingleMachine => {
+                if !scratch.is_empty() {
+                    args.push(format!("--scratch={scratch}"));
+                }
+            }
+            RavananBatch::SlurmApi {
+                endpoint,
+                jwt_file,
+                partition,
+                nice,
+            } => {
+                args.push("--batch-system=slurm-api".to_string());
+                args.push(format!("--slurm-api-endpoint={endpoint}"));
+                args.push(format!("--slurm-jwt={jwt_file}"));
+                if let Some(partition) = partition {
+                    args.push(format!("--slurm-partition={partition}"));
+                }
+                if let Some(nice) = nice {
+                    args.push(format!("--slurm-nice={nice}"));
+                }
+                if !scratch.is_empty() {
+                    args.push(format!("--scratch={scratch}"));
+                }
+            }
+        }
+        args
+    }
+}
+
+/// Reads RAVANAN_BATCH_SYSTEM, failing closed: only `single-machine` and
+/// `slurm-api` (any case, trimmed) are accepted, so a typo cannot
+/// silently leave jobs running locally. Selecting slurm-api without an
+/// endpoint or a JWT file is likewise refused at startup -- ravanan
+/// would fail every step, and a clear refusal here beats 500s later.
+fn parse_ravanan_batch(values: &HashMap<String, String>) -> RavananBatch {
+    let system = values["RAVANAN_BATCH_SYSTEM"].trim().to_ascii_lowercase();
+    match system.as_str() {
+        "" | "single-machine" => RavananBatch::SingleMachine,
+        "slurm-api" => {
+            let endpoint = values["RAVANAN_SLURM_API_ENDPOINT"].trim();
+            if endpoint.is_empty() {
+                panic!(
+                    "RAVANAN_BATCH_SYSTEM=slurm-api needs RAVANAN_SLURM_API_ENDPOINT \
+                     (e.g. http://127.0.0.1:6820)"
+                );
+            }
+            let jwt_file = values["RAVANAN_SLURM_JWT"].trim();
+            if jwt_file.is_empty() {
+                panic!(
+                    "RAVANAN_BATCH_SYSTEM=slurm-api needs RAVANAN_SLURM_JWT \
+                     (path to a file containing the JWT)"
+                );
+            }
+            let partition = Some(values["RAVANAN_SLURM_PARTITION"].trim().to_string())
+                .filter(|p| !p.is_empty());
+            let nice = Some(values["RAVANAN_SLURM_NICE"].trim().to_string())
+                .filter(|n| !n.is_empty())
+                .map(|n| {
+                    n.parse().unwrap_or_else(|_| {
+                        panic!("RAVANAN_SLURM_NICE must be an integer, not {n:?}")
+                    })
+                });
+            RavananBatch::SlurmApi {
+                endpoint: endpoint.to_string(),
+                jwt_file: jwt_file.to_string(),
+                partition,
+                nice,
+            }
+        }
+        other => panic!(
+            "RAVANAN_BATCH_SYSTEM must be \"single-machine\" or \"slurm-api\", not {other:?}"
+        ),
+    }
+}
+
 /// The proxy appends "search/" to this base; without a trailing slash
 /// the result would be ".../apisearch/". Operators may set either form.
 fn normalize_base_url(mut url: String) -> String {
@@ -366,6 +508,12 @@ mod tests {
             "GEMMA_RAVANAN_WORKFLOW",
             "GEMMA_RAVANAN_MANIFEST",
             "RAVANAN_CACHE_DIR",
+            "RAVANAN_BATCH_SYSTEM",
+            "RAVANAN_SLURM_API_ENDPOINT",
+            "RAVANAN_SLURM_JWT",
+            "RAVANAN_SLURM_PARTITION",
+            "RAVANAN_SLURM_NICE",
+            "RAVANAN_SCRATCH",
             "RQTL_WRAPPER_PATH",
             "RQTL_CACHE_DIR",
         ] {
@@ -492,4 +640,164 @@ mod tests {
         restore("XAPIAN_DB_PATH", prior_path);
         restore("SEARCH_ENGINE", prior_engine);
     }
+
+    /// A values map with every ravanan batch key present (empty means
+    /// unset), so the panic assertions below fail for the reason under
+    /// test rather than on a missing HashMap entry.
+    fn ravanan_values(system: &str, endpoint: &str, jwt: &str, nice: &str) -> HashMap<String, String> {
+        let mut values = HashMap::new();
+        values.insert("RAVANAN_BATCH_SYSTEM".to_string(), system.to_string());
+        values.insert(
+            "RAVANAN_SLURM_API_ENDPOINT".to_string(),
+            endpoint.to_string(),
+        );
+        values.insert("RAVANAN_SLURM_JWT".to_string(), jwt.to_string());
+        values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new());
+        values.insert("RAVANAN_SLURM_NICE".to_string(), nice.to_string());
+        values
+    }
+
+    /// The ravanan backend is single-machine unless asked for slurm, and
+    /// single-machine adds no switches of its own beyond --scratch.
+    #[test]
+    fn ravanan_batch_defaults_to_single_machine() {
+        let mut values = HashMap::new();
+        values.insert("RAVANAN_BATCH_SYSTEM".to_string(), String::new());
+        assert_eq!(parse_ravanan_batch(&values), RavananBatch::SingleMachine);
+        assert_eq!(
+            RavananBatch::SingleMachine.ravanan_args(""),
+            Vec::<String>::new()
+        );
+        assert_eq!(
+            RavananBatch::SingleMachine.ravanan_args("/scratch"),
+            vec!["--scratch=/scratch".to_string()]
+        );
+    }
+
+    /// Selecting slurm builds exactly the switches ravanan's slurm-api
+    /// backend needs, and nothing when they are optional and unset.
+    #[test]
+    fn ravanan_batch_slurm_args() {
+        let mut values = HashMap::new();
+        values.insert("RAVANAN_BATCH_SYSTEM".to_string(), " Slurm-API ".to_string());
+        values.insert(
+            "RAVANAN_SLURM_API_ENDPOINT".to_string(),
+            "http://127.0.0.1:6820".to_string(),
+        );
+        values.insert(
+            "RAVANAN_SLURM_JWT".to_string(),
+            "/etc/gn4/slurm-jwt".to_string(),
+        );
+        values.insert("RAVANAN_SLURM_PARTITION".to_string(), String::new());
+        values.insert("RAVANAN_SLURM_NICE".to_string(), String::new());
+        let batch = parse_ravanan_batch(&values);
+        assert_eq!(
+            batch,
+            RavananBatch::SlurmApi {
+                endpoint: "http://127.0.0.1:6820".to_string(),
+                jwt_file: "/etc/gn4/slurm-jwt".to_string(),
+                partition: None,
+                nice: None,
+            }
+        );
+        assert_eq!(
+            batch.ravanan_args("/srv/gn4-ravanan"),
+            vec![
+                "--batch-system=slurm-api".to_string(),
+                "--slurm-api-endpoint=http://127.0.0.1:6820".to_string(),
+                "--slurm-jwt=/etc/gn4/slurm-jwt".to_string(),
+                "--scratch=/srv/gn4-ravanan".to_string(),
+            ]
+        );
+
+        values.insert("RAVANAN_SLURM_PARTITION".to_string(), "debug".to_string());
+        values.insert("RAVANAN_SLURM_NICE".to_string(), "100".to_string());
+        assert_eq!(
+            parse_ravanan_batch(&values).ravanan_args(""),
+            vec![
+                "--batch-system=slurm-api".to_string(),
+                "--slurm-api-endpoint=http://127.0.0.1:6820".to_string(),
+                "--slurm-jwt=/etc/gn4/slurm-jwt".to_string(),
+                "--slurm-partition=debug".to_string(),
+                "--slurm-nice=100".to_string(),
+            ]
+        );
+    }
+
+    /// A typo in the backend name, or slurm without an endpoint or a JWT
+    /// file, refuses to start rather than silently running jobs locally
+    /// or failing every step at request time.
+    #[test]
+    #[should_panic(expected = "RAVANAN_BATCH_SYSTEM must be")]
+    fn ravanan_batch_rejects_unknown_backend() {
+        parse_ravanan_batch(&ravanan_values("slrum-api", "http://h:6820", "/jwt", ""));
+    }
+
+    #[test]
+    #[should_panic(expected = "RAVANAN_SLURM_API_ENDPOINT")]
+    fn ravanan_batch_slurm_requires_endpoint() {
+        parse_ravanan_batch(&ravanan_values("slurm-api", "", "/jwt", ""));
+    }
+
+    #[test]
+    #[should_panic(expected = "RAVANAN_SLURM_JWT")]
+    fn ravanan_batch_slurm_requires_jwt_file() {
+        parse_ravanan_batch(&ravanan_values("slurm-api", "http://127.0.0.1:6820", "", ""));
+    }
+
+    #[test]
+    #[should_panic(expected = "RAVANAN_SLURM_NICE must be an integer")]
+    fn ravanan_batch_rejects_non_integer_nice() {
+        parse_ravanan_batch(&ravanan_values("slurm-api", "http://127.0.0.1:6820", "/jwt", "high"));
+    }
+
+    /// The new keys layer like the rest: environment beats the default.
+    #[test]
+    fn ravanan_batch_keys_follow_env_over_default_layering() {
+        let keys = [
+            "RAVANAN_BATCH_SYSTEM",
+            "RAVANAN_SLURM_API_ENDPOINT",
+            "RAVANAN_SLURM_JWT",
+            "RAVANAN_SCRATCH",
+        ];
+        let prior: Vec<Option<String>> = keys.iter().map(|k| std::env::var(k).ok()).collect();
+        let restore = |key: &str, previous: Option<String>| unsafe {
+            match previous {
+                Some(value) => std::env::set_var(key, value),
+                None => std::env::remove_var(key),
+            }
+        };
+
+        unsafe {
+            std::env::remove_var("RAVANAN_BATCH_SYSTEM");
+            std::env::remove_var("RAVANAN_SLURM_API_ENDPOINT");
+            std::env::remove_var("RAVANAN_SLURM_JWT");
+            std::env::remove_var("RAVANAN_SCRATCH");
+        }
+        let config = Config::load();
+        assert_eq!(config.ravanan_batch, RavananBatch::SingleMachine);
+        assert_eq!(config.ravanan_scratch, "");
+
+        unsafe {
+            std::env::set_var("RAVANAN_BATCH_SYSTEM", "slurm-api");
+            std::env::set_var("RAVANAN_SLURM_API_ENDPOINT", "http://slurm:6820");
+            std::env::set_var("RAVANAN_SLURM_JWT", "/etc/gn4/jwt");
+            std::env::set_var("RAVANAN_SCRATCH", "/srv/scratch");
+        }
+        let config = Config::load();
+        assert_eq!(
+            config.ravanan_batch,
+            RavananBatch::SlurmApi {
+                endpoint: "http://slurm:6820".to_string(),
+                jwt_file: "/etc/gn4/jwt".to_string(),
+                partition: None,
+                nice: None,
+            }
+        );
+        assert_eq!(config.ravanan_scratch, "/srv/scratch");
+
+        for (key, previous) in keys.iter().zip(prior) {
+            restore(key, previous);
+        }
+    }
 }
diff --git a/src/gemma.rs b/src/gemma.rs
index af5113a..42a7de9 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -532,10 +532,12 @@ fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> {
 /// Backs `POST /compute/gemma/ravanan`.
 pub async fn compute_ravanan(
     ravanan_cmd: &str,
+    batch: &crate::config::RavananBatch,
     ccwl_cmd: &str,
     workflow_scm_path: &str,
     manifest_path: &str,
     cache_dir: &str,
+    scratch_dir: &str,
     genotype_file: &str,
     snps_file: &str,
     trait_values: &TraitValues,
@@ -634,13 +636,32 @@ pub async fn compute_ravanan(
     // on either side alone. Harmless to skip: it's optional, and this
     // code reads paths straight out of ravanan's own --store via its
     // stdout output record, never through --output-directory.
-    let args = vec![
+    // Backend switches: nothing for the default local runner, and
+    // --batch-system=slurm-api plus endpoint/JWT/partition/scratch for a
+    // cluster (see docs/ravanan-slurm.md). `scratch` is the cache dir
+    // unless configured otherwise: with slurm, ravanan stages each job's
+    // scratch there, and the compute nodes have to see the same path.
+    let scratch = if scratch_dir.is_empty() {
+        cache_dir
+    } else {
+        scratch_dir
+    };
+    let mut args = vec![
         store_arg.as_str(),
         manifest_arg.as_str(),
         compiled_workflow_arg.as_str(),
         inputs_yaml_arg.as_str(),
     ];
-    let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" "));
+    let batch_args = batch.ravanan_args(scratch);
+    args.extend(batch_args.iter().map(String::as_str));
+    let ravanan_note = format!(
+        "ravanan ({ravanan_cmd}, {} backend): {}",
+        match batch {
+            crate::config::RavananBatch::SingleMachine => "single-machine",
+            crate::config::RavananBatch::SlurmApi { .. } => "slurm-api",
+        },
+        args.join(" ")
+    );
     let chromosomes_note = format!(
         "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \
          (--chromosomes/--no-parallel per job, so GNU parallel is never \
diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs
index 32d1818..e096150 100644
--- a/src/routes/gemma.rs
+++ b/src/routes/gemma.rs
@@ -204,10 +204,12 @@ pub async fn compute_gemma_ravanan(
     let config = Config::load();
     gemma::compute_ravanan(
         &config.ravanan_cmd,
+        &config.ravanan_batch,
         &config.ccwl_cmd,
         &config.gemma_ravanan_workflow,
         &config.gemma_ravanan_manifest,
         &config.ravanan_cache_dir,
+        &config.ravanan_scratch,
         &request.genotype_file,
         &request.snps_file,
         &request.trait_values,