about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 16:44:07 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-11-14 16:44:07 +0000
commita89781c2eaf4b503f416cc2aba2420cd15fa5289 (patch)
tree1685790d7fde5b45da4f280210579e56fc0ae3d7 /src
parent05907254d44ccde0aa0c1f536d2ba838d0645402 (diff)
downloadgenenetwork4-a89781c2eaf4b503f416cc2aba2420cd15fa5289.tar.gz
cache: say when ravanan's cache cannot survive a restart, and document what does
Ravanan's reuse key is (script, inputs) and both live in its --store under
RAVANAN_CACHE_DIR, so the cache's lifetime is that directory's, not the
container's.  Measured: 13.6 s cold, then 3.6 s for the identical request
after restarting the container, with the store unchanged at 14 files --
Slurm's own spool state is lost and irrelevant to reuse.

The default cache dir is under /tmp, which is neither durable nor visible
inside a container that bind-mounts one host path, so a job cannot read
the interned inputs.  The reply now carries a note when the cache dir is
under /tmp, naming the fix, rather than leaving it to be discovered as a
slow or failing run.

docs/ravanan-slurm.md gains a "What survives a restart" table (ravanan
store, scratch, token registry, gemma-wrapper cache, Slurm state) and a
path checklist; the README points at it.
Diffstat (limited to 'src')
-rw-r--r--src/gemma.rs17
1 files changed, 17 insertions, 0 deletions
diff --git a/src/gemma.rs b/src/gemma.rs
index b6f3779..02a2456 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -1482,11 +1482,28 @@ pub async fn compute_ravanan(
 
     streaming::mark_complete(&log_path);
 
+    // Ravanan's reuse lives in this directory (--store), so whether the cache
+    // survives a container restart -- and whether a job inside the container
+    // can even see the interned inputs -- depends entirely on where it is.
+    // The default is under /tmp, which is neither durable nor visible to a
+    // container that only bind-mounts one host path; say so on the first run
+    // rather than letting an operator discover it as a 45-minute rebuild.
+    let mut ephemeral_note: Vec<String> = Vec::new();
+    if cache_dir.starts_with("/tmp/") || cache_dir == "/tmp" {
+        ephemeral_note.push(format!(
+            "ravanan's cache is under {cache_dir}: that is not durable (a reboot drops it) \
+             and a Slurm container can only see paths under its bind-mounted share, so jobs \
+             may not find the interned inputs. Set RAVANAN_CACHE_DIR to a durable path the \
+             nodes can see -- e.g. <shared mount>/gn4-ravanan-cache."
+        ));
+    }
+
     let mut notes = vec![
         ravanan_note,
         chromosomes_note,
         format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"),
     ];
+    notes.extend(ephemeral_note);
     if let Some(note) = variance_components_note(&components) {
         notes.push(note);
     }