about summary refs log tree commit diff
path: root/src
diff options
context:
space:
mode:
authorPjotr Prins <pjotr.public01@thebird.nl>2026-10-19 10:50:05 +0000
committerPjotr Prins <pjotr.public01@thebird.nl>2026-10-19 10:50:05 +0000
commitcda325b6334ef36e3f49501cfd77d68a7bd6a295 (patch)
tree56003d3b3cccb1cf4f12e36bb6065844ba5f2219 /src
parent15c735742f35e4688b6448d3c7e6922c78ec0746 (diff)
parentd504b5da9f5deeece9ba2ff4dcd8baf3dfa4dc32 (diff)
downloadgenenetwork4-cda325b6334ef36e3f49501cfd77d68a7bd6a295.tar.gz
Merge branch 'gemma-cwl-parallel'
# Conflicts:
#	src/gemma.rs
#	src/rqtl.rs
#	src/streaming.rs
Diffstat (limited to 'src')
-rw-r--r--src/config.rs65
-rw-r--r--src/gemma.rs355
-rw-r--r--src/lib.rs1
-rw-r--r--src/response.rs28
-rw-r--r--src/routes/gemma.rs117
-rw-r--r--src/routes/rqtl.rs12
-rw-r--r--src/rqtl.rs27
7 files changed, 556 insertions, 49 deletions
diff --git a/src/config.rs b/src/config.rs
index 7d8893c..47cb4bf 100644
--- a/src/config.rs
+++ b/src/config.rs
@@ -74,6 +74,34 @@ pub struct Config {
     /// results between calls (its own --cache-dir switch). Defaults to
     /// a "gn4-gemma" directory under the system temp dir.
     pub gemma_cache_dir: String,
+    /// The ravanan executable to invoke for POST /compute/gemma/ravanan
+    /// (see src/gemma.rs's compute_ravanan). Defaults to "ravanan",
+    /// resolved via PATH.
+    pub ravanan_cmd: String,
+    /// The ccwl (https://ccwl.systemreboot.net/) executable used to
+    /// compile `gemma_ravanan_workflow` to CWL before each ravanan run
+    /// (see src/gemma.rs's compute_ravanan). Defaults to "ccwl",
+    /// resolved via PATH.
+    pub ccwl_cmd: String,
+    /// Path to the ccwl source POST /compute/gemma/ravanan compiles to
+    /// CWL and runs under ravanan. Not a CWL file itself -- the
+    /// compiled .cwl is a generated, throwaway artifact (written under
+    /// `ravanan_cache_dir` on every call), never checked into the repo.
+    /// Defaults to that path relative to the working directory the
+    /// server is started from, same convention as `rqtl_wrapper_path`.
+    pub gemma_ravanan_workflow: String,
+    /// Path to the Guix manifest ravanan resolves its jobs' commands
+    /// against (its own --guix-manifest switch, required -- ravanan
+    /// refuses to run without one). Defaults to that path relative to
+    /// the working directory the server is started from, same
+    /// convention as `gemma_ravanan_workflow`.
+    pub gemma_ravanan_manifest: String,
+    /// Directory POST /compute/gemma/ravanan uses for phenotype files,
+    /// run logs, and ravanan's own --store and --output-directory.
+    /// Separate from `gemma_cache_dir` so the two runners don't share
+    /// state. Defaults to a "gn4-gemma-ravanan" directory under the
+    /// system temp dir.
+    pub ravanan_cache_dir: String,
     /// Path to scripts/rqtl_wrapper.R (copied here from GeneNetwork3)
     /// for POST /compute/rqtl. Defaults to that path relative to the
     /// working directory the server is started from.
@@ -136,6 +164,23 @@ impl Config {
                 .to_string_lossy()
                 .into_owned(),
         );
+        values.insert("RAVANAN_CMD".to_string(), "ravanan".to_string());
+        values.insert("CCWL_CMD".to_string(), "ccwl".to_string());
+        values.insert(
+            "GEMMA_RAVANAN_WORKFLOW".to_string(),
+            "cwl/gemma/gemma-loco.scm".to_string(),
+        );
+        values.insert(
+            "GEMMA_RAVANAN_MANIFEST".to_string(),
+            "cwl/gemma/manifest.scm".to_string(),
+        );
+        values.insert(
+            "RAVANAN_CACHE_DIR".to_string(),
+            std::env::temp_dir()
+                .join("gn4-gemma-ravanan")
+                .to_string_lossy()
+                .into_owned(),
+        );
         values.insert(
             "RQTL_WRAPPER_PATH".to_string(),
             "scripts/rqtl_wrapper.R".to_string(),
@@ -175,6 +220,11 @@ impl Config {
             "SEARCH_ENGINE",
             "GEMMA_WRAPPER_CMD",
             "GEMMA_CACHE_DIR",
+            "RAVANAN_CMD",
+            "CCWL_CMD",
+            "GEMMA_RAVANAN_WORKFLOW",
+            "GEMMA_RAVANAN_MANIFEST",
+            "RAVANAN_CACHE_DIR",
             "RQTL_WRAPPER_PATH",
             "RQTL_CACHE_DIR",
         ] {
@@ -206,6 +256,11 @@ impl Config {
             dev_token_password: values["GN_AUTH_DEV_PASSWORD"].clone(),
             gemma_wrapper_cmd: values["GEMMA_WRAPPER_CMD"].clone(),
             gemma_cache_dir: values["GEMMA_CACHE_DIR"].clone(),
+            ravanan_cmd: values["RAVANAN_CMD"].clone(),
+            ccwl_cmd: values["CCWL_CMD"].clone(),
+            gemma_ravanan_workflow: values["GEMMA_RAVANAN_WORKFLOW"].clone(),
+            gemma_ravanan_manifest: values["GEMMA_RAVANAN_MANIFEST"].clone(),
+            ravanan_cache_dir: values["RAVANAN_CACHE_DIR"].clone(),
             rqtl_wrapper_path: values["RQTL_WRAPPER_PATH"].clone(),
             rqtl_cache_dir: values["RQTL_CACHE_DIR"].clone(),
         }
@@ -306,6 +361,11 @@ mod tests {
             "SEARCH_ENGINE",
             "GEMMA_WRAPPER_CMD",
             "GEMMA_CACHE_DIR",
+            "RAVANAN_CMD",
+            "CCWL_CMD",
+            "GEMMA_RAVANAN_WORKFLOW",
+            "GEMMA_RAVANAN_MANIFEST",
+            "RAVANAN_CACHE_DIR",
             "RQTL_WRAPPER_PATH",
             "RQTL_CACHE_DIR",
         ] {
@@ -324,6 +384,11 @@ mod tests {
         assert_eq!(config.port, 3000);
         assert_eq!(config.gemma_wrapper_cmd, "gemma-wrapper");
         assert!(config.gemma_cache_dir.ends_with("gn4-gemma"));
+        assert_eq!(config.ravanan_cmd, "ravanan");
+        assert_eq!(config.ccwl_cmd, "ccwl");
+        assert_eq!(config.gemma_ravanan_workflow, "cwl/gemma/gemma-loco.scm");
+        assert_eq!(config.gemma_ravanan_manifest, "cwl/gemma/manifest.scm");
+        assert!(config.ravanan_cache_dir.ends_with("gn4-gemma-ravanan"));
         assert_eq!(config.rqtl_wrapper_path, "scripts/rqtl_wrapper.R");
         assert!(config.rqtl_cache_dir.ends_with("gn4-rqtl"));
         assert_eq!(config.auth_server_url, "http://127.0.0.1:8081/");
diff --git a/src/gemma.rs b/src/gemma.rs
index 2c7f08e..af5113a 100644
--- a/src/gemma.rs
+++ b/src/gemma.rs
@@ -43,6 +43,12 @@ pub enum GemmaError {
     /// before any subprocess runs, so this never silently truncates
     /// another run's in-progress log.
     RunIdInUse(String),
+    /// `ravanan` (see `compute_ravanan`) exited non-zero; carries its
+    /// stderr.
+    RavananFailed(String),
+    /// `ravanan`'s own stdout (the CWL outputs record) was not valid
+    /// JSON, or didn't carry the output this code expects.
+    RavananOutput(String),
 }
 
 impl std::fmt::Display for GemmaError {
@@ -65,6 +71,10 @@ impl std::fmt::Display for GemmaError {
             GemmaError::RunIdInUse(id) => {
                 write!(f, "run id '{id}' is already in use; pick a different id")
             }
+            GemmaError::RavananFailed(stderr) => write!(f, "ravanan failed: {stderr}"),
+            GemmaError::RavananOutput(detail) => {
+                write!(f, "could not read ravanan's output: {detail}")
+            }
         }
     }
 }
@@ -208,36 +218,34 @@ fn assoc_file_basenames(record: &Value) -> Vec<&str> {
         .unwrap_or_default()
 }
 
-/// One row of a GEMMA `.assoc.txt` file, keyed by its header column
-/// names. Columns vary with `-lmm` (stock 1/2/3 run one test, 4 runs
-/// all three; this build's `9`, GeneNetwork's convention, was observed
-/// to carry `p_lrt`), so this stays a generic map rather than a fixed
-/// struct.
-pub type AssocRow = serde_json::Map<String, Value>;
-
 /// Parse a GEMMA `.assoc.txt` file (tab-separated, header row first)
-/// into one row per marker. A value that parses as a number is kept
-/// numeric; "NA" and anything else stays a string.
-fn parse_assoc_file(contents: &str) -> Vec<AssocRow> {
+/// into its column names and one row per marker, each row a
+/// `Value::Array` in that same column order (not keyed by column name:
+/// columns vary with `-lmm` -- stock 1/2/3 run one test, 4 runs all
+/// three; this build's `9`, GeneNetwork's convention, was observed to
+/// carry `p_lrt` -- so callers needing a name match it up against the
+/// returned column list themselves). A value that parses as a number is
+/// kept numeric; "NA" and anything else stays a string.
+fn parse_assoc_file(contents: &str) -> (Vec<String>, Vec<Vec<Value>>) {
     let mut lines = contents.lines();
     let Some(header) = lines.next() else {
-        return Vec::new();
+        return (Vec::new(), Vec::new());
     };
-    let columns: Vec<&str> = header.split('\t').collect();
-    lines
+    let columns: Vec<String> = header.split('\t').map(str::to_string).collect();
+    let rows = lines
         .filter(|line| !line.is_empty())
         .map(|line| {
-            let mut row = AssocRow::new();
-            for (column, value) in columns.iter().zip(line.split('\t')) {
-                let json_value = value
-                    .parse::<f64>()
-                    .map(|n| serde_json::json!(n))
-                    .unwrap_or_else(|_| Value::String(value.to_string()));
-                row.insert((*column).to_string(), json_value);
-            }
-            row
+            line.split('\t')
+                .map(|value| {
+                    value
+                        .parse::<f64>()
+                        .map(|n| serde_json::json!(n))
+                        .unwrap_or_else(|_| Value::String(value.to_string()))
+                })
+                .collect()
         })
-        .collect()
+        .collect();
+    (columns, rows)
 }
 
 /// Extract `archive` (a `.tar.xz`, relative to `cache_dir`) into
@@ -265,6 +273,25 @@ async fn extract_archive(cache_dir: &str, archive: &str, dest_dir: &str) -> Resu
     Ok(())
 }
 
+/// Read and parse every `basenames` file under `extract_dir` (one per
+/// `--loco` chromosome), returning the column names (from the first
+/// file; every chromosome's `.assoc.txt` shares the same header) and
+/// every file's rows concatenated, each row a `Value::Array`.
+fn parse_assoc_files(extract_dir: &Path, basenames: Vec<&str>) -> Result<(Vec<String>, Vec<Value>), GemmaError> {
+    let mut columns = Vec::new();
+    let mut rows = Vec::new();
+    for basename in basenames {
+        let contents = std::fs::read_to_string(extract_dir.join(basename))
+            .map_err(GemmaError::AssocFile)?;
+        let (file_columns, file_rows) = parse_assoc_file(&contents);
+        if columns.is_empty() {
+            columns = file_columns;
+        }
+        rows.extend(file_rows.into_iter().map(Value::Array));
+    }
+    Ok((columns, rows))
+}
+
 fn nanos_since_epoch() -> u128 {
     std::time::SystemTime::now()
         .duration_since(std::time::UNIX_EPOCH)
@@ -272,10 +299,13 @@ fn nanos_since_epoch() -> u128 {
         .unwrap_or(0)
 }
 
-/// The mapped QTL rows, plus the exact gemma-wrapper invocations used
-/// to produce them (see `compute`'s `notes` below), so a caller can see
-/// precisely what was run without reading server logs.
+/// The mapped QTL rows (each a `Value::Array` in `columns`' order, not
+/// a keyed object -- see `parse_assoc_file`), plus the exact
+/// gemma-wrapper invocations used to produce them (see `compute`'s
+/// `notes` below), so a caller can see precisely what was run without
+/// reading server logs.
 pub struct GemmaOutput {
+    pub columns: Vec<String>,
     pub rows: Value,
     pub notes: Vec<String>,
 }
@@ -425,15 +455,11 @@ pub async fn compute(
         std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?;
         extract_archive(cache_dir, archive, &extract_dir.to_string_lossy()).await?;
 
-        let mut rows = Vec::new();
-        for basename in basenames {
-            let contents = std::fs::read_to_string(extract_dir.join(basename))
-                .map_err(GemmaError::AssocFile)?;
-            rows.extend(parse_assoc_file(&contents).into_iter().map(Value::Object));
-        }
+        let (columns, rows) = parse_assoc_files(&extract_dir, basenames)?;
         let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless
 
         Ok(GemmaOutput {
+            columns,
             rows: Value::Array(rows),
             notes: vec![kinship_note, gwa_note],
         })
@@ -452,6 +478,253 @@ pub async fn compute(
     }
 }
 
+/// Read a scattered `File[]` output's list of `path`s from ravanan's
+/// stdout (the CWL outputs record for a scattered output:
+/// `{"<output id>": [{"class": "File", "path": ..., ...}, ...], ...}`,
+/// one entry per `cwl/gemma/gemma-loco.scm` chromosome).
+fn ravanan_output_paths<'a>(outputs: &'a Value, id: &str) -> Result<Vec<&'a str>, GemmaError> {
+    let files = outputs
+        .get(id)
+        .and_then(Value::as_array)
+        .ok_or_else(|| GemmaError::RavananOutput(format!("no \"{id}\" array in ravanan's output")))?;
+    files
+        .iter()
+        .map(|file| {
+            file.get("path").and_then(Value::as_str).ok_or_else(|| {
+                GemmaError::RavananOutput(format!("no .path on one of \"{id}\"'s entries"))
+            })
+        })
+        .collect()
+}
+
+/// The distinct chromosome names in `snps_file` (whitespace-separated:
+/// rs id, position, chromosome), in first-seen order -- reproduces
+/// gemma-wrapper's own `get_chromosomes`, used here to build
+/// `compute_ravanan`'s `chromosomes` scatter input (CWL's `scatter`
+/// needs a concrete array at invocation time, which can't be
+/// discovered by a step inside the workflow itself without another
+/// round of plumbing -- see `cwl/gemma/gemma-loco.scm`'s own doc
+/// comment on this input).
+fn chromosomes_in(snps_file: &str) -> Result<Vec<String>, GemmaError> {
+    let contents = std::fs::read_to_string(snps_file).map_err(GemmaError::Io)?;
+    let mut seen = std::collections::HashSet::new();
+    let mut chromosomes = Vec::new();
+    for line in contents.lines() {
+        if let Some(chr) = line.split_whitespace().nth(2) {
+            if seen.insert(chr.to_string()) {
+                chromosomes.push(chr.to_string());
+            }
+        }
+    }
+    Ok(chromosomes)
+}
+
+/// The same GEMMA `--loco` kinship-then-GWA computation as `compute`,
+/// but run via `ravanan` (https://github.com/vepiteski/ravanan) driving
+/// a CWL workflow compiled from the ccwl (https://ccwl.systemreboot.net/)
+/// source at `workflow_scm_path` (see `cwl/gemma/gemma-loco.scm`)
+/// instead of this module's own direct subprocess calls: one
+/// gemma-wrapper invocation per chromosome, scattered by ravanan
+/// itself (`--chromosomes`/`--no-parallel`, see
+/// `cwl/gemma/gemma-loco.scm`'s `kinship` and `gwa` commands) rather
+/// than gemma-wrapper's own internal GNU parallel fan-out -- an alternative
+/// runner for the same computation, so the two can be compared.
+/// Backs `POST /compute/gemma/ravanan`.
+pub async fn compute_ravanan(
+    ravanan_cmd: &str,
+    ccwl_cmd: &str,
+    workflow_scm_path: &str,
+    manifest_path: &str,
+    cache_dir: &str,
+    genotype_file: &str,
+    snps_file: &str,
+    trait_values: &TraitValues,
+    lmm: u8,
+    run_id: Option<&str>,
+) -> Result<GemmaOutput, GemmaError> {
+    std::fs::create_dir_all(cache_dir).map_err(GemmaError::Io)?;
+    let unique = format!("{}-{}", std::process::id(), nanos_since_epoch());
+    let pheno_path = Path::new(cache_dir).join(format!("{unique}-pheno.txt"));
+    write_pheno_file(&pheno_path, trait_values).map_err(GemmaError::Io)?;
+
+    let log_path = streaming::create_run_log(cache_dir, run_id)?;
+
+    // gemma-loco.scm is the maintained source; the CWL ravanan actually
+    // runs is a generated, throwaway artifact compiled fresh on every
+    // call (cheap: ccwl compile is pure macro expansion, no Guix build
+    // involved) and never checked into the repo -- see the doc comment
+    // on Config::gemma_ravanan_workflow.
+    let compiled_workflow_path =
+        Path::new(cache_dir).join(format!("{unique}-gemma-loco.cwl"));
+    let ccwl_output_arg = format!("--output={}", compiled_workflow_path.display());
+    let ccwl_args = vec!["compile", ccwl_output_arg.as_str(), workflow_scm_path];
+    let ccwl_output = streaming::run_streaming(ccwl_cmd, &ccwl_args, None)
+        .await
+        .map_err(GemmaError::Io)?;
+    if !ccwl_output.status.success() {
+        return Err(GemmaError::CommandFailed {
+            step: "ccwl compile",
+            stderr: String::from_utf8_lossy(&ccwl_output.stderr).into_owned(),
+        });
+    }
+    let compiled_workflow_arg = compiled_workflow_path.to_string_lossy().into_owned();
+
+    let store_dir = Path::new(cache_dir).join("ravanan-store");
+    // ravanan's --store needs these three subdirectories to already
+    // exist; it does not create them itself.
+    for subdir in ["files", "logs", "data"] {
+        std::fs::create_dir_all(store_dir.join(subdir)).map_err(GemmaError::Io)?;
+    }
+
+    let chromosomes = chromosomes_in(snps_file)?;
+    if chromosomes.is_empty() {
+        return Err(GemmaError::NoAssocFiles);
+    }
+    let chromosomes_yaml = chromosomes
+        .iter()
+        .map(|c| format!("\"{c}\""))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    // One persistent cache directory per chromosome, reused across
+    // requests, so gemma-wrapper's content-hash cache actually gets hit
+    // on repeated identical requests -- unlike "." (each job's own
+    // hermetic CWL working directory under ravanan's --store, fresh
+    // every run), which never accumulates a cache at all. Keyed by
+    // chromosome, not shared, because gemma-wrapper's cache-hit hash
+    // does not vary with --chromosomes: a shared directory would let
+    // one chromosome's job silently reuse another's cached result.
+    let wrapper_cache_dir = Path::new(cache_dir).join("gemma-wrapper-cache");
+    let mut chr_cache_dirs = Vec::new();
+    for chr in &chromosomes {
+        let chr_cache_dir = wrapper_cache_dir.join(format!("chr-{chr}"));
+        std::fs::create_dir_all(&chr_cache_dir).map_err(GemmaError::Io)?;
+        chr_cache_dirs.push(chr_cache_dir);
+    }
+    let cache_dirs_yaml = chr_cache_dirs
+        .iter()
+        .map(|d| format!("\"{}\"", d.display()))
+        .collect::<Vec<_>>()
+        .join(", ");
+
+    let inputs_yaml_path = Path::new(cache_dir).join(format!("{unique}-ravanan-inputs.yaml"));
+    std::fs::write(
+        &inputs_yaml_path,
+        format!(
+            "genotype_file: {{class: File, path: {genotype_file}}}\n\
+             pheno_file: {{class: File, path: {}}}\n\
+             snps_file: {{class: File, path: {snps_file}}}\n\
+             lmm: \"{lmm}\"\n\
+             chromosomes: [{chromosomes_yaml}]\n\
+             cache_dirs: [{cache_dirs_yaml}]\n",
+            pheno_path.display(),
+        ),
+    )
+    .map_err(GemmaError::Io)?;
+
+    let store_arg = format!("--store={}", store_dir.display());
+    let manifest_arg = format!("--guix-manifest={manifest_path}");
+    let inputs_yaml_arg = inputs_yaml_path.to_string_lossy().into_owned();
+    // No --output-directory: the gwa step's per-chromosome archive
+    // outputs all share gemma-wrapper's own content-hash basename (its
+    // hash doesn't vary with --chromosomes), and ravanan's
+    // --output-directory step symlinks every scattered output into one
+    // flat directory by basename, which collides across chromosomes --
+    // a ravanan/gemma-wrapper interaction bug, not something fixable
+    // on either side alone. Harmless to skip: it's optional, and this
+    // code reads paths straight out of ravanan's own --store via its
+    // stdout output record, never through --output-directory.
+    let args = vec![
+        store_arg.as_str(),
+        manifest_arg.as_str(),
+        compiled_workflow_arg.as_str(),
+        inputs_yaml_arg.as_str(),
+    ];
+    let ravanan_note = format!("ravanan: {ravanan_cmd} {}", args.join(" "));
+    let chromosomes_note = format!(
+        "Scattered gemma-wrapper over {} chromosome(s) via CWL scatter \
+         (--chromosomes/--no-parallel per job, so GNU parallel is never \
+         invoked), each against its own persistent, chromosome-keyed \
+         --cache-dir under {}: {}",
+        chromosomes.len(),
+        wrapper_cache_dir.display(),
+        chromosomes.join(", "),
+    );
+
+    let output = streaming::run_streaming(ravanan_cmd, &args, log_path.as_deref())
+        .await
+        .map_err(GemmaError::Io)?;
+    if !output.status.success() {
+        return Err(GemmaError::RavananFailed(
+            String::from_utf8_lossy(&output.stderr).into_owned(),
+        ));
+    }
+    let outputs: Value = serde_json::from_str(&String::from_utf8_lossy(&output.stdout))
+        .map_err(|e| GemmaError::RavananOutput(format!("could not parse ravanan's stdout: {e}")))?;
+    let gwa_json_paths = ravanan_output_paths(&outputs, "gwa_json")?;
+    let archive_paths = ravanan_output_paths(&outputs, "archive")?;
+    if gwa_json_paths.len() != archive_paths.len() {
+        return Err(GemmaError::RavananOutput(
+            "\"gwa_json\" and \"archive\" arrays have different lengths".to_string(),
+        ));
+    }
+
+    // One archive per chromosome now (gwa.cwl's scatter), not one
+    // shared archive as compute() and this function's own kinship step
+    // get -- each gets its own extract dir and its basenames' rows are
+    // concatenated across all of them.
+    let mut columns = Vec::new();
+    let mut rows = Vec::new();
+    for (i, (gwa_json_path, archive_path)) in gwa_json_paths.iter().zip(archive_paths.iter()).enumerate() {
+        let gwa_record: Value = serde_json::from_str(
+            &std::fs::read_to_string(gwa_json_path).map_err(GemmaError::AssocFile)?,
+        )
+        .map_err(|e| GemmaError::WrapperError {
+            step: "gwa",
+            detail: format!("could not parse gemma-wrapper's --json output: {e}"),
+        })?;
+        let basenames = assoc_file_basenames(&gwa_record);
+        if basenames.is_empty() {
+            return Err(GemmaError::NoAssocFiles);
+        }
+
+        let extract_dir = Path::new(cache_dir).join(format!("{unique}-ravanan-extract-{i}"));
+        std::fs::create_dir_all(&extract_dir).map_err(GemmaError::Io)?;
+        let tar_output = streaming::run_streaming(
+            "tar",
+            &["xJf", archive_path, "-C", &extract_dir.to_string_lossy()],
+            None,
+        )
+        .await
+        .map_err(GemmaError::Io)?;
+        if !tar_output.status.success() {
+            return Err(GemmaError::CommandFailed {
+                step: "extract-archive",
+                stderr: String::from_utf8_lossy(&tar_output.stderr).into_owned(),
+            });
+        }
+
+        let (file_columns, file_rows) = parse_assoc_files(&extract_dir, basenames)?;
+        if columns.is_empty() {
+            columns = file_columns;
+        }
+        rows.extend(file_rows);
+        let _ = std::fs::remove_dir_all(&extract_dir); // best effort; a leftover scratch dir is harmless
+    }
+
+    streaming::mark_complete(&log_path);
+
+    Ok(GemmaOutput {
+        columns,
+        rows: Value::Array(rows),
+        notes: vec![
+            ravanan_note,
+            chromosomes_note,
+            format!("workflow: {workflow_scm_path} (compiled to {compiled_workflow_arg})"),
+        ],
+    })
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -468,17 +741,23 @@ mod tests {
     #[test]
     fn parse_assoc_file_parses_header_and_numeric_rows() {
         let contents = "chr\trs\tps\tp_wald\n1\trs001\t3010274\t0.05\n1\trs002\t3492195\tNA\n";
-        let rows = parse_assoc_file(contents);
+        let (columns, rows) = parse_assoc_file(contents);
+        assert_eq!(columns, vec!["chr", "rs", "ps", "p_wald"]);
         assert_eq!(rows.len(), 2);
-        assert_eq!(rows[0]["chr"], serde_json::json!(1.0));
-        assert_eq!(rows[0]["rs"], serde_json::json!("rs001"));
-        assert_eq!(rows[0]["p_wald"], serde_json::json!(0.05));
-        assert_eq!(rows[1]["p_wald"], serde_json::json!("NA"));
+        assert_eq!(rows[0], vec![
+            serde_json::json!(1.0),
+            serde_json::json!("rs001"),
+            serde_json::json!(3010274.0),
+            serde_json::json!(0.05),
+        ]);
+        assert_eq!(rows[1][3], serde_json::json!("NA"));
     }
 
     #[test]
     fn parse_assoc_file_empty_contents_yields_no_rows() {
-        assert_eq!(parse_assoc_file("").len(), 0);
+        let (columns, rows) = parse_assoc_file("");
+        assert_eq!(columns.len(), 0);
+        assert_eq!(rows.len(), 0);
     }
 
     #[test]
diff --git a/src/lib.rs b/src/lib.rs
index 5731147..b7407ea 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -83,6 +83,7 @@ pub fn app() -> Router {
         .routes(routes!(routes::search::search))
         .routes(routes!(routes::correlation::correlation))
         .routes(routes!(routes::gemma::compute_gemma))
+        .routes(routes!(routes::gemma::compute_gemma_ravanan))
         .routes(routes!(routes::gemma::gemma_status))
         .routes(routes!(routes::rqtl::compute_rqtl))
         .routes(routes!(routes::rqtl::rqtl_status));
diff --git a/src/response.rs b/src/response.rs
index cb2adce..e85362a 100644
--- a/src/response.rs
+++ b/src/response.rs
@@ -38,6 +38,21 @@ pub struct Meta {
     pub took_ms: u64,
     /// The request's id, also in the `x-request-id` response header.
     pub request_id: Option<String>,
+    /// Column names for a `data` that is a list of rows, each row a
+    /// positional array rather than a keyed object (e.g.
+    /// `/compute/gemma`'s GEMMA `.assoc.txt` rows) -- cuts repeated key
+    /// names out of a large response. Absent for endpoints whose `data`
+    /// is already keyed or isn't tabular.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub columns: Option<Vec<String>>,
+    /// How many rows `data` carries, alongside `columns` (same
+    /// condition for being present).
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub num_rows: Option<usize>,
+    /// `columns.len()`, alongside `columns`, so a caller doesn't need
+    /// to count it itself.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub num_cols: Option<usize>,
 }
 
 /// A success response: the payload, facts about the response, notes on
@@ -75,6 +90,16 @@ impl<T: Serialize + utoipa::ToSchema> Envelope<T> {
         self
     }
 
+    /// Sets `meta.columns`/`meta.num_cols`/`meta.num_rows` for a `data`
+    /// of `num_rows` positional rows, each `columns.len()` long (see
+    /// `Meta::columns`).
+    pub fn with_columns(mut self, columns: Vec<String>, num_rows: usize) -> Self {
+        self.meta.num_cols = Some(columns.len());
+        self.meta.columns = Some(columns);
+        self.meta.num_rows = Some(num_rows);
+        self
+    }
+
     /// Marks the response `partial` and records why in `notes`.
     pub fn partial(mut self, note: &str) -> Self {
         self.meta.state = State::Partial;
@@ -148,6 +173,9 @@ impl Ctx {
                 state: State::Complete,
                 took_ms: self.info.took_ms(),
                 request_id: self.info.id(),
+                columns: None,
+                num_rows: None,
+                num_cols: None,
             },
             notes: Vec::new(),
             next: Vec::new(),
diff --git a/src/routes/gemma.rs b/src/routes/gemma.rs
index 0b6b748..32d1818 100644
--- a/src/routes/gemma.rs
+++ b/src/routes/gemma.rs
@@ -67,7 +67,7 @@ trait values, and returns the resulting rows of GEMMA's own \
     params(ComputeGemmaParams),
     request_body = GemmaRequest,
     responses(
-        (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as a JSON array of row objects", body = Envelope<serde_json::Value>),
+        (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as positional arrays (column names in meta.columns)", body = Envelope<serde_json::Value>),
         Unauthorized,
         (status = 409, description = "id is already in use by another run", body = ApiError),
         (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError),
@@ -122,9 +122,13 @@ pub async fn compute_gemma(
     )
     .await
     .map(|output| {
-        let mut envelope = ctx.envelope(output.rows).with_note(
-            "Rows come from GEMMA's own .assoc.txt output; column names depend on the lmm value used.",
-        );
+        let num_rows = output.rows.as_array().map(Vec::len).unwrap_or(0);
+        let mut envelope = ctx
+            .envelope(output.rows)
+            .with_columns(output.columns, num_rows)
+            .with_note(
+                "Rows come from GEMMA's own .assoc.txt output, positional (see meta.columns for names); column names depend on the lmm value used.",
+            );
         for note in &output.notes {
             envelope = envelope.with_note(note);
         }
@@ -143,6 +147,100 @@ pub async fn compute_gemma(
     })
 }
 
+#[utoipa::path(post, path = "/compute/gemma/ravanan",
+    summary = "Map a QTL with GEMMA, run via ravanan instead of gemma-wrapper directly",
+    description = "Same computation and request/response shape as POST \
+/compute/gemma (gemma-wrapper's K-matrix step, then its LMM association \
+step, both --loco), but driven by ravanan \
+(https://github.com/vepiteski/ravanan) running a CWL workflow compiled \
+from the ccwl source at cwl/gemma/gemma-loco.scm instead of this server's own direct subprocess \
+calls -- an alternative runner for the same two gemma-wrapper steps, \
+for comparing the two (correctness, timing). Exploratory: no caching of \
+ravanan's own store between requests beyond its --store directory, and \
+?id=/status polling tails ravanan's own stderr rather than gemma-wrapper's.",
+    params(ComputeGemmaParams),
+    request_body = GemmaRequest,
+    responses(
+        (status = 200, description = "Mapped QTL rows from GEMMA's .assoc.txt output, as positional arrays (column names in meta.columns)", body = Envelope<serde_json::Value>),
+        Unauthorized,
+        (status = 409, description = "id is already in use by another run", body = ApiError),
+        (status = 422, description = "lmm not one of the known-working values (1, 2, 3, 4, 9), or id not alnum/-/_ (max 64 chars)", body = ApiError),
+        (status = 502, description = "ravanan or gemma-wrapper failed", body = ApiError)
+    ))]
+#[gnauth(token_only)]
+pub async fn compute_gemma_ravanan(
+    ctx: Ctx,
+    _token: AuthToken,
+    Query(params): Query<ComputeGemmaParams>,
+    Json(request): Json<GemmaRequest>,
+) -> Result<Json<Envelope<serde_json::Value>>, ApiError> {
+    let rid = ctx.request_info();
+    let lmm = request.lmm.unwrap_or(9);
+    if !matches!(lmm, 1 | 2 | 3 | 4 | 9) {
+        return Err(ApiError::new(
+            StatusCode::UNPROCESSABLE_ENTITY,
+            "invalid_lmm",
+            "Invalid lmm",
+            format!(
+                "lmm must be one of 1, 2, 3, 4, 9 (got {lmm}); 1=Wald, 2=LRT, 3=score, 4=all three, 9=GeneNetwork's convention (observed to behave like LRT)."
+            ),
+            "compute_gemma_ravanan",
+        )
+        .with_request_id(rid));
+    }
+    if let Some(id) = &params.id {
+        if !streaming::is_valid_run_id(id) {
+            return Err(ApiError::new(
+                StatusCode::UNPROCESSABLE_ENTITY,
+                "invalid_run_id",
+                "Invalid id",
+                "id must be 1-64 ASCII letters, digits, '-' or '_'.".to_string(),
+                "compute_gemma_ravanan",
+            )
+            .with_request_id(rid));
+        }
+    }
+
+    let config = Config::load();
+    gemma::compute_ravanan(
+        &config.ravanan_cmd,
+        &config.ccwl_cmd,
+        &config.gemma_ravanan_workflow,
+        &config.gemma_ravanan_manifest,
+        &config.ravanan_cache_dir,
+        &request.genotype_file,
+        &request.snps_file,
+        &request.trait_values,
+        lmm,
+        params.id.as_deref(),
+    )
+    .await
+    .map(|output| {
+        let num_rows = output.rows.as_array().map(Vec::len).unwrap_or(0);
+        let mut envelope = ctx
+            .envelope(output.rows)
+            .with_columns(output.columns, num_rows)
+            .with_note(
+                "Rows come from GEMMA's own .assoc.txt output, positional (see meta.columns for names); column names depend on the lmm value used.",
+            );
+        for note in &output.notes {
+            envelope = envelope.with_note(note);
+        }
+        Json(envelope)
+    })
+    .map_err(|e| match e {
+        GemmaError::RunIdInUse(id) => ApiError::new(
+            StatusCode::CONFLICT,
+            "run_id_in_use",
+            "Run id already in use",
+            format!("id '{id}' is already in use; pick a different id."),
+            "compute_gemma_ravanan",
+        )
+        .with_request_id(rid.clone()),
+        other => gemma_error_to_api_error(other, rid.clone()),
+    })
+}
+
 #[utoipa::path(get, path = "/compute/gemma/status/{run_id}",
     summary = "Poll a running (or finished) /compute/gemma call's progress",
     description = "Returns the lines gemma-wrapper has written to stderr \
@@ -177,7 +275,16 @@ pub async fn gemma_status(
     }
     let config = Config::load();
     let peek = params.peek.unwrap_or(0);
-    streaming::status(&config.gemma_cache_dir, &run_id, peek)
+    // A run_id may belong to either runner (POST /compute/gemma or
+    // POST /compute/gemma/ravanan), which log to different cache
+    // dirs; try gemma's first, then fall back to ravanan's.
+    let result = match streaming::status(&config.gemma_cache_dir, &run_id, peek) {
+        Err(StatusError::NotFound) => {
+            streaming::status(&config.ravanan_cache_dir, &run_id, peek)
+        }
+        other => other,
+    };
+    result
         .map(|(lines, pointer)| {
             Json(ctx.envelope(RunStatus {
                 run_id: run_id.clone(),
diff --git a/src/routes/rqtl.rs b/src/routes/rqtl.rs
index 92ee21c..2961124 100644
--- a/src/routes/rqtl.rs
+++ b/src/routes/rqtl.rs
@@ -73,7 +73,7 @@ concurrently -- same shape as /compute/gemma.",
     params(ComputeRqtlParams),
     request_body = RqtlRequest,
     responses(
-        (status = 200, description = "Mapped QTL markers (and, if requested, permutation thresholds)", body = Envelope<serde_json::Value>),
+        (status = 200, description = "Mapped QTL markers, as positional arrays (column names in meta.columns), and, if requested, permutation thresholds", body = Envelope<serde_json::Value>),
         Unauthorized,
         (status = 409, description = "id is already in use by another run", body = ApiError),
         (status = 422, description = "id not alnum/-/_ (max 64 chars)", body = ApiError),
@@ -147,9 +147,13 @@ pub async fn compute_rqtl(
     .await
     .map(|output| {
         let notes = output.notes.clone();
-        let mut envelope = ctx.envelope(rqtl::to_json(output)).with_note(
-            "markers come from R/qtl's scanone output; permutation is null unless nperm > 0 was requested.",
-        );
+        let num_rows = output.markers.len();
+        let mut envelope = ctx
+            .envelope(rqtl::to_json(output))
+            .with_columns(rqtl::MARKER_COLUMNS.iter().map(|s| s.to_string()).collect(), num_rows)
+            .with_note(
+                "markers come from R/qtl's scanone output, positional (see meta.columns for names); permutation is null unless nperm > 0 was requested.",
+            );
         for note in &notes {
             envelope = envelope.with_note(note);
         }
diff --git a/src/rqtl.rs b/src/rqtl.rs
index 691cdbe..53d6d03 100644
--- a/src/rqtl.rs
+++ b/src/rqtl.rs
@@ -391,12 +391,35 @@ pub async fn compute(
     }
 }
 
+/// `MarkerResult`'s fixed fields, in the order `to_json` lays out each
+/// marker row -- the names a caller matches `markers`' positional rows
+/// up against (a 4-way cross's `extra` effect columns, if present,
+/// trail these and aren't named here; see `MarkerResult`'s own doc
+/// comment).
+pub const MARKER_COLUMNS: [&str; 4] = ["name", "chr", "pos", "lod"];
+
 /// `RqtlOutput` as the single JSON value the route returns -- see
 /// `crate::gemma`'s own note on why `serde_json::Value`, not a `Vec`,
-/// is what utoipa needs here.
+/// is what utoipa needs here. `markers` rows are positional (see
+/// `MARKER_COLUMNS`), the same space-saving shape `crate::gemma::compute`
+/// uses for its own (much larger) marker rows.
 pub fn to_json(output: RqtlOutput) -> Value {
+    let markers: Vec<Value> = output
+        .markers
+        .into_iter()
+        .map(|m| {
+            let mut row = vec![
+                Value::String(m.name),
+                Value::String(m.chr),
+                serde_json::json!(m.pos),
+                serde_json::json!(m.lod),
+            ];
+            row.extend(m.extra.into_iter().map(Value::String));
+            Value::Array(row)
+        })
+        .collect();
     serde_json::json!({
-        "markers": output.markers,
+        "markers": markers,
         "permutation": output.permutation,
     })
 }