//! Shared infrastructure for compute endpoints that run a long external //! process and want to show progress while it's still running: tail a //! subprocess's stderr into a per-run log file as it runs, and let a //! second connection poll that log while the first is still blocked. //! Used by `crate::gemma` and `crate::rqtl`, which both call a //! different external tool (gemma-wrapper, Rscript) but need exactly //! this same shape. Mirrors GeneNetwork2/3's own equivalent //! (gn3/computations/streaming.py, gn3/api/streaming.py), used there //! for R/qtl. use std::io; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use std::process::{Output, Stdio}; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; use tokio::process::Command; /// `?peek=` on a status-polling endpoint: the byte offset already /// read; 0 to read from the start. #[derive(Deserialize, utoipa::IntoParams)] pub struct RunStatusParams { pub peek: Option, } /// A status-polling endpoint's response: new log lines since `peek`, /// and the new offset to pass as `peek` next time. #[derive(Serialize, utoipa::ToSchema)] pub struct RunStatus { pub run_id: String, /// New lines of the subprocess's stderr since `peek`. The last /// line, once the run has finished, is `COMPLETION_MARKER`, or /// `FAILURE_MARKER: ` if it failed. pub lines: Vec, /// The new byte offset; pass this back as `peek` next poll. pub pointer: u64, } /// A short token of ASCII letters, digits, `-` or `_` (no path /// separators or `.`). `run_id` is joined straight onto a cache /// directory to name the log file `status` reads back, so an unchecked /// value would let a caller write/read outside it -- the same /// unchecked-`run_id`-as-path-component GeneNetwork3 does for its own /// `/api/stream/{run_id}`, not knowingly replicated here. pub fn is_valid_run_id(run_id: &str) -> bool { !run_id.is_empty() && run_id.len() <= 64 && run_id .chars() .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') } /// The line appended to a run's log once it finishes successfully, so /// a `status` poller can tell "still running" apart from "done" without /// a separate field -- same convention GeneNetwork3 uses ("Done with /// GN3 Computation" in gn3/api/rqtl2.py), under this project's own /// name. pub const COMPLETION_MARKER: &str = "GN4: computation complete"; /// The line appended to a run's log when the computation failed, so a /// `status` poller can tell a failed run apart from a still-running /// one (whose log is legitimately quiet for long stretches during /// healthy GEMMA/R computations). Without it, a poller watches a /// frozen log forever while the POST caller already got its error. pub const FAILURE_MARKER: &str = "GN4: computation failed"; /// Why `create_run_log` couldn't start a run's log. #[derive(Debug)] pub enum RunLogError { /// `run_id` already has a log file: a run with that id is in /// progress, or finished and was never polled/cleaned up. AlreadyInUse(String), Io(io::Error), } /// Atomically create a fresh, empty log file for `run_id` under /// `cache_dir` (`None` if `run_id` is `None` -- no streaming wanted), /// returning its path. `create_new` fails if the file already exists, /// so a reused id errors out here rather than silently truncating /// another run's (possibly still in-progress) log -- there is no /// separate check-then-write step to race. pub fn create_run_log( cache_dir: &str, run_id: Option<&str>, ) -> Result, RunLogError> { let Some(id) = run_id else { return Ok(None) }; let path = Path::new(cache_dir).join(format!("{id}.log")); match std::fs::OpenOptions::new() .write(true) .create_new(true) .open(&path) { Ok(_) => Ok(Some(path)), Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { Err(RunLogError::AlreadyInUse(id.to_string())) } Err(e) => Err(RunLogError::Io(e)), } } /// Append `COMPLETION_MARKER` to `log_path`, if there is one. Best /// effort: a failure here (e.g. the log was deleted mid-run) is not /// worth failing an otherwise-successful computation over. pub fn mark_complete(log_path: &Option) { if let Some(path) = log_path { if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { let _ = writeln!(f, "{COMPLETION_MARKER}"); } } } /// Append lines to `log_path`, if there is one. Best effort, like the /// markers: a caller that asked for a log gets the server's own summary lines /// (e.g. `pve:` per chromosome) in the same stream as the child's output. pub fn append_lines(log_path: &Option, lines: &[String]) { let Some(path) = log_path else { return }; if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { for line in lines { let _ = writeln!(f, "{line}"); } } } /// Append `FAILURE_MARKER: ` to `log_path`, if there is one. /// Best effort, like [`mark_complete`]: a failure here (e.g. the log /// was deleted mid-run) must never mask the computation error it is /// reporting. pub fn mark_failed(log_path: &Option, error: &E) { if let Some(path) = log_path { if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) { let _ = writeln!(f, "{FAILURE_MARKER}: {error}"); } } } /// Run `cmd`, optionally tailing its stderr into `log_path` one line at /// a time as it runs, and return the same `Output` shape /// `Command::output()` would have (full stdout, full stderr, exit /// status) -- callers that don't pass `log_path` see identical /// behaviour to the plain, non-streaming call it replaces. /// /// stderr, not stdout, is what's tailed. For gemma-wrapper, all of its /// human-readable `--json`-mode progress goes to stderr, with stdout /// reserved for the one final JSON record (see `crate::gemma`'s own /// note on this). R/qtl's wrapper script is simpler: its `cat(...)` and /// `verbose_print(...)` calls just go to stdout by default, but run /// under `Rscript` with nothing redirecting it, so stderr still carries /// R's own progress/warning output (package load messages, warnings) -- /// tailing stderr here is a deliberate, uniform choice across both /// tools' call sites, not a guarantee every line of every tool's /// progress ends up in the log. pub async fn run_streaming( cmd: &str, args: &[&str], log_path: Option<&Path>, ) -> io::Result { run_streaming_with_env(cmd, args, log_path, &[]).await } /// [`run_streaming`] with a few environment variables set for the child -- /// Julia needs `JULIA_DEPOT_PATH`, and a wrapper that exports it would hide /// which depot a result came from. pub async fn run_streaming_with_env( cmd: &str, args: &[&str], log_path: Option<&Path>, environment: &[(&str, &str)], ) -> io::Result { let mut child = Command::new(cmd) .envs(environment.iter().copied()) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; let stdout_handle = child.stdout.take().expect("stdout was piped"); let stderr_handle = child.stderr.take().expect("stderr was piped"); let stdout_task = tokio::spawn(async move { let mut buf = Vec::new(); let mut reader = stdout_handle; let _ = reader.read_to_end(&mut buf).await; buf }); let log_path = log_path.map(Path::to_path_buf); let stderr_task = tokio::spawn(async move { let mut buf = Vec::new(); let mut lines = BufReader::new(stderr_handle).lines(); while let Ok(Some(line)) = lines.next_line().await { if let Some(path) = &log_path { if let Ok(mut f) = std::fs::OpenOptions::new() .create(true) .append(true) .open(path) { let _ = writeln!(f, "{line}"); } } buf.extend_from_slice(line.as_bytes()); buf.push(b'\n'); } buf }); let status = child.wait().await?; let stdout = stdout_task.await.unwrap_or_default(); let stderr = stderr_task.await.unwrap_or_default(); Ok(Output { status, stdout, stderr, }) } /// Why `status` couldn't read a run's log. #[derive(Debug)] pub enum StatusError { /// No log file for this `run_id` (unknown id, or the run hasn't /// written anything yet -- there is a brief window between /// `create_run_log` and the first subprocess call). NotFound, Io(io::Error), } /// Lines appended to `run_id`'s log since byte offset `peek`, and the /// new offset to pass as `peek` next time. Mirrors GeneNetwork3's /// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py, spelled /// "peak" there), stateless in the same way: the caller tracks `peek`, /// not the server. pub fn status(cache_dir: &str, run_id: &str, peek: u64) -> Result<(Vec, u64), StatusError> { let path = Path::new(cache_dir).join(format!("{run_id}.log")); let mut file = std::fs::File::open(&path).map_err(|e| { if e.kind() == io::ErrorKind::NotFound { StatusError::NotFound } else { StatusError::Io(e) } })?; file.seek(SeekFrom::Start(peek)).map_err(StatusError::Io)?; let mut buf = String::new(); file.read_to_string(&mut buf).map_err(StatusError::Io)?; let pointer = peek + buf.len() as u64; let lines = buf.lines().map(str::to_string).collect(); Ok((lines, pointer)) } #[cfg(test)] mod tests { use super::*; #[test] fn mark_failed_appends_failure_marker_with_the_error() { // Each test gets its own directory: these run in parallel, and // the reused-id test below removes its directory when it is done, // which used to delete this one's log mid-assertion. let dir = std::env::temp_dir() .join(format!("gn4-streaming-fail-marker-{}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); let log = create_run_log(dir.to_str().unwrap(), Some("fail-marker-test")).unwrap(); mark_failed(&log, &"Rscript died with status 1".to_string()); let contents = std::fs::read_to_string(log.as_ref().expect("log path")).expect("read log"); assert!( contents.contains(FAILURE_MARKER), "log must carry the failure marker: {contents}" ); assert!( contents.contains("Rscript died with status 1"), "log must carry the error text: {contents}" ); std::fs::remove_dir_all(&dir).unwrap(); } #[test] fn is_valid_run_id_accepts_alnum_dash_underscore() { for ok in ["abc123", "a-b_c", "A1", "x"] { assert!(is_valid_run_id(ok), "{ok}"); } } #[test] fn is_valid_run_id_rejects_path_separators_and_empty() { for bad in ["", "../etc/passwd", "a/b", "a.b", "a b", &"x".repeat(65)] { assert!(!is_valid_run_id(bad), "{bad}"); } } #[test] fn create_run_log_none_run_id_yields_no_path() { assert!(matches!(create_run_log("/tmp", None), Ok(None))); } #[test] fn create_run_log_rejects_a_reused_id() { let cache_dir = std::env::temp_dir().join(format!("gn4-streaming-dup-id-{}", std::process::id())); let cache_dir = cache_dir.to_str().unwrap(); std::fs::create_dir_all(cache_dir).unwrap(); let id = "dup-run"; assert!(matches!(create_run_log(cache_dir, Some(id)), Ok(Some(_)))); let err = create_run_log(cache_dir, Some(id)).unwrap_err(); assert!(matches!(err, RunLogError::AlreadyInUse(ref i) if i == id)); std::fs::remove_dir_all(cache_dir).unwrap(); } #[test] fn status_reads_from_peek_and_returns_a_new_pointer() { let path = std::env::temp_dir().join(format!("gn4-streaming-status-test-{}", std::process::id())); let cache_dir = path.to_str().unwrap(); std::fs::create_dir_all(cache_dir).unwrap(); std::fs::write( Path::new(cache_dir).join("run1.log"), "line one\nline two\n", ) .unwrap(); let (lines, pointer) = status(cache_dir, "run1", 0).unwrap(); assert_eq!(lines, vec!["line one", "line two"]); std::fs::OpenOptions::new() .append(true) .open(Path::new(cache_dir).join("run1.log")) .unwrap() .write_all(b"line three\n") .unwrap(); let (more, _) = status(cache_dir, "run1", pointer).unwrap(); assert_eq!(more, vec!["line three"]); std::fs::remove_dir_all(cache_dir).unwrap(); } #[test] fn status_unknown_run_id_is_not_found() { let err = status("/tmp", "no-such-gn4-streaming-run-id", 0).unwrap_err(); assert!(matches!(err, StatusError::NotFound)); } }