about summary refs log tree commit diff
path: root/src/streaming.rs
blob: bba5c54b88ea3e7325e741e66a0bd62e1fb46f4c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
//! Shared infrastructure for compute endpoints that run a long external
//! process and want to show progress while it's still running: tail a
//! subprocess's stderr into a per-run log file as it runs, and let a
//! second connection poll that log while the first is still blocked.
//! Used by `crate::gemma` and `crate::rqtl`, which both call a
//! different external tool (gemma-wrapper, Rscript) but need exactly
//! this same shape. Mirrors GeneNetwork2/3's own equivalent
//! (gn3/computations/streaming.py, gn3/api/streaming.py), used there
//! for R/qtl.

use std::io;
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::{Path, PathBuf};
use std::process::{Output, Stdio};

use serde::{Deserialize, Serialize};
use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader};
use tokio::process::Command;

/// `?peek=` on a status-polling endpoint: the byte offset already
/// read; 0 to read from the start.
#[derive(Deserialize, utoipa::IntoParams)]
pub struct RunStatusParams {
    pub peek: Option<u64>,
}

/// A status-polling endpoint's response: new log lines since `peek`,
/// and the new offset to pass as `peek` next time.
#[derive(Serialize, utoipa::ToSchema)]
pub struct RunStatus {
    pub run_id: String,
    /// New lines of the subprocess's stderr since `peek`. The last
    /// line, once the run has finished, is `COMPLETION_MARKER`, or
    /// `FAILURE_MARKER: <error>` if it failed.
    pub lines: Vec<String>,
    /// The new byte offset; pass this back as `peek` next poll.
    pub pointer: u64,
}

/// A short token of ASCII letters, digits, `-` or `_` (no path
/// separators or `.`). `run_id` is joined straight onto a cache
/// directory to name the log file `status` reads back, so an unchecked
/// value would let a caller write/read outside it -- the same
/// unchecked-`run_id`-as-path-component GeneNetwork3 does for its own
/// `/api/stream/{run_id}`, not knowingly replicated here.
pub fn is_valid_run_id(run_id: &str) -> bool {
    !run_id.is_empty()
        && run_id.len() <= 64
        && run_id
            .chars()
            .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
}

/// The line appended to a run's log once it finishes successfully, so
/// a `status` poller can tell "still running" apart from "done" without
/// a separate field -- same convention GeneNetwork3 uses ("Done with
/// GN3 Computation" in gn3/api/rqtl2.py), under this project's own
/// name.
pub const COMPLETION_MARKER: &str = "GN4: computation complete";

/// The line appended to a run's log when the computation failed, so a
/// `status` poller can tell a failed run apart from a still-running
/// one (whose log is legitimately quiet for long stretches during
/// healthy GEMMA/R computations). Without it, a poller watches a
/// frozen log forever while the POST caller already got its error.
pub const FAILURE_MARKER: &str = "GN4: computation failed";

/// Why `create_run_log` couldn't start a run's log.
#[derive(Debug)]
pub enum RunLogError {
    /// `run_id` already has a log file: a run with that id is in
    /// progress, or finished and was never polled/cleaned up.
    AlreadyInUse(String),
    Io(io::Error),
}

/// Atomically create a fresh, empty log file for `run_id` under
/// `cache_dir` (`None` if `run_id` is `None` -- no streaming wanted),
/// returning its path. `create_new` fails if the file already exists,
/// so a reused id errors out here rather than silently truncating
/// another run's (possibly still in-progress) log -- there is no
/// separate check-then-write step to race.
pub fn create_run_log(
    cache_dir: &str,
    run_id: Option<&str>,
) -> Result<Option<PathBuf>, RunLogError> {
    let Some(id) = run_id else { return Ok(None) };
    let path = Path::new(cache_dir).join(format!("{id}.log"));
    match std::fs::OpenOptions::new()
        .write(true)
        .create_new(true)
        .open(&path)
    {
        Ok(_) => Ok(Some(path)),
        Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {
            Err(RunLogError::AlreadyInUse(id.to_string()))
        }
        Err(e) => Err(RunLogError::Io(e)),
    }
}

/// Append `COMPLETION_MARKER` to `log_path`, if there is one. Best
/// effort: a failure here (e.g. the log was deleted mid-run) is not
/// worth failing an otherwise-successful computation over.
pub fn mark_complete(log_path: &Option<PathBuf>) {
    if let Some(path) = log_path {
        if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) {
            let _ = writeln!(f, "{COMPLETION_MARKER}");
        }
    }
}

/// Append lines to `log_path`, if there is one. Best effort, like the
/// markers: a caller that asked for a log gets the server's own summary lines
/// (e.g. `pve:` per chromosome) in the same stream as the child's output.
pub fn append_lines(log_path: &Option<PathBuf>, lines: &[String]) {
    let Some(path) = log_path else { return };
    if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) {
        for line in lines {
            let _ = writeln!(f, "{line}");
        }
    }
}

/// Append `FAILURE_MARKER: <error>` to `log_path`, if there is one.
/// Best effort, like [`mark_complete`]: a failure here (e.g. the log
/// was deleted mid-run) must never mask the computation error it is
/// reporting.
pub fn mark_failed<E: std::fmt::Display>(log_path: &Option<PathBuf>, error: &E) {
    if let Some(path) = log_path {
        if let Ok(mut f) = std::fs::OpenOptions::new().append(true).open(path) {
            let _ = writeln!(f, "{FAILURE_MARKER}: {error}");
        }
    }
}

/// Run `cmd`, optionally tailing its stderr into `log_path` one line at
/// a time as it runs, and return the same `Output` shape
/// `Command::output()` would have (full stdout, full stderr, exit
/// status) -- callers that don't pass `log_path` see identical
/// behaviour to the plain, non-streaming call it replaces.
///
/// stderr, not stdout, is what's tailed. For gemma-wrapper, all of its
/// human-readable `--json`-mode progress goes to stderr, with stdout
/// reserved for the one final JSON record (see `crate::gemma`'s own
/// note on this). R/qtl's wrapper script is simpler: its `cat(...)` and
/// `verbose_print(...)` calls just go to stdout by default, but run
/// under `Rscript` with nothing redirecting it, so stderr still carries
/// R's own progress/warning output (package load messages, warnings) --
/// tailing stderr here is a deliberate, uniform choice across both
/// tools' call sites, not a guarantee every line of every tool's
/// progress ends up in the log.
pub async fn run_streaming(
    cmd: &str,
    args: &[&str],
    log_path: Option<&Path>,
) -> io::Result<Output> {
    run_streaming_with_env(cmd, args, log_path, &[]).await
}

/// [`run_streaming`] with a few environment variables set for the child --
/// Julia needs `JULIA_DEPOT_PATH`, and a wrapper that exports it would hide
/// which depot a result came from.
pub async fn run_streaming_with_env(
    cmd: &str,
    args: &[&str],
    log_path: Option<&Path>,
    environment: &[(&str, &str)],
) -> io::Result<Output> {
    let mut child = Command::new(cmd)
        .envs(environment.iter().copied())
        .args(args)
        .stdout(Stdio::piped())
        .stderr(Stdio::piped())
        .spawn()?;
    let stdout_handle = child.stdout.take().expect("stdout was piped");
    let stderr_handle = child.stderr.take().expect("stderr was piped");

    let stdout_task = tokio::spawn(async move {
        let mut buf = Vec::new();
        let mut reader = stdout_handle;
        let _ = reader.read_to_end(&mut buf).await;
        buf
    });

    let log_path = log_path.map(Path::to_path_buf);
    let stderr_task = tokio::spawn(async move {
        let mut buf = Vec::new();
        let mut lines = BufReader::new(stderr_handle).lines();
        while let Ok(Some(line)) = lines.next_line().await {
            if let Some(path) = &log_path {
                if let Ok(mut f) = std::fs::OpenOptions::new()
                    .create(true)
                    .append(true)
                    .open(path)
                {
                    let _ = writeln!(f, "{line}");
                }
            }
            buf.extend_from_slice(line.as_bytes());
            buf.push(b'\n');
        }
        buf
    });

    let status = child.wait().await?;
    let stdout = stdout_task.await.unwrap_or_default();
    let stderr = stderr_task.await.unwrap_or_default();
    Ok(Output {
        status,
        stdout,
        stderr,
    })
}

/// Why `status` couldn't read a run's log.
#[derive(Debug)]
pub enum StatusError {
    /// No log file for this `run_id` (unknown id, or the run hasn't
    /// written anything yet -- there is a brief window between
    /// `create_run_log` and the first subprocess call).
    NotFound,
    Io(io::Error),
}

/// Lines appended to `run_id`'s log since byte offset `peek`, and the
/// new offset to pass as `peek` next time. Mirrors GeneNetwork3's
/// `GET /api/stream/{run_id}?peak=` (gn3/api/streaming.py, spelled
/// "peak" there), stateless in the same way: the caller tracks `peek`,
/// not the server.
pub fn status(cache_dir: &str, run_id: &str, peek: u64) -> Result<(Vec<String>, u64), StatusError> {
    let path = Path::new(cache_dir).join(format!("{run_id}.log"));
    let mut file = std::fs::File::open(&path).map_err(|e| {
        if e.kind() == io::ErrorKind::NotFound {
            StatusError::NotFound
        } else {
            StatusError::Io(e)
        }
    })?;
    file.seek(SeekFrom::Start(peek)).map_err(StatusError::Io)?;
    let mut buf = String::new();
    file.read_to_string(&mut buf).map_err(StatusError::Io)?;
    let pointer = peek + buf.len() as u64;
    let lines = buf.lines().map(str::to_string).collect();
    Ok((lines, pointer))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn mark_failed_appends_failure_marker_with_the_error() {
        // Each test gets its own directory: these run in parallel, and
        // the reused-id test below removes its directory when it is done,
        // which used to delete this one's log mid-assertion.
        let dir = std::env::temp_dir()
            .join(format!("gn4-streaming-fail-marker-{}", std::process::id()));
        std::fs::create_dir_all(&dir).unwrap();
        let log = create_run_log(dir.to_str().unwrap(), Some("fail-marker-test")).unwrap();
        mark_failed(&log, &"Rscript died with status 1".to_string());
        let contents = std::fs::read_to_string(log.as_ref().expect("log path")).expect("read log");
        assert!(
            contents.contains(FAILURE_MARKER),
            "log must carry the failure marker: {contents}"
        );
        assert!(
            contents.contains("Rscript died with status 1"),
            "log must carry the error text: {contents}"
        );
        std::fs::remove_dir_all(&dir).unwrap();
    }

    #[test]
    fn is_valid_run_id_accepts_alnum_dash_underscore() {
        for ok in ["abc123", "a-b_c", "A1", "x"] {
            assert!(is_valid_run_id(ok), "{ok}");
        }
    }

    #[test]
    fn is_valid_run_id_rejects_path_separators_and_empty() {
        for bad in ["", "../etc/passwd", "a/b", "a.b", "a b", &"x".repeat(65)] {
            assert!(!is_valid_run_id(bad), "{bad}");
        }
    }

    #[test]
    fn create_run_log_none_run_id_yields_no_path() {
        assert!(matches!(create_run_log("/tmp", None), Ok(None)));
    }

    #[test]
    fn create_run_log_rejects_a_reused_id() {
        let cache_dir =
            std::env::temp_dir().join(format!("gn4-streaming-dup-id-{}", std::process::id()));
        let cache_dir = cache_dir.to_str().unwrap();
        std::fs::create_dir_all(cache_dir).unwrap();
        let id = "dup-run";

        assert!(matches!(create_run_log(cache_dir, Some(id)), Ok(Some(_))));
        let err = create_run_log(cache_dir, Some(id)).unwrap_err();
        assert!(matches!(err, RunLogError::AlreadyInUse(ref i) if i == id));

        std::fs::remove_dir_all(cache_dir).unwrap();
    }

    #[test]
    fn status_reads_from_peek_and_returns_a_new_pointer() {
        let path =
            std::env::temp_dir().join(format!("gn4-streaming-status-test-{}", std::process::id()));
        let cache_dir = path.to_str().unwrap();
        std::fs::create_dir_all(cache_dir).unwrap();
        std::fs::write(
            Path::new(cache_dir).join("run1.log"),
            "line one\nline two\n",
        )
        .unwrap();

        let (lines, pointer) = status(cache_dir, "run1", 0).unwrap();
        assert_eq!(lines, vec!["line one", "line two"]);

        std::fs::OpenOptions::new()
            .append(true)
            .open(Path::new(cache_dir).join("run1.log"))
            .unwrap()
            .write_all(b"line three\n")
            .unwrap();
        let (more, _) = status(cache_dir, "run1", pointer).unwrap();
        assert_eq!(more, vec!["line three"]);

        std::fs::remove_dir_all(cache_dir).unwrap();
    }

    #[test]
    fn status_unknown_run_id_is_not_found() {
        let err = status("/tmp", "no-such-gn4-streaming-run-id", 0).unwrap_err();
        assert!(matches!(err, StatusError::NotFound));
    }
}