;;; gn4ai.scm -- a GN4AI (genenetwork4) system container ;;; ;;; A store-sharing container, run by the run-guix-container daemon, with the ;;; service on :3003 and ssh on :2223 (3000/3001 are the host's own instances, ;;; 8081 is the gn-auth stub, 2222 is another container's ssh, 3306 is MariaDB). ;;; ;;; MariaDB and gn-auth stay *outside*: `--network` shares the host's network ;;; namespace, so 127.0.0.1 inside the container is the host's loopback and ;;; SQL_URI still resolves. Nothing here reaches into the database. ;;; ;;; Build and start (MariaDB and the gn-auth stub must be up first): ;;; ;;; base=/home/wrk/tmp/ai-root/factory ;;; ;;; # the GN4 binary, from genenetwork4/guix.scm, pinned against GC ;;; cd $base/code/genenetwork/genenetwork4 ;;; guix build -f guix.scm --root=$base/gn4-root \ ;;; -L $base/code/guix-bioinformatics --cores=48 -M 8 ;;; ;;; cd $base/code/gn-machines ;;; guix system container -d --network \ ;;; -L $base/code/guix-bioinformatics -L $base/code/gn-machines \ ;;; --expose=$base/gn4-data=$base/gn4-data \ ;;; --expose=$base/gn4-index=$base/gn4-index \ ;;; --expose=$base/code=$base/code \ ;;; --expose=/home/wrk/.julia=/home/wrk/.julia \ ;;; --share=$base/gn4-container=$base/gn4-container \ ;;; gn4ai.scm # -> /gnu/store/...-run-container.drv ;;; guix build /gnu/store/...-run-container.drv ;;; $base/code/run-guix-container/bin/run-guix-container.rb \ ;;; --daemon 127.0.0.1:8989 --fingerprint ;;; ;;; Then, from the host: curl http://127.0.0.1:3003/version ;;; and over ssh: ssh -p 2223 -i ~/hello-container-key root@127.0.0.1 ;;; ;;; Paths: ;;; * read-only (`--expose`): the genotype files (42 GB), the Xapian index ;;; (97 GB), the code tree (for the scripts the API runs, the gn-bnw ;;; driver, run-guix-container), and the Julia depot; ;;; * read-write (`--share`): $base/gn4-container -- the env file, HOME, ;;; TMPDIR, the token registry, baskets, the Wikidata cache and every ;;; engine cache. A container's own root is ephemeral, so nothing that ;;; must survive a restart lives in it. ;;; ;;; `--expose`/`--share` drop a host path that does not exist *silently*, so ;;; every path above has to be created before START: `--share` of ;;; $base/gn4-container is the only one that must be writable, and the check ;;; is `ls` inside the container. ;;; ;;; Slurm is deliberately NOT in here. A batch scheduler inside the container ;;; that faces the internet would take the controller, the compute daemon and ;;; munge into the webserver's failure domain; it would add slurm, slurmrestd ;;; and munge accounts to a container that otherwise has none; and it would ;;; make every slurm configuration change a webserver rebuild. GN4 needs only ;;; the client half -- ravanan and ccwl -- pointed at a slurmrestd elsewhere ;;; (--slurm-api-endpoint), which is also the shape production has: the cluster ;;; is another box, not a neighbour process. The test fixture for that endpoint ;;; is specials/slurm-container.scm, deployable on a machine of its own exactly ;;; like this one. (There is a practical argument in the other direction -- ;;; run-guix-container supervises one container at a time, so the two cannot be ;;; up together on one host -- and that argues for fixing the daemon or using a ;;; second box, not for merging them.) ;;; ;;; Configuration is in $base/gn4-container/gn4.env (mode 0600, sourced by the ;;; wrapper below), not in this file: it carries SQL_URI's password, and store ;;; paths are world-readable. (use-modules (gnu) (gnu services base) (gnu services shepherd) (gnu services sysctl) (genenetwork services container-ssh) (genenetwork packages) ; shepherd-for-containers (guix gexp)) ;; The service's runtime: python for the plotters, gemma-wrapper (and gemma ;; behind it) for /compute/gemma, R and the R packages the scan scripts load. (use-package-modules base bash admin linux python cran statistics) ;; The GN4 binary as built above and pinned by --root, so a `guix gc` cannot ;; take it away underneath the container. A store-sharing container: the OS ;; here is minimal and every program it runs is a reference into the host ;; store, which is what makes rebuilds cheap and this file short. ;; The store path, *not* the GC-root symlink (`.../gn4-root/bin/genenetwork4`): ;; only the store and the listed paths are mounted inside the container, so a ;; symlink under `factory/` does not exist there. The `--root` from the build ;; above is what keeps this path alive across a `guix gc`. (define %gn4-binary "/gnu/store/y0drkwyhib43sg8micrnhd227mmhp1mf-genenetwork4-git-0.1.0/bin/genenetwork4") ;; gemma-wrapper in a directory of its own -- a Guix profile on the shared ;; gn4-container tree -- rather than in the container's package profile: it is ;; an *engine*, not a tool for whoever ssh-es in, and gn4.env names this ;; directory in GEMMA_DIR (GEMMA_WRAPPER_CMD still points at the program), so ;; there is exactly one copy and one version ;; in play. A directory, not a single store path, because the wrapper shells ;; out to its neighbours: `time -v gemma ...` and `env tar ...` both failed ;; with just the wrapper pinned (measured: Errno::ENOENT inside ;; Open3.popen_run, then "env: 'tar': No such file or directory"), and the ;; profile puts gemma, time, tar, parallel and xz next to it on PATH. ;; The profile lives on the share so that it survives the container, and it ;; pins its own closure, so a `guix gc` cannot take the engine away. (define %gemma-wrapper-dir "/home/wrk/tmp/ai-root/factory/gn4-container/gemma-wrapper/bin") (define %gn4-env-file "/home/wrk/tmp/ai-root/factory/gn4-container/gn4.env") (define %gn4-log-file "/home/wrk/tmp/ai-root/factory/gn4-container/log/gn4ai.log") ;; The wrapper: source the environment file, then exec. `set -a` exports ;; everything it defines, which is how a file of plain KEY=VALUE lines becomes ;; the service's environment without any of it being baked into the store. ;; Working directory: the code tree, as on the host, because the API runs the R ;; and python scripts that live in it. (define %gn4-code-dir "/home/wrk/tmp/ai-root/factory/code/genenetwork/genenetwork4") (define %gn4-start (program-file "gn4ai-start" #~(begin (execl #$(file-append bash "/bin/bash") "bash" "-c" (string-append "set -eu; set -a; . " #$%gn4-env-file "; set +a; cd " #$%gn4-code-dir "; exec " #$%gn4-binary))))) (define gn4ai-service (shepherd-service (provision '(gn4ai)) ;; After the network: the service needs 127.0.0.1 (MariaDB, gn-auth). In a ;; container the network is the host's namespace, up before shepherd runs. (requirement '(user-processes)) (start #~(make-forkexec-constructor (list #$%gn4-start) ;; GEMMA_DIR is the wrapper's own switch (--gemma-dir/GEMMA_DIR): it ;; resolves gemma, time, tar, parallel and xz from there. Set here ;; as well as in gn4.env, so the .scm stays the one place that says ;; where the engine lives. #:environment-variables (list (string-append "GEMMA_DIR=" #$%gemma-wrapper-dir) (string-append "GEMMA_WRAPPER_DIR=" #$%gemma-wrapper-dir) (string-append "PATH=" #$%gemma-wrapper-dir ":/run/current-system/profile/bin:/run/current-system/profile/sbin")) #:log-file #$%gn4-log-file)) (stop #~(make-kill-destructor)) ;; A crash should not leave the container "running" with a dead service; the ;; log file above is where to look, and it is on the shared path. (respawn? #t))) (operating-system (host-name "gn4ai-container") (timezone "Etc/UTC") (locale "en_US.utf8") (bootloader (bootloader-configuration (bootloader grub-bootloader) (targets '("/dev/sdX")))) (file-systems (cons (file-system (device (file-system-label "does-not-matter")) (mount-point "/") (type "ext4")) %base-file-systems)) ;; Run PID 1 with the self-pipe signal variant -- see ;; shepherd-for-containers in (genenetwork packages), and the same block in ;; cgit-container.scm, genecup-container.scm and the rest. shepherd 1.0.9 as ;; PID 1 reads SIGCHLD through signalfd(2), and in the container daemon's ;; sandbox that wake-up never fires: children are reaped but the process ;; monitor never learns, so any service start that waits for a child's exit ;; status (spawn-command, system*, invoke -- the stock nginx's `nginx -t` ;; one-shot is the classic) blocks forever in "starting". forkexec-based ;; starts like gn4ai's happen to survive that, which is exactly why this is ;; easy to leave out -- and then the next one-shot added here hangs the boot. ;; The self-pipe mechanism works in this environment. (essential-services (modify-services (operating-system-default-essential-services this-operating-system) (shepherd-root-service-type config => (shepherd-configuration (inherit config) (shepherd shepherd-for-containers))))) ;; The binary is invoked by absolute store path, but the *engines it shells ;; out to* are looked up on PATH -- so this is not a package list, it is the ;; service's runtime, and leaving one out turns a working endpoint into a 502 ;; naming the missing program (measured: /compute/bnw answered "python3: No ;; such file or directory", /compute/gemma "failed to run gemma-wrapper"). ;; bash/coreutils/grep/procps/shadow are also what a person ssh-ing in ;; expects; 'herd' arrives with the shepherd service. (packages (list bash coreutils grep procps shadow python ; /plot/*, /compute/bulklmm, the gn-bnw driver r ; Rscript: /compute/rqtl, /compute/qtl2, /compute/fitqtl r-qtl r-qtl2 r-optparse r-stringr r-stringi r-digest r-jsonlite)) ;; gemma-wrapper is *not* here: it is pinned by %gemma-wrapper above and ;; named by GEMMA_WRAPPER_CMD in gn4.env, so the container profile does not ;; carry a second copy that could drift from the one the API uses. (services (append (list ;; A plain shepherd-service, not a service-type: `simple-service` is how ;; one is added to the shepherd root (the same shape the sysctl/udev ;; no-ops below use). (simple-service 'gn4ai shepherd-root-service-type (list gn4ai-service))) ;; SSH on 2223: password-less root login with hello-container-key and the ;; pinned ECDSA host key, shared with the other containers. This procedure ;; returns a *list* of services, so it is `append`ed, not wrapped in one. (container-ssh-service #:port-number 2223) (list ;; In a container sysctl (writes /proc/sys) and udev (coldplug on the ;; static /dev the runtime provides) hang the boot: user-processes waits ;; for urandom-seed, which waits for udev. No-ops instead. (simple-service 'container-sysctl shepherd-root-service-type (list (shepherd-service (provision '(sysctl)) (one-shot? #t) (start #~(const #t))))) (simple-service 'container-udev shepherd-root-service-type (list (shepherd-service (provision '(udev)) (one-shot? #t) (start #~(const #t)))))) (modify-services %base-services ;; The guix service runs `guix archive --generate-key' at every boot, ;; which stalls a container, and a minimal container needs no daemon. (delete guix-service-type) (delete sysctl-service-type) (delete udev-service-type)))))