1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
|
;;; gn4ai.scm -- a GN4AI (genenetwork4) system container
;;;
;;; A store-sharing container, run by the run-guix-container daemon, with the
;;; service on :3003 and ssh on :2223 (3000/3001 are the host's own instances,
;;; 8081 is the gn-auth stub, 2222 is another container's ssh, 3306 is MariaDB).
;;;
;;; MariaDB and gn-auth stay *outside*: `--network` shares the host's network
;;; namespace, so 127.0.0.1 inside the container is the host's loopback and
;;; SQL_URI still resolves. Nothing here reaches into the database.
;;;
;;; Build and start (MariaDB and the gn-auth stub must be up first):
;;;
;;; base=/home/wrk/tmp/ai-root/factory
;;;
;;; # the GN4 binary, from genenetwork4/guix.scm, pinned against GC
;;; cd $base/code/genenetwork/genenetwork4
;;; guix build -f guix.scm --root=$base/gn4-root \
;;; -L $base/code/guix-bioinformatics --cores=48 -M 8
;;;
;;; cd $base/code/gn-machines
;;; guix system container -d --network \
;;; -L $base/code/guix-bioinformatics -L $base/code/gn-machines \
;;; --expose=$base/gn4-data=$base/gn4-data \
;;; --expose=$base/gn4-index=$base/gn4-index \
;;; --expose=$base/code=$base/code \
;;; --expose=/home/wrk/.julia=/home/wrk/.julia \
;;; --share=$base/gn4-container=$base/gn4-container \
;;; gn4ai.scm # -> /gnu/store/...-run-container.drv
;;; guix build /gnu/store/...-run-container.drv
;;; $base/code/run-guix-container/bin/run-guix-container.rb \
;;; --daemon 127.0.0.1:8989 --fingerprint <word> <store hash>
;;;
;;; Then, from the host: curl http://127.0.0.1:3003/version
;;; and over ssh: ssh -p 2223 -i ~/hello-container-key root@127.0.0.1
;;;
;;; Paths:
;;; * read-only (`--expose`): the genotype files (42 GB), the Xapian index
;;; (97 GB), the code tree (for the scripts the API runs, the gn-bnw
;;; driver, run-guix-container), and the Julia depot;
;;; * read-write (`--share`): $base/gn4-container -- the env file, HOME,
;;; TMPDIR, the token registry, baskets, the Wikidata cache and every
;;; engine cache. A container's own root is ephemeral, so nothing that
;;; must survive a restart lives in it.
;;;
;;; `--expose`/`--share` drop a host path that does not exist *silently*, so
;;; every path above has to be created before START: `--share` of
;;; $base/gn4-container is the only one that must be writable, and the check
;;; is `ls` inside the container.
;;;
;;; Slurm is deliberately NOT in here. A batch scheduler inside the container
;;; that faces the internet would take the controller, the compute daemon and
;;; munge into the webserver's failure domain; it would add slurm, slurmrestd
;;; and munge accounts to a container that otherwise has none; and it would
;;; make every slurm configuration change a webserver rebuild. GN4 needs only
;;; the client half -- ravanan and ccwl -- pointed at a slurmrestd elsewhere
;;; (--slurm-api-endpoint), which is also the shape production has: the cluster
;;; is another box, not a neighbour process. The test fixture for that endpoint
;;; is specials/slurm-container.scm, deployable on a machine of its own exactly
;;; like this one. (There is a practical argument in the other direction --
;;; run-guix-container supervises one container at a time, so the two cannot be
;;; up together on one host -- and that argues for fixing the daemon or using a
;;; second box, not for merging them.)
;;;
;;; Configuration is in $base/gn4-container/gn4.env (mode 0600, sourced by the
;;; wrapper below), not in this file: it carries SQL_URI's password, and store
;;; paths are world-readable.
(use-modules (gnu)
(gnu services base)
(gnu services shepherd)
(gnu services sysctl)
(genenetwork services container-ssh)
(genenetwork packages) ; shepherd-for-containers
(guix gexp))
;; The service's runtime: python for the plotters, gemma-wrapper (and gemma
;; behind it) for /compute/gemma, R and the R packages the scan scripts load.
(use-package-modules base bash admin linux python cran statistics)
;; The GN4 binary as built above and pinned by --root, so a `guix gc` cannot
;; take it away underneath the container. A store-sharing container: the OS
;; here is minimal and every program it runs is a reference into the host
;; store, which is what makes rebuilds cheap and this file short.
;; The store path, *not* the GC-root symlink (`.../gn4-root/bin/genenetwork4`):
;; only the store and the listed paths are mounted inside the container, so a
;; symlink under `factory/` does not exist there. The `--root` from the build
;; above is what keeps this path alive across a `guix gc`.
(define %gn4-binary
"/gnu/store/y0drkwyhib43sg8micrnhd227mmhp1mf-genenetwork4-git-0.1.0/bin/genenetwork4")
;; gemma-wrapper in a directory of its own -- a Guix profile on the shared
;; gn4-container tree -- rather than in the container's package profile: it is
;; an *engine*, not a tool for whoever ssh-es in, and gn4.env names this
;; directory in GEMMA_DIR (GEMMA_WRAPPER_CMD still points at the program), so
;; there is exactly one copy and one version
;; in play. A directory, not a single store path, because the wrapper shells
;; out to its neighbours: `time -v gemma ...` and `env tar ...` both failed
;; with just the wrapper pinned (measured: Errno::ENOENT inside
;; Open3.popen_run, then "env: 'tar': No such file or directory"), and the
;; profile puts gemma, time, tar, parallel and xz next to it on PATH.
;; The profile lives on the share so that it survives the container, and it
;; pins its own closure, so a `guix gc` cannot take the engine away.
(define %gemma-wrapper-dir
"/home/wrk/tmp/ai-root/factory/gn4-container/gemma-wrapper/bin")
(define %gn4-env-file
"/home/wrk/tmp/ai-root/factory/gn4-container/gn4.env")
(define %gn4-log-file
"/home/wrk/tmp/ai-root/factory/gn4-container/log/gn4ai.log")
;; The wrapper: source the environment file, then exec. `set -a` exports
;; everything it defines, which is how a file of plain KEY=VALUE lines becomes
;; the service's environment without any of it being baked into the store.
;; Working directory: the code tree, as on the host, because the API runs the R
;; and python scripts that live in it.
(define %gn4-code-dir
"/home/wrk/tmp/ai-root/factory/code/genenetwork/genenetwork4")
(define %gn4-start
(program-file
"gn4ai-start"
#~(begin
(execl #$(file-append bash "/bin/bash")
"bash" "-c"
(string-append "set -eu; set -a; . " #$%gn4-env-file
"; set +a; cd " #$%gn4-code-dir
"; exec " #$%gn4-binary)))))
(define gn4ai-service
(shepherd-service
(provision '(gn4ai))
;; After the network: the service needs 127.0.0.1 (MariaDB, gn-auth). In a
;; container the network is the host's namespace, up before shepherd runs.
(requirement '(user-processes))
(start #~(make-forkexec-constructor
(list #$%gn4-start)
;; GEMMA_DIR is the wrapper's own switch (--gemma-dir/GEMMA_DIR): it
;; resolves gemma, time, tar, parallel and xz from there. Set here
;; as well as in gn4.env, so the .scm stays the one place that says
;; where the engine lives.
#:environment-variables
(list (string-append "GEMMA_DIR=" #$%gemma-wrapper-dir)
(string-append "GEMMA_WRAPPER_DIR=" #$%gemma-wrapper-dir)
(string-append "PATH=" #$%gemma-wrapper-dir
":/run/current-system/profile/bin:/run/current-system/profile/sbin"))
#:log-file #$%gn4-log-file))
(stop #~(make-kill-destructor))
;; A crash should not leave the container "running" with a dead service; the
;; log file above is where to look, and it is on the shared path.
(respawn? #t)))
(operating-system
(host-name "gn4ai-container")
(timezone "Etc/UTC")
(locale "en_US.utf8")
(bootloader (bootloader-configuration
(bootloader grub-bootloader)
(targets '("/dev/sdX"))))
(file-systems (cons (file-system
(device (file-system-label "does-not-matter"))
(mount-point "/")
(type "ext4"))
%base-file-systems))
;; Run PID 1 with the self-pipe signal variant -- see
;; shepherd-for-containers in (genenetwork packages), and the same block in
;; cgit-container.scm, genecup-container.scm and the rest. shepherd 1.0.9 as
;; PID 1 reads SIGCHLD through signalfd(2), and in the container daemon's
;; sandbox that wake-up never fires: children are reaped but the process
;; monitor never learns, so any service start that waits for a child's exit
;; status (spawn-command, system*, invoke -- the stock nginx's `nginx -t`
;; one-shot is the classic) blocks forever in "starting". forkexec-based
;; starts like gn4ai's happen to survive that, which is exactly why this is
;; easy to leave out -- and then the next one-shot added here hangs the boot.
;; The self-pipe mechanism works in this environment.
(essential-services
(modify-services (operating-system-default-essential-services
this-operating-system)
(shepherd-root-service-type config =>
(shepherd-configuration
(inherit config)
(shepherd shepherd-for-containers)))))
;; The binary is invoked by absolute store path, but the *engines it shells
;; out to* are looked up on PATH -- so this is not a package list, it is the
;; service's runtime, and leaving one out turns a working endpoint into a 502
;; naming the missing program (measured: /compute/bnw answered "python3: No
;; such file or directory", /compute/gemma "failed to run gemma-wrapper").
;; bash/coreutils/grep/procps/shadow are also what a person ssh-ing in
;; expects; 'herd' arrives with the shepherd service.
(packages (list bash coreutils grep procps shadow
python ; /plot/*, /compute/bulklmm, the gn-bnw driver
r ; Rscript: /compute/rqtl, /compute/qtl2, /compute/fitqtl
r-qtl r-qtl2 r-optparse r-stringr r-stringi r-digest r-jsonlite))
;; gemma-wrapper is *not* here: it is pinned by %gemma-wrapper above and
;; named by GEMMA_WRAPPER_CMD in gn4.env, so the container profile does not
;; carry a second copy that could drift from the one the API uses.
(services
(append
(list
;; A plain shepherd-service, not a service-type: `simple-service` is how
;; one is added to the shepherd root (the same shape the sysctl/udev
;; no-ops below use).
(simple-service 'gn4ai shepherd-root-service-type (list gn4ai-service)))
;; SSH on 2223: password-less root login with hello-container-key and the
;; pinned ECDSA host key, shared with the other containers. This procedure
;; returns a *list* of services, so it is `append`ed, not wrapped in one.
(container-ssh-service #:port-number 2223)
(list
;; In a container sysctl (writes /proc/sys) and udev (coldplug on the
;; static /dev the runtime provides) hang the boot: user-processes waits
;; for urandom-seed, which waits for udev. No-ops instead.
(simple-service 'container-sysctl
shepherd-root-service-type
(list (shepherd-service
(provision '(sysctl))
(one-shot? #t)
(start #~(const #t)))))
(simple-service 'container-udev
shepherd-root-service-type
(list (shepherd-service
(provision '(udev))
(one-shot? #t)
(start #~(const #t))))))
(modify-services %base-services
;; The guix service runs `guix archive --generate-key' at every boot,
;; which stalls a container, and a minimal container needs no daemon.
(delete guix-service-type)
(delete sysctl-service-type)
(delete udev-service-type)))))
|