1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
|
;;; genenetwork-machines --- Guix configuration for genenetwork machines
;;; Copyright © 2026 Pjotr Prins <pjotr.public01@thebird.nl>
;;;
;;; This file is part of genenetwork-machines.
;;;
;;; genenetwork-machines is free software: you can redistribute it
;;; and/or modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; genenetwork-machines is distributed in the hope that it will be
;;; useful, but WITHOUT ANY WARRANTY; without even the implied
;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
;;; See the GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with genenetwork-machines. If not, see
;;; <https://www.gnu.org/licenses/>.
;;; A self-contained Slurm cluster (slurmctld, slurmd, slurmrestd with
;;; JWT auth) as a VM, to test ravanan's slurm-api batch system --
;;; one node, one partition, no accounting database.
;;;
;;; Same cluster as ../specials/slurm-container.scm, as a VM instead
;;; of a `guix system container`: a VM has its own real kernel (so a
;;; genuine /sys/fs/cgroup and /var/run, no shared-kernel container
;;; quirks) and its own writable disk (state, including host SSH
;;; keys, survives reboots), and runs with an ordinary unprivileged
;;; `qemu-system-x86_64` -- no root, no privileged daemon needed, just
;;; KVM if available for speed (there is none in some sandboxes; it
;;; still boots under software emulation, just slower). Still, every
;;; workaround from the container version is kept here too, since
;;; they are not actually container-specific:
;;;
;;; - CgroupPlugin=disabled: untested here whether a VM's own real
;;; cgroup hierarchy would let slurmd use it instead -- not tried,
;;; to keep this a direct port of the already-verified container
;;; recipe.
;;; - wait-for-munge-then-exec: shepherd's (requirement '(munged))
;;; only waits for munged's fork to succeed, not for it to finish
;;; initializing and create its socket, regardless of container vs
;;; VM.
;;; - The jwt-bootstrap.token fixed-key dance: AuthType has to be
;;; auth/jwt (not auth/munge) for rest_auth/jwt's per-request
;;; identity switch to work at all (auth/munge unconditionally
;;; refuses it), but auth/jwt's own init() only loads the
;;; verification key for slurmctld/slurmdbd and treats slurmd as a
;;; "client command" needing a token just to start -- a Slurm
;;; code-level restriction with nothing to do with privilege or
;;; sandboxing, so it is reproduced here unchanged. See the
;;; commit introducing specials/slurm-container.scm for the
;;; one-off Python snippet that computed %jwt-fixed-bootstrap-token.
;;; - slurmrestd running as its own dedicated "slurmrestd" account:
;;; it refuses to run as root or as SlurmUser, and the
;;; SLURMRESTD_SECURITY=disable_user_check/become_user escape
;;; hatches are compiled out of ordinary release builds.
;;;
;;; What a VM *does* change: slurmd can still never verify a job
;;; launch RPC under AuthType=auth/jwt (same Slurm code-level gate),
;;; so jobs submitted over the REST API still will not actually run
;;; -- this is not a container limitation to begin with, so moving to
;;; a VM does not fix it either.
;;;
;;; Build (no -L needed; slurm/munge come from the main channel):
;;;
;;; guix system image -t qcow2 vms/slurm-vm.scm
;;;
;;; Run (hostfwd maps host ports to guest ssh/slurmrestd; add
;;; -enable-kvm if /dev/kvm exists, for usable speed):
;;;
;;; cp /gnu/store/...-image.qcow2 /tmp/slurm-vm.qcow2
;;; chmod u+w /tmp/slurm-vm.qcow2
;;; qemu-system-x86_64 -m 1536 -drive file=/tmp/slurm-vm.qcow2,format=qcow2 \
;;; -netdev user,id=n0,hostfwd=tcp:127.0.0.1:2224-:22,hostfwd=tcp:127.0.0.1:6820-:6820 \
;;; -device virtio-net-pci,netdev=n0 -nographic
;;;
;;; Then from the host, straight away (no manual setup step needed):
;;;
;;; ssh -p 2224 -i ../../run-guix-container/etc/hello-container-key root@127.0.0.1
;;; curl -H "X-SLURM-USER-TOKEN: <%jwt-fixed-bootstrap-token>" \
;;; -H "X-SLURM-USER-NAME: testuser" \
;;; http://127.0.0.1:6820/slurm/v0.0.41/ping
(use-modules (gnu)
(gnu packages admin)
(gnu packages base)
(gnu packages parallel)
(gnu services base)
(gnu services networking)
(gnu services shepherd)
(gnu services ssh)
(guix gexp)
(rnrs bytevectors)
(srfi srfi-1))
;; The main channel's plain `slurm' is 23.11.10 and has no
;; CgroupPlugin=disabled escape hatch (added in a later release) --
;; use the 25.05 series instead (already exported by (gnu packages
;; parallel), restd+jwt included, same as `slurm').
(define slurm slurm-25.05)
(define %slurm-etc "/etc/slurm")
(define %slurm-conf (string-append %slurm-etc "/slurm.conf"))
(define %jwt-key (string-append %slurm-etc "/jwt_hs256.key"))
(define %munge-key "/etc/munge/munge.key")
(define %cgroup-conf (string-append %slurm-etc "/cgroup.conf"))
(define cgroup-conf-file
(plain-file "cgroup.conf" "CgroupPlugin=disabled\n"))
;; See the top-level comment on wait-for-munge-then-exec and the
;; jwt-bootstrap.token dance in ../specials/slurm-container.scm for
;; the full rationale -- unchanged here, neither is container-specific.
(define (wait-for-munge-then-exec name binary . args)
(program-file
(string-append "wait-for-munge-then-" name)
#~(begin
(use-modules (ice-9 rdelim))
(let loop ((n 0))
(unless (or (file-exists? "/var/run/munge/munge.socket.2")
(> n 60))
(sleep 1)
(loop (+ n 1))))
(let ((token-file "/etc/slurm/jwt-bootstrap.token"))
(when (file-exists? token-file)
(setenv "SLURM_JWT" (call-with-input-file token-file read-line))))
(apply execl #$binary #$binary '#$args))))
(define slurm-conf-file
(plain-file "slurm.conf"
(string-append
"ClusterName=slurm-vm\n"
"SlurmctldHost=localhost\n"
"SlurmUser=slurm\n"
"\n"
;; AuthType is auth/jwt, not auth/munge: rest_auth/jwt's
;; per-request identity switch (auth_g_thread_config) is
;; unconditionally refused by auth/munge ("auth/munge
;; does not accept user provided auth token") -- it
;; only works when AuthType is itself auth/jwt. munge
;; is still used for CredType (job step credentials).
"AuthType=auth/jwt\n"
"AuthAltParameters=jwt_key=" %jwt-key "\n"
"\n"
"SlurmctldPort=6817\n"
"SlurmdPort=6818\n"
"SlurmdSpoolDir=/var/spool/slurmd\n"
"StateSaveLocation=/var/spool/slurmctld\n"
"SlurmctldLogFile=/var/log/slurm/slurmctld.log\n"
"SlurmdLogFile=/var/log/slurm/slurmd.log\n"
"\n"
"ProctrackType=proctrack/linuxproc\n"
"TaskPlugin=task/none\n"
"SchedulerType=sched/backfill\n"
"SelectType=select/cons_tres\n"
"SelectTypeParameters=CR_Core\n"
"ReturnToService=2\n"
"MpiDefault=none\n"
"\n"
;; No slurmdbd/mariadb in this throwaway VM -- just
;; enough to prove ravanan's slurm-api submit/poll
;; cycle runs against a real cluster.
"AccountingStorageType=accounting_storage/none\n"
"JobAcctGatherType=jobacct_gather/none\n"
"\n"
"NodeName=slurm-vm NodeAddr=127.0.0.1 CPUs=1 RealMemory=1024 State=UNKNOWN\n"
"PartitionName=debug Nodes=slurm-vm Default=YES MaxTime=INFINITE State=UP\n")))
;; Fixed JWT key + a token pre-computed against it offline, for the
;; same reason as in ../specials/slurm-container.scm: AuthType=auth/jwt
;; needs a bootstrap token in place before the very first boot, and
;; there is no live cluster yet to mint one from. Fine for a throwaway
;; test VM, not a real deployment: do not reuse this key or token
;; anywhere real.
(define %jwt-fixed-key-bytes
(u8-list->bytevector (iota 32)))
;; Token for "testuser" (sun claim), 5-year expiry, HS256 against
;; %jwt-fixed-key-bytes -- see the commit introducing
;; ../specials/slurm-container.scm for the one-off Python snippet.
(define %jwt-fixed-bootstrap-token
"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NTAxOTY2MTYsImlhdCI6MTc5MjUxNjYxNiwic3VuIjoidGVzdHVzZXIifQ.zsTqK2yvFbIQP-2hJwia6JsqtBmtxvNZjNvqAGdJfJ4")
;; Generate the munge key once, at first activation (it is only used
;; for CredType job-step credentials here, not for cluster-wide RPC
;; auth, so it does not need to be fixed across boots -- and unlike
;; the container, this VM's disk is writable and persists across
;; reboots, so "once" really means once), write the fixed JWT key and
;; its pre-computed bootstrap token, and lay out the rest of
;; /etc/slurm, /etc/munge, and the various spool/log dirs.
(define slurm-keys-activation
#~(begin
(use-modules (guix build utils))
(use-modules (ice-9 binary-ports))
(define (write-random-key path n-bytes)
(unless (file-exists? path)
(let ((bytes (call-with-input-file "/dev/urandom"
(lambda (urandom) (get-bytevector-n urandom n-bytes))
#:binary #t)))
(call-with-output-file path
(lambda (port) (put-bytevector port bytes))
#:binary #t))
(chmod path #o400)))
(mkdir-p "/etc/munge")
(mkdir-p "/var/run/munge")
(mkdir-p "/var/log/munge")
(mkdir-p "/var/lib/munge")
(mkdir-p #$%slurm-etc)
(mkdir-p "/var/spool/slurmd")
(mkdir-p "/var/spool/slurmctld")
(mkdir-p "/var/log/slurm")
(write-random-key #$%munge-key 1024)
(unless (file-exists? #$%jwt-key)
(call-with-output-file #$%jwt-key
(lambda (port) (put-bytevector port '#$%jwt-fixed-key-bytes))
#:binary #t)
(chmod #$%jwt-key #o400))
(call-with-output-file "/etc/slurm/jwt-bootstrap.token"
(lambda (port) (display #$%jwt-fixed-bootstrap-token port)))
(copy-file #$cgroup-conf-file #$%cgroup-conf)
(copy-file #$slurm-conf-file #$%slurm-conf)
;; slurmctld (running as root, dropping to SlurmUser) checks
;; StateSaveLocation/SlurmdSpoolDir ownership *before* dropping
;; privileges -- they must already belong to SlurmUser.
;; Likewise munge's own key/state must belong to the munge user.
(let ((slurm-uid (passwd:uid (getpwnam "slurm")))
(slurm-gid (passwd:gid (getpwnam "slurm")))
(munge-uid (passwd:uid (getpwnam "munge")))
(munge-gid (passwd:gid (getpwnam "munge"))))
(chown "/var/spool/slurmctld" slurm-uid slurm-gid)
(chown "/var/spool/slurmd" slurm-uid slurm-gid)
(chown "/var/log/slurm" slurm-uid slurm-gid)
(chown "/etc/munge" munge-uid munge-gid)
(chown #$%munge-key munge-uid munge-gid)
(chown "/var/run/munge" munge-uid munge-gid)
(chown "/var/log/munge" munge-uid munge-gid)
(chown "/var/lib/munge" munge-uid munge-gid))))
(operating-system
(host-name "slurm-vm")
(timezone "Europe/Amsterdam")
(locale "en_US.utf8")
(kernel-arguments '("console=ttyS0,115200"))
(bootloader (bootloader-configuration
(bootloader grub-bootloader)
(targets '("/dev/vda"))))
;; The qcow2 image type creates a root partition labelled
;; "Guix_image" (hardcoded in gnu/system/image.scm).
(file-systems (cons (file-system
(device (file-system-label "Guix_image"))
(mount-point "/")
(type "ext4"))
%base-file-systems))
(users (cons* (user-account
(name "root")
(uid 0)
(group "root")
;; Password "hello", matching the other VMs in this
;; directory -- for the serial console; ssh uses the
;; shared hello-container-key below instead.
(password (crypt "hello" "$6$abc$0123456789abcdef$")))
(user-account
(name "munge")
(system? #t)
(group "munge")
(comment "munge daemon")
(home-directory "/var/lib/munge")
(shell (file-append shadow "/sbin/nologin")))
(user-account
(name "slurm")
(system? #t)
(group "slurm")
(comment "SlurmUser -- slurmctld's own identity, distinct from root")
(home-directory "/var/lib/slurm")
(shell (file-append shadow "/sbin/nologin")))
(user-account
(name "slurmrestd")
(system? #t)
(group "slurmrestd")
(comment "Runs slurmrestd -- must be neither root nor SlurmUser")
(home-directory "/var/lib/slurmrestd")
(shell (file-append shadow "/sbin/nologin")))
;; A real, ordinary user to submit test jobs as via
;; the REST API.
(user-account
(name "testuser")
(comment "Submits test jobs via the REST API")
(password (crypt "hello" "$6$abc$0123456789abcdef$"))
(group "users")
(supplementary-groups '("users")))
%base-user-accounts))
(groups (cons* (user-group (name "munge") (system? #t))
(user-group (name "slurm") (system? #t))
(user-group (name "slurmrestd") (system? #t))
%base-groups))
(packages (cons* slurm munge %base-packages))
(services
(append
(list
;; Password-less root login with the hello-container-key shared
;; by the other VMs/containers in this repository, on the
;; guest's port 22; the VM runner forwards a host port to it.
(service openssh-service-type
(openssh-configuration
(permit-root-login #t)
(password-authentication? #f)
(authorized-keys
`(("root" ,(local-file
"../../run-guix-container/etc/hello-container-key.pub"
"authorized_keys"))))))
;; QEMU user-mode network stack: guest gets 10.0.2.15, gateway
;; 10.0.2.2 (see %qemu-static-networking in gnu/services/base.scm).
(service static-networking-service-type (list %qemu-static-networking))
;; Serial console login, for debugging when ssh does not work.
(service agetty-service-type
(agetty-configuration
(extra-options '("-L"))
(baud-rate "115200")
(term "vt100")
(tty "ttyS0")))
(simple-service 'slurm-keys-and-conf
activation-service-type
slurm-keys-activation)
(simple-service 'munged
shepherd-root-service-type
(list (shepherd-service
(provision '(munged))
(requirement '(user-processes))
(start #~(make-forkexec-constructor
(list #$(file-append munge "/sbin/munged")
"--force"
(string-append "--key-file=" #$%munge-key)
"--socket=/var/run/munge/munge.socket.2"
"--pid-file=/var/run/munge/munged.pid"
"--seed-file=/var/lib/munge/munged.seed")
#:log-file "/var/log/slurm/munged.log"))
(stop #~(make-kill-destructor)))))
(simple-service 'slurmctld
shepherd-root-service-type
(list (shepherd-service
(provision '(slurmctld))
(requirement '(munged))
(start #~(make-forkexec-constructor
(list #$(wait-for-munge-then-exec
"slurmctld"
(file-append slurm "/sbin/slurmctld")
"-D" "-c" "-f" %slurm-conf))
#:log-file "/var/log/slurm/slurmctld-stdout.log"))
(stop #~(make-kill-destructor)))))
(simple-service 'slurmd
shepherd-root-service-type
(list (shepherd-service
(provision '(slurmd))
(requirement '(munged slurmctld))
(start #~(make-forkexec-constructor
(list #$(wait-for-munge-then-exec
"slurmd"
(file-append slurm "/sbin/slurmd")
"-D" "-c" "-f" %slurm-conf))
#:log-file "/var/log/slurm/slurmd-stdout.log"))
(stop #~(make-kill-destructor)))))
(simple-service 'slurmrestd
shepherd-root-service-type
(list (shepherd-service
(provision '(slurmrestd))
(requirement '(munged slurmctld slurmd))
(start #~(make-forkexec-constructor
(list #$(wait-for-munge-then-exec
"slurmrestd"
(file-append slurm "/sbin/slurmrestd")
"-f" %slurm-conf
"-a" "rest_auth/jwt"
"-s" "openapi/slurmctld"
"0.0.0.0:6820"))
;; Run as a dedicated, unprivileged
;; "slurmrestd" account -- see the
;; top-level comment above.
#:user "slurmrestd"
#:group "slurmrestd"
#:log-file "/var/log/slurm/slurmrestd-stdout.log"))
(stop #~(make-kill-destructor)))))
)
(modify-services %base-services
;; No guix-daemon in the VM: it is not needed (images are built
;; on the host), and its service would pull guix, guix-daemon
;; etc. into the system profile.
(delete guix-service-type)))))
|