;;; genenetwork-machines --- Guix configuration for genenetwork machines ;;; Copyright © 2026 Pjotr Prins ;;; ;;; This file is part of genenetwork-machines. ;;; ;;; genenetwork-machines is free software: you can redistribute it ;;; and/or modify it under the terms of the GNU General Public License ;;; as published by the Free Software Foundation, either version 3 of ;;; the License, or (at your option) any later version. ;;; ;;; genenetwork-machines is distributed in the hope that it will be ;;; useful, but WITHOUT ANY WARRANTY; without even the implied ;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. ;;; See the GNU General Public License for more details. ;;; ;;; You should have received a copy of the GNU General Public License ;;; along with genenetwork-machines. If not, see ;;; . ;;; A self-contained Slurm cluster (slurmctld, slurmd, slurmrestd with ;;; JWT auth) as a VM, to test ravanan's slurm-api batch system -- ;;; one node, one partition, no accounting database. ;;; ;;; Same cluster as ../specials/slurm-container.scm, as a VM instead ;;; of a `guix system container`: a VM has its own real kernel (so a ;;; genuine /sys/fs/cgroup and /var/run, no shared-kernel container ;;; quirks) and its own writable disk (state, including host SSH ;;; keys, survives reboots), and runs with an ordinary unprivileged ;;; `qemu-system-x86_64` -- no root, no privileged daemon needed, just ;;; KVM if available for speed (there is none in some sandboxes; it ;;; still boots under software emulation, just slower). Still, every ;;; workaround from the container version is kept here too, since ;;; they are not actually container-specific: ;;; ;;; - CgroupPlugin=disabled: untested here whether a VM's own real ;;; cgroup hierarchy would let slurmd use it instead -- not tried, ;;; to keep this a direct port of the already-verified container ;;; recipe. ;;; - wait-for-munge-then-exec: shepherd's (requirement '(munged)) ;;; only waits for munged's fork to succeed, not for it to finish ;;; initializing and create its socket, regardless of container vs ;;; VM. ;;; - The jwt-bootstrap.token fixed-key dance: AuthType has to be ;;; auth/jwt (not auth/munge) for rest_auth/jwt's per-request ;;; identity switch to work at all (auth/munge unconditionally ;;; refuses it), but auth/jwt's own init() only loads the ;;; verification key for slurmctld/slurmdbd and treats slurmd as a ;;; "client command" needing a token just to start -- a Slurm ;;; code-level restriction with nothing to do with privilege or ;;; sandboxing, so it is reproduced here unchanged. See the ;;; commit introducing specials/slurm-container.scm for the ;;; one-off Python snippet that computed %jwt-fixed-bootstrap-token. ;;; - slurmrestd running as its own dedicated "slurmrestd" account: ;;; it refuses to run as root or as SlurmUser, and the ;;; SLURMRESTD_SECURITY=disable_user_check/become_user escape ;;; hatches are compiled out of ordinary release builds. ;;; ;;; What a VM *does* change: slurmd can still never verify a job ;;; launch RPC under AuthType=auth/jwt (same Slurm code-level gate), ;;; so jobs submitted over the REST API still will not actually run ;;; -- this is not a container limitation to begin with, so moving to ;;; a VM does not fix it either. ;;; ;;; Build (no -L needed; slurm/munge come from the main channel): ;;; ;;; guix system image -t qcow2 vms/slurm-vm.scm ;;; ;;; Run (hostfwd maps host ports to guest ssh/slurmrestd; add ;;; -enable-kvm if /dev/kvm exists, for usable speed): ;;; ;;; cp /gnu/store/...-image.qcow2 /tmp/slurm-vm.qcow2 ;;; chmod u+w /tmp/slurm-vm.qcow2 ;;; qemu-system-x86_64 -m 1536 -drive file=/tmp/slurm-vm.qcow2,format=qcow2 \ ;;; -netdev user,id=n0,hostfwd=tcp:127.0.0.1:2224-:22,hostfwd=tcp:127.0.0.1:6820-:6820 \ ;;; -device virtio-net-pci,netdev=n0 -nographic ;;; ;;; Then from the host, straight away (no manual setup step needed): ;;; ;;; ssh -p 2224 -i ../../run-guix-container/etc/hello-container-key root@127.0.0.1 ;;; curl -H "X-SLURM-USER-TOKEN: <%jwt-fixed-bootstrap-token>" \ ;;; -H "X-SLURM-USER-NAME: testuser" \ ;;; http://127.0.0.1:6820/slurm/v0.0.41/ping (use-modules (gnu) (gnu packages admin) (gnu packages base) (gnu packages parallel) (gnu services base) (gnu services networking) (gnu services shepherd) (gnu services ssh) (guix gexp) (rnrs bytevectors) (srfi srfi-1)) ;; The main channel's plain `slurm' is 23.11.10 and has no ;; CgroupPlugin=disabled escape hatch (added in a later release) -- ;; use the 25.05 series instead (already exported by (gnu packages ;; parallel), restd+jwt included, same as `slurm'). (define slurm slurm-25.05) (define %slurm-etc "/etc/slurm") (define %slurm-conf (string-append %slurm-etc "/slurm.conf")) (define %jwt-key (string-append %slurm-etc "/jwt_hs256.key")) (define %munge-key "/etc/munge/munge.key") (define %cgroup-conf (string-append %slurm-etc "/cgroup.conf")) (define cgroup-conf-file (plain-file "cgroup.conf" "CgroupPlugin=disabled\n")) ;; See the top-level comment on wait-for-munge-then-exec and the ;; jwt-bootstrap.token dance in ../specials/slurm-container.scm for ;; the full rationale -- unchanged here, neither is container-specific. (define (wait-for-munge-then-exec name binary . args) (program-file (string-append "wait-for-munge-then-" name) #~(begin (use-modules (ice-9 rdelim)) (let loop ((n 0)) (unless (or (file-exists? "/var/run/munge/munge.socket.2") (> n 60)) (sleep 1) (loop (+ n 1)))) (let ((token-file "/etc/slurm/jwt-bootstrap.token")) (when (file-exists? token-file) (setenv "SLURM_JWT" (call-with-input-file token-file read-line)))) (apply execl #$binary #$binary '#$args)))) (define slurm-conf-file (plain-file "slurm.conf" (string-append "ClusterName=slurm-vm\n" "SlurmctldHost=localhost\n" "SlurmUser=slurm\n" "\n" ;; AuthType is auth/jwt, not auth/munge: rest_auth/jwt's ;; per-request identity switch (auth_g_thread_config) is ;; unconditionally refused by auth/munge ("auth/munge ;; does not accept user provided auth token") -- it ;; only works when AuthType is itself auth/jwt. munge ;; is still used for CredType (job step credentials). "AuthType=auth/jwt\n" "AuthAltParameters=jwt_key=" %jwt-key "\n" "\n" "SlurmctldPort=6817\n" "SlurmdPort=6818\n" "SlurmdSpoolDir=/var/spool/slurmd\n" "StateSaveLocation=/var/spool/slurmctld\n" "SlurmctldLogFile=/var/log/slurm/slurmctld.log\n" "SlurmdLogFile=/var/log/slurm/slurmd.log\n" "\n" "ProctrackType=proctrack/linuxproc\n" "TaskPlugin=task/none\n" "SchedulerType=sched/backfill\n" "SelectType=select/cons_tres\n" "SelectTypeParameters=CR_Core\n" "ReturnToService=2\n" "MpiDefault=none\n" "\n" ;; No slurmdbd/mariadb in this throwaway VM -- just ;; enough to prove ravanan's slurm-api submit/poll ;; cycle runs against a real cluster. "AccountingStorageType=accounting_storage/none\n" "JobAcctGatherType=jobacct_gather/none\n" "\n" "NodeName=slurm-vm NodeAddr=127.0.0.1 CPUs=1 RealMemory=1024 State=UNKNOWN\n" "PartitionName=debug Nodes=slurm-vm Default=YES MaxTime=INFINITE State=UP\n"))) ;; Fixed JWT key + a token pre-computed against it offline, for the ;; same reason as in ../specials/slurm-container.scm: AuthType=auth/jwt ;; needs a bootstrap token in place before the very first boot, and ;; there is no live cluster yet to mint one from. Fine for a throwaway ;; test VM, not a real deployment: do not reuse this key or token ;; anywhere real. (define %jwt-fixed-key-bytes (u8-list->bytevector (iota 32))) ;; Token for "testuser" (sun claim), 5-year expiry, HS256 against ;; %jwt-fixed-key-bytes -- see the commit introducing ;; ../specials/slurm-container.scm for the one-off Python snippet. (define %jwt-fixed-bootstrap-token "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE5NTAxOTY2MTYsImlhdCI6MTc5MjUxNjYxNiwic3VuIjoidGVzdHVzZXIifQ.zsTqK2yvFbIQP-2hJwia6JsqtBmtxvNZjNvqAGdJfJ4") ;; Generate the munge key once, at first activation (it is only used ;; for CredType job-step credentials here, not for cluster-wide RPC ;; auth, so it does not need to be fixed across boots -- and unlike ;; the container, this VM's disk is writable and persists across ;; reboots, so "once" really means once), write the fixed JWT key and ;; its pre-computed bootstrap token, and lay out the rest of ;; /etc/slurm, /etc/munge, and the various spool/log dirs. (define slurm-keys-activation #~(begin (use-modules (guix build utils)) (use-modules (ice-9 binary-ports)) (define (write-random-key path n-bytes) (unless (file-exists? path) (let ((bytes (call-with-input-file "/dev/urandom" (lambda (urandom) (get-bytevector-n urandom n-bytes)) #:binary #t))) (call-with-output-file path (lambda (port) (put-bytevector port bytes)) #:binary #t)) (chmod path #o400))) (mkdir-p "/etc/munge") (mkdir-p "/var/run/munge") (mkdir-p "/var/log/munge") (mkdir-p "/var/lib/munge") (mkdir-p #$%slurm-etc) (mkdir-p "/var/spool/slurmd") (mkdir-p "/var/spool/slurmctld") (mkdir-p "/var/log/slurm") (write-random-key #$%munge-key 1024) (unless (file-exists? #$%jwt-key) (call-with-output-file #$%jwt-key (lambda (port) (put-bytevector port '#$%jwt-fixed-key-bytes)) #:binary #t) (chmod #$%jwt-key #o400)) (call-with-output-file "/etc/slurm/jwt-bootstrap.token" (lambda (port) (display #$%jwt-fixed-bootstrap-token port))) (copy-file #$cgroup-conf-file #$%cgroup-conf) (copy-file #$slurm-conf-file #$%slurm-conf) ;; slurmctld (running as root, dropping to SlurmUser) checks ;; StateSaveLocation/SlurmdSpoolDir ownership *before* dropping ;; privileges -- they must already belong to SlurmUser. ;; Likewise munge's own key/state must belong to the munge user. (let ((slurm-uid (passwd:uid (getpwnam "slurm"))) (slurm-gid (passwd:gid (getpwnam "slurm"))) (munge-uid (passwd:uid (getpwnam "munge"))) (munge-gid (passwd:gid (getpwnam "munge")))) (chown "/var/spool/slurmctld" slurm-uid slurm-gid) (chown "/var/spool/slurmd" slurm-uid slurm-gid) (chown "/var/log/slurm" slurm-uid slurm-gid) (chown "/etc/munge" munge-uid munge-gid) (chown #$%munge-key munge-uid munge-gid) (chown "/var/run/munge" munge-uid munge-gid) (chown "/var/log/munge" munge-uid munge-gid) (chown "/var/lib/munge" munge-uid munge-gid)))) (operating-system (host-name "slurm-vm") (timezone "Europe/Amsterdam") (locale "en_US.utf8") (kernel-arguments '("console=ttyS0,115200")) (bootloader (bootloader-configuration (bootloader grub-bootloader) (targets '("/dev/vda")))) ;; The qcow2 image type creates a root partition labelled ;; "Guix_image" (hardcoded in gnu/system/image.scm). (file-systems (cons (file-system (device (file-system-label "Guix_image")) (mount-point "/") (type "ext4")) %base-file-systems)) (users (cons* (user-account (name "root") (uid 0) (group "root") ;; Password "hello", matching the other VMs in this ;; directory -- for the serial console; ssh uses the ;; shared hello-container-key below instead. (password (crypt "hello" "$6$abc$0123456789abcdef$"))) (user-account (name "munge") (system? #t) (group "munge") (comment "munge daemon") (home-directory "/var/lib/munge") (shell (file-append shadow "/sbin/nologin"))) (user-account (name "slurm") (system? #t) (group "slurm") (comment "SlurmUser -- slurmctld's own identity, distinct from root") (home-directory "/var/lib/slurm") (shell (file-append shadow "/sbin/nologin"))) (user-account (name "slurmrestd") (system? #t) (group "slurmrestd") (comment "Runs slurmrestd -- must be neither root nor SlurmUser") (home-directory "/var/lib/slurmrestd") (shell (file-append shadow "/sbin/nologin"))) ;; A real, ordinary user to submit test jobs as via ;; the REST API. (user-account (name "testuser") (comment "Submits test jobs via the REST API") (password (crypt "hello" "$6$abc$0123456789abcdef$")) (group "users") (supplementary-groups '("users"))) %base-user-accounts)) (groups (cons* (user-group (name "munge") (system? #t)) (user-group (name "slurm") (system? #t)) (user-group (name "slurmrestd") (system? #t)) %base-groups)) (packages (cons* slurm munge %base-packages)) (services (append (list ;; Password-less root login with the hello-container-key shared ;; by the other VMs/containers in this repository, on the ;; guest's port 22; the VM runner forwards a host port to it. (service openssh-service-type (openssh-configuration (permit-root-login #t) (password-authentication? #f) (authorized-keys `(("root" ,(local-file "../../run-guix-container/etc/hello-container-key.pub" "authorized_keys")))))) ;; QEMU user-mode network stack: guest gets 10.0.2.15, gateway ;; 10.0.2.2 (see %qemu-static-networking in gnu/services/base.scm). (service static-networking-service-type (list %qemu-static-networking)) ;; Serial console login, for debugging when ssh does not work. (service agetty-service-type (agetty-configuration (extra-options '("-L")) (baud-rate "115200") (term "vt100") (tty "ttyS0"))) (simple-service 'slurm-keys-and-conf activation-service-type slurm-keys-activation) (simple-service 'munged shepherd-root-service-type (list (shepherd-service (provision '(munged)) (requirement '(user-processes)) (start #~(make-forkexec-constructor (list #$(file-append munge "/sbin/munged") "--force" (string-append "--key-file=" #$%munge-key) "--socket=/var/run/munge/munge.socket.2" "--pid-file=/var/run/munge/munged.pid" "--seed-file=/var/lib/munge/munged.seed") #:log-file "/var/log/slurm/munged.log")) (stop #~(make-kill-destructor))))) (simple-service 'slurmctld shepherd-root-service-type (list (shepherd-service (provision '(slurmctld)) (requirement '(munged)) (start #~(make-forkexec-constructor (list #$(wait-for-munge-then-exec "slurmctld" (file-append slurm "/sbin/slurmctld") "-D" "-c" "-f" %slurm-conf)) #:log-file "/var/log/slurm/slurmctld-stdout.log")) (stop #~(make-kill-destructor))))) (simple-service 'slurmd shepherd-root-service-type (list (shepherd-service (provision '(slurmd)) (requirement '(munged slurmctld)) (start #~(make-forkexec-constructor (list #$(wait-for-munge-then-exec "slurmd" (file-append slurm "/sbin/slurmd") "-D" "-c" "-f" %slurm-conf)) #:log-file "/var/log/slurm/slurmd-stdout.log")) (stop #~(make-kill-destructor))))) (simple-service 'slurmrestd shepherd-root-service-type (list (shepherd-service (provision '(slurmrestd)) (requirement '(munged slurmctld slurmd)) (start #~(make-forkexec-constructor (list #$(wait-for-munge-then-exec "slurmrestd" (file-append slurm "/sbin/slurmrestd") "-f" %slurm-conf "-a" "rest_auth/jwt" "-s" "openapi/slurmctld" "0.0.0.0:6820")) ;; Run as a dedicated, unprivileged ;; "slurmrestd" account -- see the ;; top-level comment above. #:user "slurmrestd" #:group "slurmrestd" #:log-file "/var/log/slurm/slurmrestd-stdout.log")) (stop #~(make-kill-destructor))))) ) (modify-services %base-services ;; No guix-daemon in the VM: it is not needed (images are built ;; on the host), and its service would pull guix, guix-daemon ;; etc. into the system profile. (delete guix-service-type)))))