;;; genenetwork-machines --- Guix configuration for genenetwork machines
;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com>
;;;
;;; This file is part of genenetwork-machines.
;;;
;;; genenetwork-machines is free software: you can redistribute it
;;; and/or modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; genenetwork-machines is distributed in the hope that it will be
;;; useful, but WITHOUT ANY WARRANTY; without even the implied
;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
;;; See the GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with genenetwork-machines. If not, see
;;; <https://www.gnu.org/licenses/>.
;;; This is the production genenetwork container currently deployed on
;;; tux04.
(use-modules (gnu)
(genenetwork services genenetwork)
(gnu build linux-container)
((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs))
((gnu packages admin) #:select (shepherd shadow))
((gnu packages certs) #:select (nss-certs))
((gnu packages bash) #:select (bash))
((gnu packages check) #:select (python-pylint python-hypothesis))
((gnu packages python-check) #:select (python-mypy python-mypy-extensions))
((gnu packages python-web) #:select (gunicorn python-flask))
((gnu packages version-control) #:select (git-minimal))
((gn packages guile) #:select (gn-guile))
(gn services databases)
(gnu services databases)
(guix modules)
(gnu services shepherd)
(guix search-paths)
(guix least-authority)
(guix packages)
(guix profiles)
(guix records)
(forge socket)
(forge utils)
(srfi srfi-1)
(ice-9 match))
(define-record-type* <genenetwork-configuration>
genenetwork-configuration make-genenetwork-configuration
genenetwork-configuration?
(gn2-port genenetwork-configuration-gn2-port
(default 8082))
(gn3-port genenetwork-configuration-gn3-port
(default 8083))
(gn-auth-port genenetwork-configuration-gn-auth-port
(default 8084))
(gn2-secrets genenetwork-configuration-gn2-secrets
(default "/etc/genenetwork/conf/gn2/secrets.py"))
(gn3-secrets genenetwork-configuration-gn3-secrets
(default "/etc/genenetwork/conf/gn3/secrets"))
(gn-auth-secrets genenetwork-configuration-gn-auth-secrets
(default "/etc/genenetwork"))
(genotype-files genenetwork-configuration-genotype-files
(default "/var/genenetwork/genotype-files"))
(sparql-endpoint genenetwork-configuration-sparql-endpoint
(default "http://localhost:7082/sparql"))
(data-directory genenetwork-data-directory
(default "/var/genenetwork"))
(xapian-db-path genenetwork-xapian-db-path
(default "/var/lib/xapian"))
(auth-db-path genenetwork-auth-db-path
(default "/var/genenetwork/auth.db"))
(llm-db-path genenetwork-llm-db-path
(default "/var/lib/genenetwork-sqlite/llm.db"))
(gn-guile-port genenetwork-configuration-gn-guile-port
(default 8091))
(gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout
(default "/var/lib/gn-docs")))
(define (genenetwork2-gexp config)
"Return a G-expression that runs the latest genenetwork2 development
server described by CONFIG, a <genenetwork-configuration> object."
(match-record config <genenetwork-configuration>
(gn2-port gn3-port genotype-files)
(with-packages (list coreutils git-minimal gunicorn nss-certs)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils)
(ice-9 match))
;; Override the genenetwork3 used by genenetwork2.
(setenv "GN3_PYTHONPATH" "/genenetwork3")
;; Set other environment variables required by
;; genenetwork2.
(setenv "GN2_PROFILE" #$(profile
(content (package->development-manifest genenetwork2))
(allow-collisions? #t)))
(setenv
"GN2_SETTINGS"
#$(mixed-text-file "gn2.conf"
"GN2_SECRETS=\"/etc/genenetwork/conf/gn2/secrets.py\"\n"
"AI_SEARCH_ENABLED=True\n"
"GN3_LOCAL_URL=\""
(string-append "http://localhost:"
(number->string gn3-port))
"\"\n"
"GN_SERVER_URL=\"http://localhost:8083/api/\"\n"
"AUTH_SERVER_URL=\"http://localhost:8084/\"\n"
"SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n"
"SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn2/private.pem\"\n"
"AUTH_SERVER_SSL_PUBLIC_KEY=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/gn-auth.pem\"\n"))
;; Start genenetwork2.
(with-directory-excursion "/genenetwork2"
(invoke #$(file-append bash "/bin/sh")
"bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-dev")))))))
(define (genenetwork3-gexp config)
"Return a G-expression that runs the latest genenetwork3 development
server described by CONFIG, a <genenetwork-configuration> object."
(match-record config <genenetwork-configuration>
(gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path)
(with-manifest (package->development-manifest genenetwork3)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils)
(ice-9 match))
;; Configure genenetwork3.
(setenv "GN3_CONF"
#$(mixed-text-file "gn3.conf"
"SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n"
"DATA_DIR=\"" data-directory "\"\n"
"AUTH_SERVER_URL=\"http://localhost:8084/\"\n"
"XAPIAN_DB_PATH=\"" xapian-db-path "\"\n"
"AUTH_DB=\"" auth-db-path "\"\n"
"LLM_DB_PATH=\"" llm-db-path "\"\n"))
(setenv "HOME" "/tmp")
(setenv "GN3_SECRETS" #$gn3-secrets)
(setenv "RSCRIPT" #$(file-append
(profile
(content (package->development-manifest genenetwork3))
(allow-collisions? #t))
"/bin/Rscript"))
(setenv "FLASK_ENV" "development")
(setenv "FLASK_DEBUG" "1")
;; Run genenetwork3.
(with-directory-excursion "/genenetwork3"
(invoke #$(file-append python-flask "/bin/flask")
"run"
#$(string-append "--port=" (number->string gn3-port)))))))))
(define (gn-auth-gexp config)
"Return a G-expression that runs the latest gn-auth development
server described by CONFIG, a <genenetwork-configuration> object."
(match-record config <genenetwork-configuration>
(gn-auth-port auth-db-path gn-auth-secrets)
(with-manifest (package->development-manifest gn-auth)
(with-packages (list git-minimal nss-certs)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils)
(ice-9 match))
;; Configure gn-auth.
(setenv "GN_AUTH_CONF"
#$(mixed-text-file
"gn-auth.conf"
"LOGLEVEL=\"DEBUG\"\n"
"SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n"
"AUTH_DB=\"" auth-db-path "\"\n"
"GN_AUTH_SECRETS=\"/etc/genenetwork/conf/gn-auth/secrets.py\"\n"
"CLIENTS_SSL_PUBLIC_KEYS_DIR=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/\"\n"
"SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn-auth/private.pem\"\n"))
(setenv "HOME" "/tmp")
(setenv "AUTHLIB_INSECURE_TRANSPORT" "true")
;; Run gn-auth.
(with-directory-excursion "/gn-auth"
(invoke #$(file-append gunicorn "/bin/gunicorn")
"-b" #$(string-append "localhost:" (number->string gn-auth-port))
"--workers" "8"
"gn_auth.wsgi:app"))))))))
(define (genenetwork-activation config)
(match-record config <genenetwork-configuration>
(gn2-secrets gn3-secrets auth-db-path gn-auth-secrets)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
;; Set ownership of files.
(for-each (lambda (file)
(chown file
(passwd:uid (getpw "genenetwork"))
(passwd:gid (getpw "genenetwork"))))
(cons* #$gn3-secrets
(append (find-files #$gn2-secrets
#:directories? #t)
(find-files "/var/lib/gn-docs"
#:directories? #t)
(find-files #$(dirname auth-db-path)
#:directories? #t)
(find-files #$gn-auth-secrets
#:directories? #t))))
;; Prevent other users from reading secret files.
(for-each (lambda (file)
(chmod file #o600))
(append (list #$gn3-secrets)
(find-files #$gn2-secrets
#:directories? #f)
(find-files #$gn-auth-secrets
#:directories? #f)))))))
(define (gn-guile-gexp gn-guile-port)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
(let ((current-repo-path (string-append (getcwd) "/gn-docs")))
(when (file-exists? current-repo-path)
(delete-file-recursively current-repo-path))
(setenv "CURRENT_REPO_PATH" current-repo-path)
(invoke #$(file-append git-minimal "/bin/git")
"clone" "--depth" "1" (getenv "CGIT_REPO_PATH")))
(invoke #$(file-append gn-guile "/bin/gn-guile")
(number->string #$gn-guile-port)))))
(define (genenetwork-shepherd-services config)
"Return shepherd services to run the genenetwork development server
described by CONFIG, a <genenetwork-configuration> object."
(match-record config <genenetwork-configuration>
(gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port)
(list (shepherd-service
(documentation "Run gn-guile server.")
(provision '(gn-guile))
(requirement '(networking))
(modules '((ice-9 match)
(srfi srfi-1)))
(start
(let* ((gn-guile-settings
`(("CGIT_REPO_PATH" ,gn-doc-git-checkout)
("LC_ALL" "en_US.UTF-8")
("GIT_COMMITTER_NAME" "genenetwork")
("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org"))))
#~(make-forkexec-constructor
(list #$(least-authority-wrapper
(program-file "gn-guile"
(gn-guile-gexp gn-guile-port))
#:name "gn-guile-pola-wrapper"
#:preserved-environment-variables
(map first gn-guile-settings)
#:mappings (list (file-system-mapping
(source gn-doc-git-checkout)
(target source)
(writable? #t)))
#:namespaces (delq 'net %namespaces))
"127.0.0.1" #$(number->string gn-guile-port))
#:user "genenetwork"
#:group "genenetwork"
#:environment-variables
(map (match-lambda
((spec value)
(string-append spec "=" value)))
'#$gn-guile-settings)
#:log-file "/var/log/gn-guile.log")))
(stop #~(make-kill-destructor)))
(shepherd-service
(documentation "Run GeneNetwork 2 development server.")
(provision '(genenetwork2))
;; FIXME: The genenetwork2 service should depend on redis.
(requirement '(networking genenetwork3))
(modules '((guix search-paths)
(ice-9 match)
(srfi srfi-1)))
(start
(let* ((gn2-manifest (packages->manifest (list genenetwork2)))
(gn2-profile (profile
(content gn2-manifest)
(allow-collisions? #t)))
(gn2-settings
`(("SERVER_PORT" ,(number->string gn2-port))
("GENENETWORK_FILES" ,genotype-files)
("HOME" "/tmp")
("LC_ALL" "en_US.UTF-8")
("NO_REDIS" "no-redis")
("RUST_BACKTRACE" "1"))))
(with-imported-modules (source-module-closure '((guix search-paths)))
#~(make-forkexec-constructor
(list #$(least-authority-wrapper
(program-file "genenetwork2"
(genenetwork2-gexp config))
#:name "genenetwork2-pola-wrapper"
#:preserved-environment-variables
(append '("REQUESTS_CA_BUNDLE")
(map first gn2-settings)
(map search-path-specification-variable
(manifest-search-paths gn2-manifest)))
;; If we mapped only the mysqld.sock
;; socket file, it would break when the
;; external mysqld server is restarted.
#:mappings (list (file-system-mapping
(source genotype-files)
(target source))
(file-system-mapping
(source "/genenetwork2")
(target source)
(writable? #t))
(file-system-mapping
(source "/run/mysqld")
(target source)
(writable? #t))
;; XXXX: FIXME: R/Qtl generates
;; files in "/tmp" and
;; "/tmp/gn2". These files are
;; accessed by gn3 for R/Qtl
;; mapping
(file-system-mapping
(source "/tmp")
(target source)
(writable? #t))
(file-system-mapping
(source gn2-secrets)
(target source)
(writable? #t)))
#:namespaces (delq 'net %namespaces))
"127.0.0.1" #$(number->string gn2-port))
#:user "genenetwork"
#:group "genenetwork"
#:environment-variables
(append
'("REQUESTS_CA_BUNDLE="
#$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt"))
(map (match-lambda
((spec . value)
(string-append (search-path-specification-variable spec)
"="
value)))
(evaluate-search-paths
(map sexp->search-path-specification
'#$(map search-path-specification->sexp
(manifest-search-paths gn2-manifest)))
(list #$gn2-profile)))
(map (match-lambda
((spec value)
(string-append spec "=" value)))
'#$gn2-settings))
#:log-file "/var/log/genenetwork2.log"))))
(stop #~(make-kill-destructor)))
(shepherd-service
(documentation "Run GeneNetwork 3 development server.")
(provision '(genenetwork3))
(requirement '(networking))
(start #~(make-forkexec-constructor
(list #$(least-authority-wrapper
(program-file "genenetwork3"
(genenetwork3-gexp config))
#:name "genenetwork3-pola-wrapper"
#:mappings (list (file-system-mapping
(source "/genenetwork3")
(target source)
(writable? #t))
(file-system-mapping
(source "/run/mysqld")
(target source)
(writable? #t))
(file-system-mapping
(source "/tmp")
(target source)
(writable? #t))
(file-system-mapping
(source data-directory)
(target source))
(file-system-mapping
(source xapian-db-path)
(target source))
(file-system-mapping
(source "/etc/genenetwork/conf/gn3")
(target source)
(writable? #t))
(file-system-mapping
(source auth-db-path)
(target source)
(writable? #t))
(file-system-mapping
(source llm-db-path)
(target source)
(writable? #t)))
#:namespaces (delq 'net %namespaces))
"127.0.0.1" #$(number->string gn3-port))
#:user "genenetwork"
#:group "genenetwork"
#:log-file "/var/log/genenetwork3.log"))
(stop #~(make-kill-destructor)))
(shepherd-service
(documentation "Run gn-auth development server.")
(provision '(gn-auth))
(requirement '(networking))
(start #~(make-forkexec-constructor
(list #$(least-authority-wrapper
(program-file "gn-auth"
(gn-auth-gexp config))
#:name "gn-auth-pola-wrapper"
;; If we mapped only the mysqld.sock
;; socket file, it would break when the
;; external mysqld server is restarted.
#:mappings (list (file-system-mapping
(source "/run/mysqld")
(target source)
(writable? #t))
(file-system-mapping
(source "/gn-auth")
(target source)
(writable? #t))
(file-system-mapping
(source "/run/mysqld")
(target source)
(writable? #t))
(file-system-mapping
(source data-directory)
(target source))
(file-system-mapping
(source auth-db-path)
(target source)
(writable? #t))
(file-system-mapping
(source gn-auth-secrets)
(target source)
(writable? #t)))
#:namespaces (delq 'net %namespaces))
"127.0.0.1" #$(number->string gn-auth-port))
#:user "genenetwork"
#:group "genenetwork"
#:log-file "/var/log/gn-auth.log"))
(stop #~(make-kill-destructor))))))
(define %genenetwork-accounts
(list (user-group
(name "genenetwork")
(system? #t))
(user-account
(name "genenetwork")
(group "genenetwork")
(system? #t)
(comment "GeneNetwork user")
(home-directory "/var/empty")
(shell (file-append shadow "/sbin/nologin")))))
(define genenetwork-service-type
(service-type
(name 'genenetwork)
(description "Run GeneNetwork development servers and CI.")
(extensions
(list (service-extension account-service-type
(const %genenetwork-accounts))
(service-extension activation-service-type
genenetwork-activation)
(service-extension shepherd-root-service-type
genenetwork-shepherd-services)))
(default-value (genenetwork-configuration))))
(operating-system
(host-name "genenetwork-work-container")
(timezone "UTC")
(locale "en_US.utf8")
(bootloader (bootloader-configuration
(bootloader grub-bootloader)
(targets (list "/dev/sdX"))))
(file-systems %base-file-systems)
(users %base-user-accounts)
(sudoers-file
(mixed-text-file "sudoers"
"@include " %sudoers-specification
"\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n"))
(packages (cons* genenetwork2 python-hypothesis gn-auth
python-mypy python-mypy-extensions python-pylint
%base-packages))
(services (cons* (service virtuoso-service-type
(virtuoso-configuration
(server-port 7081)
(http-server-port 7082)
(dirs-allowed "/var/lib/virtuoso/data")
(database-file "/var/lib/virtuoso/genenetwork-virtuoso.db")
(transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx")))
(service redis-service-type
(redis-configuration
(bind "127.0.0.1")
(port 6379)
(working-directory "/var/lib/redis")))
(service genenetwork-service-type)
%base-services)))
