diff options
| -rw-r--r-- | genenetwork-local-container.scm | 492 | ||||
| -rwxr-xr-x | genenetwork-local-container.sh | 81 |
2 files changed, 573 insertions, 0 deletions
diff --git a/genenetwork-local-container.scm b/genenetwork-local-container.scm new file mode 100644 index 0000000..1b683ea --- /dev/null +++ b/genenetwork-local-container.scm @@ -0,0 +1,492 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +;;; This is the production genenetwork container currently deployed on +;;; tux04. + +(use-modules (gnu) + (genenetwork services genenetwork) + (gnu build linux-container) + ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) + ((gnu packages admin) #:select (shepherd shadow)) + ((gnu packages certs) #:select (nss-certs)) + ((gnu packages bash) #:select (bash)) + ((gnu packages check) #:select (python-pylint python-hypothesis)) + ((gnu packages python-check) #:select (python-mypy python-mypy-extensions)) + ((gnu packages python-web) #:select (gunicorn python-flask)) + ((gnu packages version-control) #:select (git-minimal)) + ((gn packages guile) #:select (gn-guile)) + (gn services databases) + (gnu services databases) + (guix modules) + (gnu services shepherd) + (guix search-paths) + (guix least-authority) + (guix packages) + (guix profiles) + (guix records) + (forge socket) + (forge utils) + (srfi srfi-1) + (ice-9 match)) + +(define-record-type* <genenetwork-configuration> + genenetwork-configuration make-genenetwork-configuration + genenetwork-configuration? + (gn2-port genenetwork-configuration-gn2-port + (default 8082)) + (gn3-port genenetwork-configuration-gn3-port + (default 8083)) + (gn-auth-port genenetwork-configuration-gn-auth-port + (default 8084)) + (gn2-secrets genenetwork-configuration-gn2-secrets + (default "/etc/genenetwork/conf/gn2/secrets.py")) + (gn3-secrets genenetwork-configuration-gn3-secrets + (default "/etc/genenetwork/conf/gn3/secrets")) + (gn-auth-secrets genenetwork-configuration-gn-auth-secrets + (default "/etc/genenetwork")) + (genotype-files genenetwork-configuration-genotype-files + (default "/var/genenetwork/genotype-files")) + (sparql-endpoint genenetwork-configuration-sparql-endpoint + (default "http://localhost:7082/sparql")) + (data-directory genenetwork-data-directory + (default "/var/genenetwork")) + (xapian-db-path genenetwork-xapian-db-path + (default "/var/lib/xapian")) + (auth-db-path genenetwork-auth-db-path + (default "/var/genenetwork/auth.db")) + (llm-db-path genenetwork-llm-db-path + (default "/var/lib/genenetwork-sqlite/llm.db")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/var/lib/gn-docs"))) + +(define (genenetwork2-gexp config) + "Return a G-expression that runs the latest genenetwork2 development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (gn2-port gn3-port genotype-files) + (with-packages (list coreutils git-minimal gunicorn nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 match)) + + ;; Override the genenetwork3 used by genenetwork2. + (setenv "GN3_PYTHONPATH" "/genenetwork3") + ;; Set other environment variables required by + ;; genenetwork2. + (setenv "GN2_PROFILE" #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t))) + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"/etc/genenetwork/conf/gn2/secrets.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "GN3_LOCAL_URL=\"" + (string-append "http://localhost:" + (number->string gn3-port)) + "\"\n" + "GN_SERVER_URL=\"http://localhost:8083/api/\"\n" + "AUTH_SERVER_URL=\"http://localhost:8084/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn2/private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/gn-auth.pem\"\n")) + + ;; Start genenetwork2. + (with-directory-excursion "/genenetwork2" + (invoke #$(file-append bash "/bin/sh") + "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-dev"))))))) + +(define (genenetwork3-gexp config) + "Return a G-expression that runs the latest genenetwork3 development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (with-manifest (package->development-manifest genenetwork3) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 match)) + + ;; Configure genenetwork3. + (setenv "GN3_CONF" + #$(mixed-text-file "gn3.conf" + "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" + "DATA_DIR=\"" data-directory "\"\n" + "AUTH_SERVER_URL=\"http://localhost:8084/\"\n" + "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" + "AUTH_DB=\"" auth-db-path "\"\n" + "LLM_DB_PATH=\"" llm-db-path "\"\n")) + (setenv "HOME" "/tmp") + (setenv "GN3_SECRETS" #$gn3-secrets) + (setenv "RSCRIPT" #$(file-append + (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t)) + "/bin/Rscript")) + (setenv "FLASK_ENV" "development") + (setenv "FLASK_DEBUG" "1") + ;; Run genenetwork3. + (with-directory-excursion "/genenetwork3" + (invoke #$(file-append python-flask "/bin/flask") + "run" + #$(string-append "--port=" (number->string gn3-port))))))))) + +(define (gn-auth-gexp config) + "Return a G-expression that runs the latest gn-auth development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (gn-auth-port auth-db-path gn-auth-secrets) + (with-manifest (package->development-manifest gn-auth) + (with-packages (list git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 match)) + ;; Configure gn-auth. + (setenv "GN_AUTH_CONF" + #$(mixed-text-file + "gn-auth.conf" + "LOGLEVEL=\"DEBUG\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "AUTH_DB=\"" auth-db-path "\"\n" + "GN_AUTH_SECRETS=\"/etc/genenetwork/conf/gn-auth/secrets.py\"\n" + "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/\"\n" + "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn-auth/private.pem\"\n")) + (setenv "HOME" "/tmp") + (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") + ;; Run gn-auth. + (with-directory-excursion "/gn-auth" + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn-auth-port)) + "--workers" "8" + "gn_auth.wsgi:app")))))))) + +(define (genenetwork-activation config) + (match-record config <genenetwork-configuration> + (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + ;; Set ownership of files. + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (cons* #$gn3-secrets + (append (find-files #$gn2-secrets + #:directories? #t) + (find-files "/var/lib/gn-docs" + #:directories? #t) + (find-files #$(dirname auth-db-path) + #:directories? #t) + (find-files #$gn-auth-secrets + #:directories? #t)))) + ;; Prevent other users from reading secret files. + (for-each (lambda (file) + (chmod file #o600)) + (append (list #$gn3-secrets) + (find-files #$gn2-secrets + #:directories? #f) + (find-files #$gn-auth-secrets + #:directories? #f))))))) + +(define (gn-guile-gexp gn-guile-port) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + + +(define (genenetwork-shepherd-services config) + "Return shepherd services to run the genenetwork development server +described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) + (list (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/gn-guile.log"))) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run GeneNetwork 2 development server.") + (provision '(genenetwork2)) + ;; FIXME: The genenetwork2 service should depend on redis. + (requirement '(networking genenetwork3)) + (modules '((guix search-paths) + (ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn2-manifest (packages->manifest (list genenetwork2))) + (gn2-profile (profile + (content gn2-manifest) + (allow-collisions? #t))) + (gn2-settings + `(("SERVER_PORT" ,(number->string gn2-port)) + ("GENENETWORK_FILES" ,genotype-files) + ("HOME" "/tmp") + ("LC_ALL" "en_US.UTF-8") + ("NO_REDIS" "no-redis") + ("RUST_BACKTRACE" "1")))) + (with-imported-modules (source-module-closure '((guix search-paths))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork2" + (genenetwork2-gexp config)) + #:name "genenetwork2-pola-wrapper" + #:preserved-environment-variables + (append '("REQUESTS_CA_BUNDLE") + (map first gn2-settings) + (map search-path-specification-variable + (manifest-search-paths gn2-manifest))) + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source genotype-files) + (target source)) + (file-system-mapping + (source "/genenetwork2") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source gn2-secrets) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn2-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (append + '("REQUESTS_CA_BUNDLE=" + #$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) + (map (match-lambda + ((spec . value) + (string-append (search-path-specification-variable spec) + "=" + value))) + (evaluate-search-paths + (map sexp->search-path-specification + '#$(map search-path-specification->sexp + (manifest-search-paths gn2-manifest))) + (list #$gn2-profile))) + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn2-settings)) + #:log-file "/var/log/genenetwork2.log")))) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run GeneNetwork 3 development server.") + (provision '(genenetwork3)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork3" + (genenetwork3-gexp config)) + #:name "genenetwork3-pola-wrapper" + #:mappings (list (file-system-mapping + (source "/genenetwork3") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source data-directory) + (target source)) + (file-system-mapping + (source xapian-db-path) + (target source)) + (file-system-mapping + (source "/etc/genenetwork/conf/gn3") + (target source) + (writable? #t)) + (file-system-mapping + (source auth-db-path) + (target source) + (writable? #t)) + (file-system-mapping + (source llm-db-path) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn3-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/genenetwork3.log")) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run gn-auth development server.") + (provision '(gn-auth)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-auth" + (gn-auth-gexp config)) + #:name "gn-auth-pola-wrapper" + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source "/gn-auth") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source data-directory) + (target source)) + (file-system-mapping + (source auth-db-path) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-auth-secrets) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-auth-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/gn-auth.log")) + (stop #~(make-kill-destructor)))))) + +(define %genenetwork-accounts + (list (user-group + (name "genenetwork") + (system? #t)) + (user-account + (name "genenetwork") + (group "genenetwork") + (system? #t) + (comment "GeneNetwork user") + (home-directory "/var/empty") + (shell (file-append shadow "/sbin/nologin"))))) + + +(define genenetwork-service-type + (service-type + (name 'genenetwork) + (description "Run GeneNetwork development servers and CI.") + (extensions + (list (service-extension account-service-type + (const %genenetwork-accounts)) + (service-extension activation-service-type + genenetwork-activation) + (service-extension shepherd-root-service-type + genenetwork-shepherd-services))) + (default-value (genenetwork-configuration)))) + + +(operating-system + (host-name "genenetwork-work-container") + (timezone "UTC") + (locale "en_US.utf8") + (bootloader (bootloader-configuration + (bootloader grub-bootloader) + (targets (list "/dev/sdX")))) + (file-systems %base-file-systems) + (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* genenetwork2 python-hypothesis gn-auth + python-mypy python-mypy-extensions python-pylint + %base-packages)) + (services (cons* (service virtuoso-service-type + (virtuoso-configuration + (server-port 7081) + (http-server-port 7082) + (dirs-allowed "/var/lib/virtuoso/data") + (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") + (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) + (service redis-service-type + (redis-configuration + (bind "127.0.0.1") + (port 6379) + (working-directory "/var/lib/redis"))) + (service genenetwork-service-type) + %base-services))) diff --git a/genenetwork-local-container.sh b/genenetwork-local-container.sh new file mode 100755 index 0000000..07d0662 --- /dev/null +++ b/genenetwork-local-container.sh @@ -0,0 +1,81 @@ +#! /bin/sh -e + +# genenetwork-machines --- Guix configuration for genenetwork machines +# Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +# +# This file is part of genenetwork-machines. +# +# genenetwork-machines is free software: you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# genenetwork-machines is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with genenetwork-machines. If not, see +# <https://www.gnu.org/licenses/>. + +# Build and install genenetwork production container on tux02. + +system_directories=("$HOME"/genenetwork/var/log + "$HOME"/genenetwork/var/genenetwork + "$HOME"/genenetwork/etc/genenetwork/conf + "$HOME"/genenetwork/etc/genenetwork + "$HOME"/genenetwork/var/lib/redis + "$HOME"/genenetwork/var/lib/virtuoso + "$HOME"/genenetwork/var/lib/xapian + "$HOME"/genenetwork/var/lib/genenetwork-sqlite + "$HOME"/genenetwork/var/lib/genenetwork-gnqa + /tmp/local-container) + +for dir in "${system_directories[@]}"; do + if [[ ! -d $dir ]]; then + mkdir -p $dir + fi +done + +gn_projects=(genenetwork2 genenetwork3) +for project in "${gn_projects[@]}"; do + dir="${HOME}/genenetwork/${project}" + if [[ ! -d $dir ]]; then + git clone "git@github.com:genenetwork/${project}" $dir + fi +done + +if [[ ! -d "${HOME}/genenetwork/gn-auth" ]]; then + git clone "https://git.genenetwork.org/gn-auth" "${HOME}/genenetwork/gn-auth" +fi + +if [[ ! -d "${HOME}/genenetwork/gn-docs" ]]; then + git clone --bare "https://git.genenetwork.org/gn-docs" "${HOME}/genenetwork/gn-docs" +fi + +container_script=$(guix system container \ + --network \ + --load-path=. \ + --verbosity=3 \ + --share="$HOME"/genenetwork/var/log=/var/log \ + --share="$HOME"/genenetwork/var/genenetwork=/var/genenetwork \ + --share="$HOME"/genenetwork/etc/genenetwork/conf=/etc/genenetwork/conf \ + --share="$HOME"/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share="$HOME"/genenetwork/var/lib/redis=/var/lib/redis \ + --share="$HOME"/genenetwork/var/lib/virtuoso/=/var/lib/virtuoso \ + --share="$HOME"/genenetwork/genenetwork2=/genenetwork2 \ + --share="$HOME"/genenetwork/genenetwork3=/genenetwork3 \ + --share="$HOME"/genenetwork/gn-auth=/gn-auth \ + --share="$HOME"/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share="$HOME"/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share="$HOME"/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ + --share=/tmp/local-container=/tmp \ + --share="$HOME"/genenetwork/gn-docs=/var/lib/gn-docs \ + --share=/run/mysqld=/run/mysqld \ + genenetwork-local-container.scm) + +echo $container_script +sudo ln --force --symbolic $container_script /usr/local/bin/genenetwork-local-container +sudo ln --force --symbolic /usr/local/bin/genenetwork-local-container /var/guix/gcroots + |