path: root/genenetwork-local-container.scm

;;; genenetwork-machines --- Guix configuration for genenetwork machines
;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com>
;;;
;;; This file is part of genenetwork-machines.
;;;
;;; genenetwork-machines is free software: you can redistribute it
;;; and/or modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; genenetwork-machines is distributed in the hope that it will be
;;; useful, but WITHOUT ANY WARRANTY; without even the implied
;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
;;; See the GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with genenetwork-machines.  If not, see
;;; <https://www.gnu.org/licenses/>.

;;; This is the production genenetwork container currently deployed on
;;; tux04.

(use-modules (gnu)
             (genenetwork services genenetwork)
             (gnu build linux-container)
             ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs))
             ((gnu packages admin) #:select (shepherd shadow))
             ((gnu packages certs) #:select (nss-certs))
             ((gnu packages bash) #:select (bash))
             ((gnu packages check) #:select (python-pylint python-hypothesis))
             ((gnu packages python-check) #:select (python-mypy python-mypy-extensions))
             ((gnu packages python-web) #:select (gunicorn python-flask))
             ((gnu packages version-control) #:select (git-minimal))
             ((gn packages guile) #:select (gn-guile))
             (gn services databases)
             (gnu services databases)
             (guix modules)
             (gnu services shepherd)
             (guix search-paths)
             (guix least-authority)
             (guix packages)
             (guix profiles)
             (guix records)
             (forge socket)
             (forge utils)
             (srfi srfi-1)
             (ice-9 match))

(define-record-type* <genenetwork-configuration>
  genenetwork-configuration make-genenetwork-configuration
  genenetwork-configuration?
  (gn2-port genenetwork-configuration-gn2-port
            (default 8082))
  (gn3-port genenetwork-configuration-gn3-port
            (default 8083))
  (gn-auth-port genenetwork-configuration-gn-auth-port
            (default 8084))
  (gn2-secrets genenetwork-configuration-gn2-secrets
               (default "/etc/genenetwork/conf/gn2/secrets.py"))
  (gn3-secrets genenetwork-configuration-gn3-secrets
               (default "/etc/genenetwork/conf/gn3/secrets"))
  (gn-auth-secrets genenetwork-configuration-gn-auth-secrets
                   (default "/etc/genenetwork"))
  (genotype-files genenetwork-configuration-genotype-files
                  (default "/var/genenetwork/genotype-files"))
  (sparql-endpoint genenetwork-configuration-sparql-endpoint
                   (default "http://localhost:7082/sparql"))
  (data-directory genenetwork-data-directory
                  (default "/var/genenetwork"))
  (xapian-db-path genenetwork-xapian-db-path
                  (default "/var/lib/xapian"))
  (auth-db-path genenetwork-auth-db-path
                (default "/var/genenetwork/auth.db"))
  (llm-db-path genenetwork-llm-db-path
               (default "/var/lib/genenetwork-sqlite/llm.db"))
  (gn-guile-port genenetwork-configuration-gn-guile-port
                 (default 8091))
  (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout
                       (default "/var/lib/gn-docs")))

(define (genenetwork2-gexp config)
  "Return a G-expression that runs the latest genenetwork2 development
server described by CONFIG, a <genenetwork-configuration> object."
  (match-record config <genenetwork-configuration>
    (gn2-port gn3-port genotype-files)
    (with-packages (list coreutils git-minimal gunicorn nss-certs)
      (with-imported-modules '((guix build utils))
        #~(begin
            (use-modules (guix build utils)
                         (ice-9 match))

            ;; Override the genenetwork3 used by genenetwork2.
            (setenv "GN3_PYTHONPATH" "/genenetwork3")
            ;; Set other environment variables required by
            ;; genenetwork2.
            (setenv "GN2_PROFILE" #$(profile
                                     (content (package->development-manifest genenetwork2))
                                     (allow-collisions? #t)))
            (setenv
             "GN2_SETTINGS"
             #$(mixed-text-file "gn2.conf"
                                "GN2_SECRETS=\"/etc/genenetwork/conf/gn2/secrets.py\"\n"
                                "AI_SEARCH_ENABLED=True\n"
                                "GN3_LOCAL_URL=\""
                                (string-append "http://localhost:"
                                               (number->string gn3-port))
                                "\"\n"
                                "GN_SERVER_URL=\"http://localhost:8083/api/\"\n"
                                "AUTH_SERVER_URL=\"http://localhost:8084/\"\n"
                                "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n"
                                "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn2/private.pem\"\n"
                                "AUTH_SERVER_SSL_PUBLIC_KEY=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/gn-auth.pem\"\n"))

            ;; Start genenetwork2.
            (with-directory-excursion "/genenetwork2"
              (invoke #$(file-append bash "/bin/sh")
                      "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-dev")))))))

(define (genenetwork3-gexp config)
  "Return a G-expression that runs the latest genenetwork3 development
server described by CONFIG, a <genenetwork-configuration> object."
  (match-record config <genenetwork-configuration>
                (gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path)
    (with-manifest (package->development-manifest genenetwork3)
      (with-imported-modules '((guix build utils))
        #~(begin
            (use-modules (guix build utils)
                         (ice-9 match))

            ;; Configure genenetwork3.
            (setenv "GN3_CONF"
                    #$(mixed-text-file "gn3.conf"
                                       "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n"
                                       "DATA_DIR=\"" data-directory "\"\n"
                                       "AUTH_SERVER_URL=\"http://localhost:8084/\"\n"
                                       "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n"
                                       "AUTH_DB=\"" auth-db-path "\"\n"
                                       "LLM_DB_PATH=\"" llm-db-path "\"\n"))
            (setenv "HOME" "/tmp")
            (setenv "GN3_SECRETS" #$gn3-secrets)
            (setenv "RSCRIPT" #$(file-append
                                 (profile
                                  (content (package->development-manifest genenetwork3))
                                  (allow-collisions? #t))
                                 "/bin/Rscript"))
            (setenv "FLASK_ENV" "development")
            (setenv "FLASK_DEBUG" "1")
            ;; Run genenetwork3.
            (with-directory-excursion "/genenetwork3"
              (invoke #$(file-append python-flask "/bin/flask")
                      "run"
                      #$(string-append "--port=" (number->string gn3-port)))))))))

(define (gn-auth-gexp config)
  "Return a G-expression that runs the latest gn-auth development
server described by CONFIG, a <genenetwork-configuration> object."
  (match-record config <genenetwork-configuration>
    (gn-auth-port auth-db-path gn-auth-secrets)
    (with-manifest (package->development-manifest gn-auth)
      (with-packages (list git-minimal nss-certs)
        (with-imported-modules '((guix build utils))
          #~(begin
              (use-modules (guix build utils)
                           (ice-9 match))
              ;; Configure gn-auth.
              (setenv "GN_AUTH_CONF"
                      #$(mixed-text-file
                         "gn-auth.conf"
                         "LOGLEVEL=\"DEBUG\"\n"
                         "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n"
                         "AUTH_DB=\"" auth-db-path "\"\n"
                         "GN_AUTH_SECRETS=\"/etc/genenetwork/conf/gn-auth/secrets.py\"\n"
                         "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/\"\n"
                         "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn-auth/private.pem\"\n"))
              (setenv "HOME" "/tmp")
              (setenv "AUTHLIB_INSECURE_TRANSPORT" "true")
              ;; Run gn-auth.
              (with-directory-excursion "/gn-auth"
                (invoke #$(file-append gunicorn "/bin/gunicorn")
                        "-b" #$(string-append "localhost:" (number->string gn-auth-port))
                        "--workers" "8"
                        "gn_auth.wsgi:app"))))))))

(define (genenetwork-activation config)
  (match-record config <genenetwork-configuration>
    (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets)
    (with-imported-modules '((guix build utils))
      #~(begin
          (use-modules (guix build utils))

          ;; Set ownership of files.
          (for-each (lambda (file)
                      (chown file
                             (passwd:uid (getpw "genenetwork"))
                             (passwd:gid (getpw "genenetwork"))))
                    (cons* #$gn3-secrets
                           (append (find-files #$gn2-secrets
                                               #:directories? #t)
                                   (find-files "/var/lib/gn-docs"
                                               #:directories? #t)
                                   (find-files #$(dirname auth-db-path)
                                               #:directories? #t)
                                   (find-files #$gn-auth-secrets
                                               #:directories? #t))))
          ;; Prevent other users from reading secret files.
          (for-each (lambda (file)
                      (chmod file #o600))
                    (append (list #$gn3-secrets)
                     (find-files #$gn2-secrets
                                 #:directories? #f)
                     (find-files #$gn-auth-secrets
                                 #:directories? #f)))))))

(define (gn-guile-gexp gn-guile-port)
  (with-imported-modules '((guix build utils))
    #~(begin
        (use-modules (guix build utils))
        (let ((current-repo-path (string-append (getcwd) "/gn-docs")))
          (when (file-exists? current-repo-path)
            (delete-file-recursively current-repo-path))
          (setenv "CURRENT_REPO_PATH" current-repo-path)
          (invoke #$(file-append git-minimal "/bin/git")
                  "clone" "--depth" "1" (getenv "CGIT_REPO_PATH")))
        (invoke #$(file-append gn-guile "/bin/gn-guile")
                (number->string #$gn-guile-port)))))


(define (genenetwork-shepherd-services config)
  "Return shepherd services to run the genenetwork development server
described by CONFIG, a <genenetwork-configuration> object."
  (match-record config <genenetwork-configuration>
                (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port)
    (list (shepherd-service
           (documentation "Run gn-guile server.")
           (provision '(gn-guile))
           (requirement '(networking))
           (modules '((ice-9 match)
                      (srfi srfi-1)))
           (start
            (let* ((gn-guile-settings
                    `(("CGIT_REPO_PATH" ,gn-doc-git-checkout)
                      ("LC_ALL" "en_US.UTF-8")
                      ("GIT_COMMITTER_NAME" "genenetwork")
                      ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org"))))
              #~(make-forkexec-constructor
	         (list #$(least-authority-wrapper
                          (program-file "gn-guile"
                                        (gn-guile-gexp gn-guile-port))
                          #:name "gn-guile-pola-wrapper"
                          #:preserved-environment-variables
                          (map first gn-guile-settings)
                          #:mappings (list (file-system-mapping
                                            (source gn-doc-git-checkout)
                                            (target source)
                                            (writable? #t)))
                          #:namespaces (delq 'net %namespaces))
                       "127.0.0.1" #$(number->string gn-guile-port))
                 #:user "genenetwork"
                 #:group "genenetwork"
                 #:environment-variables
                 (map (match-lambda
                        ((spec value)
                         (string-append spec "=" value)))
                      '#$gn-guile-settings)
	         #:log-file "/var/log/gn-guile.log")))
           (stop #~(make-kill-destructor)))
          (shepherd-service
           (documentation "Run GeneNetwork 2 development server.")
           (provision '(genenetwork2))
           ;; FIXME: The genenetwork2 service should depend on redis.
           (requirement '(networking genenetwork3))
           (modules '((guix search-paths)
                      (ice-9 match)
                      (srfi srfi-1)))
           (start
            (let* ((gn2-manifest (packages->manifest (list genenetwork2)))
                   (gn2-profile (profile
                                 (content gn2-manifest)
                                 (allow-collisions? #t)))
                   (gn2-settings
                    `(("SERVER_PORT" ,(number->string gn2-port))
                      ("GENENETWORK_FILES" ,genotype-files)
                      ("HOME" "/tmp")
                      ("LC_ALL" "en_US.UTF-8")
                      ("NO_REDIS" "no-redis")
                      ("RUST_BACKTRACE" "1"))))
              (with-imported-modules (source-module-closure '((guix search-paths)))
                #~(make-forkexec-constructor
                   (list #$(least-authority-wrapper
                            (program-file "genenetwork2"
                                          (genenetwork2-gexp config))
                            #:name "genenetwork2-pola-wrapper"
                            #:preserved-environment-variables
                            (append '("REQUESTS_CA_BUNDLE")
                                    (map first gn2-settings)
                                    (map search-path-specification-variable
                                         (manifest-search-paths gn2-manifest)))
                            ;; If we mapped only the mysqld.sock
                            ;; socket file, it would break when the
                            ;; external mysqld server is restarted.
                            #:mappings (list (file-system-mapping
                                              (source genotype-files)
                                              (target source))
                                             (file-system-mapping
                                              (source "/genenetwork2")
                                              (target source)
                                              (writable? #t))
                                             (file-system-mapping
                                              (source "/run/mysqld")
                                              (target source)
                                              (writable? #t))
                                             ;; XXXX: FIXME: R/Qtl generates
                                             ;; files in "/tmp" and
                                             ;; "/tmp/gn2".  These files are
                                             ;; accessed by gn3 for R/Qtl
                                             ;; mapping
                                             (file-system-mapping
                                              (source "/tmp")
                                              (target source)
                                              (writable? #t))
                                             (file-system-mapping
                                              (source gn2-secrets)
                                              (target source)
                                              (writable? #t)))
                            #:namespaces (delq 'net %namespaces))
                         "127.0.0.1" #$(number->string gn2-port))
                   #:user "genenetwork"
                   #:group "genenetwork"
                   #:environment-variables
                   (append
                    '("REQUESTS_CA_BUNDLE="
                      #$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt"))
                    (map (match-lambda
                           ((spec . value)
                            (string-append (search-path-specification-variable spec)
                                           "="
                                           value)))
                         (evaluate-search-paths
                          (map sexp->search-path-specification
                               '#$(map search-path-specification->sexp
                                       (manifest-search-paths gn2-manifest)))
                          (list #$gn2-profile)))
                    (map (match-lambda
                           ((spec value)
                            (string-append spec "=" value)))
                         '#$gn2-settings))
                   #:log-file "/var/log/genenetwork2.log"))))
           (stop #~(make-kill-destructor)))
          (shepherd-service
           (documentation "Run GeneNetwork 3 development server.")
           (provision '(genenetwork3))
           (requirement '(networking))
           (start #~(make-forkexec-constructor
                     (list #$(least-authority-wrapper
                              (program-file "genenetwork3"
                                            (genenetwork3-gexp config))
                              #:name "genenetwork3-pola-wrapper"
                              #:mappings (list (file-system-mapping
                                                (source "/genenetwork3")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source "/run/mysqld")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source "/tmp")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source data-directory)
                                                (target source))
                                               (file-system-mapping
                                                (source xapian-db-path)
                                                (target source))
                                               (file-system-mapping
                                                (source "/etc/genenetwork/conf/gn3")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source auth-db-path)
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source llm-db-path)
                                                (target source)
                                                (writable? #t)))
                              #:namespaces (delq 'net %namespaces))
                           "127.0.0.1" #$(number->string gn3-port))
                     #:user "genenetwork"
                     #:group "genenetwork"
                     #:log-file "/var/log/genenetwork3.log"))
           (stop #~(make-kill-destructor)))
          (shepherd-service
           (documentation "Run gn-auth development server.")
           (provision '(gn-auth))
           (requirement '(networking))
           (start #~(make-forkexec-constructor
                     (list #$(least-authority-wrapper
                              (program-file "gn-auth"
                                            (gn-auth-gexp config))
                              #:name "gn-auth-pola-wrapper"
                              ;; If we mapped only the mysqld.sock
                              ;; socket file, it would break when the
                              ;; external mysqld server is restarted.
                              #:mappings (list (file-system-mapping
                                                (source "/run/mysqld")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source "/gn-auth")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source "/run/mysqld")
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source data-directory)
                                                (target source))
                                               (file-system-mapping
                                                (source auth-db-path)
                                                (target source)
                                                (writable? #t))
                                               (file-system-mapping
                                                (source gn-auth-secrets)
                                                (target source)
                                                (writable? #t)))
                              #:namespaces (delq 'net %namespaces))
                           "127.0.0.1" #$(number->string gn-auth-port))
                     #:user "genenetwork"
                     #:group "genenetwork"
                     #:log-file "/var/log/gn-auth.log"))
           (stop #~(make-kill-destructor))))))

(define %genenetwork-accounts
  (list (user-group
         (name "genenetwork")
         (system? #t))
        (user-account
         (name "genenetwork")
         (group "genenetwork")
         (system? #t)
         (comment "GeneNetwork user")
         (home-directory "/var/empty")
         (shell (file-append shadow "/sbin/nologin")))))


(define genenetwork-service-type
  (service-type
   (name 'genenetwork)
   (description "Run GeneNetwork development servers and CI.")
   (extensions
    (list (service-extension account-service-type
                             (const %genenetwork-accounts))
          (service-extension activation-service-type
                             genenetwork-activation)
          (service-extension shepherd-root-service-type
                             genenetwork-shepherd-services)))
   (default-value (genenetwork-configuration))))


(operating-system
  (host-name "genenetwork-work-container")
  (timezone "UTC")
  (locale "en_US.utf8")
  (bootloader (bootloader-configuration
               (bootloader grub-bootloader)
               (targets (list "/dev/sdX"))))
  (file-systems %base-file-systems)
  (users %base-user-accounts)
  (sudoers-file
   (mixed-text-file "sudoers"
                    "@include " %sudoers-specification
                    "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n"))
  (packages (cons* genenetwork2 python-hypothesis gn-auth
                   python-mypy python-mypy-extensions python-pylint
                   %base-packages))
  (services (cons* (service virtuoso-service-type
                            (virtuoso-configuration
                             (server-port 7081)
                             (http-server-port 7082)
                             (dirs-allowed "/var/lib/virtuoso/data")
                             (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db")
                             (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx")))
                   (service redis-service-type
                            (redis-configuration
                             (bind "127.0.0.1")
                             (port 6379)
                             (working-directory "/var/lib/redis")))
                   (service genenetwork-service-type)
                   %base-services)))