diff options
| -rw-r--r-- | .guix-channel | 55 | ||||
| -rw-r--r-- | README.org | 32 | ||||
| -rw-r--r-- | fallback.scm | 3 | ||||
| -rwxr-xr-x | genenetwork-development-deploy.sh | 4 | ||||
| -rw-r--r-- | genenetwork-development.scm | 786 | ||||
| -rw-r--r-- | genenetwork/services/genecup.scm | 121 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 239 | ||||
| -rw-r--r-- | genenetwork/services/mouse-longevity.scm | 33 | ||||
| -rw-r--r-- | guix/gn-machines/genenetwork.scm | 270 | ||||
| -rw-r--r-- | guix/gn-machines/services/monitoring.scm | 68 | ||||
| -rwxr-xr-x | production-deploy.sh | 36 | ||||
| -rw-r--r-- | production.scm | 26 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 10 | ||||
| -rw-r--r-- | public-sparql.scm | 13 | ||||
| -rw-r--r-- | services/README.md | 17 | ||||
| -rw-r--r-- | services/gn-guile.scm | 52 | ||||
| -rw-r--r-- | services/opensmtpd.scm | 21 | ||||
| -rwxr-xr-x | singularity-head-deploy.sh | 12 | ||||
| -rwxr-xr-x | singularity-worker-deploy.sh | 37 | ||||
| -rw-r--r-- | singularity.scm | 42 | ||||
| -rw-r--r-- | specials/gndev.scm | 1 | ||||
| -rw-r--r-- | test-r-container.scm | 111 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 17 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
24 files changed, 1625 insertions, 383 deletions
diff --git a/.guix-channel b/.guix-channel new file mode 100644 index 0000000..a15d3ca --- /dev/null +++ b/.guix-channel @@ -0,0 +1,55 @@ +(channel + (version 0) + (directory "guix") + (dependencies + (channel + (name guix) + (url "https://codeberg.org/guix/guix") + (branch "master") + (commit "0a4740705090acc4c8a10d4f53afc58c9f62e980") + (introduction + (channel-introduction + (version 0) + (commit "9edb3f66fd807b096b48283debdcddccfea34bad") + (signer + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) + (channel + (name guix-forge) + (url "https://git.systemreboot.net/guix-forge/") + (branch "main") + (commit "e43fd9a4d73654d3876e2c698af7da89f3408f89") + (introduction + (channel-introduction + (version 0) + (commit "0432e37b20dd678a02efee21adf0b9525a670310") + (signer + "7F73 0343 F2F0 9F3C 77BF 79D3 2E25 EE8B 6180 2BB3")))) + (channel + (name guix-bioinformatics) + (url "https://git.genenetwork.org/guix-bioinformatics") + (commit "9b0955f14ec725990abb1f6af3b9f171e4943f77")) + ;; Until https://issues.guix.gnu.org/68797 is resolved, we need to + ;; explicitly list guix-past and guix-rust-past-crates—the + ;; dependencies of the guix-bioinformatics channel—here. + (channel + (name guix-past) + (url "https://codeberg.org/guix-science/guix-past") + (branch "master") + (commit "473c942b509ab3ead35159d27dfbf2031a36cd4d") + (introduction + (channel-introduction + (version 0) + (commit "c3bc94ee752ec545e39c1b8a29f739405767b51c") + (signer + "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5")))) + (channel + (name guix-rust-past-crates) + (url "https://codeberg.org/guix/guix-rust-past-crates.git") + (branch "trunk") + (commit "b8b7ffbd1cec9f56f93fae4da3a74163bbc9c570") + (introduction + (channel-introduction + (version 0) + (commit "1db24ca92c28255b28076792b93d533eabb3dc6a") + (signer + "F4C2 D1DF 3FDE EA63 D1D3 0776 ACC6 6D09 CA52 8292")))))) diff --git a/README.org b/README.org index 892ab23..2aa45db 100644 --- a/README.org +++ b/README.org @@ -3,19 +3,40 @@ containers. The git repo lives at https://git.genenetwork.org/gn-machines/ -* GeneNetwork development container +For CI/CD see -The GeneNetwork development container is currently run on /tux02/. It runs -continuous integration and continuous deployment services for -genenetwork2, genenetwork3 and several other associated projects. +=> https://issues.genenetwork.org/topics/systems/ci-cd + +For philosophy and (KISS) incremental minimalistic development containers, see: + +=> https://issues.genenetwork.org/topics/systems/debug-and-developing-code-with-genenetwork-system-container + +These are stored in the './minimal' directory of this repo. See the [[./minimal/README.md][README]]. + +* GeneNetwork development container (aka CI/CD) + +The GeneNetwork development container is currently run on /tux02/. +It runs continuous integration and continuous deployment services for genenetwork2, genenetwork3 and several other associated projects. To build and install the container, you will need the [[https://gitlab.com/genenetwork/guix-bioinformatics][guix-bioinformatics]] and [[https://git.systemreboot.net/guix-forge/][guix-forge]] channels. Once these channels are pulled and available, on /tux02/, run + #+BEGIN_SRC shell $ ./genenetwork-development-deploy.sh #+END_SRC +It will try to create symlinks at the end. You can do this as root (too). + +Note we current run as aruni user and the latest guix is the user profile. + +#+BEGIN_SRC shell +aruni@tux02:~/gn-machines$ which guix +/home/aruni/.config/guix/current/bin/guix +aruni@tux02:~/gn-machines$ guix describe +Generation 46 Mar 27 2025 23:39:42 (current) +#+END_SRC + /tux02/ is configured with a systemd service to run this container. Restart it. #+BEGIN_SRC shell @@ -31,6 +52,7 @@ To build and install the container, you will need the guix-bioinformatics channel. Once guix-bioinformatics is pulled and available, on /tux01/, run #+begin_src shell + $ ./virtuoso-deploy.sh #+end_src @@ -55,7 +77,7 @@ You can get a shell into the container with something like: sudo guix container exec 89086 /run/current-system/profile/bin/bash --login #+END_SRC -When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or on your can get with something like: +When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or by inspecting the output of the following ~ps~ command: #+BEGIN_SRC sh ps -u root -f --forest | grep -A4 '/usr/local/bin/genenetwork-development-container' | grep 'shepherd' diff --git a/fallback.scm b/fallback.scm index e222238..8153789 100644 --- a/fallback.scm +++ b/fallback.scm @@ -29,7 +29,7 @@ (forge socket)) (operating-system - (host-name "genenetwork-fallback") + (host-name "fallback") (timezone "UTC") (locale "en_US.utf8") (bootloader (bootloader-configuration @@ -61,6 +61,7 @@ (gn-auth-server-name "fallback-auth.genenetwork.org") (gn2-port 8892) (gn3-port 8893) + (gn-auth-port 8894) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export/data/genenetwork-sqlite/auth.db") (xapian-db "/export/data/genenetwork-xapian") diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index b251033..083441f 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -29,9 +29,10 @@ # /etc/genenetwork/conf instead of merely exposing it. container_script=$(guix system container --network \ --verbosity=3 \ - --load-path=. \ + --load-path=./guix/ \ --share=/home/git/public \ --share=/var/guix/daemon-socket=/var/host-guix/daemon-socket \ + --share=/export2/guix-containers/genenetwork-development/var/cache=/var/cache \ --share=/export2/guix-containers/genenetwork-development/var/lib/acme=/var/lib/acme \ --share=/export2/guix-containers/genenetwork-development/var/lib/laminar=/var/lib/laminar \ --share=/export2/guix-containers/genenetwork-development/var/lib/tissue=/var/lib/tissue \ @@ -42,6 +43,7 @@ container_script=$(guix system container --network \ --expose=/export/data/genenetwork \ --share=/export/data/genenetwork-xapian \ --share=/export/data/genenetwork-sqlite \ + --share=/export/data/lmdb \ --share=/var/run/mysqld=/run/mysqld \ --share=/export/data/gn-docs/ \ genenetwork-development.scm) diff --git a/genenetwork-development.scm b/genenetwork-development.scm index f841255..e2bf71d 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,28 +21,30 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) + (gn-machines services monitoring) + ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-guile gn-integration-tests)) (gn services databases) - ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) - ((gnu packages base) #:select (gnu-make tar)) + ((gnu packages base) #:select (gnu-make tar coreutils-minimal)) ((gnu packages bash) #:select (bash)) - ((gnu packages bioinformatics) #:select (ccwl) #:prefix guix:) - ((gnu packages certs) #:select (nss-certs)) - ((gnu packages check) #:select (python-pylint)) + ((gnu packages nss) #:select (nss-certs)) + ((gnu packages python) #:select (python)) + ((gnu packages check) #:select (python-pylint python-pytest)) + ((gnu packages curl) #:select (curl)) ((gnu packages ci) #:select (laminar)) ((gnu packages compression) #:select (gzip)) - ((gnu packages databases) #:select (mariadb virtuoso-ose)) + ((gnu packages databases) #:select (mariadb redis)) + ((gnu packages databases) #:select (virtuoso-ose)) ((gnu packages gnupg) #:select (guile-gcrypt)) ((gnu packages graphviz) #:select (graphviz)) ((gnu packages guile) #:select (guile-3.0 guile-git guile-zlib)) - ((gnu packages guile-xyz) #:select (guile-dbd-mysql guile-dbi guile-dsv guile-hashing - guile-ini guile-lib guile-libyaml guile-smc guile-xapian)) + ((gnu packages guile-xyz) #:select (guile-dbd-mysql guile-dbi guile-dsv guile-hashing guile-uuid + guile-ini guile-lib guile-libyaml guile-smc guile-xapian)) ((gnu packages guile-xyz) #:select (guile-sparql) #:prefix guix:) ((gnu packages haskell-apps) #:select (shellcheck)) ((gnu packages python-check) #:select (python-mypy)) - ((gnu packages python-web) #:select (gunicorn)) + ((gnu packages python-web) #:select (gunicorn python-requests)) ((gnu packages rdf) #:select (raptor2)) ((gnu packages tls) #:select (openssl)) ((gnu packages version-control) #:select (git-minimal)) @@ -70,6 +72,7 @@ (guix utils) (forge acme) (forge cgit) + (forge fcgiwrap) (forge forge) (forge laminar) (forge nginx) @@ -84,10 +87,6 @@ (define %guix-daemon-uri "/var/host-guix/daemon-socket/socket") -(define %default-guix-channel-with-substitutes - (channel-with-substitutes-available %default-guix-channel - "https://ci.guix.gnu.org")) - ;; We cannot refer to sudo in the store since that sudo does not have ;; the setuid bit set. See "(guix) Setuid Programs". (define sudo @@ -122,9 +121,13 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "/home/git/public/gn-auth")) + (default "https://git.genenetwork.org/gn-auth")) (gn-libs-repository genenetwork-configuration-gn-libs-repository - (default "/home/git/public/gn-libs")) + (default "https://git.genenetwork.org/gn-libs")) + (gn-guile-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-guile")) + (repositories-checkout-directory genenetwork-repositories-checkout-directory + (default "/home/genenetwork")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -149,10 +152,16 @@ be imported into G-expressions." (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path (default "/export/data/genenetwork-sqlite/llm.db")) + (lmdb-data-path genenetwork-lmdb-data-path + (default "/export/data/lmdb")) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) + (repositories genenetwork-configuration-repositories + (default "/export/data/repositories")) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout - (default "/export/data/gn-docs"))) + (default "/export/data/gn-docs")) + (gn-integration-tests-repository genenetwork-gn-integration-tests-repository + (default "https://git.genenetwork.org/gn-integration-tests"))) ;;; @@ -167,7 +176,7 @@ be imported into G-expressions." (ci-jobs (let ((channels (list (channel (name 'gn-bioinformatics) (url "https://git.genenetwork.org/guix-bioinformatics") - (branch "master"))))) + (branch "main"))))) (list (forge-laminar-job (name "guix-bioinformatics") (run (guix-channel-job-gexp channels @@ -192,7 +201,7 @@ executed." (match-record config <genenetwork-configuration> (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) - (with-packages (list bash coreutils git-minimal nss-certs) + (with-packages (list bash coreutils git-minimal nss-certs genenetwork2) #~(begin (use-modules (guix build utils) (srfi srfi-26)) @@ -224,10 +233,6 @@ executed." (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) - ;; This is a dummy SERVER_PORT to placate - ;; bin/genenetwork2. TODO: Fix bin/genenetwork2 so that - ;; this is not needed. - (setenv "SERVER_PORT" "8080") ;; Use a profile with all dependencies except ;; genenetwork3. (setenv "GN2_PROFILE" @@ -241,7 +246,30 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + ;; This file is cosmetic + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"/tmp/secret.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:120\"\n" + "GN3_GUILE_SERVER_URL=\"http://localhost:120\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"/tmp/ssl-private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/tmp/gn-auth-ssl-public-key.pem\"\n")) + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" "/export/data/genenetwork/genotype_files") + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/tmp") (chdir "genenetwork2") ;; XXXX: FIXME: R/Qtl tests fail because files are generated in ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 @@ -277,7 +305,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -332,16 +360,13 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port gn-guile-port gn-guile-repository) (list (forge-project (name "genenetwork2") (repository gn2-repository) (ci-jobs (list (forge-laminar-job (name "genenetwork2") - (run (genenetwork2-tests - config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "-m" "pytest"))) + (run (genenetwork2-tests config (list "python3" "-m" "pytest"))) ;; If unit tests pass, redeploy genenetwork2 and ;; trigger Mechanical Rob. (after (with-imported-modules '((guix build utils)) @@ -357,8 +382,7 @@ genenetwork3 source from the latest commit of @var{project}." (name "genenetwork2-mechanical-rob") (run (genenetwork2-tests config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "test/requests/test-website.py" + (list "python3" "test/requests/test-website.py" "--all" (string-append "http://localhost:" (number->string gn2-port))))) (trigger? #f)))) (ci-jobs-trigger 'webhook)) @@ -417,7 +441,20 @@ genenetwork3 source from the latest commit of @var{project}." #:variables (list (variable-specification (module '(gn-libs)) (name 'gn-libs))) - #:guix-daemon-uri %guix-daemon-uri)))))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook)) + (forge-project + (name "gn-guile") + (repository gn-guile-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-guile") + (run (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$sudo + #$(file-append shepherd "/bin/herd") + "restart" "gn-guile"))))))) + (ci-jobs-trigger 'webhook)) (forge-project (name "gn-auth") (repository gn-auth-repository) @@ -441,9 +478,17 @@ genenetwork3 source from the latest commit of @var{project}." (invoke #$sudo #$(file-append shepherd "/bin/herd") "restart" "gn-auth") + (invoke #$(file-append laminar "/bin/laminarc") ;; queue smoke tests + "queue" "gn-auth-smoke-tests") (invoke #$(file-append laminar "/bin/laminarc") "queue" "genenetwork2")))))) (forge-laminar-job + (name "gn-auth-smoke-tests") + (run (gn-integration-tests-gexp + config + (list "pytest" "-m" "gn_auth and smoke" "-v"))) + (trigger? #f)) + (forge-laminar-job (name "gn-auth-all-tests") (run (guix-channel-job-gexp (list (channel @@ -453,13 +498,14 @@ genenetwork3 source from the latest commit of @var{project}." #:variables (list (variable-specification (module '(gn-auth)) (name 'gn-auth-all-tests))) - #:guix-daemon-uri %guix-daemon-uri))))))))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook))))) (define (genenetwork2-cd-gexp config) "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files) + (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port repositories-checkout-directory) (with-packages (list coreutils git-minimal gunicorn nss-certs) (with-imported-modules '((guix build utils)) #~(begin @@ -476,49 +522,85 @@ server described by CONFIG, a <genenetwork-configuration> object." (hline) (invoke "git" "log" "--max-count" "1") (hline)) - - ;; Clone the latest genenetwork2 and genenetwork3 + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "TERM" "xterm-256color") + ;; Clone the latest genenetwork2 ;; repositories. - (invoke "git" "clone" "--depth" "1" #$gn2-repository) - (with-directory-excursion "genenetwork2" - (show-head-commit)) - (invoke "git" "clone" "--depth" "1" #$gn3-repository) - (with-directory-excursion "genenetwork3" - (show-head-commit)) - - ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" - (string-append (getcwd) "/genenetwork3")) - ;; Set other environment variables required by - ;; genenetwork2. - (setenv "GN2_PROFILE" #$(profile - (content (package->development-manifest genenetwork2)) - (allow-collisions? #t))) - (setenv - "GN2_SETTINGS" - #$(mixed-text-file "gn2.conf" - "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" - "AI_SEARCH_ENABLED=True\n" - "GN3_LOCAL_URL=\"" - (string-append "http://localhost:" - (number->string gn3-port)) - "\"\n" - "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" - "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" - "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" - "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) - - ;; Start genenetwork2. - (with-directory-excursion "genenetwork2" - (invoke #$(file-append bash "/bin/sh") - "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-prod"))))))) + (let ((gn2-checkout (string-append #$repositories-checkout-directory "/genenetwork2")) + (gn3-checkout (string-append #$repositories-checkout-directory "/genenetwork3")) + (gn-libs-checkout (string-append #$repositories-checkout-directory "/gn-libs"))) + (with-directory-excursion + #$repositories-checkout-directory + (when (file-exists? gn2-checkout) + (delete-file-recursively gn2-checkout)) + (invoke "git" "clone" "--depth" "1" #$gn2-repository)) + + ;; Override the genenetwork3 used by genenetwork2. + (setenv "GN3_PYTHONPATH" gn3-checkout) + ;; Set other environment variables required by + ;; genenetwork2. + (setenv "GN2_PROFILE" #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + gn-libs-checkout + ":" + (getenv "GN3_PYTHONPATH") + ":" + (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages"))) + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" #$genotype-files) + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/home/genenetwork") + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"" + (string-append "http://localhost:" + (number->string gn3-port)) + "\"\n" + "GN_GUILE_SERVER_URL=\"" + (string-append "http://localhost:" + (number->string gn-guile-port)) + "\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) + + ;; Start genenetwork2. + (with-directory-excursion + gn2-checkout + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) + "--workers" "20" + "--keep-alive" "6000" + "--max-requests" "100" + "--max-requests-jitter" "30" + "--timeout" "1200" + "--log-level" "debug" + "gn2.wsgi")))))))) (define (genenetwork3-cd-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path gn-libs-repository repositories-checkout-directory) (with-manifest (package->development-manifest genenetwork3) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -537,41 +619,60 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest genenetwork3 repository. - (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "GN3_PROFILE" #$(profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN3_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) ;; Configure genenetwork3. (setenv "GN3_CONF" #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "LMDB_DATA_PATH=\"" lmdb-data-path "\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) - (setenv "RSCRIPT" #$(file-append - (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t)) - "/bin/Rscript")) - ;; Run genenetwork3. - (with-directory-excursion "genenetwork3" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn3-port)) - "--workers" "8" - "gn3.app:create_app()" - ;; gunicorn's default 30 second timeout is - ;; insufficient for Fahamu AI endpoints and - ;; results in worker timeout errors. - "--timeout" "1200")))))))) + (setenv "RSCRIPT" (string-append + (getenv "GN3_PROFILE") + "/bin/Rscript")) + + (let ((gn3-checkout (string-append #$repositories-checkout-directory "/genenetwork3")) + (gn-libs-checkout (string-append #$repositories-checkout-directory "/gn-libs"))) + (with-directory-excursion + #$repositories-checkout-directory + ;; Clone the latest genenetwork3 repository. + (when (file-exists? "genenetwork3") + (delete-file-recursively "genenetwork3")) + (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (when (file-exists? "gn-libs") + (delete-file-recursively "gn-libs")) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository)) + (setenv "PYTHONPATH" (string-append gn-libs-checkout ":$PYTHONPATH")) + (with-directory-excursion + gn3-checkout + (show-head-commit) + ;; Run genenetwork3. + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn3-port)) + "--workers" "8" + "gn3.app:create_app()" + ;; gunicorn's default 30 second timeout is + ;; insufficient for Fahamu AI endpoints and + ;; results in worker timeout errors. + "--timeout" "1200" + "--log-level" "debug"))))))))) (define (gn-auth-cd-gexp config) "Return a G-expression that runs the latest gn-auth development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn-auth-repository gn-auth-port auth-db-path gn-auth-secrets) + (gn-auth-repository gn-libs-repository gn-auth-port auth-db-path gn-auth-secrets repositories-checkout-directory) (with-manifest (package->development-manifest gn-auth) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -590,43 +691,124 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest gn-auth repository. - (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) - ;; Configure gn-auth. - (setenv "GN_AUTH_CONF" - #$(mixed-text-file "gn-auth.conf" - "AUTH_DB=\"" auth-db-path "\"\n" - "GN_AUTH_SECRETS=\"" gn-auth-secrets "/gn-auth-secrets.py\"\n" - "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"" gn-auth-secrets "/clients-public-keys\"\n" - "SSL_PRIVATE_KEY=\"" gn-auth-secrets "/gn-auth-ssl-private-key.pem\"\n")) - (setenv "HOME" "/tmp") - (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") - ;; Run gn-auth. - (with-directory-excursion "gn-auth" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn-auth-port)) - "--workers" "8" - "gn_auth.wsgi:app")))))))) - -(define (gn-guile-gexp gn-guile-port) - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) - (when (file-exists? current-repo-path) - (delete-file-recursively current-repo-path)) - (setenv "CURRENT_REPO_PATH" current-repo-path) - (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) - (invoke #$(file-append gn-guile "/bin/gn-guile") - (number->string #$gn-guile-port))))) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + + (let ((gn-auth-checkout (string-append #$repositories-checkout-directory "/gn-auth")) + (gn-libs-checkout (string-append #$repositories-checkout-directory "/gn-libs"))) + (with-directory-excursion + #$repositories-checkout-directory + ;; Clone the latest gn-auth repository. + (when (file-exists? gn-auth-checkout) + (delete-file-recursively gn-auth-checkout)) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + (when (file-exists? gn-libs-checkout) + (delete-file-recursively gn-libs-checkout)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository)) + + ;; Configure gn-auth. + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + gn-libs-checkout + ":" + (string-append + (getenv "GN_AUTH_PROFILE") + ":" + "/lib/python3.11/site-packages"))) + (setenv "GN_AUTH_CONF" + #$(mixed-text-file "gn-auth.conf" + "AUTH_DB=\"" auth-db-path "\"\n" + "GN_AUTH_SECRETS=\"" gn-auth-secrets "/gn-auth-secrets.py\"\n" + "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"" gn-auth-secrets "/clients-public-keys\"\n" + "SSL_PRIVATE_KEY=\"" gn-auth-secrets "/gn-auth-ssl-private-key.pem\"\n")) + (setenv "HOME" "/tmp") + (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") + ;; Run gn-auth. + (with-directory-excursion gn-auth-checkout + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn-auth-port)) + "--workers" "8" + "--log-level" "debug" + "gn_auth.wsgi:app"))))))))) + +(define (gn-guile-gexp gn-guile-port repositories-checkout-directory) + (with-packages + (list coreutils git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke "git" "log" "--max-count" "1") + (hline)) + ;; KLUDGE: Here we set all the certificates properly. In gn-guile, + ;; we make request to external services. Here's an example: + ;; curl http://localhost:8091/gene/aliases/Shh + ;; + ;; Without certs, we run into: + ;; 2025-07-22 08:27:11 GET /gene/aliases/Shh + ;; [...] + ;; 2025-07-22 08:27:19 signer-not-found invalid + (setenv "GN_GUILE_PROFILE" #$(profile + (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (setenv "SSL_CERT_DIR" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs")) + (setenv "SSL_CERT_FILE" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GIT_SSL_CAINFO" (getenv "SSL_CERT_FILE")) + (setenv "CURL_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "SPARQL-ENDPOINT" "http://localhost:9082/sparql/") + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (let ((current-repo-path (string-append #$repositories-checkout-directory "/gn-docs")) + (gn-guile-checkout (string-append #$repositories-checkout-directory "/gn-guile"))) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (with-directory-excursion + #$repositories-checkout-directory + ;; All edits go to the current-repo-path; so we need it to + ;; be persistent. + (unless (file-exists? current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" + (string-append "file://" (getenv "CGIT_REPO_PATH")) current-repo-path)) + + (when (file-exists? "gn-guile") + (delete-file-recursively "gn-guile")) + (invoke "git" "clone" "--depth" "1" "https://git.genenetwork.org/gn-guile" "gn-guile") + (with-directory-excursion + "gn-guile" + (setenv "HOME" gn-guile-checkout) ;; why? + (show-head-commit) + (invoke #$(file-append gn-guile "/bin/gn-guile") + "--port" + (number->string #$gn-guile-port) + "--gn-docs-local-checkout" + current-repo-path + "--gn-docs-remote-url" + (getenv "CGIT_REPO_PATH") + "--gn-docs-working-branch" + "gn-cd-branch")))))))) (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port repositories-checkout-directory) (list (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -642,13 +824,17 @@ described by CONFIG, a <genenetwork-configuration> object." #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) + (gn-guile-gexp gn-guile-port repositories-checkout-directory)) #:name "gn-guile-pola-wrapper" #:preserved-environment-variables (map first gn-guile-settings) #:mappings (list (file-system-mapping (source gn-doc-git-checkout) (target source) + (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) @@ -664,11 +850,13 @@ described by CONFIG, a <genenetwork-configuration> object." (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) - ;; FIXME: The genenetwork2 service should depend on redis. - (requirement '(networking genenetwork3)) + (requirement '(networking redis)) (modules '((guix search-paths) (ice-9 match) (srfi srfi-1))) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn2 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start (let* ((gn2-manifest (packages->manifest (list genenetwork2))) (gn2-profile (profile @@ -698,6 +886,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source genotype-files) (target source)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source "/run/mysqld") (target source) @@ -743,6 +935,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run GeneNetwork 3 development server.") (provision '(genenetwork3)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn3 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "genenetwork3" @@ -755,6 +950,14 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source lmdb-data-path) + (target source) + (writable? #t)) ;; XXXX: FIXME: R/Qtl generates ;; files in "/tmp" and ;; "/tmp/gn2". These files are @@ -775,11 +978,11 @@ described by CONFIG, a <genenetwork-configuration> object." (target source) (writable? #t)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) - (file-system-mapping - (source llm-db-path) + (file-system-mapping + (source (dirname llm-db-path)) (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) @@ -792,6 +995,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run gn-auth development server.") (provision '(gn-auth)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn-auth to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-auth" @@ -804,11 +1010,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -831,15 +1041,29 @@ described by CONFIG, a <genenetwork-configuration> object." (group "genenetwork") (system? #t) (comment "GeneNetwork user") - (home-directory "/var/empty") + (home-directory "/home/genenetwork") (shell (file-append shadow "/sbin/nologin"))))) (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets) + (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets gn-doc-git-checkout) (with-imported-modules '((guix build utils)) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (ice-9 ftw)) + ;; KLUDGE: Guix now stores inferior profiles under + ;; /var/guix/profiles/per-user (commit + ;; d12c4452a49b355369636de1dfc766b5bad6437b). The 'laminar' + ;; user’s directory is not created automatically in our + ;; pinned Guix revision, which causes CI jobs using + ;; inferiors to fail with permission errors. + ;; XXXX: FIXME: Explicitly create the directory for + ;; now. Remove this once we update the pinned Guix commit. + (unless (file-exists? "/var/guix/profiles/per-user/laminar") + (mkdir-p "/var/guix/profiles/per-user/laminar") + (chown "/var/guix/profiles/per-user/laminar" + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) ;; Set ownership of files. (for-each (lambda (file) @@ -849,12 +1073,13 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) - (find-files "/export/data/gn-docs" + (find-files gn-doc-git-checkout #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets #:directories? #t)))) + ;; Prevent other users from reading secret files. (for-each (lambda (file) (chmod file #o600)) @@ -884,27 +1109,6 @@ described by CONFIG, a <genenetwork-configuration> object." ;;; transform-genenetwork-database ;;; - -;; Unreleased version of ccwl that is required by -;; transform-genenetwork-database for its graphql library. -(define ccwl - (let ((commit "02677a508b407779f5991a230341e016deb7f69b") - (revision "0")) - (package - (inherit guix:ccwl) - (name "ccwl") - (version (git-version (package-version guix:ccwl) revision commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/arunisaac/ccwl") - (commit commit))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1kxry8y0pibl0x789jrzqkkh2s59ajyinfvrgvd00gkbqldih82r"))))))) - ;; guile-sparql tests are broken. Disable them temporarily. The issue ;; has been reported upstream at ;; https://github.com/roelj/guile-sparql/issues/6 @@ -949,7 +1153,7 @@ described by CONFIG, a <genenetwork-configuration> object." (define (transform-genenetwork-database-gexp connection-settings virtuoso-data-dir repository) (with-imported-modules '((guix build utils)) - (with-packages (list ccwl git-minimal gnu-make guile-3.0 guile-dbd-mysql + (with-packages (list git-minimal gnu-make guile-3.0 guile-dbd-mysql guile-uuid guile-dbi guile-hashing guile-libyaml guile-sparql guile-zlib nss-certs virtuoso-ose raptor2) #~(begin @@ -976,12 +1180,6 @@ described by CONFIG, a <genenetwork-configuration> object." (copy-recursively build-directory #$virtuoso-data-dir) ;; Load RDF into virtuoso. (invoke "./pre-inst-env" "./load-rdf.scm" #$connection-settings) - ;; Visualize schema and archive results. - (invoke "./pre-inst-env" "./visualize-schema.scm" #$connection-settings) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "sql.dot" (string-append "-o" (getenv "ARCHIVE") "/sql.svg")) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "rdf.dot" (string-append "-o" (getenv "ARCHIVE") "/rdf.svg")) (delete-file-recursively build-directory))))))) (define transform-genenetwork-database-project @@ -995,7 +1193,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'transform-genenetwork-database) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))) (forge-laminar-job (name "transform-genenetwork-database") @@ -1041,7 +1239,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lmdb) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1061,7 +1259,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-gsl) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1081,7 +1279,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lapack) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1274,6 +1472,30 @@ gn-auth." ";") "proxy_set_header Host $host;"))))))) +(define (gn-guile-reverse-proxy-server-block) + "Return an <nginx-server-configuration> object to reverse proxy +gn-guile to display RDF pages." + (nginx-server-configuration + (server-name '("rdf.genenetwork.org")) + (locations + (list + ;; SPARQL web point + (nginx-location-configuration + (uri "/sparql") + (body (list (list (string-append "proxy_pass http://localhost:" + (number->string %virtuoso-sparql-port) + "/sparql;") + "proxy_set_header Host $host;" + "proxy_set_header X-Forwarded-For $remote_addr;" + "proxy_set_header X-Forwarded-Proto $scheme;")))) + ;; Default RDF page served from gn-guile + (nginx-location-configuration + (uri "/") + (body (list (string-append "proxy_pass http://localhost:" + (number->string %gn-guile-port) + ";") + "proxy_set_header Host $host;"))))))) + (define set-build-directory-permissions-gexp (with-imported-modules '((guix build utils)) #~(begin @@ -1298,6 +1520,186 @@ gn-auth." (define %gn-auth-port 9094) ;; Port on which virtuoso's SPARQL endpoint is listening (define %virtuoso-sparql-port 9082) +;; Port on which gn-guile is listening +(define %gn-guile-port 8091) + +(define %genenetwork-configuration + (genenetwork-configuration + (gn2-port %genenetwork2-port) + (gn3-port %genenetwork3-port) + (gn-auth-port %gn-auth-port) + (gn2-secrets "/etc/genenetwork/conf/gn2") + (gn3-secrets "/etc/genenetwork/conf/gn3/secrets.py") + (gn-auth-secrets "/etc/genenetwork/conf/gn-auth") + (genotype-files "/export/data/genenetwork/genotype_files") + (sparql-endpoint (string-append "http://localhost:" + (number->string %virtuoso-sparql-port) + "/sparql")) + (data-directory "/export/data/genenetwork") + (xapian-db-path %xapian-directory))) + + + + +;; +;; gn-integration-tests +;; +(define (gn-auth-test-flask config) + "Return a program-file that wraps flask with the gn-auth environment +derived from CONFIG. All arguments are forwarded to flask, allowing +the caller to invoke any flask CLI command (create-test-users, +delete-test-users, etc.) as the genenetwork user via sudo." + (match-record config <genenetwork-configuration> + (auth-db-path gn-auth-secrets gn-auth-repository repositories-checkout-directory) + (let* ((gn-auth-profile (profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (gn-auth-conf (mixed-text-file + "gn-auth-test.conf" + "AUTH_DB=\"" auth-db-path "\"\n" + "GN_AUTH_SECRETS=\"" gn-auth-secrets + "/gn-auth-secrets.py\"\n" + "CLIENTS_SSL_PUBLIC_KEYS_DIR=\"" gn-auth-secrets + "/clients-public-keys\"\n" + "SSL_PRIVATE_KEY=\"" gn-auth-secrets + "/gn-auth-ssl-private-key.pem\"\n")) + (gn-libs-checkout (string-append repositories-checkout-directory "/gn-libs")) + (gn-auth-checkout (string-append repositories-checkout-directory "/gn-auth"))) + (program-file + "gn-auth-test-flask" + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke #$(file-append git-minimal "/bin/git") + "log" "--max-count" "1") + (hline)) + (with-directory-excursion #$gn-libs-checkout + (show-head-commit)) + (with-directory-excursion #$gn-auth-checkout + (show-head-commit)) + + (setenv "PYTHONPATH" + (string-append #$gn-libs-checkout ":" + #$gn-auth-checkout ":" + #$gn-auth-profile + "/lib/python3.11/site-packages")) + (setenv "REQUESTS_CA_BUNDLE" + (string-append #$gn-auth-profile + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN_AUTH_PROFILE" #$gn-auth-profile) + (setenv "GN_AUTH_CONF" #$gn-auth-conf) + (setenv "HOME" "/tmp") + (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") + (apply invoke + (string-append #$gn-auth-profile "/bin/flask") + (cons* "--app" "gn_auth.wsgi:app" + (cdr (program-arguments)))))))))) + +(define %gn-auth-test-flask + (gn-auth-test-flask %genenetwork-configuration)) + + +(define* (gn-integration-tests-gexp config test-command + #:key + (setup '()) + (teardown '())) + (match-record config <genenetwork-configuration> + (gn-integration-tests-repository repositories-checkout-directory) + (let ((gn-libs-checkout (string-append repositories-checkout-directory "/gn-libs")) + (gn-auth-checkout (string-append repositories-checkout-directory "/gn-auth")) + (gn3-checkout (string-append repositories-checkout-directory "/genenetwork3")) + (gn2-checkout (string-append repositories-checkout-directory "/genenetwork2")) + (gn-guile-checkout (string-append repositories-checkout-directory "/gn-guile"))) + (with-imported-modules '((guix build utils)) + (with-packages (list nss-certs git-minimal) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke #$(file-append git-minimal "/bin/git") + "--no-pager" "log" "--max-count" "1") + (hline)) + + + ;; laminar user cannot `cd' into some (or all) of these directories + ;; (with-directory-excursion #$gn-libs-checkout (show-head-commit)) + ;; (with-directory-excursion #$gn-auth-checkout (show-head-commit)) + ;; (with-directory-excursion #$gn-guile-checkout (show-head-commit)) + ;; (with-directory-excursion #$gn3-checkout (show-head-commit)) + ;; (with-directory-excursion #$gn2-checkout (show-head-commit)) + + (let* ((orig-dir (getcwd)) + (tmp-dir (mkdtemp "/tmp/gn-integration-tests.XXXXXX")) + (tests-profile #$(profile + (content (package->development-manifest gn-integration-tests)) + (allow-collisions? #t))) + (py-version + #$(version-major+minor (package-version python))) + (site-packages + (string-append tests-profile "/lib/python" + py-version "/site-packages"))) + (chdir tmp-dir) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" #$gn-integration-tests-repository) + (with-directory-excursion "gn-integration-tests" + (show-head-commit)) + (chdir "gn-integration-tests") + + (setenv "GN_INTEGRATION_TESTS_PROFILE" tests-profile) + (setenv "PATH" (string-append tests-profile "/bin:" + (getenv "PATH"))) + (if (getenv "PYTHONPATH") + (setenv + "PYTHONPATH" + (string-append site-packages ":" (getenv "PYTHONPATH"))) + (setenv "PYTHONPATH" site-packages)) + (dynamic-wind + (lambda () ;; Run setup if provided + (for-each + (lambda (setup-cmd) (apply invoke setup-cmd)) + '#$setup)) + (lambda () (apply invoke '#$test-command)) ;; Run actual tests + (lambda () ;; Always run teardowns to clean up. + (for-each + (lambda (teardown-cmd) (apply invoke teardown-cmd)) + '#$teardown) + (chdir orig-dir) + (delete-file-recursively tmp-dir)))))))))) + +(define (override-fcgiwrap-extension cgit-service) + (service + (service-type + (inherit (service-kind cgit-service)) + (extensions + (map (lambda (ext) + (if (eq? (service-extension-target ext) fcgiwrap-service-type) + (service-extension + (service-extension-target ext) + (compose list + (lambda (cfg) + (fcgiwrap-instance (inherit (car cfg)) + (processes 4))) + (service-extension-compute ext))) + ext)) + (service-type-extensions (service-kind cgit-service))))) + (service-value cgit-service))) + (operating-system (host-name "genenetwork-development") @@ -1308,13 +1710,14 @@ gn-auth." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) - (packages %base-packages) + (packages (cons* curl coreutils-minimal %base-packages)) (sudoers-file (mixed-text-file "sudoers" "@include " %sudoers-specification ;; Permit the laminar user to restart genenetwork2 ;; and genenetwork3. "\nlaminar ALL = NOPASSWD: " + (file-append shepherd "/bin/herd") " restart gn-guile, " (file-append shepherd "/bin/herd") " restart genenetwork2, " (file-append shepherd "/bin/herd") " start genenetwork3, " (file-append shepherd "/bin/herd") " stop genenetwork3, " @@ -1323,7 +1726,11 @@ gn-auth." (file-append shepherd "/bin/herd") " stop gn-auth, " (file-append shepherd "/bin/herd") " restart gn-auth\n" ;; Permit the acme user to restart nginx. - "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n" + ;; Permit the laminar user to run gn-auth test setup/teardown + ;; commands as the genenetwork user. + "\nlaminar ALL = (genenetwork) NOPASSWD: " + %gn-auth-test-flask "\n")) (services (cons* (service forge-service-type (forge-configuration (projects (list transform-genenetwork-database-project @@ -1332,10 +1739,17 @@ gn-auth." guile-lapack-project guile-lmdb-project guix-bioinformatics-project)))) - (service cgit-service-type - (cgit-configuration - (server-name "git.genenetwork.org") - (repository-directory "/home/git/public"))) + (override-fcgiwrap-extension + (service cgit-service-type + (cgit-configuration + (server-name "git.genenetwork.org") + (repository-directory "/home/git/public") + (extra-options + (list (cons "cache-root" "/var/cache/cgit") + (cons "cache-size" "1000") + (cons "cache-root-ttl" "5") + (cons "cache-repo-ttl" "5") + (cons "cache-dynamic-ttl" "5")))))) (service laminar-service-type (laminar-configuration (title "GeneNetwork CI") @@ -1374,20 +1788,7 @@ gn-auth." (server-port 9081) (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) - (service genenetwork-service-type - (genenetwork-configuration - (gn2-port %genenetwork2-port) - (gn3-port %genenetwork3-port) - (gn-auth-port %gn-auth-port) - (gn2-secrets "/etc/genenetwork/conf/gn2") - (gn3-secrets "/etc/genenetwork/conf/gn3/secrets.py") - (gn-auth-secrets "/etc/genenetwork/conf/gn-auth") - (genotype-files "/export/data/genenetwork/genotype_files") - (sparql-endpoint (string-append "http://localhost:" - (number->string %virtuoso-sparql-port) - "/sparql")) - (data-directory "/export/data/genenetwork") - (xapian-db-path %xapian-directory))) + (service genenetwork-service-type %genenetwork-configuration) (simple-service 'set-build-directory-permissions activation-service-type set-build-directory-permissions-gexp) @@ -1418,9 +1819,18 @@ gn-auth." %genenetwork2-port %genenetwork3-port) (laminar-reverse-proxy-server-block "localhost:9089" %webhook-port - (list 'gn-bioinformatics)) + (list 'guix + 'guix-past + 'guix-forge + 'gn-machines + 'guix-bioinformatics + 'guix-rust-past-crates)) (tissue-reverse-proxy-server-block) - (gn-auth-reverse-proxy-server-block))))) + (gn-auth-reverse-proxy-server-block) + (gn-guile-reverse-proxy-server-block))))) + (service guile-sheepdog-service-type + (guile-sheepdog-configuration + (settings-file "/etc/genenetwork/conf/sheepdog.scm"))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) diff --git a/genenetwork/services/genecup.scm b/genenetwork/services/genecup.scm new file mode 100644 index 0000000..6ee2812 --- /dev/null +++ b/genenetwork/services/genecup.scm @@ -0,0 +1,121 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services genecup) + #:use-module (guix) + #:use-module (gnu) + #:use-module (guix git) + #:use-module (guix modules) + #:use-module (guix profiles) + #:use-module (guix records) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:use-module (forge utils) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gn packages machine-learning) + #:use-module (gnu packages certs) + #:use-module (gnu packages curl) + #:use-module (gn packages python) + #:use-module (gnu packages admin) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module ((gnu packages python) #:select (python)) + #:use-module (gnu packages python-xyz) + #:use-module (guix build python-build-system) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages python-web) + #:use-module (gnu services shepherd) + #:use-module (gnu packages python-science) + #:use-module (gnu packages machine-learning) + #:use-module ((gnu packages admin) #:select (shepherd)) + #:export (genecup-configuration + genecup-configuration? + genecup-configuration-package + genecup-service-type)) + +(define-record-type* <genecup-configuration> + genecup-configuration make-genecup-configuration + genecup-configuration? + (package genecup-configuration-package ; <package> + (default genecup-latest-with-tensorflow-native))) + +(define (genecup-activation config) + (match-record config <genecup-configuration> + (package) + (with-packages + (list python + python-nltk + nss-certs ; Needed for downloading data with nltk.downloader. + curl) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((nltk-data "/var/cache/nltk_data/") + (data-dir "/export/ratspub/tmp")) + (unless (file-exists? nltk-data) + (begin + (mkdir-p nltk-data) + (chdir nltk-data) + (invoke #$(file-append python "/bin/python3") "-m" "nltk.downloader" "-d" nltk-data "punkt"))) + (unless (file-exists? (string-append data-dir "/userspub.sqlite")) + (begin + (install-file #$(file-append package "/userspub.sqlite") data-dir) + (chmod (string-append data-dir "/userspub.sqlite") #o554))))))))) + +(define genecup-shepherd-service + (match-lambda + (($ <genecup-configuration> package) + (with-imported-modules (source-module-closure + '((guix search-paths) + (gnu build shepherd) + (gnu system file-systems))) + (list (shepherd-service + (provision '(genecup)) + (requirement '(networking)) + (modules '((gnu build shepherd) + (gnu system file-systems))) + (start + #~(make-forkexec-constructor + (list #$(file-append package "/server.py")) + #:directory #$package + #:environment-variables + (list + "NLTK_DATA=/var/cache/nltk_data" + (string-append "EDIRECT_PUBMED_MASTER=" + #$(file-append package "/minipubmed")) + (string-append "PERL_LWP_SSL_CA_FILE=" + #$(file-append (profile (content (packages->manifest (list curl nss-certs)))) + "/etc/ssl/certs/ca-certificates.crt"))) + #:log-file "/var/log/genecup.log")) + (stop #~(make-kill-destructor)))))))) + +(define genecup-service-type + (service-type + (name 'genecup) + (extensions + (list + (service-extension activation-service-type + genecup-activation) + (service-extension shepherd-root-service-type + genecup-shepherd-service))) + (default-value (genecup-configuration)) + (description + "Run a GeneCup Webserver."))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index a403f21..1e63ad2 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -20,8 +20,7 @@ ;;; <https://www.gnu.org/licenses/>. (define-module (genenetwork services genenetwork) - #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) - #:use-module ((gn packages guile) #:select (gn-guile)) + #:use-module ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-guile)) #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) #:use-module ((gnu packages admin) #:select (shadow shepherd)) @@ -48,6 +47,7 @@ #:use-module (forge utils) #:use-module (srfi srfi-1) #:use-module (ice-9 match) + #:use-module (gnu packages ssh) #:export (genenetwork-service-type genenetwork-configuration genenetwork-configuration? @@ -57,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -117,10 +118,20 @@ (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) - (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout - (default "/export/data/gn-docs")) + (gn-guile-ssh-identity-file genenetwork-configuration-gn-guile-ssh-identity-file + (default "/opt/home/gn-guile/.ssh/id_ed25519")) + (gn-guile-known-hosts-file genenetwork-configuration-gn-guile-known-hosts-file + (default "/opt/home/gn-guile/.ssh/known_hosts")) + (gn-guile-working-dir genenetwork-configuration-gn-guile-working-dir + (default "/export/data")) + (gn-doc-remote-uri genenetwork-configuration-gn-doc-remote-uri + (default "/export/data/gn-docs.git")) + (gn-docs-working-branch genenetwork-configuration-gn-docs-working-branch + (default "gn-production-branch")) (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory (default "/export/data/virtuoso/ttl")) (gn-tmpdir genenetwork-configuration-gn-tmpdir @@ -150,6 +161,12 @@ (default "https://genenetwork.org")) (sessions-dir gn-uploader-sessions-dir (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (gn-tmpdir gn-uploader-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (genotype-files-directory gn-uploader-configuration-genotype-files-directory + (default "/var/genenetwork/genotype-files")) (log-level gn-uploader-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -254,9 +271,35 @@ (chmod file #o644)) (find-files #$xapian-directory))))))))) +(define (samples-count-script-gexp config) + (match-record config <genenetwork-configuration> + (genenetwork2 sql-uri) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PYTHONPATH" + (string-append + #$(file-append genenetwork2 + "/lib/python" + (python-version (package-version python)) + "/site-packages") + ":" + #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "/lib/python" + #$(python-version (package-version python)) + "/site-packages")) + + (invoke #$(file-append python "/bin/python3") + "-m" + "gn2.scripts.sample_count" + #$sql-uri))))) + (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-guile-working-dir gn2-sessions-dir gn-guile-ssh-identity-file) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -319,8 +362,10 @@ (chown file (passwd:uid (getpw "genenetwork")) (passwd:gid (getpw "genenetwork")))) - (find-files #$gn-doc-git-checkout - #:directories? #t)))))) + (append (find-files #$gn-guile-working-dir + #:directories? #t) + (find-files #$(dirname (dirname gn-guile-ssh-identity-file)) + #:directories? #t))))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -351,7 +396,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -370,6 +415,8 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) @@ -386,6 +433,8 @@ object." (configuration-file-gexp `(("AUTH_DB" ,auth-db) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) @@ -411,7 +460,7 @@ object." (sockets (list (forge-ip-socket (port gn2-port)))) (wsgi-app-module "gn2.wsgi") - (workers 20) + (workers 10) (timeout 1200) (environment-variables (list (environment-variable @@ -470,7 +519,7 @@ object." (sockets (list (forge-ip-socket (port gn3-port)))) (wsgi-app-module "gn3.app:create_app()") - (workers 20) + (workers 10) ;; gunicorn's default 30 second timeout is insufficient ;; for Fahamu AI endpoints and results in worker timeout ;; errors. @@ -518,7 +567,7 @@ object." (source xapian-db) (target source)) (file-system-mapping - (source llm-db-path) + (source (dirname llm-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -533,7 +582,8 @@ object." (sockets (list (forge-ip-socket (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") - (workers 20) + (workers 10) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -552,9 +602,9 @@ object." (source gn-auth-conf) (target source)) (file-system-mapping - (source auth-db) - (target source) - (writable? #t)) + (source (dirname auth-db)) + (target source) + (writable? #t)) (file-system-mapping (source gn-auth-secrets) (target source) @@ -611,24 +661,58 @@ a @code{<genenetwork-configuration>} record." (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron-gexp" (build-xapian-index-cron-gexp config)) - #:user "root"))) + #:user "root") + #~(job '(next-minute-from (next-hour) '(17)) ;17th minute of every hour + #$(program-file "samples-count-script-gexp" + (samples-count-script-gexp config))))) -(define (gn-guile-gexp gn-guile-port) - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) - (when (file-exists? current-repo-path) - (delete-file-recursively current-repo-path)) - (setenv "CURRENT_REPO_PATH" current-repo-path) - (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) - (invoke #$(file-append gn-guile "/bin/gn-guile") - (number->string #$gn-guile-port))))) +(define (gn-guile-gexp config) + (values + config + (match-record config <genenetwork-configuration> + (gn-guile-port gn-guile gn-guile-ssh-identity-file gn-guile-known-hosts-file gn-doc-remote-uri gn-docs-working-branch) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (gn-docs-local-checkout (string-append (pk "CWD" (getcwd)) "/gn-docs")) + (ssh-command #$(file-append openssh-sans-x "/bin/ssh")) + (ssh-config-file #$(mixed-text-file "gn-guile-ssh-config" + "Host git.genenetwork.org\n" + "\tUser git\n" + "\tIdentitiesOnly yes\n" + "\tIdentityFile " gn-guile-ssh-identity-file "\n" + "\tUserKnownHostsFile " gn-guile-known-hosts-file))) + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + (setenv "GIT_SSH_COMMAND" (pk "GIT_SSH_COMMAND ===> " + (string-append ssh-command + " -F " + ssh-config-file))) + (setenv "GIT_COMMITTER_NAME" "genenetwork") + (setenv "GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org") -(define (gn-guile-shepherd-service config) + (when (file-exists? gn-docs-local-checkout) + (delete-file-recursively gn-docs-local-checkout)) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" "--branch" #$gn-docs-working-branch #$gn-doc-remote-uri)) + (invoke #$(file-append gn-guile "/bin/gn-guile") + "--port" + (number->string #$gn-guile-port) + "--gn-docs-local-checkout" + (string-append (getcwd) "/gn-docs") + "--gn-docs-remote-url" + #$gn-doc-remote-uri + "--gn-docs-working-branch" + #$gn-docs-working-branch)))))) + +(define (gn-guile-shepherd-service config program-gexp) (match-record config <genenetwork-configuration> - (gn-doc-git-checkout gn-guile-port) + (gn-guile gn-guile-port gn-guile-ssh-identity-file gn-guile-known-hosts-file gn-guile-working-dir) (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -636,32 +720,31 @@ a @code{<genenetwork-configuration>} record." (modules '((ice-9 match) (srfi srfi-1))) (start - (let* ((gn-guile-settings - `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) - ("LC_ALL" "en_US.UTF-8") - ("GIT_COMMITTER_NAME" "genenetwork") - ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) - #~(make-forkexec-constructor - (list #$(least-authority-wrapper - (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) - #:name "gn-guile-pola-wrapper" - #:preserved-environment-variables - (map first gn-guile-settings) - #:mappings (list (file-system-mapping - (source gn-doc-git-checkout) - (target source) - (writable? #t))) - #:namespaces (delq 'net %namespaces)) - "127.0.0.1" #$(number->string gn-guile-port)) - #:user "genenetwork" - #:group "genenetwork" - #:environment-variables - (map (match-lambda - ((spec value) - (string-append spec "=" value))) - '#$gn-guile-settings) - #:log-file "/var/log/gn-guile.log"))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" program-gexp) + #:name "gn-guile-pola-wrapper" + #:directory gn-guile-working-dir + #:mappings (list (file-system-mapping + (source gn-guile-working-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-guile-ssh-identity-file) + (target source) + (writable? #f)) + (file-system-mapping + (source gn-guile-known-hosts-file) + (target source) + (writable? #f))) + #:namespaces (delq 'user (delq 'net %namespaces)) + #:user "genenetwork" + #:group "genenetwork") + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "root" + #:group "root" + #:log-file "/var/log/gn-guile.log" + #:environment-variables (user-environment-variables))) (stop #~(make-kill-destructor))))) (define genenetwork-service-type @@ -678,13 +761,13 @@ a @code{<genenetwork-configuration>} record." (service-extension forge-nginx-service-type genenetwork-nginx-server-blocks) (service-extension shepherd-root-service-type - (compose list gn-guile-shepherd-service)) + (compose list gn-guile-shepherd-service gn-guile-gexp)) (service-extension mcron-service-type genenetwork-mcron-jobs))) (default-value (genenetwork-configuration)))) (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory sessions-dir) + (secrets data-directory sessions-dir sqlite-databases-directory gn-tmpdir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -693,8 +776,10 @@ a @code{<genenetwork-configuration>} record." (chown file (passwd:uid (getpw "gunicorn-gn-uploader")) (passwd:gid (getpw "gunicorn-gn-uploader")))) - (append (list #$secrets) + (append (list #$(dirname secrets)) (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) @@ -706,27 +791,35 @@ a @code{<genenetwork-configuration>} record." (passwd:uid (getpw "gunicorn-gn-uploader")) (passwd:gid (getpw "gunicorn-gn-uploader")))) (append (list #$data-directory) + (find-files #$(dirname secrets) + #:directories? #t) (find-files #$data-directory + #:directories? #t) + (find-files #$(string-append gn-tmpdir "/gn-uploader-scratch") #:directories? #t))))))) (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory gn-tmpdir genotype-files-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping (source "/run/mysqld") (target source) (writable? #t))) + (gn-uploader-scratch (string-append gn-tmpdir "/gn-uploader-scratch")) (gn-uploader-conf (computed-file "gn-uploader.conf" (configuration-file-gexp `(("UPLOADER_SECRETS" ,secrets) ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory + ("UPLOADS_DIRECTORY" ,(string-append gn-uploader-scratch "/uploads")) + ("SCRATCH_DIRECTORY" ,gn-uploader-scratch) ("AUTH_SERVER_URL" ,auth-server-url) ("GN2_SERVER_URL" ,gn2-server-url) - ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir))))) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")) + ("GENOTYPE_FILES_DIRECTORY" ,genotype-files-directory))))) (gn-uploader-profile (profile (content (package->development-manifest gn-uploader)) (allow-collisions? #t))) @@ -737,7 +830,8 @@ a @code{<genenetwork-configuration>} record." (sockets (list (forge-ip-socket (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") - (workers 20) + (workers 10) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -756,8 +850,9 @@ a @code{<genenetwork-configuration>} record." (source gn-uploader-conf) (target source)) (file-system-mapping - (source secrets) - (target source)) + (source (dirname secrets)) + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source) @@ -771,7 +866,19 @@ a @code{<genenetwork-configuration>} record." (file-system-mapping (source sessions-dir) (target source) - (writable? #t)))) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-scratch) + (target source) + (writable? #t)) + (file-system-mapping + (source genotype-files-directory) + (target source) + (writable? #f)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))))))) diff --git a/genenetwork/services/mouse-longevity.scm b/genenetwork/services/mouse-longevity.scm new file mode 100644 index 0000000..c9b977d --- /dev/null +++ b/genenetwork/services/mouse-longevity.scm @@ -0,0 +1,33 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services mouse-longevity) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gnu services) + #:export (mouse-longevity-service)) + +(define* (mouse-longevity-service #:optional (package mouse-longevity-app)) + (simple-service 'mouse-longevity + rshiny-service-type + (list + (rshiny-configuration + (package package) + (binary "mouse-longevity-app"))))) diff --git a/guix/gn-machines/genenetwork.scm b/guix/gn-machines/genenetwork.scm new file mode 100644 index 0000000..fce7e63 --- /dev/null +++ b/guix/gn-machines/genenetwork.scm @@ -0,0 +1,270 @@ +(define-module (gn-machines genenetwork) + #:use-module (guix) + #:use-module (guix packages) + #:use-module (guix git-download) + #:use-module (guix build-system pyproject) + #:use-module ((guix licenses) #:prefix license:) + + #:use-module ((gnu packages nss) #:select (nss-certs)) + #:use-module ((gnu packages ssh) #:select (openssh-sans-x)) + #:use-module ((gnu packages guile-xyz) #:select (guile-config)) + #:use-module ((gnu packages check) #:select (python-pytest)) + #:use-module ((gnu packages python-check) #:select (python-vulture)) + #:use-module ((gnu packages python-web) #:select (python-requests)) + + #:use-module ((gn packages genenetwork) + #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-libs rust-qtlreaper) + #:prefix gn:) + #:use-module((gn packages guile) + #:select (gn-guile guile-sheepdog) + #:prefix gng:)) + +(define-public genenetwork2 + (let ((commit "7145b2fe9cf63e17326c2e838b8b6753a8a00cb1") + (revision "4")) + (package + (inherit gn:genenetwork2) + (name "genenetwork2") + (version (git-version (package-version gn:genenetwork2) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork2.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1masn0xaazf26kg27srihdg4164hgzh11ml66sc16ssnzic620mx")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork2) + (replace "gn-libs" gn-libs) + (replace "genenetwork3" genenetwork3)))))) + +(define-public rust-qtlreaper + (let ((commit "a62886a52a980474b410a06d4ba92a219c484ec9") + (revision "1")) + (package + (inherit gn:rust-qtlreaper) + (name "rust-qtlreaper") + (version (git-version (package-version gn:rust-qtlreaper) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/rust-qtlreaper.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0jhmfzin96nsadsgspnl4kk4znq365x0srx8hc2madrshnnkzz36"))))))) + +(define-public genenetwork3 + (let ((commit "f09ab5ba5c50653e0d92a835a08b871680af12c9") + (revision "5")) + (package + (inherit gn:genenetwork3) + (name "genenetwork3") + (version (git-version (package-version gn:genenetwork3) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork3.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1xlrllx7f4077kphidkpqzy0xamdbvywp25qzh3lbfbfi5ki00vi")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork3) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-auth + (let ((commit "8cd2d82ceb0bf7ffc6c742b56fd2039cd742e508") + (revision "1")) + (package + (inherit gn:gn-auth) + (name "gn-auth") + (version (git-version (package-version gn:gn-auth) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-auth") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "14khr5xnhiadh59x9mwrrp47vg79xi471c108l1i8scii4n34gv9"))))) + (build-system pyproject-build-system) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-auth) + (replace "gn-libs" gn-libs))) + (native-inputs + (modify-inputs (package-native-inputs gn:gn-auth) + (append python-vulture)))))) + +(define-public gn-uploader + (let ((commit "112aa267709b130447d8ca387631f5dbeba02366") + (revision "0")) + (package + (inherit gn:gn-uploader) + (name "gn-uploader") + (version (git-version (package-version gn:gn-uploader) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-uploader") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "073jy7cxkmjk881yp6kccsd6mg2gkdg2ky059m4m6p2kqnlkdh82"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-uploader) + (replace "gn-libs" gn-libs) + (replace "rust-qtlreaper" rust-qtlreaper)))))) + +(define-public gn-libs + (let ((commit "22fe4293947499628f87f36561c96a3b3b070166") + (revision "03")) + (package + (inherit gn:gn-libs) + (name "gn-libs") + (version (git-version (package-version gn:gn-libs) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-libs") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1pfy24abv958lg6qac1y58rrvrbswb1fp0y0hm7q7zyjd4jy7a45"))))))) + +(define-public gn-guile + (let ((commit "0e8e605434b37251e6729121c77afa963cebef6a") + (revision "0")) + (package + (inherit gng:gn-guile) + (name "gn-guile") + (version (git-version "4.0.1" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-guile/") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "0xy8c4x0hfi9za47prf59a9qjc983ygl2g34ndi9imjy36aa4pib")))) + (arguments + (list + #:not-compiled-file-regexp "(guix|guix/.*)[.]scm$" + #:modules '((srfi srfi-1) + (guix build guile-build-system) + (guix build utils)) + #:phases + #~(modify-phases %standard-phases + (add-after 'unpack 'patch-git + (lambda* (#:key inputs #:allow-other-keys) + (let ((git (search-input-file inputs "/bin/git"))) + (substitute* "web/view/markdown.scm" + (("\"git\"") (string-append "\"" git "\"")) + (("git -C") (string-append git " -C")))))) + (add-after 'patch-source-shebangs 'patch-gn-guile-source-shebangs + ;; there is still the shell-script `lmdb-publishdata-export' that + ;; needs to be patched the usual way, so we have 2 different + ;; source-shebang patching phases. + (lambda* (#:key inputs #:allow-other-keys) + (substitute* "bin/gn-guile" + (("^exec guile") + (string-append "exec " + (search-input-file inputs "/bin/guile")))))) + (add-after 'build 'install-scripts + (lambda* _ + (mkdir-p "bin") + (copy-file "scripts/lmdb-publishdata-export.scm" + "bin/lmdb-publishdata-export") + (for-each (lambda (script-path) + (install-file script-path + (string-append #$output "/bin"))) + (list "bin/gn-guile" + "bin/lmdb-publishdata-export")))) + (add-after 'install-scripts 'wrap + (lambda* (#:key inputs #:allow-other-keys) + (let((effective-version (target-guile-effective-version)) + (guile-path + (dirname (search-input-file inputs "/bin/guile")))) + (for-each (lambda (script-name) + (wrap-program + (string-append #$output "/bin/" script-name) + `("PATH" prefix (,guile-path ,(getenv "PATH"))) + `("GUILE_LOAD_PATH" prefix + ("./" ;; for CD. Should not affect production. + ,(string-append #$output + "/share/guile/site/" + effective-version) + ,(getenv "GUILE_LOAD_PATH"))) + `("GUILE_LOAD_COMPILED_PATH" prefix + (,(string-append #$output "/lib/guile" + effective-version + "/site-ccache") + ,(getenv "GUILE_LOAD_COMPILED_PATH"))) + `("GUILE_AUTO_COMPILE" ":" = ("0")))) + (list "lmdb-publishdata-export" + "gn-guile")))))))) + (propagated-inputs + ;; TODO: remove openssh-sans-x on next update to newer + ;; guix-bioinformatics commit, i.e. newer than commit + ;; `9b0955f14ec725990abb1f6af3b9f171e4943f77'. + (modify-inputs (package-propagated-inputs gng:gn-guile) + (prepend openssh-sans-x guile-config)))))) + +(define-public guile-sheepdog + (let ((commit "1426617d58f305a4126bb867202843e8cf7dd4b2") + (revision "0")) + (package + (inherit gng:guile-sheepdog) + (name "guile-sheepdog") + (version (git-version "4.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/BonfaceKilz/guile-sheepdog.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1z0xzg11p75s2hk312akxlg2h5278w2abma27dhzjf981g3lcqvr"))))))) + +(define-public gn-integration-tests + (let ((commit "64f779a275cde734ba9bbe7f808c34d3a9217f32") + (revision "0")) + (package + (name "gn-integration-tests") + (version (git-version "0.1.0" revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-integration-tests") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0l9114170l4s7sha5zwcvvwf113b1qk1f082p998qlimfh8hfq8a")))) + (build-system pyproject-build-system) + (arguments + (list #:tests? #f)) ; tests require a live GeneNetwork deployment + (propagated-inputs + (list nss-certs python-pytest python-requests)) + (home-page "https://git.genenetwork.org/gn-integration-tests/about/") + (synopsis "Integration tests for the GeneNetwork stack") + (description + "End-to-end smoke and integration tests for the GeneNetwork web +services (genenetwork2, genenetwork3, gn-auth). Tests run against a live +deployment; target URLs are configured via environment variables.") + (license license:agpl3+)))) diff --git a/guix/gn-machines/services/monitoring.scm b/guix/gn-machines/services/monitoring.scm new file mode 100644 index 0000000..7fa59c9 --- /dev/null +++ b/guix/gn-machines/services/monitoring.scm @@ -0,0 +1,68 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (gn-machines services monitoring) + #:use-module (gnu) + #:use-module (gnu services shepherd) + #:use-module (gnu services databases) + #:use-module ((gn-machines genenetwork) #:select (guile-sheepdog)) + #:use-module (guix gexp) + #:use-module (guix records) + #:use-module (ice-9 match) + #:export (guile-sheepdog-configuration + guile-sheepdog-configuration? + guile-sheepdog-configuration-settings-file + guile-sheepdog-configuration-package + guile-sheepdog-service-type)) + +(define-record-type* <guile-sheepdog-configuration> + guile-sheepdog-configuration + make-guile-sheepdog-configuration + guile-sheepdog-configuration? + (settings-file guile-sheepdog-configuration-settings-file + (default "/etc/conn.scm")) + (package guile-sheepdog-configuration-package (default guile-sheepdog))) + +(define (guile-sheepdog-gexp config) + (match-record config <guile-sheepdog-configuration> (settings-file package) + (program-file + "guile-sheepdog" + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$(file-append package "/bin/guile-sheepdog") #$settings-file)))))) + +(define (guile-sheepdog-shepherd-service config) + (shepherd-service + (documentation "Run Sheepdog") + (provision '(guile-sheepdog)) + (requirement '(networking redis)) + (start #~(make-forkexec-constructor + (list #$(guile-sheepdog-gexp config)) + #:log-file "/var/log/sheepdog.log")) + (stop #~(make-kill-destructor)))) + +(define guile-sheepdog-service-type + (service-type + (name 'guile-sheepdog) + (description "Run sheepdog monitor") + (extensions + (list (service-extension shepherd-root-service-type + (compose list guile-sheepdog-shepherd-service)))) + (default-value (guile-sheepdog-configuration)))) diff --git a/production-deploy.sh b/production-deploy.sh index a88fcb8..793fd34 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -3,6 +3,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> # Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> +# Copyright © 2026 Munyoki Kilyungi <me@bonfacemunyoki.com> # # This file is part of genenetwork-machines. # @@ -24,23 +25,28 @@ container_script=$(guix system container \ --network \ - --load-path=. \ + --load-path=./guix/ \ + --load-path=./ \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ - --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ + --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/genenetwork/var/log=/var/log \ + --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \ --share=/var/run/mysqld=/run/mysqld \ - --share=/export/data/gn-docs/ \ - --share=/export2/guix-containers/genenetwork/tmp=/opt/gn/tmp \ - --expose=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ - --share=/export2/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ - --share=/export2/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \ + --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile=/var/lib/genenetwork/gn-guile \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile/ssh=/opt/home/gn-guile/.ssh \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 9e629f0..302e7ef 100644 --- a/production.scm +++ b/production.scm @@ -49,8 +49,8 @@ (server-port 9892) (http-server-port 9893) (dirs-allowed (list "/export/data/virtuoso")) - (number-of-buffers 4000000) - (maximum-dirty-buffers 3000000) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) (service forge-nginx-service-type @@ -84,9 +84,25 @@ (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") (gn-auth-secrets "/etc/genenetwork/gn-auth") - (auth-db "/var/lib/genenetwork-sqlite/auth.db") - (llm-db-path "/var/lib/genenetwork-gnqa/llm.db") + (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db") + (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db") (gn3-alias-server-port 9800) (gn-tmpdir "/opt/gn/tmp") - (log-level 'debug))) + (gn-doc-remote-uri "git@git.genenetwork.org:/home/git/public/gn-docs") + (gn-guile-working-dir "/var/lib/genenetwork/gn-guile/") + (gn-docs-working-branch "gn2-production-branch") + (gn-guile-ssh-identity-file "/opt/home/gn-guile/.ssh/id-ed25519-gn2-production-on-tux04") + (log-level 'info))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "uploader.genenetwork.org") + (port 9897) + (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (data-directory "/var/lib/genenetwork/uploader/data") + (auth-server-url "https://auth.genenetwork.org/") + (gn2-server-url "https://genenetwork.org") + (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader") + (log-level 'info))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index 4ecacc6..bd8b938 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -22,11 +22,11 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/public-sparql/tmp=/tmp \ - --share=/export2/guix-containers/public-sparql/var/log=/var/log \ - --share=/export2/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ + --share=/export/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/public-sparql/tmp=/tmp \ + --share=/export/guix-containers/public-sparql/var/log=/var/log \ + --share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index 6fbf75c..87ef843 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -20,6 +20,7 @@ (use-modules (gnu) (gn services databases) (gnu services web) + ((gnu packages admin) #:select (shepherd)) (forge nginx) (forge socket)) @@ -50,16 +51,22 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) - (number-of-buffers 4000000) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) (dirs-allowed (list "/export/data/virtuoso")) - (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/public-virtuoso.db") - (transaction-file "/var/lib/virtuoso/public-virtuoso.trx"))) + (transaction-file "/var/lib/virtuoso/public-virtuoso.trx") + (error-log-file "/var/lib/public-virtuoso-errors.log") + (syslog "1"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket diff --git a/services/README.md b/services/README.md deleted file mode 100644 index d0d1c01..0000000 --- a/services/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Services - -This directory contains small and simple services that may be run independently. -Note that composition is not the goal. -For example, databases and web proxies are handled outsite the containers. -Use these services for simple deployment and ad hoc testing. -One nice aspect of small system containers is that you can run these easily on your laptop. - -IMPORTANT: more complex services do not belong in this directory. - -# Examples - -## gn-guile - -`gn-guile` is our next generation service (gn4?). It provides a REST API, at this point, and some portals, such as https://aging.genenetwork.org/. - -See the [gn-guile](./gn-guile.scm) system definition example. diff --git a/services/gn-guile.scm b/services/gn-guile.scm deleted file mode 100644 index 2f675a8..0000000 --- a/services/gn-guile.scm +++ /dev/null @@ -1,52 +0,0 @@ -;; This is an example definition for the gn-guile/GN4 service -;; -;; Run with -;; -;; export runner=$(guix system container gn-guile.scm) -;; -;; as root -;; -;; sudo bash -c $runner -;; echo $runner -;; -;; make a note of pid and -;; -;; sudo bash -c "nsenter -a -t 4050285" -;; -;; now you should be inside the container (note bash should be in the container!) - -(use-modules (gnu) - (guix records) - (forge utils)) - -(define-record-type* <gn-guile-configuration> - gn-guile-configuration make-gn-guile-configuration - gn-guile-configuration? - (gn2-repository gn-guile-configuration-gn2-repository - (default "https://github.com/genenetwork/genenetwork2")) - (gn2-port gn-guile-configuration-gn2-port - (default 8082))) - - -(define gn-guile-service-type - (service-type - (name 'gn-guile) - (description "gn-guile/GN4 webservice") - (extensions '()) - )) - -(operating-system - (host-name "gn-guile") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service gn-guile-service-type - (gn-guile-configuration)) - %base-services))) diff --git a/services/opensmtpd.scm b/services/opensmtpd.scm deleted file mode 100644 index 1b1e58f..0000000 --- a/services/opensmtpd.scm +++ /dev/null @@ -1,21 +0,0 @@ -(use-modules (gnu) - (gnu services mail)) - -(operating-system - (host-name "mail") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service opensmtpd-service-type - (opensmtpd-configuration - (config-file %default-opensmtpd-config-file - ; (config-file (local-file "./my-smtpd.conf"))) - ))) - %base-services))) diff --git a/singularity-head-deploy.sh b/singularity-head-deploy.sh new file mode 100755 index 0000000..533224b --- /dev/null +++ b/singularity-head-deploy.sh @@ -0,0 +1,12 @@ +#! /bin/sh -xe + +## +## singularity deployment on octopus01 (the head node) +## + +## Install singularity in the same way as the worker nodes. +./singularity-worker-deploy.sh $(guix build -f singularity.scm) + +# Register garbage collector root to prevent `guix gc' from garbage +# collecting singularity. +sudo ln --force --symbolic /usr/local/bin/singularity /var/guix/gcroots diff --git a/singularity-worker-deploy.sh b/singularity-worker-deploy.sh new file mode 100755 index 0000000..d709758 --- /dev/null +++ b/singularity-worker-deploy.sh @@ -0,0 +1,37 @@ +#! /bin/sh -xe + +## +## singularity deployment on octopus worker nodes +## + +case $1 in + "") + echo "Usage: $0 SINGULARITY_STORE_ITEM" + exit 1 + ;; + *) + singularity=$1 + echo $singularity + ;; +esac + +# Symlink singularity executable. +sudo ln --force --symbolic $singularity/bin/singularity /usr/local/bin/singularity + +# To set up singularity, we imitate what the Guix +# singularity-service-type does. + +# Install setuid binaries. +sudo mkdir -p /usr/local/libexec/singularity/bin +for program in action mount start; +do + sudo cp $singularity/libexec/singularity/bin/$program-suid /usr/local/libexec/singularity/bin/singularity-$program-helper + sudo chmod u+s /usr/local/libexec/singularity/bin/singularity-$program-helper +done + +# Create the directories that Singularity 2.6 expects to find. +for directory in container final overlay session; +do + sudo mkdir -p /var/singularity/mnt/$directory + sudo chmod 755 /var/singularity/mnt/$directory +done diff --git a/singularity.scm b/singularity.scm new file mode 100644 index 0000000..9a4c0ce --- /dev/null +++ b/singularity.scm @@ -0,0 +1,42 @@ +(use-modules (gnu packages linux) + (guix download) + (guix packages)) + +(package + (inherit singularity) + (version (package-version singularity)) + (source (origin + (inherit (package-source singularity)) + (snippet + '(begin + ;; We put the singularity setuid binaries under + ;; /usr/local. The Guix package puts it under + ;; /run/privileged/bin. But, we cannot do that since + ;; /run is mounted noexec on octopus. + (substitute* (find-files "libexec/cli" "\\.exec$") + (("\\$SINGULARITY_libexecdir/singularity/bin/([a-z]+)-suid" + _ program) + (string-append "/usr/local/libexec/singularity/bin/singularity-" + program "-helper"))) + + ;; The remaining snippet code below is copied from the + ;; Guix package. + + ;; Do not create directories in /var. + (substitute* "Makefile.in" + (("\\$\\(MAKE\\) .*install-data-hook") "")) + + ;; The original source overrides PATH so that it + ;; points to /bin, /usr/local/bin, etc., which + ;; obviously doesn't work on Guix System. Leave PATH + ;; unchanged so we refer to the installed Coreutils, + ;; grep, etc. + (substitute* "bin/singularity.in" + (("^PATH=.*" all) + (string-append "#" all "\n"))) + + ;; These squashfs mount options are apparently no + ;; longer supported since Linux-libre 5.4.5. + (substitute* "src/lib/image/squashfs/mount.c" + (("\"errors=remount-ro\"") + "NULL"))))))) diff --git a/specials/gndev.scm b/specials/gndev.scm index 440aedb..0f9394f 100644 --- a/specials/gndev.scm +++ b/specials/gndev.scm @@ -61,6 +61,7 @@ (gn-auth-server-name "gndev-auth.genenetwork.org") (gn2-port 8992) (gn3-port 8993) + (gn-auth-port 8994) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export2/data/gndev-sqlite/auth.db") (xapian-db "/export2/data/gndev-xapian") diff --git a/test-r-container.scm b/test-r-container.scm new file mode 100644 index 0000000..3167574 --- /dev/null +++ b/test-r-container.scm @@ -0,0 +1,111 @@ +(use-modules (guix) + (gnu) + (guix git) + (guix modules) + (guix profiles) + (guix records) + (guix packages) + (guix git-download) + (guix build-system trivial) + (gnu services dbus) + (gnu services networking) + (gnu services ssh) + (gnu services web) + (gnu services certbot) + (gnu services docker) + (gnu services desktop) + (gnu services shepherd) + (gnu packages admin) + (gnu packages statistics) + (gnu packages cran) + (gnu packages curl) + (gnu packages lsof) + (srfi srfi-1) + (ice-9 match) + (genenetwork services mouse-longevity) + (gn services rshiny) + (gn packages mouse-longevity) + (forge acme) + (forge nginx) + (forge socket) + (gnu services)) + +(define %nginx-configuration + (nginx-configuration + (server-blocks + (list + ;; Redirect domains that don't explicitly support HTTP (below) to HTTPS. + (nginx-server-configuration + (listen '("8080"))) + + ;; Domains that still explicitly support plain HTTP. + (nginx-server-configuration + (listen '("80")) + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list + (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://127.0.0.1:3979;"))))) + (raw-content + (list + "proxy_set_header Host $host;" + "proxy_set_header X-Real-IP $remote_addr;" + "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" + "proxy_set_header X-Forwarded-Proto $scheme;"))))))) + +(operating-system + (host-name "testing-genenetwork-services") + (keyboard-layout (keyboard-layout "us")) + (kernel-arguments + (cons* "console=ttyS0,115200" "console=tty0" + %default-kernel-arguments)) + (bootloader (bootloader-configuration (bootloader grub-bootloader))) + (issue "This is a GeneNetwork container. Welcome!\n") + (file-systems %base-file-systems) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* lsof curl %base-packages)) + (services + (cons* + (service dhcp-client-service-type) + (service dbus-root-service-type) + (service polkit-service-type) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8080"))) + (https-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8443"))) + (server-blocks + (list + (nginx-server-configuration + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://localhost:3979;" + "proxy_set_header Host $host;")))))))))) + (service acme-service-type + (acme-configuration + (email "jgart@dismail.de") + (acme-url %letsencrypt-staging-url))) + ;; (service certbot-service-type + ;; (certbot-configuration + ;; (email "jgart@dismail.de") + ;; (certificates + ;; (list + ;; (certificate-configuration + ;; (domains '("longevity-explorer.genenetwork.org" + ;; "www.longevity-explorer.genenetwork.org"))))))) + ;; (mouse-longevity-service ) + (service rshiny-service-type + (rshiny-configuration + (package mouse-longevity-app) + (binary "mouse-longevity-app"))) + ;; (service nginx-service-type %nginx-configuration) + %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0dd2509..d18caec 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2025 Pjotr Prins <pjotr.public01@thebird.nl> # # This file is part of genenetwork-machines. # @@ -19,15 +20,25 @@ # along with genenetwork-machines. If not, see # <https://www.gnu.org/licenses/>. -# Build and install virtuoso container on tux01. +# Build and install virtuoso container. Note the shared path is the sane default. Symlink if necessary. +# See also topics/systems/debug-and-developing-code-with-genenetwork-system-container.gmi container_script=$(guix system container \ + -L ~/guix-bioinformatics \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ + --share=/export/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/data/virtuoso/ttl=/export/data/virtuoso/ttl \ virtuoso.scm) echo $container_script sudo ln --force --symbolic $container_script /usr/local/bin/virtuoso-container sudo ln --force --symbolic /usr/local/bin/virtuoso-container /var/guix/gcroots + +echo "Run virtuoso with: sudo /usr/local/bin/virtuoso-container" +echo "Enter with something like: nsenter -a -t 1692115 /run/current-system/profile/bin/bash --login" +echo "Admin: isql 8891" +echo " ld_dir('/export/data/virtuoso/ttl','test.rdf','http://pjotr.genenetwork.org/');" +echo " rdf_loader_run();" +echo " checkpoint;" +echo "Web: http://localhost:8892/sparql" diff --git a/virtuoso.scm b/virtuoso.scm index 3272f41..ae33dcd 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed (list "/export/data/virtuoso")))) + (dirs-allowed (list "/export/data/virtuoso/ttl")))) %base-services))) |