diff options
| -rw-r--r-- | .guix-channel | 55 | ||||
| -rw-r--r-- | README.org | 32 | ||||
| -rw-r--r-- | fallback.scm | 3 | ||||
| -rwxr-xr-x | genenetwork-development-deploy.sh | 5 | ||||
| -rw-r--r-- | genenetwork-development.scm | 739 | ||||
| -rw-r--r-- | genenetwork/services/genecup.scm | 121 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 464 | ||||
| -rw-r--r-- | genenetwork/services/mouse-longevity.scm | 33 | ||||
| -rw-r--r-- | guix/gn-machines/genenetwork.scm | 230 | ||||
| -rw-r--r-- | guix/gn-machines/services/monitoring.scm | 68 | ||||
| -rwxr-xr-x | production-deploy.sh | 32 | ||||
| -rw-r--r-- | production.scm | 70 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 8 | ||||
| -rw-r--r-- | public-sparql.scm | 39 | ||||
| -rw-r--r-- | services/README.md | 17 | ||||
| -rw-r--r-- | services/gn-guile.scm | 52 | ||||
| -rw-r--r-- | services/opensmtpd.scm | 21 | ||||
| -rwxr-xr-x | singularity-head-deploy.sh | 12 | ||||
| -rwxr-xr-x | singularity-worker-deploy.sh | 37 | ||||
| -rw-r--r-- | singularity.scm | 42 | ||||
| -rw-r--r-- | slurm.scm | 15 | ||||
| -rw-r--r-- | specials/gndev.scm | 1 | ||||
| -rw-r--r-- | test-r-container.scm | 111 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 26 | ||||
| -rw-r--r-- | uploader.scm | 24 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 16 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
27 files changed, 1828 insertions, 447 deletions
diff --git a/.guix-channel b/.guix-channel new file mode 100644 index 0000000..a15d3ca --- /dev/null +++ b/.guix-channel @@ -0,0 +1,55 @@ +(channel + (version 0) + (directory "guix") + (dependencies + (channel + (name guix) + (url "https://codeberg.org/guix/guix") + (branch "master") + (commit "0a4740705090acc4c8a10d4f53afc58c9f62e980") + (introduction + (channel-introduction + (version 0) + (commit "9edb3f66fd807b096b48283debdcddccfea34bad") + (signer + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) + (channel + (name guix-forge) + (url "https://git.systemreboot.net/guix-forge/") + (branch "main") + (commit "e43fd9a4d73654d3876e2c698af7da89f3408f89") + (introduction + (channel-introduction + (version 0) + (commit "0432e37b20dd678a02efee21adf0b9525a670310") + (signer + "7F73 0343 F2F0 9F3C 77BF 79D3 2E25 EE8B 6180 2BB3")))) + (channel + (name guix-bioinformatics) + (url "https://git.genenetwork.org/guix-bioinformatics") + (commit "9b0955f14ec725990abb1f6af3b9f171e4943f77")) + ;; Until https://issues.guix.gnu.org/68797 is resolved, we need to + ;; explicitly list guix-past and guix-rust-past-crates—the + ;; dependencies of the guix-bioinformatics channel—here. + (channel + (name guix-past) + (url "https://codeberg.org/guix-science/guix-past") + (branch "master") + (commit "473c942b509ab3ead35159d27dfbf2031a36cd4d") + (introduction + (channel-introduction + (version 0) + (commit "c3bc94ee752ec545e39c1b8a29f739405767b51c") + (signer + "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5")))) + (channel + (name guix-rust-past-crates) + (url "https://codeberg.org/guix/guix-rust-past-crates.git") + (branch "trunk") + (commit "b8b7ffbd1cec9f56f93fae4da3a74163bbc9c570") + (introduction + (channel-introduction + (version 0) + (commit "1db24ca92c28255b28076792b93d533eabb3dc6a") + (signer + "F4C2 D1DF 3FDE EA63 D1D3 0776 ACC6 6D09 CA52 8292")))))) diff --git a/README.org b/README.org index 892ab23..2aa45db 100644 --- a/README.org +++ b/README.org @@ -3,19 +3,40 @@ containers. The git repo lives at https://git.genenetwork.org/gn-machines/ -* GeneNetwork development container +For CI/CD see -The GeneNetwork development container is currently run on /tux02/. It runs -continuous integration and continuous deployment services for -genenetwork2, genenetwork3 and several other associated projects. +=> https://issues.genenetwork.org/topics/systems/ci-cd + +For philosophy and (KISS) incremental minimalistic development containers, see: + +=> https://issues.genenetwork.org/topics/systems/debug-and-developing-code-with-genenetwork-system-container + +These are stored in the './minimal' directory of this repo. See the [[./minimal/README.md][README]]. + +* GeneNetwork development container (aka CI/CD) + +The GeneNetwork development container is currently run on /tux02/. +It runs continuous integration and continuous deployment services for genenetwork2, genenetwork3 and several other associated projects. To build and install the container, you will need the [[https://gitlab.com/genenetwork/guix-bioinformatics][guix-bioinformatics]] and [[https://git.systemreboot.net/guix-forge/][guix-forge]] channels. Once these channels are pulled and available, on /tux02/, run + #+BEGIN_SRC shell $ ./genenetwork-development-deploy.sh #+END_SRC +It will try to create symlinks at the end. You can do this as root (too). + +Note we current run as aruni user and the latest guix is the user profile. + +#+BEGIN_SRC shell +aruni@tux02:~/gn-machines$ which guix +/home/aruni/.config/guix/current/bin/guix +aruni@tux02:~/gn-machines$ guix describe +Generation 46 Mar 27 2025 23:39:42 (current) +#+END_SRC + /tux02/ is configured with a systemd service to run this container. Restart it. #+BEGIN_SRC shell @@ -31,6 +52,7 @@ To build and install the container, you will need the guix-bioinformatics channel. Once guix-bioinformatics is pulled and available, on /tux01/, run #+begin_src shell + $ ./virtuoso-deploy.sh #+end_src @@ -55,7 +77,7 @@ You can get a shell into the container with something like: sudo guix container exec 89086 /run/current-system/profile/bin/bash --login #+END_SRC -When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or on your can get with something like: +When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or by inspecting the output of the following ~ps~ command: #+BEGIN_SRC sh ps -u root -f --forest | grep -A4 '/usr/local/bin/genenetwork-development-container' | grep 'shepherd' diff --git a/fallback.scm b/fallback.scm index e222238..8153789 100644 --- a/fallback.scm +++ b/fallback.scm @@ -29,7 +29,7 @@ (forge socket)) (operating-system - (host-name "genenetwork-fallback") + (host-name "fallback") (timezone "UTC") (locale "en_US.utf8") (bootloader (bootloader-configuration @@ -61,6 +61,7 @@ (gn-auth-server-name "fallback-auth.genenetwork.org") (gn2-port 8892) (gn3-port 8893) + (gn-auth-port 8894) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export/data/genenetwork-sqlite/auth.db") (xapian-db "/export/data/genenetwork-xapian") diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index d39bb7f..53c53d9 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -29,7 +29,7 @@ # /etc/genenetwork/conf instead of merely exposing it. container_script=$(guix system container --network \ --verbosity=3 \ - --load-path=. \ + --load-path=./guix/ \ --share=/home/git/public \ --share=/var/guix/daemon-socket=/var/host-guix/daemon-socket \ --share=/export2/guix-containers/genenetwork-development/var/lib/acme=/var/lib/acme \ @@ -42,8 +42,9 @@ container_script=$(guix system container --network \ --expose=/export/data/genenetwork \ --share=/export/data/genenetwork-xapian \ --share=/export/data/genenetwork-sqlite \ - --share=/export/genenetwork-database-dump \ + --share=/export/data/lmdb \ --share=/var/run/mysqld=/run/mysqld \ + --share=/export/data/gn-docs/ \ genenetwork-development.scm) echo $container_script diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 5fafc12..d8a9c52 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,18 +21,21 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) + (gn-machines services monitoring) + ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) (gn services databases) + ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) - ((gnu packages base) #:select (gnu-make tar)) + ((gnu packages base) #:select (gnu-make tar coreutils-minimal)) ((gnu packages bash) #:select (bash)) - ((gnu packages bioinformatics) #:select (ccwl) #:prefix guix:) - ((gnu packages certs) #:select (nss-certs)) + ((gnu packages nss) #:select (nss-certs)) ((gnu packages check) #:select (python-pylint)) + ((gnu packages curl) #:select (curl)) ((gnu packages ci) #:select (laminar)) ((gnu packages compression) #:select (gzip)) - ((gnu packages databases) #:select (mariadb virtuoso-ose)) + ((gnu packages databases) #:select (mariadb redis)) + ((gnu packages databases) #:select (virtuoso-ose)) ((gnu packages gnupg) #:select (guile-gcrypt)) ((gnu packages graphviz) #:select (graphviz)) ((gnu packages guile) #:select (guile-3.0 guile-git guile-zlib)) @@ -64,6 +67,7 @@ (guix packages) (guix profiles) (guix records) + (guix search-paths) (guix store) (guix utils) (forge acme) @@ -82,10 +86,6 @@ (define %guix-daemon-uri "/var/host-guix/daemon-socket/socket") -(define %default-guix-channel-with-substitutes - (channel-with-substitutes-available %default-guix-channel - "https://ci.guix.gnu.org")) - ;; We cannot refer to sudo in the store since that sudo does not have ;; the setuid bit set. See "(guix) Setuid Programs". (define sudo @@ -120,7 +120,11 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "https://git.genenetwork.org/gn-auth")) + (default "https://git.genenetwork.org/gn-auth")) + (gn-libs-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-libs")) + (gn-guile-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-guile")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -144,7 +148,15 @@ be imported into G-expressions." (auth-db-path genenetwork-auth-db-path (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path - (default "/export/data/genenetwork-sqlite/llm.db"))) + (default "/export/data/genenetwork-sqlite/llm.db")) + (lmdb-data-path genenetwork-lmdb-data-path + (default "/export/data/lmdb")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (repositories genenetwork-configuration-repositories + (default "/export/data/repositories")) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/export/data/gn-docs"))) ;;; @@ -159,7 +171,7 @@ be imported into G-expressions." (ci-jobs (let ((channels (list (channel (name 'gn-bioinformatics) (url "https://git.genenetwork.org/guix-bioinformatics") - (branch "master"))))) + (branch "main"))))) (list (forge-laminar-job (name "guix-bioinformatics") (run (guix-channel-job-gexp channels @@ -182,11 +194,12 @@ described by CONFIG, a <genenetwork-configuration> object. TEST-COMMAND is a list of strings specifying the command to be executed." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn3-port genotype-files) + (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) - (with-packages (list bash coreutils git-minimal nss-certs) + (with-packages (list bash coreutils git-minimal nss-certs genenetwork2) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (srfi srfi-26)) (define (hline) "Print a horizontal line 50 '=' characters long." @@ -199,16 +212,22 @@ executed." (invoke "git" "log" "--max-count" "1") (hline)) + (define (call-with-temporary-directory proc) + (let ((tmp-dir (mkdtemp "/tmp/gn.XXXXXX"))) + (dynamic-wind + (const #t) + (cut proc tmp-dir) + (cut delete-file-recursively tmp-dir)))) + (invoke "git" "clone" "--depth" "1" #$gn3-repository) (with-directory-excursion "genenetwork3" (show-head-commit)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository) + (with-directory-excursion "gn-libs" + (show-head-commit)) (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) - ;; This is a dummy SERVER_PORT to placate - ;; bin/genenetwork2. TODO: Fix bin/genenetwork2 so that - ;; this is not needed. - (setenv "SERVER_PORT" "8080") ;; Use a profile with all dependencies except ;; genenetwork3. (setenv "GN2_PROFILE" @@ -222,9 +241,38 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl") + ;; This file is cosmetic + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"/tmp/secret.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:120\"\n" + "GN3_GUILE_SERVER_URL=\"http://localhost:120\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"/tmp/ssl-private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/tmp/gn-auth-ssl-public-key.pem\"\n")) + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" "/export/data/genenetwork/genotype_files") + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/tmp") (chdir "genenetwork2") - (apply invoke '#$test-command)))))) + ;; XXXX: FIXME: R/Qtl tests fail because files are generated in + ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 + ;; so tests will fail because of permission issues. + (call-with-temporary-directory + (lambda (tmp-dir) + (setenv "TMPDIR" tmp-dir) + (apply invoke '#$test-command)))))))) (define %xapian-directory "/export/data/genenetwork-xapian") @@ -252,7 +300,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -297,7 +345,7 @@ genenetwork3 source from the latest commit of @var{project}." (system* (string-append gn3-dir "/scripts/index-genenetwork") "is-data-modified" #$%xapian-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql")))) (setenv "LAMINAR_REASON" "Nightly xapian index rebuild") (invoke #$(file-append laminar "/bin/laminarc") @@ -307,16 +355,13 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port gn-guile-port gn-guile-repository) (list (forge-project (name "genenetwork2") (repository gn2-repository) (ci-jobs (list (forge-laminar-job (name "genenetwork2") - (run (genenetwork2-tests - config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "-m" "pytest"))) + (run (genenetwork2-tests config (list "python3" "-m" "pytest"))) ;; If unit tests pass, redeploy genenetwork2 and ;; trigger Mechanical Rob. (after (with-imported-modules '((guix build utils)) @@ -332,8 +377,7 @@ genenetwork3 source from the latest commit of @var{project}." (name "genenetwork2-mechanical-rob") (run (genenetwork2-tests config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "test/requests/test-website.py" + (list "python3" "test/requests/test-website.py" "--all" (string-append "http://localhost:" (number->string gn2-port))))) (trigger? #f)))) (ci-jobs-trigger 'webhook)) @@ -380,6 +424,33 @@ genenetwork3 source from the latest commit of @var{project}." (trigger? #f)))) (ci-jobs-trigger 'webhook)) (forge-project + (name "gn-libs") + (repository gn-libs-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-libs") + (run (guix-channel-job-gexp + (list (channel + (name 'gn-libs) + (url (forge-project-repository this-forge-project)) + (branch "main"))) + #:variables (list (variable-specification + (module '(gn-libs)) + (name 'gn-libs))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook)) + (forge-project + (name "gn-guile") + (repository gn-guile-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-guile") + (run (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$sudo + #$(file-append shepherd "/bin/herd") + "restart" "gn-guile"))))))) + (ci-jobs-trigger 'webhook)) + (forge-project (name "gn-auth") (repository gn-auth-repository) (ci-jobs (list (forge-laminar-job @@ -421,7 +492,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files) + (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port) (with-packages (list coreutils git-minimal gunicorn nss-certs) (with-imported-modules '((guix build utils)) #~(begin @@ -438,59 +509,79 @@ server described by CONFIG, a <genenetwork-configuration> object." (hline) (invoke "git" "log" "--max-count" "1") (hline)) - - ;; Clone the latest genenetwork2 and genenetwork3 + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "TERM" "xterm-256color") + ;; Clone the latest genenetwork2 ;; repositories. - (invoke "git" "clone" "--depth" "1" #$gn2-repository) - (with-directory-excursion "genenetwork2" - (show-head-commit)) - (invoke "git" "clone" "--depth" "1" #$gn3-repository) - (with-directory-excursion "genenetwork3" - (show-head-commit)) + (with-directory-excursion + "/home/genenetwork" + (when (file-exists? "/home/genenetwork/genenetwork2") + (delete-file-recursively "/home/genenetwork/genenetwork2")) + (invoke "git" "clone" "--depth" "1" #$gn2-repository)) ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" - (string-append (getcwd) "/genenetwork3")) + (setenv "GN3_PYTHONPATH" "/home/genenetwork/genenetwork3") ;; Set other environment variables required by ;; genenetwork2. - (setenv "SERVER_PORT" #$(number->string gn2-port)) (setenv "GN2_PROFILE" #$(profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) - (setenv "GN_SERVER_URL" "https://cd.genenetwork.org/api3/") - (setenv "GN3_LOCAL_URL" - #$(string-append "http://localhost:" - (number->string gn3-port))) - (setenv "GENENETWORK_FILES" #$genotype-files) - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl") - (setenv "HOME" "/tmp") - (setenv "NO_REDIS" "no-redis") - (setenv "RUST_BACKTRACE" "1") - + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + (getenv "GN3_PYTHONPATH") + ":" + (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages"))) + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" #$genotype-files) + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/home/genenetwork") (setenv "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" "GN3_LOCAL_URL=\"" (string-append "http://localhost:" (number->string gn3-port)) "\"\n" + "GN_GUILE_SERVER_URL=\"" + (string-append "http://localhost:" + (number->string gn-guile-port)) + "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) ;; Start genenetwork2. - (with-directory-excursion "genenetwork2" - (invoke #$(file-append bash "/bin/sh") - "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-prod"))))))) + (with-directory-excursion + "/home/genenetwork/genenetwork2" + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) + "--workers" "20" + "--keep-alive" "6000" + "--max-requests" "100" + "--max-requests-jitter" "30" + "--timeout" "1200" + "gn2.wsgi"))))))) (define (genenetwork3-cd-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path) (with-manifest (package->development-manifest genenetwork3) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -509,29 +600,48 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest genenetwork3 repository. - (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "GN3_PROFILE" #$(profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN3_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) ;; Configure genenetwork3. (setenv "GN3_CONF" #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "LMDB_DATA_PATH=\"" lmdb-data-path "\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) - ;; Run genenetwork3. - (with-directory-excursion "genenetwork3" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn3-port)) - "--workers" "8" - "gn3.app:create_app()" - ;; gunicorn's default 30 second timeout is - ;; insufficient for Fahamu AI endpoints and - ;; results in worker timeout errors. - "--timeout" "1200")))))))) + (setenv "RSCRIPT" (string-append + (getenv "GN3_PROFILE") + "/bin/Rscript")) + + (with-directory-excursion + "/home/genenetwork" + ;; Clone the latest genenetwork3 repository. + (when (file-exists? "/home/genenetwork/genenetwork3") + (delete-file-recursively "/home/genenetwork/genenetwork3")) + (invoke "git" "clone" "--depth" "1" #$gn3-repository)) + + (with-directory-excursion + "/home/genenetwork/genenetwork3" + (show-head-commit) + ;; Run genenetwork3. + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn3-port)) + "--workers" "8" + "gn3.app:create_app()" + ;; gunicorn's default 30 second timeout is + ;; insufficient for Fahamu AI endpoints and + ;; results in worker timeout errors. + "--timeout" "1200")))))))) (define (gn-auth-cd-gexp config) "Return a G-expression that runs the latest gn-auth development @@ -556,9 +666,23 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest gn-auth repository. - (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + + (with-directory-excursion + "/home/genenetwork/" + ;; Clone the latest gn-auth repository. + (when (file-exists? "/home/genenetwork/gn-auth") + (delete-file-recursively "/home/genenetwork/gn-auth")) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository)) + ;; Configure gn-auth. + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN_AUTH_CONF" #$(mixed-text-file "gn-auth.conf" "AUTH_DB=\"" auth-db-path "\"\n" @@ -568,52 +692,208 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "HOME" "/tmp") (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") ;; Run gn-auth. - (with-directory-excursion "gn-auth" + (with-directory-excursion "/home/genenetwork/gn-auth" (show-head-commit) (invoke #$(file-append gunicorn "/bin/gunicorn") "-b" #$(string-append "localhost:" (number->string gn-auth-port)) "--workers" "8" "gn_auth.wsgi:app")))))))) +(define (gn-guile-gexp gn-guile-port) + (with-packages + (list coreutils git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke "git" "log" "--max-count" "1") + (hline)) + ;; KLUDGE: Here we set all the certificates properly. In gn-guile, + ;; we make request to external services. Here's an example: + ;; curl http://localhost:8091/gene/aliases/Shh + ;; + ;; Without certs, we run into: + ;; 2025-07-22 08:27:11 GET /gene/aliases/Shh + ;; [...] + ;; 2025-07-22 08:27:19 signer-not-found invalid + (setenv "GN_GUILE_PROFILE" #$(profile + (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (setenv "SSL_CERT_DIR" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs")) + (setenv "SSL_CERT_FILE" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GIT_SSL_CAINFO" (getenv "SSL_CERT_FILE")) + (setenv "CURL_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "SPARQL-ENDPOINT" "http://localhost:9082/sparql/") + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (let ((current-repo-path "/home/genenetwork/gn-docs")) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (with-directory-excursion + "/home/genenetwork" + ;; All edits go to the current-repo-path; so we need it to + ;; be persistent. + (unless (file-exists? current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH") current-repo-path)) + + (when (file-exists? "gn-guile") + (delete-file-recursively "gn-guile")) + (invoke "git" "clone" "--depth" "1" "https://git.genenetwork.org/gn-guile") + ;; We have a gn-guile-dev wrapper script that sets a "./" in the + ;; GN_GUILE_LOAD_PATH hence allowing this to be run from the gn-guile + ;; directory. This allows gn-guile to be run from the latest + ;; upstream commits without pinning to guix. + (with-directory-excursion "gn-guile" + (show-head-commit) + (invoke #$(file-append gn-guile "/bin/gn-guile-dev") + (number->string #$gn-guile-port))))))))) + (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port) (list (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/cd/gn-guile.log"))) + (stop #~(make-kill-destructor))) + (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) - ;; FIXME: The genenetwork2 service should depend on redis. - (requirement '(networking genenetwork3)) - (start #~(make-forkexec-constructor - (list #$(least-authority-wrapper - (program-file "genenetwork2" - (genenetwork2-cd-gexp config)) - #:name "genenetwork2-pola-wrapper" - ;; If we mapped only the mysqld.sock - ;; socket file, it would break when the - ;; external mysqld server is restarted. - #:mappings (list (file-system-mapping - (source genotype-files) - (target source)) - (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t)) - (file-system-mapping - (source gn2-secrets) - (target source) - (writable? #t))) - #:namespaces (delq 'net %namespaces)) - "127.0.0.1" #$(number->string gn2-port)) - #:user "genenetwork" - #:group "genenetwork" - #:log-file "/var/log/cd/genenetwork2.log")) + (requirement '(networking redis)) + (modules '((guix search-paths) + (ice-9 match) + (srfi srfi-1))) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn2 to be disabled when it is respawned "too fast." + (respawn-delay 5) + (start + (let* ((gn2-manifest (packages->manifest (list genenetwork2))) + (gn2-profile (profile + (content gn2-manifest) + (allow-collisions? #t))) + (gn2-settings + `(("SERVER_PORT" ,(number->string gn2-port)) + ("GENENETWORK_FILES" ,genotype-files) + ("HOME" "/tmp") + ("LC_ALL" "en_US.UTF-8") + ("NO_REDIS" "no-redis") + ("RUST_BACKTRACE" "1")))) + (with-imported-modules (source-module-closure '((guix search-paths))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork2" + (genenetwork2-cd-gexp config)) + #:name "genenetwork2-pola-wrapper" + #:preserved-environment-variables + (append '("REQUESTS_CA_BUNDLE") + (map first gn2-settings) + (map search-path-specification-variable + (manifest-search-paths gn2-manifest))) + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source genotype-files) + (target source)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source gn2-secrets) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn2-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (append + '("REQUESTS_CA_BUNDLE=" + #$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) + (map (match-lambda + ((spec . value) + (string-append (search-path-specification-variable spec) + "=" + value))) + (evaluate-search-paths + (map sexp->search-path-specification + '#$(map search-path-specification->sexp + (manifest-search-paths gn2-manifest))) + (list #$gn2-profile))) + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn2-settings)) + #:log-file "/var/log/cd/genenetwork2.log")))) (stop #~(make-kill-destructor))) (shepherd-service (documentation "Run GeneNetwork 3 development server.") (provision '(genenetwork3)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn3 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "genenetwork3" @@ -626,6 +906,23 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source lmdb-data-path) + (target source) + (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) @@ -637,11 +934,11 @@ described by CONFIG, a <genenetwork-configuration> object." (target source) (writable? #t)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) - (file-system-mapping - (source llm-db-path) + (file-system-mapping + (source (dirname llm-db-path)) (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) @@ -654,6 +951,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run gn-auth development server.") (provision '(gn-auth)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn-auth to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-auth" @@ -666,11 +966,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -693,15 +997,29 @@ described by CONFIG, a <genenetwork-configuration> object." (group "genenetwork") (system? #t) (comment "GeneNetwork user") - (home-directory "/var/empty") + (home-directory "/home/genenetwork") (shell (file-append shadow "/sbin/nologin"))))) (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets) + (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets gn-doc-git-checkout) (with-imported-modules '((guix build utils)) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (ice-9 ftw)) + ;; KLUDGE: Guix now stores inferior profiles under + ;; /var/guix/profiles/per-user (commit + ;; d12c4452a49b355369636de1dfc766b5bad6437b). The 'laminar' + ;; user’s directory is not created automatically in our + ;; pinned Guix revision, which causes CI jobs using + ;; inferiors to fail with permission errors. + ;; XXXX: FIXME: Explicitly create the directory for + ;; now. Remove this once we update the pinned Guix commit. + (unless (file-exists? "/var/guix/profiles/per-user/laminar") + (mkdir-p "/var/guix/profiles/per-user/laminar") + (chown "/var/guix/profiles/per-user/laminar" + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) ;; Set ownership of files. (for-each (lambda (file) @@ -711,10 +1029,13 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) + (find-files gn-doc-git-checkout + #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets #:directories? #t)))) + ;; Prevent other users from reading secret files. (for-each (lambda (file) (chmod file #o600)) @@ -744,31 +1065,6 @@ described by CONFIG, a <genenetwork-configuration> object." ;;; transform-genenetwork-database ;;; -;; Path to genenetwork database dump export directory that has lots of -;; free space -(define %transform-genenetwork-database-export-directory - "/export/genenetwork-database-dump") - -;; Unreleased version of ccwl that is required by -;; transform-genenetwork-database for its graphql library. -(define ccwl - (let ((commit "02677a508b407779f5991a230341e016deb7f69b") - (revision "0")) - (package - (inherit guix:ccwl) - (name "ccwl") - (version (git-version (package-version guix:ccwl) revision commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/arunisaac/ccwl") - (commit commit))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1kxry8y0pibl0x789jrzqkkh2s59ajyinfvrgvd00gkbqldih82r"))))))) - ;; guile-sparql tests are broken. Disable them temporarily. The issue ;; has been reported upstream at ;; https://github.com/roelj/guile-sparql/issues/6 @@ -804,57 +1100,48 @@ described by CONFIG, a <genenetwork-configuration> object." (description "run64 is a SRFI-64 test runner for Scheme.") (license license:gpl3+))) -(define (transform-genenetwork-database project) +;; Connection settings for Virtuoso and MySQL used to load data into Virtuoso +(define %connection-settings + "/etc/genenetwork/conf/gn-transform-database/conn.scm") + +;; Path to where the data directory from which virtuoso loads all the files +(define %virtuoso-data-dir "/var/lib/data") + +(define (transform-genenetwork-database-gexp connection-settings virtuoso-data-dir repository) (with-imported-modules '((guix build utils)) - (with-packages (list ccwl git-minimal gnu-make guile-3.0 guile-dbd-mysql + (with-packages (list git-minimal gnu-make guile-3.0 guile-dbd-mysql guile-dbi guile-hashing guile-libyaml guile-sparql - guile-zlib nss-certs virtuoso-ose) - #~(begin - (use-modules (guix build utils) - (srfi srfi-26) - (ice-9 threads)) - - (invoke "git" "clone" - "--depth" "1" - #$(forge-project-repository project) - ".") - (invoke "make" "-j" (number->string (current-processor-count))) - (let ((connection-settings-file #$(string-append %transform-genenetwork-database-export-directory - "/conn.scm")) - (dump-directory #$(string-append %transform-genenetwork-database-export-directory - "/dump"))) - (when (file-exists? dump-directory) - (delete-file-recursively dump-directory)) - (mkdir-p dump-directory) - ;; Dump data to RDF. - (invoke "./pre-inst-env" "./dump.scm" - connection-settings-file - dump-directory) - ;; Validate dumped RDF, sending the error output to - ;; oblivion because we don't want to print out potentially - ;; sensitive data. - (with-error-to-file "/dev/null" - (cut invoke - #$(file-append raptor2 "/bin/rapper") - "--input" "turtle" - "--count" - (string-append dump-directory "/dump.ttl"))) - ;; Load RDF into virtuoso. - (invoke "./pre-inst-env" "./load-rdf.scm" - connection-settings-file - (string-append dump-directory "/dump.ttl")) - ;; Visualize schema and archive results. - (invoke "./pre-inst-env" "./visualize-schema.scm" - connection-settings-file) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "sql.dot" (string-append "-o" (getenv "ARCHIVE") "/sql.svg")) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "rdf.dot" (string-append "-o" (getenv "ARCHIVE") "/rdf.svg"))))))) + guile-zlib nss-certs virtuoso-ose raptor2) + #~(begin + (use-modules (guix build utils) + (srfi srfi-26) + (ice-9 threads)) + (setenv "LC_ALL" "en_US.UTF-8") + (let ((build-directory (string-append #$virtuoso-data-dir + "/build"))) + ;; Only run this job if the build directory does not + ;; exists. This ensures that no other process is + ;; running this. + (unless (file-exists? build-directory) + (invoke "git" "clone" "--depth" "1" #$repository ".") + (invoke "make" "-j" (number->string (current-processor-count))) + (invoke "./generate-ttl-files.scm" "--settings" + #$connection-settings "--output" build-directory) + ;; First clear all the files in our virtuoso directory + (for-each (lambda (file) + (unless (string-suffix? "build" (dirname file)) + (delete-file file))) + (find-files #$virtuoso-data-dir ".ttl")) + ;; Move data into the container's virtuoso data directory + (copy-recursively build-directory #$virtuoso-data-dir) + ;; Load RDF into virtuoso. + (invoke "./pre-inst-env" "./load-rdf.scm" #$connection-settings) + (delete-file-recursively build-directory))))))) (define transform-genenetwork-database-project (forge-project (name "transform-genenetwork-database") - (repository "/home/git/public/gn-transform-databases/") + (repository "/home/git/public/gn-transform-databases") (ci-jobs (list (forge-laminar-job (name "transform-genenetwork-database-tests") (run (guix-channel-job-gexp @@ -862,11 +1149,14 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'transform-genenetwork-database) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))) (forge-laminar-job (name "transform-genenetwork-database") - (run (transform-genenetwork-database this-forge-project))))))) + (run (transform-genenetwork-database-gexp + %connection-settings + %virtuoso-data-dir + "https://git.genenetwork.org/gn-transform-databases"))))))) ;;; @@ -884,6 +1174,7 @@ described by CONFIG, a <genenetwork-configuration> object." #~(begin (use-modules (guix build utils)) + (setenv "LC_ALL" "en_US.UTF-8") (invoke #$(file-append tissue "/bin/tissue") "pull" "issues.genenetwork.org")))))))) (ci-jobs-trigger 'webhook))) @@ -904,7 +1195,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lmdb) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -924,7 +1215,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-gsl) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -944,7 +1235,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lapack) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1137,6 +1428,44 @@ gn-auth." ";") "proxy_set_header Host $host;"))))))) +(define (gn-guile-reverse-proxy-server-block) + "Return an <nginx-server-configuration> object to reverse proxy +gn-guile to display RDF pages." + (nginx-server-configuration + (server-name '("rdf.genenetwork.org")) + (locations + (list + ;; SPARQL web point + (nginx-location-configuration + (uri "/sparql") + (body (list (list (string-append "proxy_pass http://localhost:" + (number->string %virtuoso-sparql-port) + "/sparql;") + "proxy_set_header Host $host;" + "proxy_set_header X-Forwarded-For $remote_addr;" + "proxy_set_header X-Forwarded-Proto $scheme;")))) + ;; Default RDF page served from gn-guile + (nginx-location-configuration + (uri "/") + (body (list (string-append "proxy_pass http://localhost:" + (number->string %gn-guile-port) + ";") + "proxy_set_header Host $host;"))))))) + +(define set-build-directory-permissions-gexp + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) + (append (find-files #$%xapian-directory + #:directories? #t) + (find-files #$%virtuoso-data-dir + #:directories? #t)))))) + ;; Port on which webhook is listening (define %webhook-port 9091) ;; Port on which genenetwork2 is listening @@ -1147,6 +1476,8 @@ gn-auth." (define %gn-auth-port 9094) ;; Port on which virtuoso's SPARQL endpoint is listening (define %virtuoso-sparql-port 9082) +;; Port on which gn-guile is listening +(define %gn-guile-port 8091) (operating-system (host-name "genenetwork-development") @@ -1157,13 +1488,14 @@ gn-auth." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) - (packages %base-packages) + (packages (cons* curl coreutils-minimal %base-packages)) (sudoers-file (mixed-text-file "sudoers" "@include " %sudoers-specification ;; Permit the laminar user to restart genenetwork2 ;; and genenetwork3. "\nlaminar ALL = NOPASSWD: " + (file-append shepherd "/bin/herd") " restart gn-guile, " (file-append shepherd "/bin/herd") " restart genenetwork2, " (file-append shepherd "/bin/herd") " start genenetwork3, " (file-append shepherd "/bin/herd") " stop genenetwork3, " @@ -1194,6 +1526,15 @@ gn-auth." (jobs (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron" build-xapian-index-cron-gexp) + #:user "laminar") + ;; Run cron once a week at midnight on Sunday morning + ;; Verify using: https://crontab.guru/#0_0_*_*_0 + #~(job "0 0 * * 0" + #$(program-file "update-virtuoso" + (transform-genenetwork-database-gexp + %connection-settings + %virtuoso-data-dir + "https://git.genenetwork.org/gn-transform-databases")) #:user "laminar"))))) (simple-service 'install-laminar-template activation-service-type @@ -1212,7 +1553,7 @@ gn-auth." (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (server-port 9081) - (dirs-allowed "/var/lib/data") + (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) (service genenetwork-service-type (genenetwork-configuration @@ -1230,18 +1571,7 @@ gn-auth." (xapian-db-path %xapian-directory))) (simple-service 'set-build-directory-permissions activation-service-type - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - - (for-each (lambda (file) - (chown file - (passwd:uid (getpw "laminar")) - (passwd:gid (getpw "laminar")))) - (append (find-files #$%xapian-directory - #:directories? #t) - (find-files #$%transform-genenetwork-database-export-directory - #:directories? #t)))))) + set-build-directory-permissions-gexp) (service tissue-service-type (tissue-configuration (socket @@ -1250,8 +1580,12 @@ gn-auth." (hosts (list (tissue-host (name "issues.genenetwork.org") - (user "laminar") - (upstream-repository "https://github.com/genenetwork/gn-gemtext-threads")))))) + (projects (list (tissue-project + (name "issues.genenetwork.org") + (user "laminar") + (base-path "/") + (upstream-repository + "https://github.com/genenetwork/gn-gemtext-threads"))))))))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket @@ -1265,9 +1599,14 @@ gn-auth." %genenetwork2-port %genenetwork3-port) (laminar-reverse-proxy-server-block "localhost:9089" %webhook-port - (list 'gn-bioinformatics)) + (list 'gn-bioinformatics + 'guix-bioinformatics)) (tissue-reverse-proxy-server-block) - (gn-auth-reverse-proxy-server-block))))) + (gn-auth-reverse-proxy-server-block) + (gn-guile-reverse-proxy-server-block))))) + (service guile-sheepdog-service-type + (guile-sheepdog-configuration + (settings-file "/etc/genenetwork/conf/sheepdog.scm"))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) diff --git a/genenetwork/services/genecup.scm b/genenetwork/services/genecup.scm new file mode 100644 index 0000000..6ee2812 --- /dev/null +++ b/genenetwork/services/genecup.scm @@ -0,0 +1,121 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services genecup) + #:use-module (guix) + #:use-module (gnu) + #:use-module (guix git) + #:use-module (guix modules) + #:use-module (guix profiles) + #:use-module (guix records) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:use-module (forge utils) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gn packages machine-learning) + #:use-module (gnu packages certs) + #:use-module (gnu packages curl) + #:use-module (gn packages python) + #:use-module (gnu packages admin) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module ((gnu packages python) #:select (python)) + #:use-module (gnu packages python-xyz) + #:use-module (guix build python-build-system) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages python-web) + #:use-module (gnu services shepherd) + #:use-module (gnu packages python-science) + #:use-module (gnu packages machine-learning) + #:use-module ((gnu packages admin) #:select (shepherd)) + #:export (genecup-configuration + genecup-configuration? + genecup-configuration-package + genecup-service-type)) + +(define-record-type* <genecup-configuration> + genecup-configuration make-genecup-configuration + genecup-configuration? + (package genecup-configuration-package ; <package> + (default genecup-latest-with-tensorflow-native))) + +(define (genecup-activation config) + (match-record config <genecup-configuration> + (package) + (with-packages + (list python + python-nltk + nss-certs ; Needed for downloading data with nltk.downloader. + curl) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((nltk-data "/var/cache/nltk_data/") + (data-dir "/export/ratspub/tmp")) + (unless (file-exists? nltk-data) + (begin + (mkdir-p nltk-data) + (chdir nltk-data) + (invoke #$(file-append python "/bin/python3") "-m" "nltk.downloader" "-d" nltk-data "punkt"))) + (unless (file-exists? (string-append data-dir "/userspub.sqlite")) + (begin + (install-file #$(file-append package "/userspub.sqlite") data-dir) + (chmod (string-append data-dir "/userspub.sqlite") #o554))))))))) + +(define genecup-shepherd-service + (match-lambda + (($ <genecup-configuration> package) + (with-imported-modules (source-module-closure + '((guix search-paths) + (gnu build shepherd) + (gnu system file-systems))) + (list (shepherd-service + (provision '(genecup)) + (requirement '(networking)) + (modules '((gnu build shepherd) + (gnu system file-systems))) + (start + #~(make-forkexec-constructor + (list #$(file-append package "/server.py")) + #:directory #$package + #:environment-variables + (list + "NLTK_DATA=/var/cache/nltk_data" + (string-append "EDIRECT_PUBMED_MASTER=" + #$(file-append package "/minipubmed")) + (string-append "PERL_LWP_SSL_CA_FILE=" + #$(file-append (profile (content (packages->manifest (list curl nss-certs)))) + "/etc/ssl/certs/ca-certificates.crt"))) + #:log-file "/var/log/genecup.log")) + (stop #~(make-kill-destructor)))))))) + +(define genecup-service-type + (service-type + (name 'genecup) + (extensions + (list + (service-extension activation-service-type + genecup-activation) + (service-extension shepherd-root-service-type + genecup-shepherd-service))) + (default-value (genecup-configuration)) + (description + "Run a GeneCup Webserver."))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index 34d70df..1e63ad2 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -20,17 +20,23 @@ ;;; <https://www.gnu.org/licenses/>. (define-module (genenetwork services genenetwork) - #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-guile)) + #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) #:use-module ((gnu packages admin) #:select (shadow shepherd)) + #:use-module ((gnu packages version-control) #:select (git-minimal)) #:use-module ((gnu packages python) #:select (python)) #:use-module (gnu services) #:use-module (gnu services web) #:use-module (gnu services mcron) + #:use-module (gnu services shepherd) #:use-module (gnu system file-systems) #:use-module (gnu system shadow) #:use-module (guix build python-build-system) + #:use-module (guix diagnostics) #:use-module (guix gexp) + #:use-module (guix i18n) + #:use-module (guix least-authority) #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix records) @@ -38,8 +44,10 @@ #:use-module (forge nginx) #:use-module (forge gunicorn) #:use-module (forge socket) + #:use-module (forge utils) #:use-module (srfi srfi-1) #:use-module (ice-9 match) + #:use-module (gnu packages ssh) #:export (genenetwork-service-type genenetwork-configuration genenetwork-configuration? @@ -49,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -83,10 +92,14 @@ (default 8083)) (gn-auth-port genenetwork-configuration-gn-auth-port (default 8084)) + (gn3-alias-server-port genenetwork-gn3-alias-server-port + (default 8000)) (sql-uri genenetwork-configuration-sql-uri (default "mysql://username:password@localhost/database")) (auth-db genenetwork-configuration-auth-db (default "/var/genenetwork/auth.db")) + (llm-db-path genenetwork-configuration-llm-db-path + (default "/var/genenetwork/llm.db")) (xapian-db genenetwork-configuration-xapian-db (default "/var/genenetwork/xapian")) (genotype-files genenetwork-configuration-genotype-files @@ -94,15 +107,38 @@ (sparql-endpoint genenetwork-configuration-sparql-endpoint (default "http://localhost:8081/sparql")) (gn-sourcecode-directory genenetwork-configuration-gn-sourcecode-directory - (default "/var/empty")) + (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) + (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir + (default "/var/genenetwork/sessions/genenetwork2")) (gn2-secrets genenetwork-configuration-gn2-secrets (default "/etc/genenetwork")) (gn3-secrets genenetwork-configuration-gn3-secrets (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets - (default "/etc/genenetwork"))) + (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-guile-ssh-identity-file genenetwork-configuration-gn-guile-ssh-identity-file + (default "/opt/home/gn-guile/.ssh/id_ed25519")) + (gn-guile-known-hosts-file genenetwork-configuration-gn-guile-known-hosts-file + (default "/opt/home/gn-guile/.ssh/known_hosts")) + (gn-guile-working-dir genenetwork-configuration-gn-guile-working-dir + (default "/export/data")) + (gn-doc-remote-uri genenetwork-configuration-gn-doc-remote-uri + (default "/export/data/gn-docs.git")) + (gn-docs-working-branch genenetwork-configuration-gn-docs-working-branch + (default "gn-production-branch")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (log-level genenetwork-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) (define-record-type* <gn-uploader-configuration> gn-uploader-configuration make-gn-uploader-configuration @@ -119,11 +155,28 @@ (default "/var/genenetwork")) (secrets gn-uploader-configuration-secrets (default "/etc/genenetwork/gn-uploader-secrets.py")) - (auth-server-url gn-uploader-auth-server-url + (auth-server-url gn-uploader-configuration-auth-server-url (default "https://auth.genenetwork.org")) - (gn2-server-url gn-uploader-gn2-server-url + (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) - (log-level gn-uploader-log-level (default "WARNING"))) + (sessions-dir gn-uploader-sessions-dir + (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (gn-tmpdir gn-uploader-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (genotype-files-directory gn-uploader-configuration-genotype-files-directory + (default "/var/genenetwork/genotype-files")) + (log-level gn-uploader-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) + +(define (sanitize-log-level log-level) + (case log-level + ((fatal error warning info debug trace) log-level) + (else + (leave (G_ "Log level ~a is invalid. It must be one of the following symbols---fatal, error, warn, info, debug or trace.~%") + log-level)))) (define %genenetwork-accounts (list (user-group @@ -142,12 +195,14 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) (gn3-profile (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t))) + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) (python3-version (python-version (package-version python)))) (with-imported-modules '((guix build utils)) #~(begin @@ -175,7 +230,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -184,7 +241,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -212,9 +271,35 @@ (chmod file #o644)) (find-files #$xapian-directory))))))))) +(define (samples-count-script-gexp config) + (match-record config <genenetwork-configuration> + (genenetwork2 sql-uri) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PYTHONPATH" + (string-append + #$(file-append genenetwork2 + "/lib/python" + (python-version (package-version python)) + "/site-packages") + ":" + #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "/lib/python" + #$(python-version (package-version python)) + "/site-packages")) + + (invoke #$(file-append python "/bin/python3") + "-m" + "gn2.scripts.sample_count" + #$sql-uri))))) + (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-guile-working-dir gn2-sessions-dir gn-guile-ssh-identity-file) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -234,11 +319,19 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets + #:directories? #t) + (find-files #$gn2-sessions-dir #:directories? #t))) - (chown #$gn3-secrets - (passwd:uid (getpw "gunicorn-genenetwork3")) - (passwd:gid (getpw "gunicorn-genenetwork3"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-genenetwork3")) + (passwd:gid (getpw "gunicorn-genenetwork3")))) + (cons #$gn3-secrets + (find-files #$(dirname llm-db-path) + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -246,7 +339,33 @@ (find-files #$gn2-secrets #:directories? #f) (find-files #$gn-auth-secrets - #:directories? #f))))))) + #:directories? #f))) + ;; Make sub-directories for various apps under gn-tmpdir and assign + ;; appropriate permissions + (for-each (match-lambda + ((subdir user) + (let ((full-path + (string-append #$gn-tmpdir "/" subdir))) + (unless (file-exists? full-path) + (mkdir full-path #o755)) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw user)) + (passwd:gid (getpw user)))) + (find-files full-path + #:directories? #t))))) + '(("gn2-tmpdir" "gunicorn-genenetwork2") + ("gn3-tmpdir" "gunicorn-genenetwork3"))) + + ;; setup correct ownership for gn-docs + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (append (find-files #$gn-guile-working-dir + #:directories? #t) + (find-files #$(dirname (dirname gn-guile-ssh-identity-file)) + #:directories? #t))))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -277,7 +396,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -287,6 +406,7 @@ object." (gn2-profile (profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) + (gn2-ca-bundle (file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) (gn2-conf (computed-file "gn2.conf" (configuration-file-gexp `(("GN2_SECRETS" ,(string-append gn2-secrets "/gn2-secrets.py")) @@ -295,35 +415,52 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("SSL_PRIVATE_KEY" ,(string-append gn2-secrets "/gn2-ssl-private-key.pem")) - ("AUTH_SERVER_SSL_PUBLIC_KEY" ,(string-append gn2-secrets "/gn-auth-ssl-public-key.pem")))))) + ("AI_SEARCH_ENABLED" "True") + ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir) + ("MAX_FORM_MEMORY_SIZE" 52428800))))) + (gn3-profile (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (gn3-ca-bundle (file-append gn3-profile "/etc/ssl/certs/ca-certificates.crt")) (gn3-conf (computed-file "gn3.conf" (configuration-file-gexp `(("AUTH_DB" ,auth-db) + ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) ("SQL_URI" ,sql-uri) - ("XAPIAN_DB_PATH" ,xapian-db))))) + ("XAPIAN_DB_PATH" ,xapian-db) + ("GENOTYPE_FILES" ,genotype-files) + ("REAPER_COMMAND" ,(file-append gn2-profile "/bin/qtlreaper")) + ("LLM_DB_PATH" ,llm-db-path))))) + (gn-auth-profile (profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (gn-auth-ca-bundle (file-append gn-auth-profile "/etc/ssl/certs/ca-certificates.crt")) (gn-auth-conf (computed-file "gn-auth.conf" (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) ("AUTH_DB" ,auth-db) - ("SQL_URI" ,sql-uri) - ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys")) - ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem"))))))) + ("SQL_URI" ,sql-uri))))) + (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir")) + (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir"))) (list (gunicorn-app (name "genenetwork2") (package genenetwork2) (sockets (list (forge-ip-socket (port gn2-port)))) (wsgi-app-module "gn2.wsgi") - (workers 20) + (workers 10) (timeout 1200) (environment-variables (list (environment-variable @@ -331,23 +468,30 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn2-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) (environment-variable (name "HOME") - (value "/tmp")))) + (value "/tmp")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn2-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn2-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -358,14 +502,24 @@ object." (file-system-mapping (source gn2-secrets) (target source) - (writable? #t))))) + (writable? #t)) + (file-system-mapping + (source gn2-ca-bundle) + (target source)) + (file-system-mapping + (source gn2-sessions-dir) + (target source) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "genenetwork3") (package genenetwork3) (sockets (list (forge-ip-socket (port gn3-port)))) (wsgi-app-module "gn3.app:create_app()") - (workers 20) + (workers 10) ;; gunicorn's default 30 second timeout is insufficient ;; for Fahamu AI endpoints and results in worker timeout ;; errors. @@ -376,13 +530,16 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn3-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) (environment-variable (name "HOME") - (value "/tmp")))) + (value "/tmp")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn3-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn3-conf) @@ -399,24 +556,34 @@ object." (file-system-mapping (source gn3-data-directory) (target source)) ; Rqtl usese this - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn3-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) (target source)) (file-system-mapping - (source auth-db) + (source (dirname llm-db-path)) (target source) - (writable? #t))))) + (writable? #t)) + (file-system-mapping + (source gn3-ca-bundle) + (target source)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "gn-auth") (package gn-auth) (sockets (list (forge-ip-socket (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") - (workers 20) + (workers 10) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -426,26 +593,35 @@ object." (value "/tmp")) (environment-variable (name "AUTHLIB_INSECURE_TRANSPORT") - (value "true")))) + (value "true")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn-auth-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn-auth-conf) (target source)) (file-system-mapping - (source auth-db) - (target source) - (writable? #t)) + (source (dirname auth-db)) + (target source) + (writable? #t)) (file-system-mapping (source gn-auth-secrets) (target source) - (writable? #t))))))))) + (writable? #t)) + (file-system-mapping + (source gn-auth-ca-bundle) + (target source)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (genenetwork-nginx-server-blocks config) "Return a list of @code{<nginx-server-configuration>} records specifying reverse proxies for the genenetwork service described by @var{config}, a @code{<genenetwork-configuration>} record." (match-record config <genenetwork-configuration> - (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port) + (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port gn3-alias-server-port) (list (nginx-server-configuration (server-name (list server-name)) (locations @@ -455,13 +631,23 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" (string-append "proxy_pass http://localhost:" (number->string gn3-port) ";") - "proxy_set_header Host $host;")))))) + "proxy_set_header Host $host;"))) + (nginx-location-configuration + (uri "/gn3/") + (body + (list "rewrite /gn3/(.*) /$1 break;" + (string-append "proxy_pass http://localhost:" + (number->string gn3-alias-server-port) + ";") + "proxy_redirect off;" + "proxy_set_header Host $host;")))))) (nginx-server-configuration (server-name (list gn-auth-server-name)) (locations @@ -475,7 +661,91 @@ a @code{<genenetwork-configuration>} record." (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron-gexp" (build-xapian-index-cron-gexp config)) - #:user "root"))) + #:user "root") + #~(job '(next-minute-from (next-hour) '(17)) ;17th minute of every hour + #$(program-file "samples-count-script-gexp" + (samples-count-script-gexp config))))) + +(define (gn-guile-gexp config) + (values + config + (match-record config <genenetwork-configuration> + (gn-guile-port gn-guile gn-guile-ssh-identity-file gn-guile-known-hosts-file gn-doc-remote-uri gn-docs-working-branch) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (gn-docs-local-checkout (string-append (pk "CWD" (getcwd)) "/gn-docs")) + (ssh-command #$(file-append openssh-sans-x "/bin/ssh")) + (ssh-config-file #$(mixed-text-file "gn-guile-ssh-config" + "Host git.genenetwork.org\n" + "\tUser git\n" + "\tIdentitiesOnly yes\n" + "\tIdentityFile " gn-guile-ssh-identity-file "\n" + "\tUserKnownHostsFile " gn-guile-known-hosts-file))) + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + (setenv "GIT_SSH_COMMAND" (pk "GIT_SSH_COMMAND ===> " + (string-append ssh-command + " -F " + ssh-config-file))) + (setenv "GIT_COMMITTER_NAME" "genenetwork") + (setenv "GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org") + + (when (file-exists? gn-docs-local-checkout) + (delete-file-recursively gn-docs-local-checkout)) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" "--branch" #$gn-docs-working-branch #$gn-doc-remote-uri)) + (invoke #$(file-append gn-guile "/bin/gn-guile") + "--port" + (number->string #$gn-guile-port) + "--gn-docs-local-checkout" + (string-append (getcwd) "/gn-docs") + "--gn-docs-remote-url" + #$gn-doc-remote-uri + "--gn-docs-working-branch" + #$gn-docs-working-branch)))))) + +(define (gn-guile-shepherd-service config program-gexp) + (match-record config <genenetwork-configuration> + (gn-guile gn-guile-port gn-guile-ssh-identity-file gn-guile-known-hosts-file gn-guile-working-dir) + (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" program-gexp) + #:name "gn-guile-pola-wrapper" + #:directory gn-guile-working-dir + #:mappings (list (file-system-mapping + (source gn-guile-working-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-guile-ssh-identity-file) + (target source) + (writable? #f)) + (file-system-mapping + (source gn-guile-known-hosts-file) + (target source) + (writable? #f))) + #:namespaces (delq 'user (delq 'net %namespaces)) + #:user "genenetwork" + #:group "genenetwork") + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "root" + #:group "root" + #:log-file "/var/log/gn-guile.log" + #:environment-variables (user-environment-variables))) + (stop #~(make-kill-destructor))))) (define genenetwork-service-type (service-type @@ -490,19 +760,27 @@ a @code{<genenetwork-configuration>} record." genenetwork-gunicorn-apps) (service-extension forge-nginx-service-type genenetwork-nginx-server-blocks) + (service-extension shepherd-root-service-type + (compose list gn-guile-shepherd-service gn-guile-gexp)) (service-extension mcron-service-type genenetwork-mcron-jobs))) (default-value (genenetwork-configuration)))) (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory) + (secrets data-directory sessions-dir sqlite-databases-directory gn-tmpdir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Let service user own their own secrets files. - (chown #$secrets - (passwd:uid (getpw "gunicorn-gn-uploader")) - (passwd:gid (getpw "gunicorn-gn-uploader"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader")))) + (append (list #$(dirname secrets)) + (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -513,36 +791,47 @@ a @code{<genenetwork-configuration>} record." (passwd:uid (getpw "gunicorn-gn-uploader")) (passwd:gid (getpw "gunicorn-gn-uploader")))) (append (list #$data-directory) + (find-files #$(dirname secrets) + #:directories? #t) (find-files #$data-directory + #:directories? #t) + (find-files #$(string-append gn-tmpdir "/gn-uploader-scratch") #:directories? #t))))))) (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory gn-tmpdir genotype-files-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t)))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-scratch (string-append gn-tmpdir "/gn-uploader-scratch")) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOADS_DIRECTORY" ,(string-append gn-uploader-scratch + "/uploads")) + ("SCRATCH_DIRECTORY" ,gn-uploader-scratch) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")) + ("GENOTYPE_FILES_DIRECTORY" ,genotype-files-directory))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) (sockets (list (forge-ip-socket (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") - (workers 20) + (workers 10) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -552,22 +841,47 @@ a @code{<genenetwork-configuration>} record." (value "/tmp")) (environment-variable (name "GN_UPLOADER_ENVIRONMENT") - (value gn-uploader-profile)))) + (value gn-uploader-profile)) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn-uploader-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn-uploader-conf) (target source)) (file-system-mapping - (source secrets) - (target source)) + (source (dirname secrets)) + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source) (writable? #t)) (file-system-mapping (source gn-uploader-profile) - (target source)))) - (extra-cli-arguments (list "--log-level" log-level))))))) + (target source)) + (file-system-mapping + (source gn-uploader-ca-bundle) + (target source)) + (file-system-mapping + (source sessions-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-scratch) + (target source) + (writable? #t)) + (file-system-mapping + (source genotype-files-directory) + (target source) + (writable? #f)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (gn-uploader-nginx-server-block config) (match-record config <gn-uploader-configuration> @@ -582,7 +896,7 @@ a @code{<genenetwork-configuration>} record." #$(file-append gn-uploader "/lib/python" (python-version (package-version python)) - "/site-packages/qc_app;"))))) + "/site-packages/uploader;"))))) (nginx-location-configuration (uri "/") (body (list (string-append "proxy_pass http://localhost:" diff --git a/genenetwork/services/mouse-longevity.scm b/genenetwork/services/mouse-longevity.scm new file mode 100644 index 0000000..c9b977d --- /dev/null +++ b/genenetwork/services/mouse-longevity.scm @@ -0,0 +1,33 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services mouse-longevity) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gnu services) + #:export (mouse-longevity-service)) + +(define* (mouse-longevity-service #:optional (package mouse-longevity-app)) + (simple-service 'mouse-longevity + rshiny-service-type + (list + (rshiny-configuration + (package package) + (binary "mouse-longevity-app"))))) diff --git a/guix/gn-machines/genenetwork.scm b/guix/gn-machines/genenetwork.scm new file mode 100644 index 0000000..e349ff3 --- /dev/null +++ b/guix/gn-machines/genenetwork.scm @@ -0,0 +1,230 @@ +(define-module (gn-machines genenetwork) + #:use-module (guix) + #:use-module (guix packages) + #:use-module (guix git-download) + + #:use-module ((gnu packages ssh) #:select (openssh-sans-x)) + #:use-module ((gnu packages guile-xyz) #:select (guile-config)) + + #:use-module ((gn packages genenetwork) + #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-libs rust-qtlreaper) + #:prefix gn:) + #:use-module((gn packages guile) + #:select (gn-guile guile-sheepdog) + #:prefix gng:)) + +(define-public genenetwork2 + (let ((commit "fc968a1f37b154830d394b656d067e456efd53ea") + (revision "4")) + (package + (inherit gn:genenetwork2) + (name "genenetwork2") + (version (git-version (package-version gn:genenetwork2) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork2.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "0l4ws9mbhyik2q0mgm6hpck9a2l4lp9d1mbr2kqx4qg6g35vq85i")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork2) + (replace "gn-libs" gn-libs) + (replace "genenetwork3" genenetwork3)))))) + +(define-public rust-qtlreaper + (let ((commit "a62886a52a980474b410a06d4ba92a219c484ec9") + (revision "1")) + (package + (inherit gn:rust-qtlreaper) + (name "rust-qtlreaper") + (version (git-version (package-version gn:rust-qtlreaper) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/rust-qtlreaper.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "0jhmfzin96nsadsgspnl4kk4znq365x0srx8hc2madrshnnkzz36"))))))) + +(define-public genenetwork3 + (let ((commit "e1cf6c46ae35ae758e247f3ed09e48e9eddc47cb") + (revision "5")) + (package + (inherit gn:genenetwork3) + (name "genenetwork3") + (version (git-version (package-version gn:genenetwork3) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork3.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "14hxrqrnv320dmfnp7zzp58w72a072pjp770crsbhjyc1cngbcbf")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork3) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-auth + (let ((commit "15c417b4c69343a0e085d0fd1c41d98ec81a1011") + (revision "1")) + (package + (inherit gn:gn-auth) + (name "gn-auth") + (version (git-version (package-version gn:gn-auth) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-auth") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "0l0fjsyf42h98mg6gyksmkjzg82v62wpfqlsda5ibfny80nak8w2"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-auth) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-uploader + (let ((commit "112aa267709b130447d8ca387631f5dbeba02366") + (revision "0")) + (package + (inherit gn:gn-uploader) + (name "gn-uploader") + (version (git-version (package-version gn:gn-uploader) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-uploader") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "073jy7cxkmjk881yp6kccsd6mg2gkdg2ky059m4m6p2kqnlkdh82"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-uploader) + (replace "gn-libs" gn-libs) + (replace "rust-qtlreaper" rust-qtlreaper)))))) + +(define-public gn-libs + (let ((commit "78c54eed593e1ddd6fb31745e101b4d9a1d2647d") + (revision "03")) + (package + (inherit gn:gn-libs) + (name "gn-libs") + (version (git-version (package-version gn:gn-libs) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-libs") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "0qcwdn0jbd39dccxa1c8a9ph6yfprwldscc4byanm0y0c68ijy98"))))))) + +(define-public gn-guile + (let ((commit "0e8e605434b37251e6729121c77afa963cebef6a") + (revision "0")) + (package + (inherit gng:gn-guile) + (name "gn-guile") + (version (git-version "4.0.1" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-guile/") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "0xy8c4x0hfi9za47prf59a9qjc983ygl2g34ndi9imjy36aa4pib")))) + (arguments + (list + #:not-compiled-file-regexp "(guix|guix/.*)[.]scm$" + #:modules '((srfi srfi-1) + (guix build guile-build-system) + (guix build utils)) + #:phases + #~(modify-phases %standard-phases + (add-after 'unpack 'patch-git + (lambda* (#:key inputs #:allow-other-keys) + (let ((git (search-input-file inputs "/bin/git"))) + (substitute* "web/view/markdown.scm" + (("\"git\"") (string-append "\"" git "\"")) + (("git -C") (string-append git " -C")))))) + (add-after 'patch-source-shebangs 'patch-gn-guile-source-shebangs + ;; there is still the shell-script `lmdb-publishdata-export' that + ;; needs to be patched the usual way, so we have 2 different + ;; source-shebang patching phases. + (lambda* (#:key inputs #:allow-other-keys) + (substitute* "bin/gn-guile" + (("^exec guile") + (string-append "exec " + (search-input-file inputs "/bin/guile")))))) + (add-after 'build 'install-scripts + (lambda* _ + (mkdir-p "bin") + (copy-file "scripts/lmdb-publishdata-export.scm" + "bin/lmdb-publishdata-export") + (for-each (lambda (script-path) + (install-file script-path + (string-append #$output "/bin"))) + (list "bin/gn-guile" + "bin/lmdb-publishdata-export")))) + (add-after 'install-scripts 'wrap + (lambda* (#:key inputs #:allow-other-keys) + (let((effective-version (target-guile-effective-version)) + (guile-path + (dirname (search-input-file inputs "/bin/guile")))) + (for-each (lambda (script-name) + (wrap-program + (string-append #$output "/bin/" script-name) + `("PATH" prefix (,guile-path ,(getenv "PATH"))) + `("GUILE_LOAD_PATH" prefix + (,(string-append #$output + "/share/guile/site/" + effective-version) + ,(getenv "GUILE_LOAD_PATH"))) + `("GUILE_LOAD_COMPILED_PATH" prefix + (,(string-append #$output "/lib/guile" + effective-version + "/site-ccache") + ,(getenv "GUILE_LOAD_COMPILED_PATH"))) + `("GUILE_AUTO_COMPILE" ":" = ("0")))) + (list "lmdb-publishdata-export" + "gn-guile")))))))) + (propagated-inputs + ;; TODO: remove openssh-sans-x on next update to newer + ;; guix-bioinformatics commit, i.e. newer than commit + ;; `9b0955f14ec725990abb1f6af3b9f171e4943f77'. + (modify-inputs (package-propagated-inputs gng:gn-guile) + (prepend openssh-sans-x guile-config)))))) + +(define-public guile-sheepdog + (let ((commit "1426617d58f305a4126bb867202843e8cf7dd4b2") + (revision "0")) + (package + (inherit gng:guile-sheepdog) + (name "guile-sheepdog") + (version (git-version "4.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/BonfaceKilz/guile-sheepdog.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1z0xzg11p75s2hk312akxlg2h5278w2abma27dhzjf981g3lcqvr"))))))) diff --git a/guix/gn-machines/services/monitoring.scm b/guix/gn-machines/services/monitoring.scm new file mode 100644 index 0000000..7fa59c9 --- /dev/null +++ b/guix/gn-machines/services/monitoring.scm @@ -0,0 +1,68 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (gn-machines services monitoring) + #:use-module (gnu) + #:use-module (gnu services shepherd) + #:use-module (gnu services databases) + #:use-module ((gn-machines genenetwork) #:select (guile-sheepdog)) + #:use-module (guix gexp) + #:use-module (guix records) + #:use-module (ice-9 match) + #:export (guile-sheepdog-configuration + guile-sheepdog-configuration? + guile-sheepdog-configuration-settings-file + guile-sheepdog-configuration-package + guile-sheepdog-service-type)) + +(define-record-type* <guile-sheepdog-configuration> + guile-sheepdog-configuration + make-guile-sheepdog-configuration + guile-sheepdog-configuration? + (settings-file guile-sheepdog-configuration-settings-file + (default "/etc/conn.scm")) + (package guile-sheepdog-configuration-package (default guile-sheepdog))) + +(define (guile-sheepdog-gexp config) + (match-record config <guile-sheepdog-configuration> (settings-file package) + (program-file + "guile-sheepdog" + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$(file-append package "/bin/guile-sheepdog") #$settings-file)))))) + +(define (guile-sheepdog-shepherd-service config) + (shepherd-service + (documentation "Run Sheepdog") + (provision '(guile-sheepdog)) + (requirement '(networking redis)) + (start #~(make-forkexec-constructor + (list #$(guile-sheepdog-gexp config)) + #:log-file "/var/log/sheepdog.log")) + (stop #~(make-kill-destructor)))) + +(define guile-sheepdog-service-type + (service-type + (name 'guile-sheepdog) + (description "Run sheepdog monitor") + (extensions + (list (service-extension shepherd-root-service-type + (compose list guile-sheepdog-shepherd-service)))) + (default-value (guile-sheepdog-configuration)))) diff --git a/production-deploy.sh b/production-deploy.sh index b4924a7..793fd34 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -2,6 +2,8 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> +# Copyright © 2026 Munyoki Kilyungi <me@bonfacemunyoki.com> # # This file is part of genenetwork-machines. # @@ -23,18 +25,28 @@ container_script=$(guix system container \ --network \ - --load-path=. \ + --load-path=./guix/ \ + --load-path=./ \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --expose=/export/data/genenetwork-xapian \ - --share=/export/data/genenetwork-sqlite \ - --expose=/export/data/genenetwork/genotype_files \ + --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/genenetwork/var/log=/var/log \ + --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \ --share=/var/run/mysqld=/run/mysqld \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \ + --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile=/var/lib/genenetwork/gn-guile \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile/ssh=/opt/home/gn-guile/.ssh \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 399c921..302e7ef 100644 --- a/production.scm +++ b/production.scm @@ -1,5 +1,6 @@ ;;; genenetwork-machines --- Guix configuration for genenetwork machines ;;; Copyright © 2022–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> ;;; ;;; This file is part of genenetwork-machines. ;;; @@ -17,6 +18,9 @@ ;;; along with genenetwork-machines. If not, see ;;; <https://www.gnu.org/licenses/>. +;;; This is the production genenetwork container currently deployed on +;;; tux04. + (use-modules (gnu) (genenetwork services genenetwork) ((gnu packages admin) #:select (shepherd)) @@ -40,37 +44,65 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service mysql-service-type - (mysql-configuration - (auto-upgrade? #f))) - (service virtuoso-service-type + (services (cons* (service virtuoso-service-type (virtuoso-configuration - (server-port 7892) - (http-server-port 7893))) + (server-port 9892) + (http-server-port 9893) + (dirs-allowed (list "/export/data/virtuoso")) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) + (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") + (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") - (port 7890))) + (port 9890))) (https-listen (forge-ip-socket (ip "0.0.0.0") - (port 7891))))) + (port 9891))))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) + (service redis-service-type + (redis-configuration + (bind "127.0.0.1") + (port 6379) + (working-directory "/var/lib/redis"))) (service genenetwork-service-type (genenetwork-configuration - (server-name "test1.genenetwork.org") - (gn-auth-server-name "test1-auth.genenetwork.org") - (gn2-port 7894) - (gn3-port 7895) - (gn-auth-port 7896) - (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") - (sparql-endpoint "http://localhost:7893/sparql") - (gn3-data-directory "/export/data/genenetwork") + (server-name "genenetwork.org") + (gn-auth-server-name "auth.genenetwork.org") + (gn2-port 9894) + (gn3-port 9895) + (gn-auth-port 9896) + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (xapian-db "/var/lib/xapian") + (sparql-endpoint "http://localhost:9893/sparql") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db") + (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db") + (gn3-alias-server-port 9800) + (gn-tmpdir "/opt/gn/tmp") + (gn-doc-remote-uri "git@git.genenetwork.org:/home/git/public/gn-docs") + (gn-guile-working-dir "/var/lib/genenetwork/gn-guile/") + (gn-docs-working-branch "gn2-production-branch") + (gn-guile-ssh-identity-file "/opt/home/gn-guile/.ssh/id-ed25519-gn2-production-on-tux04") + (log-level 'info))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "uploader.genenetwork.org") + (port 9897) + (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (data-directory "/var/lib/genenetwork/uploader/data") + (auth-server-url "https://auth.genenetwork.org/") + (gn2-server-url "https://genenetwork.org") + (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader") + (log-level 'info))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index bc4c23a..bd8b938 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -22,9 +22,11 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/public-sparql/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export/data/genenetwork-virtuoso=/var/lib/data \ + --share=/export/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/public-sparql/tmp=/tmp \ + --share=/export/guix-containers/public-sparql/var/log=/var/log \ + --share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index 9781b5d..87ef843 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -19,15 +19,17 @@ (use-modules (gnu) (gn services databases) - (gnu services web)) + (gnu services web) + ((gnu packages admin) #:select (shepherd)) + (forge nginx) + (forge socket)) -(define (virtuoso-reverse-proxy-server-block listen sparql-port) +(define (virtuoso-reverse-proxy-server-block sparql-port) "Return an <nginx-server-configuration> object listening on LISTEN to reverse proxy the Virtuoso server. SPARQL-PORT is the port virtuoso's SPARQL endpoint is listening on." (nginx-server-configuration (server-name '("sparql.genenetwork.org")) - (listen (list listen)) (locations (list (nginx-location-configuration (uri "/") @@ -35,9 +37,10 @@ SPARQL endpoint is listening on." (number->string sparql-port) ";") "proxy_set_header Host $host;"))))))) -(define %reverse-proxy-port 8990) +(define %reverse-http-proxy-port 8990) (define %virtuoso-port 8981) (define %sparql-port 8982) +(define %reverse-https-proxy-port 8993) (operating-system (host-name "sparql") @@ -48,18 +51,30 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) - (number-of-buffers 4000000) - (dirs-allowed "/var/lib/data") - (maximum-dirty-buffers 3000000))) - (service nginx-service-type - (nginx-configuration + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) + (dirs-allowed (list "/export/data/virtuoso")) + (database-file "/var/lib/virtuoso/public-virtuoso.db") + (transaction-file "/var/lib/virtuoso/public-virtuoso.trx") + (error-log-file "/var/lib/public-virtuoso-errors.log") + (syslog "1"))) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "0.0.0.0") + (port %reverse-http-proxy-port))) + (https-listen (forge-ip-socket + (ip "0.0.0.0") + (port %reverse-https-proxy-port))) (server-blocks - (list (virtuoso-reverse-proxy-server-block - (number->string %reverse-proxy-port) - %sparql-port))))) + (list (virtuoso-reverse-proxy-server-block %sparql-port))))) %base-services))) diff --git a/services/README.md b/services/README.md deleted file mode 100644 index d0d1c01..0000000 --- a/services/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Services - -This directory contains small and simple services that may be run independently. -Note that composition is not the goal. -For example, databases and web proxies are handled outsite the containers. -Use these services for simple deployment and ad hoc testing. -One nice aspect of small system containers is that you can run these easily on your laptop. - -IMPORTANT: more complex services do not belong in this directory. - -# Examples - -## gn-guile - -`gn-guile` is our next generation service (gn4?). It provides a REST API, at this point, and some portals, such as https://aging.genenetwork.org/. - -See the [gn-guile](./gn-guile.scm) system definition example. diff --git a/services/gn-guile.scm b/services/gn-guile.scm deleted file mode 100644 index 2f675a8..0000000 --- a/services/gn-guile.scm +++ /dev/null @@ -1,52 +0,0 @@ -;; This is an example definition for the gn-guile/GN4 service -;; -;; Run with -;; -;; export runner=$(guix system container gn-guile.scm) -;; -;; as root -;; -;; sudo bash -c $runner -;; echo $runner -;; -;; make a note of pid and -;; -;; sudo bash -c "nsenter -a -t 4050285" -;; -;; now you should be inside the container (note bash should be in the container!) - -(use-modules (gnu) - (guix records) - (forge utils)) - -(define-record-type* <gn-guile-configuration> - gn-guile-configuration make-gn-guile-configuration - gn-guile-configuration? - (gn2-repository gn-guile-configuration-gn2-repository - (default "https://github.com/genenetwork/genenetwork2")) - (gn2-port gn-guile-configuration-gn2-port - (default 8082))) - - -(define gn-guile-service-type - (service-type - (name 'gn-guile) - (description "gn-guile/GN4 webservice") - (extensions '()) - )) - -(operating-system - (host-name "gn-guile") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service gn-guile-service-type - (gn-guile-configuration)) - %base-services))) diff --git a/services/opensmtpd.scm b/services/opensmtpd.scm deleted file mode 100644 index 1b1e58f..0000000 --- a/services/opensmtpd.scm +++ /dev/null @@ -1,21 +0,0 @@ -(use-modules (gnu) - (gnu services mail)) - -(operating-system - (host-name "mail") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service opensmtpd-service-type - (opensmtpd-configuration - (config-file %default-opensmtpd-config-file - ; (config-file (local-file "./my-smtpd.conf"))) - ))) - %base-services))) diff --git a/singularity-head-deploy.sh b/singularity-head-deploy.sh new file mode 100755 index 0000000..533224b --- /dev/null +++ b/singularity-head-deploy.sh @@ -0,0 +1,12 @@ +#! /bin/sh -xe + +## +## singularity deployment on octopus01 (the head node) +## + +## Install singularity in the same way as the worker nodes. +./singularity-worker-deploy.sh $(guix build -f singularity.scm) + +# Register garbage collector root to prevent `guix gc' from garbage +# collecting singularity. +sudo ln --force --symbolic /usr/local/bin/singularity /var/guix/gcroots diff --git a/singularity-worker-deploy.sh b/singularity-worker-deploy.sh new file mode 100755 index 0000000..d709758 --- /dev/null +++ b/singularity-worker-deploy.sh @@ -0,0 +1,37 @@ +#! /bin/sh -xe + +## +## singularity deployment on octopus worker nodes +## + +case $1 in + "") + echo "Usage: $0 SINGULARITY_STORE_ITEM" + exit 1 + ;; + *) + singularity=$1 + echo $singularity + ;; +esac + +# Symlink singularity executable. +sudo ln --force --symbolic $singularity/bin/singularity /usr/local/bin/singularity + +# To set up singularity, we imitate what the Guix +# singularity-service-type does. + +# Install setuid binaries. +sudo mkdir -p /usr/local/libexec/singularity/bin +for program in action mount start; +do + sudo cp $singularity/libexec/singularity/bin/$program-suid /usr/local/libexec/singularity/bin/singularity-$program-helper + sudo chmod u+s /usr/local/libexec/singularity/bin/singularity-$program-helper +done + +# Create the directories that Singularity 2.6 expects to find. +for directory in container final overlay session; +do + sudo mkdir -p /var/singularity/mnt/$directory + sudo chmod 755 /var/singularity/mnt/$directory +done diff --git a/singularity.scm b/singularity.scm new file mode 100644 index 0000000..9a4c0ce --- /dev/null +++ b/singularity.scm @@ -0,0 +1,42 @@ +(use-modules (gnu packages linux) + (guix download) + (guix packages)) + +(package + (inherit singularity) + (version (package-version singularity)) + (source (origin + (inherit (package-source singularity)) + (snippet + '(begin + ;; We put the singularity setuid binaries under + ;; /usr/local. The Guix package puts it under + ;; /run/privileged/bin. But, we cannot do that since + ;; /run is mounted noexec on octopus. + (substitute* (find-files "libexec/cli" "\\.exec$") + (("\\$SINGULARITY_libexecdir/singularity/bin/([a-z]+)-suid" + _ program) + (string-append "/usr/local/libexec/singularity/bin/singularity-" + program "-helper"))) + + ;; The remaining snippet code below is copied from the + ;; Guix package. + + ;; Do not create directories in /var. + (substitute* "Makefile.in" + (("\\$\\(MAKE\\) .*install-data-hook") "")) + + ;; The original source overrides PATH so that it + ;; points to /bin, /usr/local/bin, etc., which + ;; obviously doesn't work on Guix System. Leave PATH + ;; unchanged so we refer to the installed Coreutils, + ;; grep, etc. + (substitute* "bin/singularity.in" + (("^PATH=.*" all) + (string-append "#" all "\n"))) + + ;; These squashfs mount options are apparently no + ;; longer supported since Linux-libre 5.4.5. + (substitute* "src/lib/image/squashfs/mount.c" + (("\"errors=remount-ro\"") + "NULL"))))))) diff --git a/slurm.scm b/slurm.scm index 83f4032..4f5ece0 100644 --- a/slurm.scm +++ b/slurm.scm @@ -45,6 +45,17 @@ (define slurm (package (inherit guix:slurm) + (name "slurm") + (version "24.05.3") + (source (origin + (inherit (package-source guix:slurm)) + (method url-fetch) + (uri (string-append + "https://download.schedmd.com/slurm/slurm-" + version ".tar.bz2")) + (sha256 + (base32 + "095fck6016kslggd1d9mnwahr66b1fahpmlmvdyqdbmnx49hbd5h")))) (arguments (substitute-keyword-arguments (package-arguments guix:slurm) ((#:configure-flags flags #~'()) @@ -73,9 +84,7 @@ (substitute* (string-append #$output "/etc/slurmrestd.service") ;; Set user and group to run slurmrestd as. (("# User=") "User=slurmrestd") - (("# Group=") "Group=slurmrestd") - ;; Disable listening on Unix socket by default. - ((" unix:[^ ]*") "")))))))) + (("# Group=") "Group=slurmrestd")))))))) (inputs (modify-inputs (package-inputs guix:slurm) (prepend dbus http-parser json-c libjwt diff --git a/specials/gndev.scm b/specials/gndev.scm index 440aedb..0f9394f 100644 --- a/specials/gndev.scm +++ b/specials/gndev.scm @@ -61,6 +61,7 @@ (gn-auth-server-name "gndev-auth.genenetwork.org") (gn2-port 8992) (gn3-port 8993) + (gn-auth-port 8994) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export2/data/gndev-sqlite/auth.db") (xapian-db "/export2/data/gndev-xapian") diff --git a/test-r-container.scm b/test-r-container.scm new file mode 100644 index 0000000..3167574 --- /dev/null +++ b/test-r-container.scm @@ -0,0 +1,111 @@ +(use-modules (guix) + (gnu) + (guix git) + (guix modules) + (guix profiles) + (guix records) + (guix packages) + (guix git-download) + (guix build-system trivial) + (gnu services dbus) + (gnu services networking) + (gnu services ssh) + (gnu services web) + (gnu services certbot) + (gnu services docker) + (gnu services desktop) + (gnu services shepherd) + (gnu packages admin) + (gnu packages statistics) + (gnu packages cran) + (gnu packages curl) + (gnu packages lsof) + (srfi srfi-1) + (ice-9 match) + (genenetwork services mouse-longevity) + (gn services rshiny) + (gn packages mouse-longevity) + (forge acme) + (forge nginx) + (forge socket) + (gnu services)) + +(define %nginx-configuration + (nginx-configuration + (server-blocks + (list + ;; Redirect domains that don't explicitly support HTTP (below) to HTTPS. + (nginx-server-configuration + (listen '("8080"))) + + ;; Domains that still explicitly support plain HTTP. + (nginx-server-configuration + (listen '("80")) + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list + (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://127.0.0.1:3979;"))))) + (raw-content + (list + "proxy_set_header Host $host;" + "proxy_set_header X-Real-IP $remote_addr;" + "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" + "proxy_set_header X-Forwarded-Proto $scheme;"))))))) + +(operating-system + (host-name "testing-genenetwork-services") + (keyboard-layout (keyboard-layout "us")) + (kernel-arguments + (cons* "console=ttyS0,115200" "console=tty0" + %default-kernel-arguments)) + (bootloader (bootloader-configuration (bootloader grub-bootloader))) + (issue "This is a GeneNetwork container. Welcome!\n") + (file-systems %base-file-systems) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* lsof curl %base-packages)) + (services + (cons* + (service dhcp-client-service-type) + (service dbus-root-service-type) + (service polkit-service-type) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8080"))) + (https-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8443"))) + (server-blocks + (list + (nginx-server-configuration + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://localhost:3979;" + "proxy_set_header Host $host;")))))))))) + (service acme-service-type + (acme-configuration + (email "jgart@dismail.de") + (acme-url %letsencrypt-staging-url))) + ;; (service certbot-service-type + ;; (certbot-configuration + ;; (email "jgart@dismail.de") + ;; (certificates + ;; (list + ;; (certificate-configuration + ;; (domains '("longevity-explorer.genenetwork.org" + ;; "www.longevity-explorer.genenetwork.org"))))))) + ;; (mouse-longevity-service ) + (service rshiny-service-type + (rshiny-configuration + (package mouse-longevity-app) + (binary "mouse-longevity-app"))) + ;; (service nginx-service-type %nginx-configuration) + %base-services))) diff --git a/uploader-deploy.sh b/uploader-deploy.sh index 90fd7e4..415790b 100755 --- a/uploader-deploy.sh +++ b/uploader-deploy.sh @@ -41,18 +41,22 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/uploader/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/uploader/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ - --share=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ - --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ - --expose=/export/data/genenetwork/genotype_files=/export/data/genenetwork/genotype_files \ - --expose=/export/data/uploader/genenetwork3 \ - --share=/export/data/uploader/gn-uploader \ + --share=/export/guix-containers/uploader/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/uploader/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/uploader/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/uploader/var/log=/var/log \ + --share=/export/guix-containers/uploader/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-xapian=/var/lib/xapian \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld3307=/run/mysqld \ + --share=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/uploader/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/uploader/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/data/uploader/gn-uploader \ uploader.scm) echo "${container_script}" diff --git a/uploader.scm b/uploader.scm index 62ab35f..5064426 100644 --- a/uploader.scm +++ b/uploader.scm @@ -42,11 +42,7 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service virtuoso-service-type - (virtuoso-configuration - (server-port 10892) - (http-server-port 10893))) - (service forge-nginx-service-type + (services (cons* (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") @@ -64,24 +60,26 @@ (gn2-port 10894) (gn3-port 10895) (gn-auth-port 10896) - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") - (auth-db "/export/data/genenetwork-sqlite/auth.db") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (auth-db "/var/lib/genenetwork-sqlite/auth.db") + (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:10893/sparql") - (gn3-data-directory "/export/data/uploader/genenetwork3") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (llm-db-path "/var/lib/genenetwork-gnqa/llm.db"))) (service gn-uploader-service-type (gn-uploader-configuration (gn-uploader gn-uploader) (server-name "staging-uploader.genenetwork.org") (port 10897) (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (data-directory "/export/data/uploader/gn-uploader") - (log-level "DEBUG") + (log-level 'debug) (auth-server-url "https://staging-auth.genenetwork.org/") (gn2-server-url "https://staging.genenetwork.org"))) %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0414a65..d18caec 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2025 Pjotr Prins <pjotr.public01@thebird.nl> # # This file is part of genenetwork-machines. # @@ -19,14 +20,25 @@ # along with genenetwork-machines. If not, see # <https://www.gnu.org/licenses/>. -# Build and install virtuoso container on tux01. +# Build and install virtuoso container. Note the shared path is the sane default. Symlink if necessary. +# See also topics/systems/debug-and-developing-code-with-genenetwork-system-container.gmi container_script=$(guix system container \ + -L ~/guix-bioinformatics \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/data/virtuoso/ttl=/export/data/virtuoso/ttl \ virtuoso.scm) echo $container_script sudo ln --force --symbolic $container_script /usr/local/bin/virtuoso-container sudo ln --force --symbolic /usr/local/bin/virtuoso-container /var/guix/gcroots + +echo "Run virtuoso with: sudo /usr/local/bin/virtuoso-container" +echo "Enter with something like: nsenter -a -t 1692115 /run/current-system/profile/bin/bash --login" +echo "Admin: isql 8891" +echo " ld_dir('/export/data/virtuoso/ttl','test.rdf','http://pjotr.genenetwork.org/');" +echo " rdf_loader_run();" +echo " checkpoint;" +echo "Web: http://localhost:8892/sparql" diff --git a/virtuoso.scm b/virtuoso.scm index edcd575..ae33dcd 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed "/var/lib/virtuoso"))) + (dirs-allowed (list "/export/data/virtuoso/ttl")))) %base-services))) |