diff options
| -rw-r--r-- | .guix-channel | 53 | ||||
| -rw-r--r-- | README.org | 32 | ||||
| -rw-r--r-- | fallback.scm | 3 | ||||
| -rwxr-xr-x | genenetwork-development-deploy.sh | 3 | ||||
| -rw-r--r-- | genenetwork-development.scm | 444 | ||||
| -rw-r--r-- | genenetwork/services/genecup.scm | 121 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 266 | ||||
| -rw-r--r-- | genenetwork/services/mouse-longevity.scm | 33 | ||||
| -rw-r--r-- | guix/gn-machines/genenetwork.scm | 145 | ||||
| -rw-r--r-- | guix/gn-machines/services/monitoring.scm | 68 | ||||
| -rwxr-xr-x | production-deploy.sh | 33 | ||||
| -rw-r--r-- | production.scm | 31 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 9 | ||||
| -rw-r--r-- | public-sparql.scm | 15 | ||||
| -rw-r--r-- | services/README.md | 17 | ||||
| -rw-r--r-- | services/gn-guile.scm | 52 | ||||
| -rw-r--r-- | services/opensmtpd.scm | 21 | ||||
| -rw-r--r-- | specials/gndev.scm | 1 | ||||
| -rw-r--r-- | test-r-container.scm | 111 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 26 | ||||
| -rw-r--r-- | uploader.scm | 24 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 16 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
23 files changed, 1185 insertions, 341 deletions
diff --git a/.guix-channel b/.guix-channel new file mode 100644 index 0000000..b8f3873 --- /dev/null +++ b/.guix-channel @@ -0,0 +1,53 @@ +(channel + (version 0) + (directory "guix") + (dependencies + (channel + (name guix) + (url "https://codeberg.org/guix/guix") + (branch "master") + (commit "0a4740705090acc4c8a10d4f53afc58c9f62e980") + (introduction + (channel-introduction + (version 0) + (commit "9edb3f66fd807b096b48283debdcddccfea34bad") + (signer + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) + (channel + (name guix-forge) + (url "https://git.systemreboot.net/guix-forge/") + (branch "main") + (commit "e43fd9a4d73654d3876e2c698af7da89f3408f89") + (introduction + (channel-introduction + (version 0) + (commit "0432e37b20dd678a02efee21adf0b9525a670310") + (signer + "7F73 0343 F2F0 9F3C 77BF 79D3 2E25 EE8B 6180 2BB3")))) + (channel + (name guix-bioinformatics) + (url "https://git.genenetwork.org/guix-bioinformatics") + (commit "9b0955f14ec725990abb1f6af3b9f171e4943f77")) + ;; Until https://issues.guix.gnu.org/68797 is resolved, we need to + ;; explicitly list guix-past and guix-rust-past-crates—the + ;; dependencies of the guix-bioinformatics channel—here. + (channel + (name guix-past) + (url "https://codeberg.org/guix-science/guix-past") + (branch "master") + (introduction + (channel-introduction + (version 0) + (commit "c3bc94ee752ec545e39c1b8a29f739405767b51c") + (signer + "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5")))) + (channel + (name guix-rust-past-crates) + (url "https://codeberg.org/guix/guix-rust-past-crates.git") + (branch "trunk") + (introduction + (channel-introduction + (version 0) + (commit "1db24ca92c28255b28076792b93d533eabb3dc6a") + (signer + "F4C2 D1DF 3FDE EA63 D1D3 0776 ACC6 6D09 CA52 8292")))))) diff --git a/README.org b/README.org index 892ab23..2aa45db 100644 --- a/README.org +++ b/README.org @@ -3,19 +3,40 @@ containers. The git repo lives at https://git.genenetwork.org/gn-machines/ -* GeneNetwork development container +For CI/CD see -The GeneNetwork development container is currently run on /tux02/. It runs -continuous integration and continuous deployment services for -genenetwork2, genenetwork3 and several other associated projects. +=> https://issues.genenetwork.org/topics/systems/ci-cd + +For philosophy and (KISS) incremental minimalistic development containers, see: + +=> https://issues.genenetwork.org/topics/systems/debug-and-developing-code-with-genenetwork-system-container + +These are stored in the './minimal' directory of this repo. See the [[./minimal/README.md][README]]. + +* GeneNetwork development container (aka CI/CD) + +The GeneNetwork development container is currently run on /tux02/. +It runs continuous integration and continuous deployment services for genenetwork2, genenetwork3 and several other associated projects. To build and install the container, you will need the [[https://gitlab.com/genenetwork/guix-bioinformatics][guix-bioinformatics]] and [[https://git.systemreboot.net/guix-forge/][guix-forge]] channels. Once these channels are pulled and available, on /tux02/, run + #+BEGIN_SRC shell $ ./genenetwork-development-deploy.sh #+END_SRC +It will try to create symlinks at the end. You can do this as root (too). + +Note we current run as aruni user and the latest guix is the user profile. + +#+BEGIN_SRC shell +aruni@tux02:~/gn-machines$ which guix +/home/aruni/.config/guix/current/bin/guix +aruni@tux02:~/gn-machines$ guix describe +Generation 46 Mar 27 2025 23:39:42 (current) +#+END_SRC + /tux02/ is configured with a systemd service to run this container. Restart it. #+BEGIN_SRC shell @@ -31,6 +52,7 @@ To build and install the container, you will need the guix-bioinformatics channel. Once guix-bioinformatics is pulled and available, on /tux01/, run #+begin_src shell + $ ./virtuoso-deploy.sh #+end_src @@ -55,7 +77,7 @@ You can get a shell into the container with something like: sudo guix container exec 89086 /run/current-system/profile/bin/bash --login #+END_SRC -When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or on your can get with something like: +When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or by inspecting the output of the following ~ps~ command: #+BEGIN_SRC sh ps -u root -f --forest | grep -A4 '/usr/local/bin/genenetwork-development-container' | grep 'shepherd' diff --git a/fallback.scm b/fallback.scm index e222238..8153789 100644 --- a/fallback.scm +++ b/fallback.scm @@ -29,7 +29,7 @@ (forge socket)) (operating-system - (host-name "genenetwork-fallback") + (host-name "fallback") (timezone "UTC") (locale "en_US.utf8") (bootloader (bootloader-configuration @@ -61,6 +61,7 @@ (gn-auth-server-name "fallback-auth.genenetwork.org") (gn2-port 8892) (gn3-port 8893) + (gn-auth-port 8894) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export/data/genenetwork-sqlite/auth.db") (xapian-db "/export/data/genenetwork-xapian") diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index b251033..53c53d9 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -29,7 +29,7 @@ # /etc/genenetwork/conf instead of merely exposing it. container_script=$(guix system container --network \ --verbosity=3 \ - --load-path=. \ + --load-path=./guix/ \ --share=/home/git/public \ --share=/var/guix/daemon-socket=/var/host-guix/daemon-socket \ --share=/export2/guix-containers/genenetwork-development/var/lib/acme=/var/lib/acme \ @@ -42,6 +42,7 @@ container_script=$(guix system container --network \ --expose=/export/data/genenetwork \ --share=/export/data/genenetwork-xapian \ --share=/export/data/genenetwork-sqlite \ + --share=/export/data/lmdb \ --share=/var/run/mysqld=/run/mysqld \ --share=/export/data/gn-docs/ \ genenetwork-development.scm) diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 7d3a14b..bfd2fc1 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,19 +21,21 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) + (gn-machines services monitoring) + ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) (gn services databases) ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) - ((gnu packages base) #:select (gnu-make tar)) + ((gnu packages base) #:select (gnu-make tar coreutils-minimal)) ((gnu packages bash) #:select (bash)) - ((gnu packages bioinformatics) #:select (ccwl) #:prefix guix:) - ((gnu packages certs) #:select (nss-certs)) + ((gnu packages nss) #:select (nss-certs)) ((gnu packages check) #:select (python-pylint)) + ((gnu packages curl) #:select (curl)) ((gnu packages ci) #:select (laminar)) ((gnu packages compression) #:select (gzip)) - ((gnu packages databases) #:select (mariadb virtuoso-ose)) + ((gnu packages databases) #:select (mariadb redis)) + ((gnu packages databases) #:select (virtuoso-ose)) ((gnu packages gnupg) #:select (guile-gcrypt)) ((gnu packages graphviz) #:select (graphviz)) ((gnu packages guile) #:select (guile-3.0 guile-git guile-zlib)) @@ -84,10 +86,6 @@ (define %guix-daemon-uri "/var/host-guix/daemon-socket/socket") -(define %default-guix-channel-with-substitutes - (channel-with-substitutes-available %default-guix-channel - "https://ci.guix.gnu.org")) - ;; We cannot refer to sudo in the store since that sudo does not have ;; the setuid bit set. See "(guix) Setuid Programs". (define sudo @@ -122,7 +120,11 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "https://git.genenetwork.org/gn-auth")) + (default "https://git.genenetwork.org/gn-auth")) + (gn-libs-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-libs")) + (gn-guile-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-guile")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -147,8 +149,12 @@ be imported into G-expressions." (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path (default "/export/data/genenetwork-sqlite/llm.db")) + (lmdb-data-path genenetwork-lmdb-data-path + (default "/export/data/lmdb")) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) + (repositories genenetwork-configuration-repositories + (default "/export/data/repositories")) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout (default "/export/data/gn-docs"))) @@ -188,9 +194,9 @@ described by CONFIG, a <genenetwork-configuration> object. TEST-COMMAND is a list of strings specifying the command to be executed." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn3-port genotype-files) + (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) - (with-packages (list bash coreutils git-minimal nss-certs) + (with-packages (list bash coreutils git-minimal nss-certs genenetwork2) #~(begin (use-modules (guix build utils) (srfi srfi-26)) @@ -216,13 +222,12 @@ executed." (invoke "git" "clone" "--depth" "1" #$gn3-repository) (with-directory-excursion "genenetwork3" (show-head-commit)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository) + (with-directory-excursion "gn-libs" + (show-head-commit)) (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) - ;; This is a dummy SERVER_PORT to placate - ;; bin/genenetwork2. TODO: Fix bin/genenetwork2 so that - ;; this is not needed. - (setenv "SERVER_PORT" "8080") ;; Use a profile with all dependencies except ;; genenetwork3. (setenv "GN2_PROFILE" @@ -236,7 +241,30 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + ;; This file is cosmetic + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"/tmp/secret.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:120\"\n" + "GN3_GUILE_SERVER_URL=\"http://localhost:120\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"/tmp/ssl-private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/tmp/gn-auth-ssl-public-key.pem\"\n")) + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" "/export/data/genenetwork/genotype_files") + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/tmp") (chdir "genenetwork2") ;; XXXX: FIXME: R/Qtl tests fail because files are generated in ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 @@ -272,7 +300,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -327,16 +355,13 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port gn-guile-port gn-guile-repository) (list (forge-project (name "genenetwork2") (repository gn2-repository) (ci-jobs (list (forge-laminar-job (name "genenetwork2") - (run (genenetwork2-tests - config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "-m" "pytest"))) + (run (genenetwork2-tests config (list "python3" "-m" "pytest"))) ;; If unit tests pass, redeploy genenetwork2 and ;; trigger Mechanical Rob. (after (with-imported-modules '((guix build utils)) @@ -352,8 +377,7 @@ genenetwork3 source from the latest commit of @var{project}." (name "genenetwork2-mechanical-rob") (run (genenetwork2-tests config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "test/requests/test-website.py" + (list "python3" "test/requests/test-website.py" "--all" (string-append "http://localhost:" (number->string gn2-port))))) (trigger? #f)))) (ci-jobs-trigger 'webhook)) @@ -400,6 +424,33 @@ genenetwork3 source from the latest commit of @var{project}." (trigger? #f)))) (ci-jobs-trigger 'webhook)) (forge-project + (name "gn-libs") + (repository gn-libs-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-libs") + (run (guix-channel-job-gexp + (list (channel + (name 'gn-libs) + (url (forge-project-repository this-forge-project)) + (branch "main"))) + #:variables (list (variable-specification + (module '(gn-libs)) + (name 'gn-libs))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook)) + (forge-project + (name "gn-guile") + (repository gn-guile-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-guile") + (run (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$sudo + #$(file-append shepherd "/bin/herd") + "restart" "gn-guile"))))))) + (ci-jobs-trigger 'webhook)) + (forge-project (name "gn-auth") (repository gn-auth-repository) (ci-jobs (list (forge-laminar-job @@ -441,7 +492,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files) + (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port) (with-packages (list coreutils git-minimal gunicorn nss-certs) (with-imported-modules '((guix build utils)) #~(begin @@ -458,33 +509,55 @@ server described by CONFIG, a <genenetwork-configuration> object." (hline) (invoke "git" "log" "--max-count" "1") (hline)) - - ;; Clone the latest genenetwork2 and genenetwork3 + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "TERM" "xterm-256color") + ;; Clone the latest genenetwork2 ;; repositories. - (invoke "git" "clone" "--depth" "1" #$gn2-repository) - (with-directory-excursion "genenetwork2" - (show-head-commit)) - (invoke "git" "clone" "--depth" "1" #$gn3-repository) - (with-directory-excursion "genenetwork3" - (show-head-commit)) + (with-directory-excursion + "/home/genenetwork" + (when (file-exists? "/home/genenetwork/genenetwork2") + (delete-file-recursively "/home/genenetwork/genenetwork2")) + (invoke "git" "clone" "--depth" "1" #$gn2-repository)) ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" - (string-append (getcwd) "/genenetwork3")) + (setenv "GN3_PYTHONPATH" "/home/genenetwork/genenetwork3") ;; Set other environment variables required by ;; genenetwork2. (setenv "GN2_PROFILE" #$(profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + (getenv "GN3_PYTHONPATH") + ":" + (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages"))) + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" #$genotype-files) + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/home/genenetwork") (setenv "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" "GN3_LOCAL_URL=\"" (string-append "http://localhost:" (number->string gn3-port)) "\"\n" + "GN_GUILE_SERVER_URL=\"" + (string-append "http://localhost:" + (number->string gn-guile-port)) + "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" @@ -492,15 +565,23 @@ server described by CONFIG, a <genenetwork-configuration> object." "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) ;; Start genenetwork2. - (with-directory-excursion "genenetwork2" - (invoke #$(file-append bash "/bin/sh") - "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-prod"))))))) + (with-directory-excursion + "/home/genenetwork/genenetwork2" + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) + "--workers" "20" + "--keep-alive" "6000" + "--max-requests" "100" + "--max-requests-jitter" "30" + "--timeout" "1200" + "gn2.wsgi"))))))) (define (genenetwork3-cd-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path) (with-manifest (package->development-manifest genenetwork3) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -519,35 +600,48 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest genenetwork3 repository. - (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "GN3_PROFILE" #$(profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN3_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) ;; Configure genenetwork3. (setenv "GN3_CONF" #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "LMDB_DATA_PATH=\"" lmdb-data-path "\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) - (setenv "RSCRIPT" #$(file-append - (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t)) - "/bin/Rscript")) - ;; Run genenetwork3. - (with-directory-excursion "genenetwork3" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn3-port)) - "--workers" "8" - "gn3.app:create_app()" - ;; gunicorn's default 30 second timeout is - ;; insufficient for Fahamu AI endpoints and - ;; results in worker timeout errors. - "--timeout" "1200")))))))) + (setenv "RSCRIPT" (string-append + (getenv "GN3_PROFILE") + "/bin/Rscript")) + + (with-directory-excursion + "/home/genenetwork" + ;; Clone the latest genenetwork3 repository. + (when (file-exists? "/home/genenetwork/genenetwork3") + (delete-file-recursively "/home/genenetwork/genenetwork3")) + (invoke "git" "clone" "--depth" "1" #$gn3-repository)) + + (with-directory-excursion + "/home/genenetwork/genenetwork3" + (show-head-commit) + ;; Run genenetwork3. + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn3-port)) + "--workers" "8" + "gn3.app:create_app()" + ;; gunicorn's default 30 second timeout is + ;; insufficient for Fahamu AI endpoints and + ;; results in worker timeout errors. + "--timeout" "1200")))))))) (define (gn-auth-cd-gexp config) "Return a G-expression that runs the latest gn-auth development @@ -572,9 +666,23 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest gn-auth repository. - (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + + (with-directory-excursion + "/home/genenetwork/" + ;; Clone the latest gn-auth repository. + (when (file-exists? "/home/genenetwork/gn-auth") + (delete-file-recursively "/home/genenetwork/gn-auth")) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository)) + ;; Configure gn-auth. + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN_AUTH_CONF" #$(mixed-text-file "gn-auth.conf" "AUTH_DB=\"" auth-db-path "\"\n" @@ -584,7 +692,7 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "HOME" "/tmp") (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") ;; Run gn-auth. - (with-directory-excursion "gn-auth" + (with-directory-excursion "/home/genenetwork/gn-auth" (show-head-commit) (invoke #$(file-append gunicorn "/bin/gunicorn") "-b" #$(string-append "localhost:" (number->string gn-auth-port)) @@ -592,23 +700,71 @@ server described by CONFIG, a <genenetwork-configuration> object." "gn_auth.wsgi:app")))))))) (define (gn-guile-gexp gn-guile-port) - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) - (when (file-exists? current-repo-path) - (delete-file-recursively current-repo-path)) - (setenv "CURRENT_REPO_PATH" current-repo-path) - (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) - (invoke #$(file-append gn-guile "/bin/gn-guile") - (number->string #$gn-guile-port))))) + (with-packages + (list coreutils git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke "git" "log" "--max-count" "1") + (hline)) + ;; KLUDGE: Here we set all the certificates properly. In gn-guile, + ;; we make request to external services. Here's an example: + ;; curl http://localhost:8091/gene/aliases/Shh + ;; + ;; Without certs, we run into: + ;; 2025-07-22 08:27:11 GET /gene/aliases/Shh + ;; [...] + ;; 2025-07-22 08:27:19 signer-not-found invalid + (setenv "GN_GUILE_PROFILE" #$(profile + (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (setenv "SSL_CERT_DIR" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs")) + (setenv "SSL_CERT_FILE" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GIT_SSL_CAINFO" (getenv "SSL_CERT_FILE")) + (setenv "CURL_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "SPARQL-ENDPOINT" "http://localhost:9082/sparql/") + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (let ((current-repo-path "/home/genenetwork/gn-docs")) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (with-directory-excursion + "/home/genenetwork" + ;; All edits go to the current-repo-path; so we need it to + ;; be persistent. + (unless (file-exists? current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH") current-repo-path)) + + (when (file-exists? "gn-guile") + (delete-file-recursively "gn-guile")) + (invoke "git" "clone" "--depth" "1" "https://git.genenetwork.org/gn-guile") + ;; We have a gn-guile-dev wrapper script that sets a "./" in the + ;; GN_GUILE_LOAD_PATH hence allowing this to be run from the gn-guile + ;; directory. This allows gn-guile to be run from the latest + ;; upstream commits without pinning to guix. + (with-directory-excursion "gn-guile" + (show-head-commit) + (invoke #$(file-append gn-guile "/bin/gn-guile-dev") + (number->string #$gn-guile-port))))))))) (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port) (list (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -631,6 +787,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source gn-doc-git-checkout) (target source) + (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) @@ -646,11 +806,13 @@ described by CONFIG, a <genenetwork-configuration> object." (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) - ;; FIXME: The genenetwork2 service should depend on redis. - (requirement '(networking genenetwork3)) + (requirement '(networking redis)) (modules '((guix search-paths) (ice-9 match) (srfi srfi-1))) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn2 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start (let* ((gn2-manifest (packages->manifest (list genenetwork2))) (gn2-profile (profile @@ -680,6 +842,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source genotype-files) (target source)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source "/run/mysqld") (target source) @@ -725,6 +891,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run GeneNetwork 3 development server.") (provision '(genenetwork3)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn3 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "genenetwork3" @@ -737,6 +906,14 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source lmdb-data-path) + (target source) + (writable? #t)) ;; XXXX: FIXME: R/Qtl generates ;; files in "/tmp" and ;; "/tmp/gn2". These files are @@ -757,11 +934,11 @@ described by CONFIG, a <genenetwork-configuration> object." (target source) (writable? #t)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) - (file-system-mapping - (source llm-db-path) + (file-system-mapping + (source (dirname llm-db-path)) (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) @@ -774,6 +951,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run gn-auth development server.") (provision '(gn-auth)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn-auth to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-auth" @@ -786,11 +966,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -813,15 +997,29 @@ described by CONFIG, a <genenetwork-configuration> object." (group "genenetwork") (system? #t) (comment "GeneNetwork user") - (home-directory "/var/empty") + (home-directory "/home/genenetwork") (shell (file-append shadow "/sbin/nologin"))))) (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets) + (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets gn-doc-git-checkout) (with-imported-modules '((guix build utils)) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (ice-9 ftw)) + ;; KLUDGE: Guix now stores inferior profiles under + ;; /var/guix/profiles/per-user (commit + ;; d12c4452a49b355369636de1dfc766b5bad6437b). The 'laminar' + ;; user’s directory is not created automatically in our + ;; pinned Guix revision, which causes CI jobs using + ;; inferiors to fail with permission errors. + ;; XXXX: FIXME: Explicitly create the directory for + ;; now. Remove this once we update the pinned Guix commit. + (unless (file-exists? "/var/guix/profiles/per-user/laminar") + (mkdir-p "/var/guix/profiles/per-user/laminar") + (chown "/var/guix/profiles/per-user/laminar" + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) ;; Set ownership of files. (for-each (lambda (file) @@ -831,12 +1029,13 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) - (find-files "/export/data/gn-docs" + (find-files gn-doc-git-checkout #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets #:directories? #t)))) + ;; Prevent other users from reading secret files. (for-each (lambda (file) (chmod file #o600)) @@ -866,27 +1065,6 @@ described by CONFIG, a <genenetwork-configuration> object." ;;; transform-genenetwork-database ;;; - -;; Unreleased version of ccwl that is required by -;; transform-genenetwork-database for its graphql library. -(define ccwl - (let ((commit "02677a508b407779f5991a230341e016deb7f69b") - (revision "0")) - (package - (inherit guix:ccwl) - (name "ccwl") - (version (git-version (package-version guix:ccwl) revision commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/arunisaac/ccwl") - (commit commit))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1kxry8y0pibl0x789jrzqkkh2s59ajyinfvrgvd00gkbqldih82r"))))))) - ;; guile-sparql tests are broken. Disable them temporarily. The issue ;; has been reported upstream at ;; https://github.com/roelj/guile-sparql/issues/6 @@ -931,7 +1109,7 @@ described by CONFIG, a <genenetwork-configuration> object." (define (transform-genenetwork-database-gexp connection-settings virtuoso-data-dir repository) (with-imported-modules '((guix build utils)) - (with-packages (list ccwl git-minimal gnu-make guile-3.0 guile-dbd-mysql + (with-packages (list git-minimal gnu-make guile-3.0 guile-dbd-mysql guile-dbi guile-hashing guile-libyaml guile-sparql guile-zlib nss-certs virtuoso-ose raptor2) #~(begin @@ -958,18 +1136,12 @@ described by CONFIG, a <genenetwork-configuration> object." (copy-recursively build-directory #$virtuoso-data-dir) ;; Load RDF into virtuoso. (invoke "./pre-inst-env" "./load-rdf.scm" #$connection-settings) - ;; Visualize schema and archive results. - (invoke "./pre-inst-env" "./visualize-schema.scm" #$connection-settings) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "sql.dot" (string-append "-o" (getenv "ARCHIVE") "/sql.svg")) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "rdf.dot" (string-append "-o" (getenv "ARCHIVE") "/rdf.svg")) (delete-file-recursively build-directory))))))) (define transform-genenetwork-database-project (forge-project (name "transform-genenetwork-database") - (repository "https://git.genenetwork.org/gn-transform-databases") + (repository "/home/git/public/gn-transform-databases") (ci-jobs (list (forge-laminar-job (name "transform-genenetwork-database-tests") (run (guix-channel-job-gexp @@ -977,15 +1149,14 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'transform-genenetwork-database) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))) (forge-laminar-job (name "transform-genenetwork-database") (run (transform-genenetwork-database-gexp %connection-settings %virtuoso-data-dir - "https://git.genenetwork.org/gn-transform-databases"))))) - (ci-jobs-trigger 'webhook))) + "https://git.genenetwork.org/gn-transform-databases"))))))) ;;; @@ -1024,7 +1195,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lmdb) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1044,7 +1215,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-gsl) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1064,7 +1235,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lapack) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1257,6 +1428,19 @@ gn-auth." ";") "proxy_set_header Host $host;"))))))) +(define (gn-guile-reverse-proxy-server-block) + "Return an <nginx-server-configuration> object to reverse proxy +gn-guile to display RDF pages." + (nginx-server-configuration + (server-name '("rdf.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list (string-append "proxy_pass http://localhost:" + (number->string %gn-guile-port) + ";") + "proxy_set_header Host $host;"))))))) + (define set-build-directory-permissions-gexp (with-imported-modules '((guix build utils)) #~(begin @@ -1281,6 +1465,8 @@ gn-auth." (define %gn-auth-port 9094) ;; Port on which virtuoso's SPARQL endpoint is listening (define %virtuoso-sparql-port 9082) +;; Port on which gn-guile is listening +(define %gn-guile-port 8091) (operating-system (host-name "genenetwork-development") @@ -1291,13 +1477,14 @@ gn-auth." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) - (packages %base-packages) + (packages (cons* curl coreutils-minimal %base-packages)) (sudoers-file (mixed-text-file "sudoers" "@include " %sudoers-specification ;; Permit the laminar user to restart genenetwork2 ;; and genenetwork3. "\nlaminar ALL = NOPASSWD: " + (file-append shepherd "/bin/herd") " restart gn-guile, " (file-append shepherd "/bin/herd") " restart genenetwork2, " (file-append shepherd "/bin/herd") " start genenetwork3, " (file-append shepherd "/bin/herd") " stop genenetwork3, " @@ -1355,7 +1542,7 @@ gn-auth." (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (server-port 9081) - (dirs-allowed "/var/lib/data") + (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) (service genenetwork-service-type (genenetwork-configuration @@ -1382,8 +1569,12 @@ gn-auth." (hosts (list (tissue-host (name "issues.genenetwork.org") - (user "laminar") - (upstream-repository "https://github.com/genenetwork/gn-gemtext-threads")))))) + (projects (list (tissue-project + (name "issues.genenetwork.org") + (user "laminar") + (base-path "/") + (upstream-repository + "https://github.com/genenetwork/gn-gemtext-threads"))))))))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket @@ -1397,9 +1588,14 @@ gn-auth." %genenetwork2-port %genenetwork3-port) (laminar-reverse-proxy-server-block "localhost:9089" %webhook-port - (list 'gn-bioinformatics)) + (list 'gn-bioinformatics + 'guix-bioinformatics)) (tissue-reverse-proxy-server-block) - (gn-auth-reverse-proxy-server-block))))) + (gn-auth-reverse-proxy-server-block) + (gn-guile-reverse-proxy-server-block))))) + (service guile-sheepdog-service-type + (guile-sheepdog-configuration + (settings-file "/etc/genenetwork/conf/sheepdog.scm"))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) diff --git a/genenetwork/services/genecup.scm b/genenetwork/services/genecup.scm new file mode 100644 index 0000000..6ee2812 --- /dev/null +++ b/genenetwork/services/genecup.scm @@ -0,0 +1,121 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services genecup) + #:use-module (guix) + #:use-module (gnu) + #:use-module (guix git) + #:use-module (guix modules) + #:use-module (guix profiles) + #:use-module (guix records) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:use-module (forge utils) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gn packages machine-learning) + #:use-module (gnu packages certs) + #:use-module (gnu packages curl) + #:use-module (gn packages python) + #:use-module (gnu packages admin) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module ((gnu packages python) #:select (python)) + #:use-module (gnu packages python-xyz) + #:use-module (guix build python-build-system) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages python-web) + #:use-module (gnu services shepherd) + #:use-module (gnu packages python-science) + #:use-module (gnu packages machine-learning) + #:use-module ((gnu packages admin) #:select (shepherd)) + #:export (genecup-configuration + genecup-configuration? + genecup-configuration-package + genecup-service-type)) + +(define-record-type* <genecup-configuration> + genecup-configuration make-genecup-configuration + genecup-configuration? + (package genecup-configuration-package ; <package> + (default genecup-latest-with-tensorflow-native))) + +(define (genecup-activation config) + (match-record config <genecup-configuration> + (package) + (with-packages + (list python + python-nltk + nss-certs ; Needed for downloading data with nltk.downloader. + curl) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((nltk-data "/var/cache/nltk_data/") + (data-dir "/export/ratspub/tmp")) + (unless (file-exists? nltk-data) + (begin + (mkdir-p nltk-data) + (chdir nltk-data) + (invoke #$(file-append python "/bin/python3") "-m" "nltk.downloader" "-d" nltk-data "punkt"))) + (unless (file-exists? (string-append data-dir "/userspub.sqlite")) + (begin + (install-file #$(file-append package "/userspub.sqlite") data-dir) + (chmod (string-append data-dir "/userspub.sqlite") #o554))))))))) + +(define genecup-shepherd-service + (match-lambda + (($ <genecup-configuration> package) + (with-imported-modules (source-module-closure + '((guix search-paths) + (gnu build shepherd) + (gnu system file-systems))) + (list (shepherd-service + (provision '(genecup)) + (requirement '(networking)) + (modules '((gnu build shepherd) + (gnu system file-systems))) + (start + #~(make-forkexec-constructor + (list #$(file-append package "/server.py")) + #:directory #$package + #:environment-variables + (list + "NLTK_DATA=/var/cache/nltk_data" + (string-append "EDIRECT_PUBMED_MASTER=" + #$(file-append package "/minipubmed")) + (string-append "PERL_LWP_SSL_CA_FILE=" + #$(file-append (profile (content (packages->manifest (list curl nss-certs)))) + "/etc/ssl/certs/ca-certificates.crt"))) + #:log-file "/var/log/genecup.log")) + (stop #~(make-kill-destructor)))))))) + +(define genecup-service-type + (service-type + (name 'genecup) + (extensions + (list + (service-extension activation-service-type + genecup-activation) + (service-extension shepherd-root-service-type + genecup-shepherd-service))) + (default-value (genecup-configuration)) + (description + "Run a GeneCup Webserver."))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index b863bcf..f66dcd9 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -20,7 +20,7 @@ ;;; <https://www.gnu.org/licenses/>. (define-module (genenetwork services genenetwork) - #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) #:use-module ((gn packages guile) #:select (gn-guile)) #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) @@ -57,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -109,16 +110,24 @@ (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) + (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir + (default "/var/genenetwork/sessions/genenetwork2")) (gn2-secrets genenetwork-configuration-gn2-secrets (default "/etc/genenetwork")) (gn3-secrets genenetwork-configuration-gn3-secrets (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout (default "/export/data/gn-docs")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) (log-level genenetwork-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -142,6 +151,14 @@ (default "https://auth.genenetwork.org")) (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) + (sessions-dir gn-uploader-sessions-dir + (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (gn-tmpdir gn-uploader-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (genotype-files-directory gn-uploader-configuration-genotype-files-directory + (default "/var/genenetwork/genotype-files")) (log-level gn-uploader-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -170,6 +187,8 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) @@ -203,7 +222,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -212,7 +233,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -240,9 +263,35 @@ (chmod file #o644)) (find-files #$xapian-directory))))))))) +(define (samples-count-script-gexp config) + (match-record config <genenetwork-configuration> + (genenetwork2 sql-uri) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PYTHONPATH" + (string-append + #$(file-append genenetwork2 + "/lib/python" + (python-version (package-version python)) + "/site-packages") + ":" + #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "/lib/python" + #$(python-version (package-version python)) + "/site-packages")) + + (invoke #$(file-append python "/bin/python3") + "-m" + "gn2.scripts.sample_count" + #$sql-uri))))) + (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -262,7 +311,11 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets + #:directories? #t) + (find-files #$gn2-sessions-dir #:directories? #t))) (for-each (lambda (file) (chown file @@ -278,7 +331,31 @@ (find-files #$gn2-secrets #:directories? #f) (find-files #$gn-auth-secrets - #:directories? #f))))))) + #:directories? #f))) + ;; Make sub-directories for various apps under gn-tmpdir and assign + ;; appropriate permissions + (for-each (match-lambda + ((subdir user) + (let ((full-path + (string-append #$gn-tmpdir "/" subdir))) + (unless (file-exists? full-path) + (mkdir full-path #o755)) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw user)) + (passwd:gid (getpw user)))) + (find-files full-path + #:directories? #t))))) + '(("gn2-tmpdir" "gunicorn-genenetwork2") + ("gn3-tmpdir" "gunicorn-genenetwork3"))) + + ;; setup correct ownership for gn-docs + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (find-files #$(dirname gn-doc-git-checkout) + #:directories? #t)))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -309,7 +386,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -328,12 +405,16 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("AI_SEARCH_ENABLED" "True"))))) + ("AI_SEARCH_ENABLED" "True") + ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir) + ("MAX_FORM_MEMORY_SIZE" 52428800))))) (gn3-profile (profile (content (package->development-manifest genenetwork3)) (allow-collisions? #t))) @@ -342,6 +423,8 @@ object." (configuration-file-gexp `(("AUTH_DB" ,auth-db) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) @@ -358,16 +441,16 @@ object." (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) ("AUTH_DB" ,auth-db) - ("SQL_URI" ,sql-uri) - ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys")) - ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem"))))))) + ("SQL_URI" ,sql-uri))))) + (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir")) + (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir"))) (list (gunicorn-app (name "genenetwork2") (package genenetwork2) (sockets (list (forge-ip-socket (port gn2-port)))) (wsgi-app-module "gn2.wsgi") - (workers 20) + (workers 10) (timeout 1200) (environment-variables (list (environment-variable @@ -375,7 +458,7 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn2-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) @@ -388,13 +471,17 @@ object." (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn2-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -408,7 +495,11 @@ object." (writable? #t)) (file-system-mapping (source gn2-ca-bundle) - (target source)))) + (target source)) + (file-system-mapping + (source gn2-sessions-dir) + (target source) + (writable? #t)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))) @@ -418,7 +509,7 @@ object." (sockets (list (forge-ip-socket (port gn3-port)))) (wsgi-app-module "gn3.app:create_app()") - (workers 20) + (workers 10) ;; gunicorn's default 30 second timeout is insufficient ;; for Fahamu AI endpoints and results in worker timeout ;; errors. @@ -429,7 +520,7 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn3-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) @@ -455,15 +546,18 @@ object." (file-system-mapping (source gn3-data-directory) (target source)) ; Rqtl usese this - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn3-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) (target source)) (file-system-mapping - (source llm-db-path) + (source (dirname llm-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -478,7 +572,8 @@ object." (sockets (list (forge-ip-socket (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") - (workers 20) + (workers 10) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -497,9 +592,9 @@ object." (source gn-auth-conf) (target source)) (file-system-mapping - (source auth-db) - (target source) - (writable? #t)) + (source (dirname auth-db)) + (target source) + (writable? #t)) (file-system-mapping (source gn-auth-secrets) (target source) @@ -526,7 +621,8 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" @@ -555,24 +651,38 @@ a @code{<genenetwork-configuration>} record." (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron-gexp" (build-xapian-index-cron-gexp config)) - #:user "root"))) + #:user "root") + #~(job '(next-minute-from (next-hour) '(17)) ;17th minute of every hour + #$(program-file "samples-count-script-gexp" + (samples-count-script-gexp config))))) -(define (gn-guile-gexp gn-guile-port) +(define (gn-guile-gexp gn-guile-port gn-guile-pkg bare-repo) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile-pkg)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (current-repo-path (string-append (pk "CWD" (getcwd)) "/gn-docs"))) + ;; These have to be setup manually here an not in the + ;; `gn-guile-shepherd-service' function, otherwise, they do not take + ;; effect for some reason. + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + (when (file-exists? current-repo-path) (delete-file-recursively current-repo-path)) (setenv "CURRENT_REPO_PATH" current-repo-path) (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + "clone" "--depth" "1" #$(string-append "file://" bare-repo))) (invoke #$(file-append gn-guile "/bin/gn-guile") (number->string #$gn-guile-port))))) (define (gn-guile-shepherd-service config) (match-record config <genenetwork-configuration> - (gn-doc-git-checkout gn-guile-port) + (gn-guile gn-doc-git-checkout gn-guile-port) (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -588,14 +698,15 @@ a @code{<genenetwork-configuration>} record." #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) + (gn-guile-gexp gn-guile-port gn-guile gn-doc-git-checkout)) #:name "gn-guile-pola-wrapper" + #:directory (dirname gn-doc-git-checkout) #:preserved-environment-variables (map first gn-guile-settings) #:mappings (list (file-system-mapping - (source gn-doc-git-checkout) - (target source) - (writable? #t))) + (source (dirname gn-doc-git-checkout)) + (target source) + (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) #:user "genenetwork" @@ -628,14 +739,20 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory) + (secrets data-directory sessions-dir sqlite-databases-directory gn-tmpdir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Let service user own their own secrets files. - (chown #$secrets - (passwd:uid (getpw "gunicorn-gn-uploader")) - (passwd:gid (getpw "gunicorn-gn-uploader"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader")))) + (append (list #$(dirname secrets)) + (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -647,36 +764,44 @@ a @code{<genenetwork-configuration>} record." (passwd:gid (getpw "gunicorn-gn-uploader")))) (append (list #$data-directory) (find-files #$data-directory + #:directories? #t) + (find-files #$(string-append gn-tmpdir "/gn-uploader-tmpdir") #:directories? #t))))))) (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory gn-tmpdir genotype-files-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t))) - (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-tmpdir (string-append gn-tmpdir "/gn-uploader-tmpdir")) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOAD_FOLDER" ,(string-append data-directory + "/uploads")) + ("TEMPORARY_DIRECTORY" ,gn-uploader-tmpdir) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")) + ("GENOTYPE_FILES_DIRECTORY" ,genotype-files-directory))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) (sockets (list (forge-ip-socket (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") - (workers 20) + (workers 10) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -695,8 +820,9 @@ a @code{<genenetwork-configuration>} record." (source gn-uploader-conf) (target source)) (file-system-mapping - (source secrets) - (target source)) + (source (dirname secrets)) + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source) @@ -706,7 +832,23 @@ a @code{<genenetwork-configuration>} record." (target source)) (file-system-mapping (source gn-uploader-ca-bundle) - (target source)))) + (target source)) + (file-system-mapping + (source sessions-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-tmpdir) + (target source) + (writable? #t)) + (file-system-mapping + (source genotype-files-directory) + (target source) + (writable? #f)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))))))) diff --git a/genenetwork/services/mouse-longevity.scm b/genenetwork/services/mouse-longevity.scm new file mode 100644 index 0000000..c9b977d --- /dev/null +++ b/genenetwork/services/mouse-longevity.scm @@ -0,0 +1,33 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services mouse-longevity) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gnu services) + #:export (mouse-longevity-service)) + +(define* (mouse-longevity-service #:optional (package mouse-longevity-app)) + (simple-service 'mouse-longevity + rshiny-service-type + (list + (rshiny-configuration + (package package) + (binary "mouse-longevity-app"))))) diff --git a/guix/gn-machines/genenetwork.scm b/guix/gn-machines/genenetwork.scm new file mode 100644 index 0000000..657322e --- /dev/null +++ b/guix/gn-machines/genenetwork.scm @@ -0,0 +1,145 @@ +(define-module (gn-machines genenetwork) + #:use-module ((gn packages genenetwork) + #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-libs) + #:prefix gn:) + #:use-module((gn packages guile) + #:select (gn-guile guile-sheepdog) + #:prefix gng:) + #:use-module (guix git-download) + #:use-module (guix packages)) + +(define-public genenetwork2 + (let ((commit "905cacdc33a054b463d3b98c2adcd882dc3573fe") + (revision "4")) + (package + (inherit gn:genenetwork2) + (name "genenetwork2") + (version (git-version (package-version gn:genenetwork2) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork2.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "0vjsfihj94g1cxivkvmfnixjpmmn3wsciiiry2zkqjnzb3yryy2x")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork2) + (replace "gn-libs" gn-libs) + (replace "genenetwork3" genenetwork3)))))) + +(define-public genenetwork3 + (let ((commit "34a6b37b312b2e2e92324683ec47046b6ab63fe4") + (revision "5")) + (package + (inherit gn:genenetwork3) + (name "genenetwork3") + (version (git-version (package-version gn:genenetwork3) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork3.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1mz71kc4icgcj8mjpnipbyfv3nvyrdmqdzpjy1nd3j2f9zjxypz0")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork3) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-auth + (let ((commit "bd0f453033327a392afa3194538db5d2be22969e") + (revision "1")) + (package + (inherit gn:gn-auth) + (name "gn-auth") + (version (git-version (package-version gn:gn-auth) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-auth") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "0iamdg2xbj4f5z0qd8z8jildmq2scr10s547fn2k4rac2xm0yn59"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-auth) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-uploader + (let ((commit "096fe9dfe3a582658c6bf9179b6c2662cc78711b") + (revision "0")) + (package + (inherit gn:gn-uploader) + (name "gn-uploader") + (version (git-version (package-version gn:gn-uploader) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-uploader") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "02ds5vv8x2367r95zv0ddh42cqdlwh2jjbqipfd6ppn1dms3p9yj"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-uploader) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-libs + (let ((commit "5f977e69b46e0ee99845fb69d504f299dc7429a1") + (revision "03")) + (package + (inherit gn:gn-libs) + (name "gn-libs") + (version (git-version (package-version gn:gn-libs) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-libs") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "0hvixdpmqhhby4xv2sq7wnw15gbbp3vicxxa04rmgq1fd4mj0j49"))))))) + +(define-public gn-guile + (let ((commit "454244b774ece37f04f146c74353ea4ec35d43e1") + (revision "0")) + (package + (inherit gng:gn-guile) + (name "gn-guile") + (version (git-version "4.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-guile/") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1n1zjpwfksq8bf42d7xr7lnblgsbqc0yszkbid253s943fyxzhfn"))))))) + +(define-public guile-sheepdog + (let ((commit "1426617d58f305a4126bb867202843e8cf7dd4b2") + (revision "0")) + (package + (inherit gng:guile-sheepdog) + (name "guile-sheepdog") + (version (git-version "4.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/BonfaceKilz/guile-sheepdog.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1z0xzg11p75s2hk312akxlg2h5278w2abma27dhzjf981g3lcqvr"))))))) diff --git a/guix/gn-machines/services/monitoring.scm b/guix/gn-machines/services/monitoring.scm new file mode 100644 index 0000000..7fa59c9 --- /dev/null +++ b/guix/gn-machines/services/monitoring.scm @@ -0,0 +1,68 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (gn-machines services monitoring) + #:use-module (gnu) + #:use-module (gnu services shepherd) + #:use-module (gnu services databases) + #:use-module ((gn-machines genenetwork) #:select (guile-sheepdog)) + #:use-module (guix gexp) + #:use-module (guix records) + #:use-module (ice-9 match) + #:export (guile-sheepdog-configuration + guile-sheepdog-configuration? + guile-sheepdog-configuration-settings-file + guile-sheepdog-configuration-package + guile-sheepdog-service-type)) + +(define-record-type* <guile-sheepdog-configuration> + guile-sheepdog-configuration + make-guile-sheepdog-configuration + guile-sheepdog-configuration? + (settings-file guile-sheepdog-configuration-settings-file + (default "/etc/conn.scm")) + (package guile-sheepdog-configuration-package (default guile-sheepdog))) + +(define (guile-sheepdog-gexp config) + (match-record config <guile-sheepdog-configuration> (settings-file package) + (program-file + "guile-sheepdog" + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$(file-append package "/bin/guile-sheepdog") #$settings-file)))))) + +(define (guile-sheepdog-shepherd-service config) + (shepherd-service + (documentation "Run Sheepdog") + (provision '(guile-sheepdog)) + (requirement '(networking redis)) + (start #~(make-forkexec-constructor + (list #$(guile-sheepdog-gexp config)) + #:log-file "/var/log/sheepdog.log")) + (stop #~(make-kill-destructor)))) + +(define guile-sheepdog-service-type + (service-type + (name 'guile-sheepdog) + (description "Run sheepdog monitor") + (extensions + (list (service-extension shepherd-root-service-type + (compose list guile-sheepdog-shepherd-service)))) + (default-value (guile-sheepdog-configuration)))) diff --git a/production-deploy.sh b/production-deploy.sh index bd95814..8fb3d36 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -3,6 +3,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> # Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> +# Copyright © 2026 Munyoki Kilyungi <me@bonfacemunyoki.com> # # This file is part of genenetwork-machines. # @@ -24,21 +25,27 @@ container_script=$(guix system container \ --network \ - --load-path=. \ + --load-path=./guix/ \ + --load-path=./ \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ - --share=/export/mysql/database=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ + --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/genenetwork/var/log=/var/log \ + --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \ --share=/var/run/mysqld=/run/mysqld \ - --share=/export/data/gn-docs/ \ - --share=/export2/guix-containers/genenetwork/tmp=/tmp \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \ + --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile=/var/lib/genenetwork/gn-guile \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 933320a..6339b66 100644 --- a/production.scm +++ b/production.scm @@ -44,16 +44,13 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service mysql-service-type - (mysql-configuration - (auto-upgrade? #f))) - (service virtuoso-service-type + (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port 9892) (http-server-port 9893) - (dirs-allowed "/var/lib/virtuoso") - (number-of-buffers 4000000) - (maximum-dirty-buffers 3000000) + (dirs-allowed (list "/export/data/virtuoso")) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) (service forge-nginx-service-type @@ -80,15 +77,29 @@ (gn3-port 9895) (gn-auth-port 9896) (sql-uri - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:9893/sparql") (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") (gn-auth-secrets "/etc/genenetwork/gn-auth") - (auth-db "/var/lib/genenetwork-sqlite/auth.db") - (llm-db-path "/var/lib/genenetwork-gnqa/llm.db") + (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db") + (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db") (gn3-alias-server-port 9800) + (gn-tmpdir "/opt/gn/tmp") + (gn-doc-git-checkout "/var/lib/genenetwork/gn-guile/gn-docs.git") + (log-level 'debug))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "uploader.genenetwork.org") + (port 9897) + (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (data-directory "/var/lib/genenetwork/uploader/data") + (auth-server-url "https://auth.genenetwork.org/") + (gn2-server-url "https://genenetwork.org") + (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader") (log-level 'debug))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index bee9abf..bd8b938 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -22,10 +22,11 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/public-sparql/tmp=/tmp \ - --share=/export2/guix-containers/public-sparql/var/log=/var/log \ - --share=/export2/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/public-sparql/tmp=/tmp \ + --share=/export/guix-containers/public-sparql/var/log=/var/log \ + --share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index f6efb15..87ef843 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -20,6 +20,7 @@ (use-modules (gnu) (gn services databases) (gnu services web) + ((gnu packages admin) #:select (shepherd)) (forge nginx) (forge socket)) @@ -50,16 +51,22 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) - (number-of-buffers 4000000) - (dirs-allowed "/var/lib/virtuoso") - (maximum-dirty-buffers 3000000) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) + (dirs-allowed (list "/export/data/virtuoso")) (database-file "/var/lib/virtuoso/public-virtuoso.db") - (transaction-file "/var/lib/virtuoso/public-virtuoso.trx"))) + (transaction-file "/var/lib/virtuoso/public-virtuoso.trx") + (error-log-file "/var/lib/public-virtuoso-errors.log") + (syslog "1"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket diff --git a/services/README.md b/services/README.md deleted file mode 100644 index d0d1c01..0000000 --- a/services/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Services - -This directory contains small and simple services that may be run independently. -Note that composition is not the goal. -For example, databases and web proxies are handled outsite the containers. -Use these services for simple deployment and ad hoc testing. -One nice aspect of small system containers is that you can run these easily on your laptop. - -IMPORTANT: more complex services do not belong in this directory. - -# Examples - -## gn-guile - -`gn-guile` is our next generation service (gn4?). It provides a REST API, at this point, and some portals, such as https://aging.genenetwork.org/. - -See the [gn-guile](./gn-guile.scm) system definition example. diff --git a/services/gn-guile.scm b/services/gn-guile.scm deleted file mode 100644 index 2f675a8..0000000 --- a/services/gn-guile.scm +++ /dev/null @@ -1,52 +0,0 @@ -;; This is an example definition for the gn-guile/GN4 service -;; -;; Run with -;; -;; export runner=$(guix system container gn-guile.scm) -;; -;; as root -;; -;; sudo bash -c $runner -;; echo $runner -;; -;; make a note of pid and -;; -;; sudo bash -c "nsenter -a -t 4050285" -;; -;; now you should be inside the container (note bash should be in the container!) - -(use-modules (gnu) - (guix records) - (forge utils)) - -(define-record-type* <gn-guile-configuration> - gn-guile-configuration make-gn-guile-configuration - gn-guile-configuration? - (gn2-repository gn-guile-configuration-gn2-repository - (default "https://github.com/genenetwork/genenetwork2")) - (gn2-port gn-guile-configuration-gn2-port - (default 8082))) - - -(define gn-guile-service-type - (service-type - (name 'gn-guile) - (description "gn-guile/GN4 webservice") - (extensions '()) - )) - -(operating-system - (host-name "gn-guile") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service gn-guile-service-type - (gn-guile-configuration)) - %base-services))) diff --git a/services/opensmtpd.scm b/services/opensmtpd.scm deleted file mode 100644 index 1b1e58f..0000000 --- a/services/opensmtpd.scm +++ /dev/null @@ -1,21 +0,0 @@ -(use-modules (gnu) - (gnu services mail)) - -(operating-system - (host-name "mail") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service opensmtpd-service-type - (opensmtpd-configuration - (config-file %default-opensmtpd-config-file - ; (config-file (local-file "./my-smtpd.conf"))) - ))) - %base-services))) diff --git a/specials/gndev.scm b/specials/gndev.scm index 440aedb..0f9394f 100644 --- a/specials/gndev.scm +++ b/specials/gndev.scm @@ -61,6 +61,7 @@ (gn-auth-server-name "gndev-auth.genenetwork.org") (gn2-port 8992) (gn3-port 8993) + (gn-auth-port 8994) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export2/data/gndev-sqlite/auth.db") (xapian-db "/export2/data/gndev-xapian") diff --git a/test-r-container.scm b/test-r-container.scm new file mode 100644 index 0000000..3167574 --- /dev/null +++ b/test-r-container.scm @@ -0,0 +1,111 @@ +(use-modules (guix) + (gnu) + (guix git) + (guix modules) + (guix profiles) + (guix records) + (guix packages) + (guix git-download) + (guix build-system trivial) + (gnu services dbus) + (gnu services networking) + (gnu services ssh) + (gnu services web) + (gnu services certbot) + (gnu services docker) + (gnu services desktop) + (gnu services shepherd) + (gnu packages admin) + (gnu packages statistics) + (gnu packages cran) + (gnu packages curl) + (gnu packages lsof) + (srfi srfi-1) + (ice-9 match) + (genenetwork services mouse-longevity) + (gn services rshiny) + (gn packages mouse-longevity) + (forge acme) + (forge nginx) + (forge socket) + (gnu services)) + +(define %nginx-configuration + (nginx-configuration + (server-blocks + (list + ;; Redirect domains that don't explicitly support HTTP (below) to HTTPS. + (nginx-server-configuration + (listen '("8080"))) + + ;; Domains that still explicitly support plain HTTP. + (nginx-server-configuration + (listen '("80")) + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list + (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://127.0.0.1:3979;"))))) + (raw-content + (list + "proxy_set_header Host $host;" + "proxy_set_header X-Real-IP $remote_addr;" + "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" + "proxy_set_header X-Forwarded-Proto $scheme;"))))))) + +(operating-system + (host-name "testing-genenetwork-services") + (keyboard-layout (keyboard-layout "us")) + (kernel-arguments + (cons* "console=ttyS0,115200" "console=tty0" + %default-kernel-arguments)) + (bootloader (bootloader-configuration (bootloader grub-bootloader))) + (issue "This is a GeneNetwork container. Welcome!\n") + (file-systems %base-file-systems) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* lsof curl %base-packages)) + (services + (cons* + (service dhcp-client-service-type) + (service dbus-root-service-type) + (service polkit-service-type) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8080"))) + (https-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8443"))) + (server-blocks + (list + (nginx-server-configuration + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://localhost:3979;" + "proxy_set_header Host $host;")))))))))) + (service acme-service-type + (acme-configuration + (email "jgart@dismail.de") + (acme-url %letsencrypt-staging-url))) + ;; (service certbot-service-type + ;; (certbot-configuration + ;; (email "jgart@dismail.de") + ;; (certificates + ;; (list + ;; (certificate-configuration + ;; (domains '("longevity-explorer.genenetwork.org" + ;; "www.longevity-explorer.genenetwork.org"))))))) + ;; (mouse-longevity-service ) + (service rshiny-service-type + (rshiny-configuration + (package mouse-longevity-app) + (binary "mouse-longevity-app"))) + ;; (service nginx-service-type %nginx-configuration) + %base-services))) diff --git a/uploader-deploy.sh b/uploader-deploy.sh index fdbbe0c..415790b 100755 --- a/uploader-deploy.sh +++ b/uploader-deploy.sh @@ -41,18 +41,22 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/uploader/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/uploader/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ - --share=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ - --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ - --expose=/export/data/uploader/genotype_files=/export/data/genenetwork/genotype_files \ - --expose=/export/data/uploader/genenetwork3 \ - --share=/export/data/uploader/gn-uploader \ + --share=/export/guix-containers/uploader/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/uploader/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/uploader/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/uploader/var/log=/var/log \ + --share=/export/guix-containers/uploader/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-xapian=/var/lib/xapian \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld3307=/run/mysqld \ + --share=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/uploader/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/uploader/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/data/uploader/gn-uploader \ uploader.scm) echo "${container_script}" diff --git a/uploader.scm b/uploader.scm index 62ab35f..5064426 100644 --- a/uploader.scm +++ b/uploader.scm @@ -42,11 +42,7 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service virtuoso-service-type - (virtuoso-configuration - (server-port 10892) - (http-server-port 10893))) - (service forge-nginx-service-type + (services (cons* (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") @@ -64,24 +60,26 @@ (gn2-port 10894) (gn3-port 10895) (gn-auth-port 10896) - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") - (auth-db "/export/data/genenetwork-sqlite/auth.db") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (auth-db "/var/lib/genenetwork-sqlite/auth.db") + (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:10893/sparql") - (gn3-data-directory "/export/data/uploader/genenetwork3") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (llm-db-path "/var/lib/genenetwork-gnqa/llm.db"))) (service gn-uploader-service-type (gn-uploader-configuration (gn-uploader gn-uploader) (server-name "staging-uploader.genenetwork.org") (port 10897) (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (data-directory "/export/data/uploader/gn-uploader") - (log-level "DEBUG") + (log-level 'debug) (auth-server-url "https://staging-auth.genenetwork.org/") (gn2-server-url "https://staging.genenetwork.org"))) %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0414a65..d18caec 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2025 Pjotr Prins <pjotr.public01@thebird.nl> # # This file is part of genenetwork-machines. # @@ -19,14 +20,25 @@ # along with genenetwork-machines. If not, see # <https://www.gnu.org/licenses/>. -# Build and install virtuoso container on tux01. +# Build and install virtuoso container. Note the shared path is the sane default. Symlink if necessary. +# See also topics/systems/debug-and-developing-code-with-genenetwork-system-container.gmi container_script=$(guix system container \ + -L ~/guix-bioinformatics \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/data/virtuoso/ttl=/export/data/virtuoso/ttl \ virtuoso.scm) echo $container_script sudo ln --force --symbolic $container_script /usr/local/bin/virtuoso-container sudo ln --force --symbolic /usr/local/bin/virtuoso-container /var/guix/gcroots + +echo "Run virtuoso with: sudo /usr/local/bin/virtuoso-container" +echo "Enter with something like: nsenter -a -t 1692115 /run/current-system/profile/bin/bash --login" +echo "Admin: isql 8891" +echo " ld_dir('/export/data/virtuoso/ttl','test.rdf','http://pjotr.genenetwork.org/');" +echo " rdf_loader_run();" +echo " checkpoint;" +echo "Web: http://localhost:8892/sparql" diff --git a/virtuoso.scm b/virtuoso.scm index edcd575..ae33dcd 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed "/var/lib/virtuoso"))) + (dirs-allowed (list "/export/data/virtuoso/ttl")))) %base-services))) |