diff options
| -rw-r--r-- | .guix-channel | 53 | ||||
| -rw-r--r-- | README.org | 32 | ||||
| -rw-r--r-- | fallback.scm | 3 | ||||
| -rwxr-xr-x | genenetwork-development-deploy.sh | 1 | ||||
| -rw-r--r-- | genenetwork-development.scm | 383 | ||||
| -rw-r--r-- | genenetwork/services/genecup.scm | 121 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 123 | ||||
| -rw-r--r-- | genenetwork/services/mouse-longevity.scm | 33 | ||||
| -rw-r--r-- | guix/gn-machines/genenetwork.scm | 108 | ||||
| -rwxr-xr-x | production-deploy.sh | 31 | ||||
| -rw-r--r-- | production.scm | 17 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 10 | ||||
| -rw-r--r-- | public-sparql.scm | 9 | ||||
| -rw-r--r-- | specials/gndev.scm | 1 | ||||
| -rw-r--r-- | test-r-container.scm | 111 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 17 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
17 files changed, 887 insertions, 168 deletions
diff --git a/.guix-channel b/.guix-channel new file mode 100644 index 0000000..f43f5ea --- /dev/null +++ b/.guix-channel @@ -0,0 +1,53 @@ +(channel + (version 0) + (directory "guix") + (dependencies + (channel + (name guix) + (url "https://codeberg.org/guix/guix") + (branch "master") + (commit "0a4740705090acc4c8a10d4f53afc58c9f62e980") + (introduction + (channel-introduction + (version 0) + (commit "9edb3f66fd807b096b48283debdcddccfea34bad") + (signer + "BBB0 2DDF 2CEA F6A8 0D1D E643 A2A0 6DF2 A33A 54FA")))) + (channel + (name guix-forge) + (url "https://git.systemreboot.net/guix-forge/") + (branch "main") + (commit "e43fd9a4d73654d3876e2c698af7da89f3408f89") + (introduction + (channel-introduction + (version 0) + (commit "0432e37b20dd678a02efee21adf0b9525a670310") + (signer + "7F73 0343 F2F0 9F3C 77BF 79D3 2E25 EE8B 6180 2BB3")))) + (channel + (name guix-bioinformatics) + (url "https://git.genenetwork.org/guix-bioinformatics") + (commit "903465c85c9b2ae28480b236c3364da873ca8f51")) + ;; Until https://issues.guix.gnu.org/68797 is resolved, we need to + ;; explicitly list guix-past and guix-rust-past-crates—the + ;; dependencies of the guix-bioinformatics channel—here. + (channel + (name guix-past) + (url "https://codeberg.org/guix-science/guix-past") + (branch "master") + (introduction + (channel-introduction + (version 0) + (commit "c3bc94ee752ec545e39c1b8a29f739405767b51c") + (signer + "3CE4 6455 8A84 FDC6 9DB4 0CFB 090B 1199 3D9A EBB5")))) + (channel + (name guix-rust-past-crates) + (url "https://codeberg.org/guix/guix-rust-past-crates.git") + (branch "trunk") + (introduction + (channel-introduction + (version 0) + (commit "1db24ca92c28255b28076792b93d533eabb3dc6a") + (signer + "F4C2 D1DF 3FDE EA63 D1D3 0776 ACC6 6D09 CA52 8292")))))) diff --git a/README.org b/README.org index 892ab23..2aa45db 100644 --- a/README.org +++ b/README.org @@ -3,19 +3,40 @@ containers. The git repo lives at https://git.genenetwork.org/gn-machines/ -* GeneNetwork development container +For CI/CD see -The GeneNetwork development container is currently run on /tux02/. It runs -continuous integration and continuous deployment services for -genenetwork2, genenetwork3 and several other associated projects. +=> https://issues.genenetwork.org/topics/systems/ci-cd + +For philosophy and (KISS) incremental minimalistic development containers, see: + +=> https://issues.genenetwork.org/topics/systems/debug-and-developing-code-with-genenetwork-system-container + +These are stored in the './minimal' directory of this repo. See the [[./minimal/README.md][README]]. + +* GeneNetwork development container (aka CI/CD) + +The GeneNetwork development container is currently run on /tux02/. +It runs continuous integration and continuous deployment services for genenetwork2, genenetwork3 and several other associated projects. To build and install the container, you will need the [[https://gitlab.com/genenetwork/guix-bioinformatics][guix-bioinformatics]] and [[https://git.systemreboot.net/guix-forge/][guix-forge]] channels. Once these channels are pulled and available, on /tux02/, run + #+BEGIN_SRC shell $ ./genenetwork-development-deploy.sh #+END_SRC +It will try to create symlinks at the end. You can do this as root (too). + +Note we current run as aruni user and the latest guix is the user profile. + +#+BEGIN_SRC shell +aruni@tux02:~/gn-machines$ which guix +/home/aruni/.config/guix/current/bin/guix +aruni@tux02:~/gn-machines$ guix describe +Generation 46 Mar 27 2025 23:39:42 (current) +#+END_SRC + /tux02/ is configured with a systemd service to run this container. Restart it. #+BEGIN_SRC shell @@ -31,6 +52,7 @@ To build and install the container, you will need the guix-bioinformatics channel. Once guix-bioinformatics is pulled and available, on /tux01/, run #+begin_src shell + $ ./virtuoso-deploy.sh #+end_src @@ -55,7 +77,7 @@ You can get a shell into the container with something like: sudo guix container exec 89086 /run/current-system/profile/bin/bash --login #+END_SRC -When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or on your can get with something like: +When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or by inspecting the output of the following ~ps~ command: #+BEGIN_SRC sh ps -u root -f --forest | grep -A4 '/usr/local/bin/genenetwork-development-container' | grep 'shepherd' diff --git a/fallback.scm b/fallback.scm index e222238..8153789 100644 --- a/fallback.scm +++ b/fallback.scm @@ -29,7 +29,7 @@ (forge socket)) (operating-system - (host-name "genenetwork-fallback") + (host-name "fallback") (timezone "UTC") (locale "en_US.utf8") (bootloader (bootloader-configuration @@ -61,6 +61,7 @@ (gn-auth-server-name "fallback-auth.genenetwork.org") (gn2-port 8892) (gn3-port 8893) + (gn-auth-port 8894) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export/data/genenetwork-sqlite/auth.db") (xapian-db "/export/data/genenetwork-xapian") diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index b251033..d63dcf0 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -42,6 +42,7 @@ container_script=$(guix system container --network \ --expose=/export/data/genenetwork \ --share=/export/data/genenetwork-xapian \ --share=/export/data/genenetwork-sqlite \ + --share=/export/data/lmdb \ --share=/var/run/mysqld=/run/mysqld \ --share=/export/data/gn-docs/ \ genenetwork-development.scm) diff --git a/genenetwork-development.scm b/genenetwork-development.scm index f841255..8e4e1e8 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,19 +21,20 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) + ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) (gn services databases) ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) - ((gnu packages base) #:select (gnu-make tar)) + ((gnu packages base) #:select (gnu-make tar coreutils-minimal)) ((gnu packages bash) #:select (bash)) - ((gnu packages bioinformatics) #:select (ccwl) #:prefix guix:) - ((gnu packages certs) #:select (nss-certs)) + ((gnu packages nss) #:select (nss-certs)) ((gnu packages check) #:select (python-pylint)) + ((gnu packages curl) #:select (curl)) ((gnu packages ci) #:select (laminar)) ((gnu packages compression) #:select (gzip)) - ((gnu packages databases) #:select (mariadb virtuoso-ose)) + ((gnu packages databases) #:select (mariadb redis)) + ((gnu packages databases) #:select (virtuoso-ose)) ((gnu packages gnupg) #:select (guile-gcrypt)) ((gnu packages graphviz) #:select (graphviz)) ((gnu packages guile) #:select (guile-3.0 guile-git guile-zlib)) @@ -84,10 +85,6 @@ (define %guix-daemon-uri "/var/host-guix/daemon-socket/socket") -(define %default-guix-channel-with-substitutes - (channel-with-substitutes-available %default-guix-channel - "https://ci.guix.gnu.org")) - ;; We cannot refer to sudo in the store since that sudo does not have ;; the setuid bit set. See "(guix) Setuid Programs". (define sudo @@ -122,9 +119,11 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "/home/git/public/gn-auth")) + (default "https://git.genenetwork.org/gn-auth")) (gn-libs-repository genenetwork-configuration-gn-libs-repository - (default "/home/git/public/gn-libs")) + (default "https://git.genenetwork.org/gn-libs")) + (gn-guile-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-guile")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -149,8 +148,12 @@ be imported into G-expressions." (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path (default "/export/data/genenetwork-sqlite/llm.db")) + (lmdb-data-path genenetwork-lmdb-data-path + (default "/export/data/lmdb")) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) + (repositories genenetwork-configuration-repositories + (default "/export/data/repositories")) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout (default "/export/data/gn-docs"))) @@ -192,7 +195,7 @@ executed." (match-record config <genenetwork-configuration> (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) - (with-packages (list bash coreutils git-minimal nss-certs) + (with-packages (list bash coreutils git-minimal nss-certs genenetwork2) #~(begin (use-modules (guix build utils) (srfi srfi-26)) @@ -224,10 +227,6 @@ executed." (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) - ;; This is a dummy SERVER_PORT to placate - ;; bin/genenetwork2. TODO: Fix bin/genenetwork2 so that - ;; this is not needed. - (setenv "SERVER_PORT" "8080") ;; Use a profile with all dependencies except ;; genenetwork3. (setenv "GN2_PROFILE" @@ -241,7 +240,30 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + ;; This file is cosmetic + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"/tmp/secret.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:120\"\n" + "GN3_GUILE_SERVER_URL=\"http://localhost:120\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"/tmp/ssl-private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/tmp/gn-auth-ssl-public-key.pem\"\n")) + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" "/export/data/genenetwork/genotype_files") + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/tmp") (chdir "genenetwork2") ;; XXXX: FIXME: R/Qtl tests fail because files are generated in ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 @@ -277,7 +299,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -332,16 +354,13 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port gn-guile-port gn-guile-repository) (list (forge-project (name "genenetwork2") (repository gn2-repository) (ci-jobs (list (forge-laminar-job (name "genenetwork2") - (run (genenetwork2-tests - config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "-m" "pytest"))) + (run (genenetwork2-tests config (list "python3" "-m" "pytest"))) ;; If unit tests pass, redeploy genenetwork2 and ;; trigger Mechanical Rob. (after (with-imported-modules '((guix build utils)) @@ -357,8 +376,7 @@ genenetwork3 source from the latest commit of @var{project}." (name "genenetwork2-mechanical-rob") (run (genenetwork2-tests config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "test/requests/test-website.py" + (list "python3" "test/requests/test-website.py" "--all" (string-append "http://localhost:" (number->string gn2-port))))) (trigger? #f)))) (ci-jobs-trigger 'webhook)) @@ -417,7 +435,20 @@ genenetwork3 source from the latest commit of @var{project}." #:variables (list (variable-specification (module '(gn-libs)) (name 'gn-libs))) - #:guix-daemon-uri %guix-daemon-uri)))))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook)) + (forge-project + (name "gn-guile") + (repository gn-guile-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-guile") + (run (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$sudo + #$(file-append shepherd "/bin/herd") + "restart" "gn-guile"))))))) + (ci-jobs-trigger 'webhook)) (forge-project (name "gn-auth") (repository gn-auth-repository) @@ -453,13 +484,14 @@ genenetwork3 source from the latest commit of @var{project}." #:variables (list (variable-specification (module '(gn-auth)) (name 'gn-auth-all-tests))) - #:guix-daemon-uri %guix-daemon-uri))))))))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook))))) (define (genenetwork2-cd-gexp config) "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files) + (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port) (with-packages (list coreutils git-minimal gunicorn nss-certs) (with-imported-modules '((guix build utils)) #~(begin @@ -476,33 +508,52 @@ server described by CONFIG, a <genenetwork-configuration> object." (hline) (invoke "git" "log" "--max-count" "1") (hline)) - - ;; Clone the latest genenetwork2 and genenetwork3 + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "TERM" "xterm-256color") + ;; Clone the latest genenetwork2 ;; repositories. - (invoke "git" "clone" "--depth" "1" #$gn2-repository) - (with-directory-excursion "genenetwork2" - (show-head-commit)) - (invoke "git" "clone" "--depth" "1" #$gn3-repository) - (with-directory-excursion "genenetwork3" - (show-head-commit)) + (with-directory-excursion + "/home/genenetwork" + (when (file-exists? "/home/genenetwork/genenetwork2") + (delete-file-recursively "/home/genenetwork/genenetwork2")) + (invoke "git" "clone" "--depth" "1" #$gn2-repository)) ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" - (string-append (getcwd) "/genenetwork3")) + (setenv "GN3_PYTHONPATH" "/home/genenetwork/genenetwork3") ;; Set other environment variables required by ;; genenetwork2. (setenv "GN2_PROFILE" #$(profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages")) + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" #$genotype-files) + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/home/genenetwork") (setenv "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" "GN3_LOCAL_URL=\"" (string-append "http://localhost:" (number->string gn3-port)) "\"\n" + "GN_GUILE_SERVER_URL=\"" + (string-append "http://localhost:" + (number->string gn-guile-port)) + "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" @@ -510,15 +561,23 @@ server described by CONFIG, a <genenetwork-configuration> object." "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) ;; Start genenetwork2. - (with-directory-excursion "genenetwork2" - (invoke #$(file-append bash "/bin/sh") - "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-prod"))))))) + (with-directory-excursion + "/home/genenetwork/genenetwork2" + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) + "--workers" "20" + "--keep-alive" "6000" + "--max-requests" "100" + "--max-requests-jitter" "30" + "--timeout" "1200" + "gn2.wsgi"))))))) (define (genenetwork3-cd-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path) (with-manifest (package->development-manifest genenetwork3) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -537,35 +596,48 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest genenetwork3 repository. - (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "GN3_PROFILE" #$(profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN3_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) ;; Configure genenetwork3. (setenv "GN3_CONF" #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "LMDB_DATA_PATH=\"" lmdb-data-path "\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) - (setenv "RSCRIPT" #$(file-append - (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t)) - "/bin/Rscript")) - ;; Run genenetwork3. - (with-directory-excursion "genenetwork3" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn3-port)) - "--workers" "8" - "gn3.app:create_app()" - ;; gunicorn's default 30 second timeout is - ;; insufficient for Fahamu AI endpoints and - ;; results in worker timeout errors. - "--timeout" "1200")))))))) + (setenv "RSCRIPT" (string-append + (getenv "GN3_PROFILE") + "/bin/Rscript")) + + (with-directory-excursion + "/home/genenetwork" + ;; Clone the latest genenetwork3 repository. + (when (file-exists? "/home/genenetwork/genenetwork3") + (delete-file-recursively "/home/genenetwork/genenetwork3")) + (invoke "git" "clone" "--depth" "1" #$gn3-repository)) + + (with-directory-excursion + "/home/genenetwork/genenetwork3" + (show-head-commit) + ;; Run genenetwork3. + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn3-port)) + "--workers" "8" + "gn3.app:create_app()" + ;; gunicorn's default 30 second timeout is + ;; insufficient for Fahamu AI endpoints and + ;; results in worker timeout errors. + "--timeout" "1200")))))))) (define (gn-auth-cd-gexp config) "Return a G-expression that runs the latest gn-auth development @@ -590,9 +662,23 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest gn-auth repository. - (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + + (with-directory-excursion + "/home/genenetwork/" + ;; Clone the latest gn-auth repository. + (when (file-exists? "/home/genenetwork/gn-auth") + (delete-file-recursively "/home/genenetwork/gn-auth")) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository)) + ;; Configure gn-auth. + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN_AUTH_CONF" #$(mixed-text-file "gn-auth.conf" "AUTH_DB=\"" auth-db-path "\"\n" @@ -602,7 +688,7 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "HOME" "/tmp") (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") ;; Run gn-auth. - (with-directory-excursion "gn-auth" + (with-directory-excursion "/home/genenetwork/gn-auth" (show-head-commit) (invoke #$(file-append gunicorn "/bin/gunicorn") "-b" #$(string-append "localhost:" (number->string gn-auth-port)) @@ -610,23 +696,70 @@ server described by CONFIG, a <genenetwork-configuration> object." "gn_auth.wsgi:app")))))))) (define (gn-guile-gexp gn-guile-port) - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) - (when (file-exists? current-repo-path) - (delete-file-recursively current-repo-path)) - (setenv "CURRENT_REPO_PATH" current-repo-path) - (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) - (invoke #$(file-append gn-guile "/bin/gn-guile") - (number->string #$gn-guile-port))))) + (with-packages + (list coreutils git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke "git" "log" "--max-count" "1") + (hline)) + ;; KLUDGE: Here we set all the certificates properly. In gn-guile, + ;; we make request to external services. Here's an example: + ;; curl http://localhost:8091/gene/aliases/Shh + ;; + ;; Without certs, we run into: + ;; 2025-07-22 08:27:11 GET /gene/aliases/Shh + ;; [...] + ;; 2025-07-22 08:27:19 signer-not-found invalid + (setenv "GN_GUILE_PROFILE" #$(profile + (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (setenv "SSL_CERT_DIR" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs")) + (setenv "SSL_CERT_FILE" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GIT_SSL_CAINFO" (getenv "SSL_CERT_FILE")) + (setenv "CURL_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (let ((current-repo-path "/home/genenetwork/gn-docs")) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (with-directory-excursion + "/home/genenetwork" + ;; All edits go to the current-repo-path; so we need it to + ;; be persistent. + (unless (file-exists? current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH") current-repo-path)) + + (when (file-exists? "gn-guile") + (delete-file-recursively "gn-guile")) + (invoke "git" "clone" "--depth" "1" "https://git.genenetwork.org/gn-guile") + ;; We have a gn-guile-dev wrapper script that sets a "./" in the + ;; GN_GUILE_LOAD_PATH hence allowing this to be run from the gn-guile + ;; directory. This allows gn-guile to be run from the latest + ;; upstream commits without pinning to guix. + (with-directory-excursion "gn-guile" + (show-head-commit) + (invoke #$(file-append gn-guile "/bin/gn-guile-dev") + (number->string #$gn-guile-port))))))))) (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port) (list (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -649,6 +782,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source gn-doc-git-checkout) (target source) + (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) @@ -664,11 +801,13 @@ described by CONFIG, a <genenetwork-configuration> object." (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) - ;; FIXME: The genenetwork2 service should depend on redis. - (requirement '(networking genenetwork3)) + (requirement '(networking redis)) (modules '((guix search-paths) (ice-9 match) (srfi srfi-1))) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn2 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start (let* ((gn2-manifest (packages->manifest (list genenetwork2))) (gn2-profile (profile @@ -698,6 +837,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source genotype-files) (target source)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source "/run/mysqld") (target source) @@ -743,6 +886,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run GeneNetwork 3 development server.") (provision '(genenetwork3)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn3 to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "genenetwork3" @@ -755,6 +901,14 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source lmdb-data-path) + (target source) + (writable? #t)) ;; XXXX: FIXME: R/Qtl generates ;; files in "/tmp" and ;; "/tmp/gn2". These files are @@ -775,11 +929,11 @@ described by CONFIG, a <genenetwork-configuration> object." (target source) (writable? #t)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) - (file-system-mapping - (source llm-db-path) + (file-system-mapping + (source (dirname llm-db-path)) (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) @@ -792,6 +946,9 @@ described by CONFIG, a <genenetwork-configuration> object." (documentation "Run gn-auth development server.") (provision '(gn-auth)) (requirement '(networking)) + ;; KLUDGE: The default value of 0.5 is too low, and causes + ;; gn-auth to be disabled when it is respawned "too fast." + (respawn-delay 5) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-auth" @@ -804,11 +961,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -831,15 +992,29 @@ described by CONFIG, a <genenetwork-configuration> object." (group "genenetwork") (system? #t) (comment "GeneNetwork user") - (home-directory "/var/empty") + (home-directory "/home/genenetwork") (shell (file-append shadow "/sbin/nologin"))))) (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets) + (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets gn-doc-git-checkout) (with-imported-modules '((guix build utils)) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (ice-9 ftw)) + ;; KLUDGE: Guix now stores inferior profiles under + ;; /var/guix/profiles/per-user (commit + ;; d12c4452a49b355369636de1dfc766b5bad6437b). The 'laminar' + ;; user’s directory is not created automatically in our + ;; pinned Guix revision, which causes CI jobs using + ;; inferiors to fail with permission errors. + ;; XXXX: FIXME: Explicitly create the directory for + ;; now. Remove this once we update the pinned Guix commit. + (unless (file-exists? "/var/guix/profiles/per-user/laminar") + (mkdir-p "/var/guix/profiles/per-user/laminar") + (chown "/var/guix/profiles/per-user/laminar" + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) ;; Set ownership of files. (for-each (lambda (file) @@ -849,12 +1024,13 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) - (find-files "/export/data/gn-docs" + (find-files gn-doc-git-checkout #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets #:directories? #t)))) + ;; Prevent other users from reading secret files. (for-each (lambda (file) (chmod file #o600)) @@ -884,27 +1060,6 @@ described by CONFIG, a <genenetwork-configuration> object." ;;; transform-genenetwork-database ;;; - -;; Unreleased version of ccwl that is required by -;; transform-genenetwork-database for its graphql library. -(define ccwl - (let ((commit "02677a508b407779f5991a230341e016deb7f69b") - (revision "0")) - (package - (inherit guix:ccwl) - (name "ccwl") - (version (git-version (package-version guix:ccwl) revision commit)) - (source - (origin - (method git-fetch) - (uri (git-reference - (url "https://github.com/arunisaac/ccwl") - (commit commit))) - (file-name (git-file-name name version)) - (sha256 - (base32 - "1kxry8y0pibl0x789jrzqkkh2s59ajyinfvrgvd00gkbqldih82r"))))))) - ;; guile-sparql tests are broken. Disable them temporarily. The issue ;; has been reported upstream at ;; https://github.com/roelj/guile-sparql/issues/6 @@ -949,7 +1104,7 @@ described by CONFIG, a <genenetwork-configuration> object." (define (transform-genenetwork-database-gexp connection-settings virtuoso-data-dir repository) (with-imported-modules '((guix build utils)) - (with-packages (list ccwl git-minimal gnu-make guile-3.0 guile-dbd-mysql + (with-packages (list git-minimal gnu-make guile-3.0 guile-dbd-mysql guile-dbi guile-hashing guile-libyaml guile-sparql guile-zlib nss-certs virtuoso-ose raptor2) #~(begin @@ -995,7 +1150,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'transform-genenetwork-database) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))) (forge-laminar-job (name "transform-genenetwork-database") @@ -1041,7 +1196,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lmdb) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1061,7 +1216,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-gsl) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1081,7 +1236,7 @@ described by CONFIG, a <genenetwork-configuration> object." (name 'guile-lapack) (url (forge-project-repository this-forge-project)) (branch "master")) - %default-guix-channel-with-substitutes) + %default-guix-channel) #:guix-daemon-uri %guix-daemon-uri))))) (ci-jobs-trigger 'webhook))) @@ -1308,13 +1463,14 @@ gn-auth." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) - (packages %base-packages) + (packages (cons* curl coreutils-minimal %base-packages)) (sudoers-file (mixed-text-file "sudoers" "@include " %sudoers-specification ;; Permit the laminar user to restart genenetwork2 ;; and genenetwork3. "\nlaminar ALL = NOPASSWD: " + (file-append shepherd "/bin/herd") " restart gn-guile, " (file-append shepherd "/bin/herd") " restart genenetwork2, " (file-append shepherd "/bin/herd") " start genenetwork3, " (file-append shepherd "/bin/herd") " stop genenetwork3, " @@ -1418,7 +1574,8 @@ gn-auth." %genenetwork2-port %genenetwork3-port) (laminar-reverse-proxy-server-block "localhost:9089" %webhook-port - (list 'gn-bioinformatics)) + (list 'gn-bioinformatics + 'guix-bioinformatics)) (tissue-reverse-proxy-server-block) (gn-auth-reverse-proxy-server-block))))) (service acme-service-type diff --git a/genenetwork/services/genecup.scm b/genenetwork/services/genecup.scm new file mode 100644 index 0000000..6ee2812 --- /dev/null +++ b/genenetwork/services/genecup.scm @@ -0,0 +1,121 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services genecup) + #:use-module (guix) + #:use-module (gnu) + #:use-module (guix git) + #:use-module (guix modules) + #:use-module (guix profiles) + #:use-module (guix records) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:use-module (forge utils) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gn packages machine-learning) + #:use-module (gnu packages certs) + #:use-module (gnu packages curl) + #:use-module (gn packages python) + #:use-module (gnu packages admin) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module ((gnu packages python) #:select (python)) + #:use-module (gnu packages python-xyz) + #:use-module (guix build python-build-system) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages python-web) + #:use-module (gnu services shepherd) + #:use-module (gnu packages python-science) + #:use-module (gnu packages machine-learning) + #:use-module ((gnu packages admin) #:select (shepherd)) + #:export (genecup-configuration + genecup-configuration? + genecup-configuration-package + genecup-service-type)) + +(define-record-type* <genecup-configuration> + genecup-configuration make-genecup-configuration + genecup-configuration? + (package genecup-configuration-package ; <package> + (default genecup-latest-with-tensorflow-native))) + +(define (genecup-activation config) + (match-record config <genecup-configuration> + (package) + (with-packages + (list python + python-nltk + nss-certs ; Needed for downloading data with nltk.downloader. + curl) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((nltk-data "/var/cache/nltk_data/") + (data-dir "/export/ratspub/tmp")) + (unless (file-exists? nltk-data) + (begin + (mkdir-p nltk-data) + (chdir nltk-data) + (invoke #$(file-append python "/bin/python3") "-m" "nltk.downloader" "-d" nltk-data "punkt"))) + (unless (file-exists? (string-append data-dir "/userspub.sqlite")) + (begin + (install-file #$(file-append package "/userspub.sqlite") data-dir) + (chmod (string-append data-dir "/userspub.sqlite") #o554))))))))) + +(define genecup-shepherd-service + (match-lambda + (($ <genecup-configuration> package) + (with-imported-modules (source-module-closure + '((guix search-paths) + (gnu build shepherd) + (gnu system file-systems))) + (list (shepherd-service + (provision '(genecup)) + (requirement '(networking)) + (modules '((gnu build shepherd) + (gnu system file-systems))) + (start + #~(make-forkexec-constructor + (list #$(file-append package "/server.py")) + #:directory #$package + #:environment-variables + (list + "NLTK_DATA=/var/cache/nltk_data" + (string-append "EDIRECT_PUBMED_MASTER=" + #$(file-append package "/minipubmed")) + (string-append "PERL_LWP_SSL_CA_FILE=" + #$(file-append (profile (content (packages->manifest (list curl nss-certs)))) + "/etc/ssl/certs/ca-certificates.crt"))) + #:log-file "/var/log/genecup.log")) + (stop #~(make-kill-destructor)))))))) + +(define genecup-service-type + (service-type + (name 'genecup) + (extensions + (list + (service-extension activation-service-type + genecup-activation) + (service-extension shepherd-root-service-type + genecup-shepherd-service))) + (default-value (genecup-configuration)) + (description + "Run a GeneCup Webserver."))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index a403f21..5c6b9f0 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -20,7 +20,7 @@ ;;; <https://www.gnu.org/licenses/>. (define-module (genenetwork services genenetwork) - #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) #:use-module ((gn packages guile) #:select (gn-guile)) #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) @@ -57,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -117,6 +118,8 @@ (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout @@ -150,6 +153,12 @@ (default "https://genenetwork.org")) (sessions-dir gn-uploader-sessions-dir (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (gn-tmpdir gn-uploader-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (genotype-files-directory gn-uploader-configuration-genotype-files-directory + (default "/var/genenetwork/genotype-files")) (log-level gn-uploader-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -254,6 +263,32 @@ (chmod file #o644)) (find-files #$xapian-directory))))))))) +(define (samples-count-script-gexp config) + (match-record config <genenetwork-configuration> + (genenetwork2 sql-uri) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PYTHONPATH" + (string-append + #$(file-append genenetwork2 + "/lib/python" + (python-version (package-version python)) + "/site-packages") + ":" + #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "/lib/python" + #$(python-version (package-version python)) + "/site-packages")) + + (invoke #$(file-append python "/bin/python3") + "-m" + "gn2.scripts.sample_count" + #$sql-uri))))) + (define (genenetwork-activation config) (match-record config <genenetwork-configuration> (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) @@ -319,7 +354,7 @@ (chown file (passwd:uid (getpw "genenetwork")) (passwd:gid (getpw "genenetwork")))) - (find-files #$gn-doc-git-checkout + (find-files #$(dirname gn-doc-git-checkout) #:directories? #t)))))) (define (configuration-file-gexp alist) @@ -351,7 +386,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -370,6 +405,8 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) @@ -386,6 +423,8 @@ object." (configuration-file-gexp `(("AUTH_DB" ,auth-db) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) @@ -518,7 +557,7 @@ object." (source xapian-db) (target source)) (file-system-mapping - (source llm-db-path) + (source (dirname llm-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -534,6 +573,7 @@ object." (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -552,9 +592,9 @@ object." (source gn-auth-conf) (target source)) (file-system-mapping - (source auth-db) - (target source) - (writable? #t)) + (source (dirname auth-db)) + (target source) + (writable? #t)) (file-system-mapping (source gn-auth-secrets) (target source) @@ -611,24 +651,38 @@ a @code{<genenetwork-configuration>} record." (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron-gexp" (build-xapian-index-cron-gexp config)) - #:user "root"))) + #:user "root") + #~(job '(next-minute-from (next-hour) '(17)) ;17th minute of every hour + #$(program-file "samples-count-script-gexp" + (samples-count-script-gexp config))))) -(define (gn-guile-gexp gn-guile-port) +(define (gn-guile-gexp gn-guile-port gn-guile-pkg bare-repo) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile-pkg)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (current-repo-path (string-append (pk "CWD" (getcwd)) "/gn-docs"))) + ;; These have to be setup manually here an not in the + ;; `gn-guile-shepherd-service' function, otherwise, they do not take + ;; effect for some reason. + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + (when (file-exists? current-repo-path) (delete-file-recursively current-repo-path)) (setenv "CURRENT_REPO_PATH" current-repo-path) (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + "clone" "--depth" "1" #$(string-append "file://" bare-repo))) (invoke #$(file-append gn-guile "/bin/gn-guile") (number->string #$gn-guile-port))))) (define (gn-guile-shepherd-service config) (match-record config <genenetwork-configuration> - (gn-doc-git-checkout gn-guile-port) + (gn-guile gn-doc-git-checkout gn-guile-port) (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -644,14 +698,15 @@ a @code{<genenetwork-configuration>} record." #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) + (gn-guile-gexp gn-guile-port gn-guile gn-doc-git-checkout)) #:name "gn-guile-pola-wrapper" + #:directory (dirname gn-doc-git-checkout) #:preserved-environment-variables (map first gn-guile-settings) #:mappings (list (file-system-mapping - (source gn-doc-git-checkout) - (target source) - (writable? #t))) + (source (dirname gn-doc-git-checkout)) + (target source) + (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) #:user "genenetwork" @@ -684,7 +739,7 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory sessions-dir) + (secrets data-directory sessions-dir sqlite-databases-directory gn-tmpdir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -693,8 +748,10 @@ a @code{<genenetwork-configuration>} record." (chown file (passwd:uid (getpw "gunicorn-gn-uploader")) (passwd:gid (getpw "gunicorn-gn-uploader")))) - (append (list #$secrets) + (append (list #$(dirname secrets)) (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) @@ -707,26 +764,32 @@ a @code{<genenetwork-configuration>} record." (passwd:gid (getpw "gunicorn-gn-uploader")))) (append (list #$data-directory) (find-files #$data-directory + #:directories? #t) + (find-files #$(string-append gn-tmpdir "/gn-uploader-tmpdir") #:directories? #t))))))) (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory gn-tmpdir genotype-files-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping (source "/run/mysqld") (target source) (writable? #t))) + (gn-uploader-tmpdir (string-append gn-tmpdir "/gn-uploader-tmpdir")) (gn-uploader-conf (computed-file "gn-uploader.conf" (configuration-file-gexp `(("UPLOADER_SECRETS" ,secrets) ("SQL_URI" ,sql-uri) ("UPLOAD_FOLDER" ,(string-append data-directory "/uploads")) + ("TEMPORARY_DIRECTORY" ,gn-uploader-tmpdir) ("AUTH_SERVER_URL" ,auth-server-url) ("GN2_SERVER_URL" ,gn2-server-url) - ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir))))) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")) + ("GENOTYPE_FILES_DIRECTORY" ,genotype-files-directory))))) (gn-uploader-profile (profile (content (package->development-manifest gn-uploader)) (allow-collisions? #t))) @@ -738,6 +801,7 @@ a @code{<genenetwork-configuration>} record." (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -756,8 +820,9 @@ a @code{<genenetwork-configuration>} record." (source gn-uploader-conf) (target source)) (file-system-mapping - (source secrets) - (target source)) + (source (dirname secrets)) + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source) @@ -771,7 +836,19 @@ a @code{<genenetwork-configuration>} record." (file-system-mapping (source sessions-dir) (target source) - (writable? #t)))) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-tmpdir) + (target source) + (writable? #t)) + (file-system-mapping + (source genotype-files-directory) + (target source) + (writable? #f)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))))))) diff --git a/genenetwork/services/mouse-longevity.scm b/genenetwork/services/mouse-longevity.scm new file mode 100644 index 0000000..c9b977d --- /dev/null +++ b/genenetwork/services/mouse-longevity.scm @@ -0,0 +1,33 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services mouse-longevity) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gnu services) + #:export (mouse-longevity-service)) + +(define* (mouse-longevity-service #:optional (package mouse-longevity-app)) + (simple-service 'mouse-longevity + rshiny-service-type + (list + (rshiny-configuration + (package package) + (binary "mouse-longevity-app"))))) diff --git a/guix/gn-machines/genenetwork.scm b/guix/gn-machines/genenetwork.scm new file mode 100644 index 0000000..36b3abe --- /dev/null +++ b/guix/gn-machines/genenetwork.scm @@ -0,0 +1,108 @@ +(define-module (gn-machines genenetwork) + #:use-module ((gn packages genenetwork) + #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-libs) + #:prefix gn:) + #:use-module (guix git-download) + #:use-module (guix packages)) + +(define-public genenetwork2 + (let ((commit "bde8bde1732cc042e96819bfdfd17a004b92d06f") + (revision "4")) + (package + (inherit gn:genenetwork2) + (name "genenetwork2") + (version (git-version (package-version gn:genenetwork2) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork2.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "05yslcqykvxiddlhdvfmjpmrm36149zxr7mjwq0pyhm6fg5dgzqp")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork2) + (replace "gn-libs" gn-libs) + (replace "genenetwork3" genenetwork3)))))) + +(define-public genenetwork3 + (let ((commit "810dbf22e6e921681dc187f37658c5f9874f12d7") + (revision "5")) + (package + (inherit gn:genenetwork3) + (name "genenetwork3") + (version (git-version (package-version gn:genenetwork3) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/genenetwork/genenetwork3.git") + (commit commit))) + (file-name (git-file-name name version)) + (sha256 + (base32 + "1jjv4fdjsilcjad8lpqpx2d3qaq90ygv6hcrq8di2wb63998xcvb")))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:genenetwork3) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-auth + (let ((commit "5823b0aaa411748bd361d23ad6eb07c2a3604dbf") + (revision "1")) + (package + (inherit gn:gn-auth) + (name "gn-auth") + (version (git-version (package-version gn:gn-auth) revision commit)) + (source + (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-auth") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "01c4diaw560ki51zssygl0738a9wj60hk47w7vd2dsfwnjmcilq4"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-auth) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-uploader + (let ((commit "9514647b70fe90a709b2881e50871a6e15f1236e") + (revision "0")) + (package + (inherit gn:gn-uploader) + (name "gn-uploader") + (version (git-version (package-version gn:gn-uploader) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-uploader") + (commit commit))) + (file-name (git-file-name name version)) + (hash + (content-hash + (base32 + "1gdn60vb4jk63akffkmbmgi7lil6bv8fqksdn1v2bww1z9rmiydl"))))) + (propagated-inputs + (modify-inputs (package-propagated-inputs gn:gn-uploader) + (replace "gn-libs" gn-libs)))))) + +(define-public gn-libs + (let ((commit "ef71bf1f5a1af847492ea58282511d728a837630") + (revision "03")) + (package + (inherit gn:gn-libs) + (name "gn-libs") + (version (git-version (package-version gn:gn-libs) revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-libs") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "03i1kj7jbmlkjs61a9fb4j3s78p4d5lmd8lqnfsv42wxwwyvsm8p"))))))) diff --git a/production-deploy.sh b/production-deploy.sh index a88fcb8..c8e30fe 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -26,21 +26,24 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ - --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ + --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/genenetwork/var/log=/var/log \ + --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \ --share=/var/run/mysqld=/run/mysqld \ - --share=/export/data/gn-docs/ \ - --share=/export2/guix-containers/genenetwork/tmp=/opt/gn/tmp \ - --expose=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ - --share=/export2/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ - --share=/export2/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \ + --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile=/var/lib/genenetwork/gn-guile \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 9e629f0..bfd9e48 100644 --- a/production.scm +++ b/production.scm @@ -84,9 +84,22 @@ (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") (gn-auth-secrets "/etc/genenetwork/gn-auth") - (auth-db "/var/lib/genenetwork-sqlite/auth.db") - (llm-db-path "/var/lib/genenetwork-gnqa/llm.db") + (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db") + (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db") (gn3-alias-server-port 9800) (gn-tmpdir "/opt/gn/tmp") + (gn-doc-git-checkout "/var/lib/genenetwork/gn-guile/gn-docs.git") + (log-level 'debug))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "uploader.genenetwork.org") + (port 9897) + (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (data-directory "/var/lib/genenetwork/uploader/data") + (auth-server-url "https://auth.genenetwork.org/") + (gn2-server-url "https://genenetwork.org") + (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader") (log-level 'debug))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index 4ecacc6..bd8b938 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -22,11 +22,11 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/public-sparql/tmp=/tmp \ - --share=/export2/guix-containers/public-sparql/var/log=/var/log \ - --share=/export2/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ + --share=/export/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/public-sparql/tmp=/tmp \ + --share=/export/guix-containers/public-sparql/var/log=/var/log \ + --share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index 6fbf75c..edfcd87 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -20,6 +20,7 @@ (use-modules (gnu) (gn services databases) (gnu services web) + ((gnu packages admin) #:select (shepherd)) (forge nginx) (forge socket)) @@ -50,6 +51,10 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration @@ -59,7 +64,9 @@ SPARQL endpoint is listening on." (dirs-allowed (list "/export/data/virtuoso")) (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/public-virtuoso.db") - (transaction-file "/var/lib/virtuoso/public-virtuoso.trx"))) + (transaction-file "/var/lib/virtuoso/public-virtuoso.trx") + (error-log-file "/var/lib/public-virtuoso-errors.log") + (syslog "1"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket diff --git a/specials/gndev.scm b/specials/gndev.scm index 440aedb..0f9394f 100644 --- a/specials/gndev.scm +++ b/specials/gndev.scm @@ -61,6 +61,7 @@ (gn-auth-server-name "gndev-auth.genenetwork.org") (gn2-port 8992) (gn3-port 8993) + (gn-auth-port 8994) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export2/data/gndev-sqlite/auth.db") (xapian-db "/export2/data/gndev-xapian") diff --git a/test-r-container.scm b/test-r-container.scm new file mode 100644 index 0000000..3167574 --- /dev/null +++ b/test-r-container.scm @@ -0,0 +1,111 @@ +(use-modules (guix) + (gnu) + (guix git) + (guix modules) + (guix profiles) + (guix records) + (guix packages) + (guix git-download) + (guix build-system trivial) + (gnu services dbus) + (gnu services networking) + (gnu services ssh) + (gnu services web) + (gnu services certbot) + (gnu services docker) + (gnu services desktop) + (gnu services shepherd) + (gnu packages admin) + (gnu packages statistics) + (gnu packages cran) + (gnu packages curl) + (gnu packages lsof) + (srfi srfi-1) + (ice-9 match) + (genenetwork services mouse-longevity) + (gn services rshiny) + (gn packages mouse-longevity) + (forge acme) + (forge nginx) + (forge socket) + (gnu services)) + +(define %nginx-configuration + (nginx-configuration + (server-blocks + (list + ;; Redirect domains that don't explicitly support HTTP (below) to HTTPS. + (nginx-server-configuration + (listen '("8080"))) + + ;; Domains that still explicitly support plain HTTP. + (nginx-server-configuration + (listen '("80")) + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list + (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://127.0.0.1:3979;"))))) + (raw-content + (list + "proxy_set_header Host $host;" + "proxy_set_header X-Real-IP $remote_addr;" + "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" + "proxy_set_header X-Forwarded-Proto $scheme;"))))))) + +(operating-system + (host-name "testing-genenetwork-services") + (keyboard-layout (keyboard-layout "us")) + (kernel-arguments + (cons* "console=ttyS0,115200" "console=tty0" + %default-kernel-arguments)) + (bootloader (bootloader-configuration (bootloader grub-bootloader))) + (issue "This is a GeneNetwork container. Welcome!\n") + (file-systems %base-file-systems) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* lsof curl %base-packages)) + (services + (cons* + (service dhcp-client-service-type) + (service dbus-root-service-type) + (service polkit-service-type) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8080"))) + (https-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8443"))) + (server-blocks + (list + (nginx-server-configuration + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://localhost:3979;" + "proxy_set_header Host $host;")))))))))) + (service acme-service-type + (acme-configuration + (email "jgart@dismail.de") + (acme-url %letsencrypt-staging-url))) + ;; (service certbot-service-type + ;; (certbot-configuration + ;; (email "jgart@dismail.de") + ;; (certificates + ;; (list + ;; (certificate-configuration + ;; (domains '("longevity-explorer.genenetwork.org" + ;; "www.longevity-explorer.genenetwork.org"))))))) + ;; (mouse-longevity-service ) + (service rshiny-service-type + (rshiny-configuration + (package mouse-longevity-app) + (binary "mouse-longevity-app"))) + ;; (service nginx-service-type %nginx-configuration) + %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0dd2509..d18caec 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2025 Pjotr Prins <pjotr.public01@thebird.nl> # # This file is part of genenetwork-machines. # @@ -19,15 +20,25 @@ # along with genenetwork-machines. If not, see # <https://www.gnu.org/licenses/>. -# Build and install virtuoso container on tux01. +# Build and install virtuoso container. Note the shared path is the sane default. Symlink if necessary. +# See also topics/systems/debug-and-developing-code-with-genenetwork-system-container.gmi container_script=$(guix system container \ + -L ~/guix-bioinformatics \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ + --share=/export/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/data/virtuoso/ttl=/export/data/virtuoso/ttl \ virtuoso.scm) echo $container_script sudo ln --force --symbolic $container_script /usr/local/bin/virtuoso-container sudo ln --force --symbolic /usr/local/bin/virtuoso-container /var/guix/gcroots + +echo "Run virtuoso with: sudo /usr/local/bin/virtuoso-container" +echo "Enter with something like: nsenter -a -t 1692115 /run/current-system/profile/bin/bash --login" +echo "Admin: isql 8891" +echo " ld_dir('/export/data/virtuoso/ttl','test.rdf','http://pjotr.genenetwork.org/');" +echo " rdf_loader_run();" +echo " checkpoint;" +echo "Web: http://localhost:8892/sparql" diff --git a/virtuoso.scm b/virtuoso.scm index 3272f41..ae33dcd 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed (list "/export/data/virtuoso")))) + (dirs-allowed (list "/export/data/virtuoso/ttl")))) %base-services))) |