diff options
| -rw-r--r-- | README.org | 32 | ||||
| -rw-r--r-- | fallback.scm | 3 | ||||
| -rwxr-xr-x | genenetwork-development-deploy.sh | 1 | ||||
| -rw-r--r-- | genenetwork-development.scm | 350 | ||||
| -rw-r--r-- | genenetwork/services/genecup.scm | 121 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 239 | ||||
| -rw-r--r-- | genenetwork/services/mouse-longevity.scm | 33 | ||||
| -rwxr-xr-x | production-deploy.sh | 29 | ||||
| -rw-r--r-- | production.scm | 27 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 9 | ||||
| -rw-r--r-- | public-sparql.scm | 11 | ||||
| -rw-r--r-- | specials/gndev.scm | 1 | ||||
| -rw-r--r-- | test-r-container.scm | 111 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 26 | ||||
| -rw-r--r-- | uploader.scm | 24 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 16 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
17 files changed, 835 insertions, 200 deletions
diff --git a/README.org b/README.org index 892ab23..2aa45db 100644 --- a/README.org +++ b/README.org @@ -3,19 +3,40 @@ containers. The git repo lives at https://git.genenetwork.org/gn-machines/ -* GeneNetwork development container +For CI/CD see -The GeneNetwork development container is currently run on /tux02/. It runs -continuous integration and continuous deployment services for -genenetwork2, genenetwork3 and several other associated projects. +=> https://issues.genenetwork.org/topics/systems/ci-cd + +For philosophy and (KISS) incremental minimalistic development containers, see: + +=> https://issues.genenetwork.org/topics/systems/debug-and-developing-code-with-genenetwork-system-container + +These are stored in the './minimal' directory of this repo. See the [[./minimal/README.md][README]]. + +* GeneNetwork development container (aka CI/CD) + +The GeneNetwork development container is currently run on /tux02/. +It runs continuous integration and continuous deployment services for genenetwork2, genenetwork3 and several other associated projects. To build and install the container, you will need the [[https://gitlab.com/genenetwork/guix-bioinformatics][guix-bioinformatics]] and [[https://git.systemreboot.net/guix-forge/][guix-forge]] channels. Once these channels are pulled and available, on /tux02/, run + #+BEGIN_SRC shell $ ./genenetwork-development-deploy.sh #+END_SRC +It will try to create symlinks at the end. You can do this as root (too). + +Note we current run as aruni user and the latest guix is the user profile. + +#+BEGIN_SRC shell +aruni@tux02:~/gn-machines$ which guix +/home/aruni/.config/guix/current/bin/guix +aruni@tux02:~/gn-machines$ guix describe +Generation 46 Mar 27 2025 23:39:42 (current) +#+END_SRC + /tux02/ is configured with a systemd service to run this container. Restart it. #+BEGIN_SRC shell @@ -31,6 +52,7 @@ To build and install the container, you will need the guix-bioinformatics channel. Once guix-bioinformatics is pulled and available, on /tux01/, run #+begin_src shell + $ ./virtuoso-deploy.sh #+end_src @@ -55,7 +77,7 @@ You can get a shell into the container with something like: sudo guix container exec 89086 /run/current-system/profile/bin/bash --login #+END_SRC -When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or on your can get with something like: +When you start the container, you can get a shell into the container using the ~nsenter~ command. You will need the process ID of the container, which you can see on container startup or by inspecting the output of the following ~ps~ command: #+BEGIN_SRC sh ps -u root -f --forest | grep -A4 '/usr/local/bin/genenetwork-development-container' | grep 'shepherd' diff --git a/fallback.scm b/fallback.scm index e222238..8153789 100644 --- a/fallback.scm +++ b/fallback.scm @@ -29,7 +29,7 @@ (forge socket)) (operating-system - (host-name "genenetwork-fallback") + (host-name "fallback") (timezone "UTC") (locale "en_US.utf8") (bootloader (bootloader-configuration @@ -61,6 +61,7 @@ (gn-auth-server-name "fallback-auth.genenetwork.org") (gn2-port 8892) (gn3-port 8893) + (gn-auth-port 8894) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export/data/genenetwork-sqlite/auth.db") (xapian-db "/export/data/genenetwork-xapian") diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index b251033..d63dcf0 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -42,6 +42,7 @@ container_script=$(guix system container --network \ --expose=/export/data/genenetwork \ --share=/export/data/genenetwork-xapian \ --share=/export/data/genenetwork-sqlite \ + --share=/export/data/lmdb \ --share=/var/run/mysqld=/run/mysqld \ --share=/export/data/gn-docs/ \ genenetwork-development.scm) diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 7d3a14b..abdb694 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,19 +21,21 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) + ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) (gn services databases) ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) - ((gnu packages base) #:select (gnu-make tar)) + ((gnu packages base) #:select (gnu-make tar coreutils-minimal)) ((gnu packages bash) #:select (bash)) ((gnu packages bioinformatics) #:select (ccwl) #:prefix guix:) - ((gnu packages certs) #:select (nss-certs)) + ((gnu packages nss) #:select (nss-certs)) ((gnu packages check) #:select (python-pylint)) + ((gnu packages curl) #:select (curl)) ((gnu packages ci) #:select (laminar)) ((gnu packages compression) #:select (gzip)) - ((gnu packages databases) #:select (mariadb virtuoso-ose)) + ((gnu packages databases) #:select (mariadb redis)) + ((gnu packages databases) #:select (virtuoso-ose)) ((gnu packages gnupg) #:select (guile-gcrypt)) ((gnu packages graphviz) #:select (graphviz)) ((gnu packages guile) #:select (guile-3.0 guile-git guile-zlib)) @@ -122,7 +124,11 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "https://git.genenetwork.org/gn-auth")) + (default "https://git.genenetwork.org/gn-auth")) + (gn-libs-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-libs")) + (gn-guile-repository genenetwork-configuration-gn-libs-repository + (default "https://git.genenetwork.org/gn-guile")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -147,8 +153,12 @@ be imported into G-expressions." (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path (default "/export/data/genenetwork-sqlite/llm.db")) + (lmdb-data-path genenetwork-lmdb-data-path + (default "/export/data/lmdb")) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) + (repositories genenetwork-configuration-repositories + (default "/export/data/repositories")) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout (default "/export/data/gn-docs"))) @@ -188,9 +198,9 @@ described by CONFIG, a <genenetwork-configuration> object. TEST-COMMAND is a list of strings specifying the command to be executed." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn3-port genotype-files) + (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) - (with-packages (list bash coreutils git-minimal nss-certs) + (with-packages (list bash coreutils git-minimal nss-certs genenetwork2) #~(begin (use-modules (guix build utils) (srfi srfi-26)) @@ -216,13 +226,12 @@ executed." (invoke "git" "clone" "--depth" "1" #$gn3-repository) (with-directory-excursion "genenetwork3" (show-head-commit)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository) + (with-directory-excursion "gn-libs" + (show-head-commit)) (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) - ;; This is a dummy SERVER_PORT to placate - ;; bin/genenetwork2. TODO: Fix bin/genenetwork2 so that - ;; this is not needed. - (setenv "SERVER_PORT" "8080") ;; Use a profile with all dependencies except ;; genenetwork3. (setenv "GN2_PROFILE" @@ -236,7 +245,30 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + ;; This file is cosmetic + (setenv + "GN2_SETTINGS" + #$(mixed-text-file "gn2.conf" + "GN2_SECRETS=\"/tmp/secret.py\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:120\"\n" + "GN3_GUILE_SERVER_URL=\"http://localhost:120\"\n" + "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SSL_PRIVATE_KEY=\"/tmp/ssl-private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/tmp/gn-auth-ssl-public-key.pem\"\n")) + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" "/export/data/genenetwork/genotype_files") + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/tmp") (chdir "genenetwork2") ;; XXXX: FIXME: R/Qtl tests fail because files are generated in ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 @@ -272,7 +304,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -327,16 +359,13 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port gn-guile-port gn-guile-repository) (list (forge-project (name "genenetwork2") (repository gn2-repository) (ci-jobs (list (forge-laminar-job (name "genenetwork2") - (run (genenetwork2-tests - config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "-m" "pytest"))) + (run (genenetwork2-tests config (list "python3" "-m" "pytest"))) ;; If unit tests pass, redeploy genenetwork2 and ;; trigger Mechanical Rob. (after (with-imported-modules '((guix build utils)) @@ -352,8 +381,7 @@ genenetwork3 source from the latest commit of @var{project}." (name "genenetwork2-mechanical-rob") (run (genenetwork2-tests config - (list "sh" "bin/genenetwork2" "./gn2/default_settings.py" - "-c" "test/requests/test-website.py" + (list "python3" "test/requests/tests-website.py" "--all" (string-append "http://localhost:" (number->string gn2-port))))) (trigger? #f)))) (ci-jobs-trigger 'webhook)) @@ -400,6 +428,33 @@ genenetwork3 source from the latest commit of @var{project}." (trigger? #f)))) (ci-jobs-trigger 'webhook)) (forge-project + (name "gn-libs") + (repository gn-libs-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-libs") + (run (guix-channel-job-gexp + (list (channel + (name 'gn-libs) + (url (forge-project-repository this-forge-project)) + (branch "main"))) + #:variables (list (variable-specification + (module '(gn-libs)) + (name 'gn-libs))) + #:guix-daemon-uri %guix-daemon-uri))))) + (ci-jobs-trigger 'webhook)) + (forge-project + (name "gn-guile") + (repository gn-guile-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-guile") + (run (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$sudo + #$(file-append shepherd "/bin/herd") + "restart" "gn-guile"))))))) + (ci-jobs-trigger 'webhook)) + (forge-project (name "gn-auth") (repository gn-auth-repository) (ci-jobs (list (forge-laminar-job @@ -441,7 +496,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return a G-expression that runs the latest genenetwork2 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files) + (gn2-repository gn3-repository gn2-port gn3-port gn2-secrets genotype-files gn-guile-port) (with-packages (list coreutils git-minimal gunicorn nss-certs) (with-imported-modules '((guix build utils)) #~(begin @@ -458,33 +513,52 @@ server described by CONFIG, a <genenetwork-configuration> object." (hline) (invoke "git" "log" "--max-count" "1") (hline)) - - ;; Clone the latest genenetwork2 and genenetwork3 + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "TERM" "xterm-256color") + ;; Clone the latest genenetwork2 ;; repositories. - (invoke "git" "clone" "--depth" "1" #$gn2-repository) - (with-directory-excursion "genenetwork2" - (show-head-commit)) - (invoke "git" "clone" "--depth" "1" #$gn3-repository) - (with-directory-excursion "genenetwork3" - (show-head-commit)) + (with-directory-excursion + "/home/genenetwork" + (when (file-exists? "/home/genenetwork/genenetwork2") + (delete-file-recursively "/home/genenetwork/genenetwork2")) + (invoke "git" "clone" "--depth" "1" #$gn2-repository)) ;; Override the genenetwork3 used by genenetwork2. - (setenv "GN3_PYTHONPATH" - (string-append (getcwd) "/genenetwork3")) + (setenv "GN3_PYTHONPATH" "/home/genenetwork/genenetwork3") ;; Set other environment variables required by ;; genenetwork2. (setenv "GN2_PROFILE" #$(profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "PYTHONPATH" (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages")) + (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) + (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) + (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) + (setenv "GUIX_GENENETWORK_FILES" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2")) + (setenv "GENENETWORK_FILES" #$genotype-files) + (setenv "PLINK_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/plink2")) + (setenv "GEMMA_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma")) + (setenv "GEMMA_WRAPPER_COMMAND" (string-append (getenv "GN2_PROFILE") "/bin/gemma-wrapper")) + (setenv "HOME" "/home/genenetwork") (setenv "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" "GN3_LOCAL_URL=\"" (string-append "http://localhost:" (number->string gn3-port)) "\"\n" + "GN_GUILE_SERVER_URL=\"" + (string-append "http://localhost:" + (number->string gn-guile-port)) + "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" @@ -492,15 +566,23 @@ server described by CONFIG, a <genenetwork-configuration> object." "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) ;; Start genenetwork2. - (with-directory-excursion "genenetwork2" - (invoke #$(file-append bash "/bin/sh") - "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-prod"))))))) + (with-directory-excursion + "/home/genenetwork/genenetwork2" + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "--bind" (string-append "0.0.0.0:" (number->string #$gn2-port)) + "--workers" "20" + "--keep-alive" "6000" + "--max-requests" "100" + "--max-requests-jitter" "30" + "--timeout" "1200" + "gn2.wsgi"))))))) (define (genenetwork3-cd-gexp config) "Return a G-expression that runs the latest genenetwork3 development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (gn3-repository gn3-port gn3-secrets sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path lmdb-data-path) (with-manifest (package->development-manifest genenetwork3) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) @@ -519,35 +601,48 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest genenetwork3 repository. - (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (setenv "GN3_PROFILE" #$(profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN3_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) ;; Configure genenetwork3. (setenv "GN3_CONF" #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "LMDB_DATA_PATH=\"" lmdb-data-path "\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) - (setenv "RSCRIPT" #$(file-append - (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t)) - "/bin/Rscript")) - ;; Run genenetwork3. - (with-directory-excursion "genenetwork3" - (show-head-commit) - (invoke #$(file-append gunicorn "/bin/gunicorn") - "-b" #$(string-append "localhost:" (number->string gn3-port)) - "--workers" "8" - "gn3.app:create_app()" - ;; gunicorn's default 30 second timeout is - ;; insufficient for Fahamu AI endpoints and - ;; results in worker timeout errors. - "--timeout" "1200")))))))) + (setenv "RSCRIPT" (string-append + (getenv "GN3_PROFILE") + "/bin/Rscript")) + + (with-directory-excursion + "/home/genenetwork" + ;; Clone the latest genenetwork3 repository. + (when (file-exists? "/home/genenetwork/genenetwork3") + (delete-file-recursively "/home/genenetwork/genenetwork3")) + (invoke "git" "clone" "--depth" "1" #$gn3-repository)) + + (with-directory-excursion + "/home/genenetwork/genenetwork3" + (show-head-commit) + ;; Run genenetwork3. + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn3-port)) + "--workers" "8" + "gn3.app:create_app()" + ;; gunicorn's default 30 second timeout is + ;; insufficient for Fahamu AI endpoints and + ;; results in worker timeout errors. + "--timeout" "1200")))))))) (define (gn-auth-cd-gexp config) "Return a G-expression that runs the latest gn-auth development @@ -572,9 +667,23 @@ server described by CONFIG, a <genenetwork-configuration> object." (invoke "git" "log" "--max-count" "1") (hline)) - ;; Clone the latest gn-auth repository. - (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + + (with-directory-excursion + "/home/genenetwork/" + ;; Clone the latest gn-auth repository. + (when (file-exists? "/home/genenetwork/gn-auth") + (delete-file-recursively "/home/genenetwork/gn-auth")) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository)) + ;; Configure gn-auth. + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN_AUTH_CONF" #$(mixed-text-file "gn-auth.conf" "AUTH_DB=\"" auth-db-path "\"\n" @@ -584,7 +693,7 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "HOME" "/tmp") (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") ;; Run gn-auth. - (with-directory-excursion "gn-auth" + (with-directory-excursion "/home/genenetwork/gn-auth" (show-head-commit) (invoke #$(file-append gunicorn "/bin/gunicorn") "-b" #$(string-append "localhost:" (number->string gn-auth-port)) @@ -592,23 +701,70 @@ server described by CONFIG, a <genenetwork-configuration> object." "gn_auth.wsgi:app")))))))) (define (gn-guile-gexp gn-guile-port) - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) - (when (file-exists? current-repo-path) - (delete-file-recursively current-repo-path)) - (setenv "CURRENT_REPO_PATH" current-repo-path) - (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) - (invoke #$(file-append gn-guile "/bin/gn-guile") - (number->string #$gn-guile-port))))) + (with-packages + (list coreutils git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke "git" "log" "--max-count" "1") + (hline)) + ;; KLUDGE: Here we set all the certificates properly. In gn-guile, + ;; we make request to external services. Here's an example: + ;; curl http://localhost:8091/gene/aliases/Shh + ;; + ;; Without certs, we run into: + ;; 2025-07-22 08:27:11 GET /gene/aliases/Shh + ;; [...] + ;; 2025-07-22 08:27:19 signer-not-found invalid + (setenv "GN_GUILE_PROFILE" #$(profile + (content (package->development-manifest gn-guile)) + (allow-collisions? #t))) + (setenv "SSL_CERT_DIR" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs")) + (setenv "SSL_CERT_FILE" (string-append + (getenv "GN_GUILE_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GIT_SSL_CAINFO" (getenv "SSL_CERT_FILE")) + (setenv "CURL_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) + (let ((current-repo-path "/home/genenetwork/gn-docs")) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (with-directory-excursion + "/home/genenetwork/gn-docs" + ;; All edits go to the current-repo-path; so we need it to + ;; be persistent. + (unless (file-exists? current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH") current-repo-path)) + + (when (file-exists? "gn-guile") + (delete-file-recursively "gn-guile")) + (invoke "git" "clone" "--depth" "1" "https://git.genenetwork.org/gn-guile") + ;; We have a gn-guile-dev wrapper script that sets a "./" in the + ;; GN_GUILE_LOAD_PATH hence allowing this to be run from the gn-guile + ;; directory. This allows gn-guile to be run from the latest + ;; upstream commits without pinning to guix. + (with-directory-excursion "gn-guile" + (show-head-commit) + (invoke #$(file-append gn-guile "/bin/gn-guile-dev") + (number->string #$gn-guile-port))))))))) (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path lmdb-data-path gn-doc-git-checkout gn-guile-port) (list (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -631,6 +787,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source gn-doc-git-checkout) (target source) + (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) @@ -646,8 +806,7 @@ described by CONFIG, a <genenetwork-configuration> object." (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) - ;; FIXME: The genenetwork2 service should depend on redis. - (requirement '(networking genenetwork3)) + (requirement '(networking redis gn-auth genenetwork3)) (modules '((guix search-paths) (ice-9 match) (srfi srfi-1))) @@ -680,6 +839,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:mappings (list (file-system-mapping (source genotype-files) (target source)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source "/run/mysqld") (target source) @@ -724,7 +887,7 @@ described by CONFIG, a <genenetwork-configuration> object." (shepherd-service (documentation "Run GeneNetwork 3 development server.") (provision '(genenetwork3)) - (requirement '(networking)) + (requirement '(networking gn-auth)) (start #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "genenetwork3" @@ -737,6 +900,14 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source lmdb-data-path) + (target source) + (writable? #t)) ;; XXXX: FIXME: R/Qtl generates ;; files in "/tmp" and ;; "/tmp/gn2". These files are @@ -786,11 +957,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + (file-system-mapping + (source "/home/genenetwork") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) (file-system-mapping - (source auth-db-path) + (source (dirname auth-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -813,15 +988,16 @@ described by CONFIG, a <genenetwork-configuration> object." (group "genenetwork") (system? #t) (comment "GeneNetwork user") - (home-directory "/var/empty") + (home-directory "/home/genenetwork") (shell (file-append shadow "/sbin/nologin"))))) (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets) + (gn2-secrets gn3-secrets auth-db-path gn-auth-secrets gn-doc-git-checkout) (with-imported-modules '((guix build utils)) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (ice-9 ftw)) ;; Set ownership of files. (for-each (lambda (file) @@ -831,12 +1007,13 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) - (find-files "/export/data/gn-docs" + (find-files gn-doc-git-checkout #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets #:directories? #t)))) + ;; Prevent other users from reading secret files. (for-each (lambda (file) (chmod file #o600)) @@ -969,7 +1146,7 @@ described by CONFIG, a <genenetwork-configuration> object." (define transform-genenetwork-database-project (forge-project (name "transform-genenetwork-database") - (repository "https://git.genenetwork.org/gn-transform-databases") + (repository "/home/git/public/gn-transform-databases") (ci-jobs (list (forge-laminar-job (name "transform-genenetwork-database-tests") (run (guix-channel-job-gexp @@ -984,8 +1161,7 @@ described by CONFIG, a <genenetwork-configuration> object." (run (transform-genenetwork-database-gexp %connection-settings %virtuoso-data-dir - "https://git.genenetwork.org/gn-transform-databases"))))) - (ci-jobs-trigger 'webhook))) + "https://git.genenetwork.org/gn-transform-databases"))))))) ;;; @@ -1291,13 +1467,14 @@ gn-auth." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) - (packages %base-packages) + (packages (cons* curl coreutils-minimal %base-packages)) (sudoers-file (mixed-text-file "sudoers" "@include " %sudoers-specification ;; Permit the laminar user to restart genenetwork2 ;; and genenetwork3. "\nlaminar ALL = NOPASSWD: " + (file-append shepherd "/bin/herd") " restart gn-guile, " (file-append shepherd "/bin/herd") " restart genenetwork2, " (file-append shepherd "/bin/herd") " start genenetwork3, " (file-append shepherd "/bin/herd") " stop genenetwork3, " @@ -1355,7 +1532,7 @@ gn-auth." (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (server-port 9081) - (dirs-allowed "/var/lib/data") + (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) (service genenetwork-service-type (genenetwork-configuration @@ -1382,8 +1559,12 @@ gn-auth." (hosts (list (tissue-host (name "issues.genenetwork.org") - (user "laminar") - (upstream-repository "https://github.com/genenetwork/gn-gemtext-threads")))))) + (projects (list (tissue-project + (name "issues.genenetwork.org") + (user "laminar") + (base-path "/") + (upstream-repository + "https://github.com/genenetwork/gn-gemtext-threads"))))))))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket @@ -1397,7 +1578,8 @@ gn-auth." %genenetwork2-port %genenetwork3-port) (laminar-reverse-proxy-server-block "localhost:9089" %webhook-port - (list 'gn-bioinformatics)) + (list 'gn-bioinformatics + 'guix-bioinformatics)) (tissue-reverse-proxy-server-block) (gn-auth-reverse-proxy-server-block))))) (service acme-service-type diff --git a/genenetwork/services/genecup.scm b/genenetwork/services/genecup.scm new file mode 100644 index 0000000..6ee2812 --- /dev/null +++ b/genenetwork/services/genecup.scm @@ -0,0 +1,121 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services genecup) + #:use-module (guix) + #:use-module (gnu) + #:use-module (guix git) + #:use-module (guix modules) + #:use-module (guix profiles) + #:use-module (guix records) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:use-module (forge utils) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gn packages machine-learning) + #:use-module (gnu packages certs) + #:use-module (gnu packages curl) + #:use-module (gn packages python) + #:use-module (gnu packages admin) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module ((gnu packages python) #:select (python)) + #:use-module (gnu packages python-xyz) + #:use-module (guix build python-build-system) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages python-web) + #:use-module (gnu services shepherd) + #:use-module (gnu packages python-science) + #:use-module (gnu packages machine-learning) + #:use-module ((gnu packages admin) #:select (shepherd)) + #:export (genecup-configuration + genecup-configuration? + genecup-configuration-package + genecup-service-type)) + +(define-record-type* <genecup-configuration> + genecup-configuration make-genecup-configuration + genecup-configuration? + (package genecup-configuration-package ; <package> + (default genecup-latest-with-tensorflow-native))) + +(define (genecup-activation config) + (match-record config <genecup-configuration> + (package) + (with-packages + (list python + python-nltk + nss-certs ; Needed for downloading data with nltk.downloader. + curl) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((nltk-data "/var/cache/nltk_data/") + (data-dir "/export/ratspub/tmp")) + (unless (file-exists? nltk-data) + (begin + (mkdir-p nltk-data) + (chdir nltk-data) + (invoke #$(file-append python "/bin/python3") "-m" "nltk.downloader" "-d" nltk-data "punkt"))) + (unless (file-exists? (string-append data-dir "/userspub.sqlite")) + (begin + (install-file #$(file-append package "/userspub.sqlite") data-dir) + (chmod (string-append data-dir "/userspub.sqlite") #o554))))))))) + +(define genecup-shepherd-service + (match-lambda + (($ <genecup-configuration> package) + (with-imported-modules (source-module-closure + '((guix search-paths) + (gnu build shepherd) + (gnu system file-systems))) + (list (shepherd-service + (provision '(genecup)) + (requirement '(networking)) + (modules '((gnu build shepherd) + (gnu system file-systems))) + (start + #~(make-forkexec-constructor + (list #$(file-append package "/server.py")) + #:directory #$package + #:environment-variables + (list + "NLTK_DATA=/var/cache/nltk_data" + (string-append "EDIRECT_PUBMED_MASTER=" + #$(file-append package "/minipubmed")) + (string-append "PERL_LWP_SSL_CA_FILE=" + #$(file-append (profile (content (packages->manifest (list curl nss-certs)))) + "/etc/ssl/certs/ca-certificates.crt"))) + #:log-file "/var/log/genecup.log")) + (stop #~(make-kill-destructor)))))))) + +(define genecup-service-type + (service-type + (name 'genecup) + (extensions + (list + (service-extension activation-service-type + genecup-activation) + (service-extension shepherd-root-service-type + genecup-shepherd-service))) + (default-value (genecup-configuration)) + (description + "Run a GeneCup Webserver."))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index b863bcf..9ba6213 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -57,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -109,16 +110,24 @@ (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) + (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir + (default "/var/genenetwork/sessions/genenetwork2")) (gn2-secrets genenetwork-configuration-gn2-secrets (default "/etc/genenetwork")) (gn3-secrets genenetwork-configuration-gn3-secrets (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout (default "/export/data/gn-docs")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) (log-level genenetwork-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -142,6 +151,10 @@ (default "https://auth.genenetwork.org")) (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) + (sessions-dir gn-uploader-sessions-dir + (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) (log-level gn-uploader-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -170,6 +183,8 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) @@ -203,7 +218,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -212,7 +229,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -240,9 +259,35 @@ (chmod file #o644)) (find-files #$xapian-directory))))))))) +(define (samples-count-script-gexp config) + (match-record config <genenetwork-configuration> + (genenetwork2 sql-uri) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PYTHONPATH" + (string-append + #$(file-append genenetwork2 + "/lib/python" + (python-version (package-version python)) + "/site-packages") + ":" + #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "/lib/python" + #$(python-version (package-version python)) + "/site-packages")) + + (invoke #$(file-append python "/bin/python3") + "-m" + "gn2.scripts.sample_count" + #$sql-uri))))) + (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -262,7 +307,11 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets + #:directories? #t) + (find-files #$gn2-sessions-dir #:directories? #t))) (for-each (lambda (file) (chown file @@ -278,7 +327,31 @@ (find-files #$gn2-secrets #:directories? #f) (find-files #$gn-auth-secrets - #:directories? #f))))))) + #:directories? #f))) + ;; Make sub-directories for various apps under gn-tmpdir and assign + ;; appropriate permissions + (for-each (match-lambda + ((subdir user) + (let ((full-path + (string-append #$gn-tmpdir "/" subdir))) + (unless (file-exists? full-path) + (mkdir full-path #o755)) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw user)) + (passwd:gid (getpw user)))) + (find-files full-path + #:directories? #t))))) + '(("gn2-tmpdir" "gunicorn-genenetwork2") + ("gn3-tmpdir" "gunicorn-genenetwork3"))) + + ;; setup correct ownership for gn-docs + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (find-files #$(dirname gn-doc-git-checkout) + #:directories? #t)))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -309,7 +382,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -328,12 +401,16 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("AI_SEARCH_ENABLED" "True"))))) + ("AI_SEARCH_ENABLED" "True") + ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir) + ("MAX_FORM_MEMORY_SIZE" 52428800))))) (gn3-profile (profile (content (package->development-manifest genenetwork3)) (allow-collisions? #t))) @@ -342,6 +419,8 @@ object." (configuration-file-gexp `(("AUTH_DB" ,auth-db) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) @@ -358,9 +437,9 @@ object." (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) ("AUTH_DB" ,auth-db) - ("SQL_URI" ,sql-uri) - ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys")) - ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem"))))))) + ("SQL_URI" ,sql-uri))))) + (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir")) + (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir"))) (list (gunicorn-app (name "genenetwork2") (package genenetwork2) @@ -375,7 +454,7 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn2-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) @@ -388,13 +467,17 @@ object." (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn2-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -408,7 +491,11 @@ object." (writable? #t)) (file-system-mapping (source gn2-ca-bundle) - (target source)))) + (target source)) + (file-system-mapping + (source gn2-sessions-dir) + (target source) + (writable? #t)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))) @@ -429,7 +516,7 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn3-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) @@ -455,15 +542,18 @@ object." (file-system-mapping (source gn3-data-directory) (target source)) ; Rqtl usese this - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn3-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) (target source)) (file-system-mapping - (source llm-db-path) + (source (dirname llm-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -479,6 +569,7 @@ object." (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -497,9 +588,9 @@ object." (source gn-auth-conf) (target source)) (file-system-mapping - (source auth-db) - (target source) - (writable? #t)) + (source (dirname auth-db)) + (target source) + (writable? #t)) (file-system-mapping (source gn-auth-secrets) (target source) @@ -526,7 +617,8 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" @@ -555,24 +647,38 @@ a @code{<genenetwork-configuration>} record." (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron-gexp" (build-xapian-index-cron-gexp config)) - #:user "root"))) + #:user "root") + #~(job '(next-minute-from (next-hour) '(17)) ;17th minute of every hour + #$(program-file "samples-count-script-gexp" + (samples-count-script-gexp config))))) -(define (gn-guile-gexp gn-guile-port) +(define (gn-guile-gexp gn-guile-port gn-guile-pkg bare-repo) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile-pkg)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (current-repo-path (string-append (pk "CWD" (getcwd)) "/gn-docs"))) + ;; These have to be setup manually here an not in the + ;; `gn-guile-shepherd-service' function, otherwise, they do not take + ;; effect for some reason. + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + (when (file-exists? current-repo-path) (delete-file-recursively current-repo-path)) (setenv "CURRENT_REPO_PATH" current-repo-path) (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + "clone" "--depth" "1" #$(string-append "file://" bare-repo))) (invoke #$(file-append gn-guile "/bin/gn-guile") (number->string #$gn-guile-port))))) (define (gn-guile-shepherd-service config) (match-record config <genenetwork-configuration> - (gn-doc-git-checkout gn-guile-port) + (gn-guile gn-doc-git-checkout gn-guile-port) (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -588,14 +694,15 @@ a @code{<genenetwork-configuration>} record." #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) + (gn-guile-gexp gn-guile-port gn-guile gn-doc-git-checkout)) #:name "gn-guile-pola-wrapper" + #:directory (dirname gn-doc-git-checkout) #:preserved-environment-variables (map first gn-guile-settings) #:mappings (list (file-system-mapping - (source gn-doc-git-checkout) - (target source) - (writable? #t))) + (source (dirname gn-doc-git-checkout)) + (target source) + (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) #:user "genenetwork" @@ -628,14 +735,20 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory) + (secrets data-directory sessions-dir sqlite-databases-directory) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Let service user own their own secrets files. - (chown #$secrets - (passwd:uid (getpw "gunicorn-gn-uploader")) - (passwd:gid (getpw "gunicorn-gn-uploader"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader")))) + (append (list #$(dirname secrets)) + (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -651,25 +764,27 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t))) - (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOAD_FOLDER" ,(string-append data-directory + "/uploads")) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) @@ -677,6 +792,7 @@ a @code{<genenetwork-configuration>} record." (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -695,8 +811,9 @@ a @code{<genenetwork-configuration>} record." (source gn-uploader-conf) (target source)) (file-system-mapping - (source secrets) - (target source)) + (source (dirname secrets)) + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source) @@ -706,7 +823,15 @@ a @code{<genenetwork-configuration>} record." (target source)) (file-system-mapping (source gn-uploader-ca-bundle) - (target source)))) + (target source)) + (file-system-mapping + (source sessions-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))))))) diff --git a/genenetwork/services/mouse-longevity.scm b/genenetwork/services/mouse-longevity.scm new file mode 100644 index 0000000..c9b977d --- /dev/null +++ b/genenetwork/services/mouse-longevity.scm @@ -0,0 +1,33 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services mouse-longevity) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gnu services) + #:export (mouse-longevity-service)) + +(define* (mouse-longevity-service #:optional (package mouse-longevity-app)) + (simple-service 'mouse-longevity + rshiny-service-type + (list + (rshiny-configuration + (package package) + (binary "mouse-longevity-app"))))) diff --git a/production-deploy.sh b/production-deploy.sh index bd95814..c8e30fe 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -26,19 +26,24 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ - --share=/export/mysql/database=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ - --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ + --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/genenetwork/var/log=/var/log \ + --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \ --share=/var/run/mysqld=/run/mysqld \ - --share=/export/data/gn-docs/ \ - --share=/export2/guix-containers/genenetwork/tmp=/tmp \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \ + --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/gn-guile=/var/lib/genenetwork/gn-guile \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 933320a..bfd9e48 100644 --- a/production.scm +++ b/production.scm @@ -44,14 +44,11 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service mysql-service-type - (mysql-configuration - (auto-upgrade? #f))) - (service virtuoso-service-type + (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port 9892) (http-server-port 9893) - (dirs-allowed "/var/lib/virtuoso") + (dirs-allowed (list "/export/data/virtuoso")) (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") @@ -80,15 +77,29 @@ (gn3-port 9895) (gn-auth-port 9896) (sql-uri - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:9893/sparql") (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") (gn-auth-secrets "/etc/genenetwork/gn-auth") - (auth-db "/var/lib/genenetwork-sqlite/auth.db") - (llm-db-path "/var/lib/genenetwork-gnqa/llm.db") + (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db") + (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db") (gn3-alias-server-port 9800) + (gn-tmpdir "/opt/gn/tmp") + (gn-doc-git-checkout "/var/lib/genenetwork/gn-guile/gn-docs.git") + (log-level 'debug))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "uploader.genenetwork.org") + (port 9897) + (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (data-directory "/var/lib/genenetwork/uploader/data") + (auth-server-url "https://auth.genenetwork.org/") + (gn2-server-url "https://genenetwork.org") + (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader") (log-level 'debug))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index bee9abf..bd8b938 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -22,10 +22,11 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/public-sparql/tmp=/tmp \ - --share=/export2/guix-containers/public-sparql/var/log=/var/log \ - --share=/export2/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/public-sparql/tmp=/tmp \ + --share=/export/guix-containers/public-sparql/var/log=/var/log \ + --share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index f6efb15..edfcd87 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -20,6 +20,7 @@ (use-modules (gnu) (gn services databases) (gnu services web) + ((gnu packages admin) #:select (shepherd)) (forge nginx) (forge socket)) @@ -50,16 +51,22 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) (number-of-buffers 4000000) - (dirs-allowed "/var/lib/virtuoso") + (dirs-allowed (list "/export/data/virtuoso")) (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/public-virtuoso.db") - (transaction-file "/var/lib/virtuoso/public-virtuoso.trx"))) + (transaction-file "/var/lib/virtuoso/public-virtuoso.trx") + (error-log-file "/var/lib/public-virtuoso-errors.log") + (syslog "1"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket diff --git a/specials/gndev.scm b/specials/gndev.scm index 440aedb..0f9394f 100644 --- a/specials/gndev.scm +++ b/specials/gndev.scm @@ -61,6 +61,7 @@ (gn-auth-server-name "gndev-auth.genenetwork.org") (gn2-port 8992) (gn3-port 8993) + (gn-auth-port 8994) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") (auth-db "/export2/data/gndev-sqlite/auth.db") (xapian-db "/export2/data/gndev-xapian") diff --git a/test-r-container.scm b/test-r-container.scm new file mode 100644 index 0000000..3167574 --- /dev/null +++ b/test-r-container.scm @@ -0,0 +1,111 @@ +(use-modules (guix) + (gnu) + (guix git) + (guix modules) + (guix profiles) + (guix records) + (guix packages) + (guix git-download) + (guix build-system trivial) + (gnu services dbus) + (gnu services networking) + (gnu services ssh) + (gnu services web) + (gnu services certbot) + (gnu services docker) + (gnu services desktop) + (gnu services shepherd) + (gnu packages admin) + (gnu packages statistics) + (gnu packages cran) + (gnu packages curl) + (gnu packages lsof) + (srfi srfi-1) + (ice-9 match) + (genenetwork services mouse-longevity) + (gn services rshiny) + (gn packages mouse-longevity) + (forge acme) + (forge nginx) + (forge socket) + (gnu services)) + +(define %nginx-configuration + (nginx-configuration + (server-blocks + (list + ;; Redirect domains that don't explicitly support HTTP (below) to HTTPS. + (nginx-server-configuration + (listen '("8080"))) + + ;; Domains that still explicitly support plain HTTP. + (nginx-server-configuration + (listen '("80")) + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list + (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://127.0.0.1:3979;"))))) + (raw-content + (list + "proxy_set_header Host $host;" + "proxy_set_header X-Real-IP $remote_addr;" + "proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;" + "proxy_set_header X-Forwarded-Proto $scheme;"))))))) + +(operating-system + (host-name "testing-genenetwork-services") + (keyboard-layout (keyboard-layout "us")) + (kernel-arguments + (cons* "console=ttyS0,115200" "console=tty0" + %default-kernel-arguments)) + (bootloader (bootloader-configuration (bootloader grub-bootloader))) + (issue "This is a GeneNetwork container. Welcome!\n") + (file-systems %base-file-systems) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* lsof curl %base-packages)) + (services + (cons* + (service dhcp-client-service-type) + (service dbus-root-service-type) + (service polkit-service-type) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8080"))) + (https-listen (forge-ip-socket + (ip "127.0.0.1") + (port "8443"))) + (server-blocks + (list + (nginx-server-configuration + (server-name '("longevity-explorer.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list "proxy_pass http://localhost:3979;" + "proxy_set_header Host $host;")))))))))) + (service acme-service-type + (acme-configuration + (email "jgart@dismail.de") + (acme-url %letsencrypt-staging-url))) + ;; (service certbot-service-type + ;; (certbot-configuration + ;; (email "jgart@dismail.de") + ;; (certificates + ;; (list + ;; (certificate-configuration + ;; (domains '("longevity-explorer.genenetwork.org" + ;; "www.longevity-explorer.genenetwork.org"))))))) + ;; (mouse-longevity-service ) + (service rshiny-service-type + (rshiny-configuration + (package mouse-longevity-app) + (binary "mouse-longevity-app"))) + ;; (service nginx-service-type %nginx-configuration) + %base-services))) diff --git a/uploader-deploy.sh b/uploader-deploy.sh index fdbbe0c..415790b 100755 --- a/uploader-deploy.sh +++ b/uploader-deploy.sh @@ -41,18 +41,22 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/uploader/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/uploader/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ - --share=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ - --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ - --expose=/export/data/uploader/genotype_files=/export/data/genenetwork/genotype_files \ - --expose=/export/data/uploader/genenetwork3 \ - --share=/export/data/uploader/gn-uploader \ + --share=/export/guix-containers/uploader/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/uploader/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/uploader/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/uploader/var/log=/var/log \ + --share=/export/guix-containers/uploader/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-xapian=/var/lib/xapian \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld3307=/run/mysqld \ + --share=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/uploader/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/uploader/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/data/uploader/gn-uploader \ uploader.scm) echo "${container_script}" diff --git a/uploader.scm b/uploader.scm index 62ab35f..5064426 100644 --- a/uploader.scm +++ b/uploader.scm @@ -42,11 +42,7 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service virtuoso-service-type - (virtuoso-configuration - (server-port 10892) - (http-server-port 10893))) - (service forge-nginx-service-type + (services (cons* (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") @@ -64,24 +60,26 @@ (gn2-port 10894) (gn3-port 10895) (gn-auth-port 10896) - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") - (auth-db "/export/data/genenetwork-sqlite/auth.db") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (auth-db "/var/lib/genenetwork-sqlite/auth.db") + (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:10893/sparql") - (gn3-data-directory "/export/data/uploader/genenetwork3") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (llm-db-path "/var/lib/genenetwork-gnqa/llm.db"))) (service gn-uploader-service-type (gn-uploader-configuration (gn-uploader gn-uploader) (server-name "staging-uploader.genenetwork.org") (port 10897) (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (data-directory "/export/data/uploader/gn-uploader") - (log-level "DEBUG") + (log-level 'debug) (auth-server-url "https://staging-auth.genenetwork.org/") (gn2-server-url "https://staging.genenetwork.org"))) %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0414a65..d18caec 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2025 Pjotr Prins <pjotr.public01@thebird.nl> # # This file is part of genenetwork-machines. # @@ -19,14 +20,25 @@ # along with genenetwork-machines. If not, see # <https://www.gnu.org/licenses/>. -# Build and install virtuoso container on tux01. +# Build and install virtuoso container. Note the shared path is the sane default. Symlink if necessary. +# See also topics/systems/debug-and-developing-code-with-genenetwork-system-container.gmi container_script=$(guix system container \ + -L ~/guix-bioinformatics \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/virtuoso/data/virtuoso/ttl=/export/data/virtuoso/ttl \ virtuoso.scm) echo $container_script sudo ln --force --symbolic $container_script /usr/local/bin/virtuoso-container sudo ln --force --symbolic /usr/local/bin/virtuoso-container /var/guix/gcroots + +echo "Run virtuoso with: sudo /usr/local/bin/virtuoso-container" +echo "Enter with something like: nsenter -a -t 1692115 /run/current-system/profile/bin/bash --login" +echo "Admin: isql 8891" +echo " ld_dir('/export/data/virtuoso/ttl','test.rdf','http://pjotr.genenetwork.org/');" +echo " rdf_loader_run();" +echo " checkpoint;" +echo "Web: http://localhost:8892/sparql" diff --git a/virtuoso.scm b/virtuoso.scm index edcd575..ae33dcd 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed "/var/lib/virtuoso"))) + (dirs-allowed (list "/export/data/virtuoso/ttl")))) %base-services))) |