diff options
| -rw-r--r-- | genenetwork-development.scm | 30 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 73 | ||||
| -rwxr-xr-x | production-deploy.sh | 11 | ||||
| -rw-r--r-- | production.scm | 45 | ||||
| -rw-r--r-- | slurm.scm | 15 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 2 |
6 files changed, 124 insertions, 52 deletions
diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 5fafc12..6f3cfa4 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -471,6 +471,7 @@ server described by CONFIG, a <genenetwork-configuration> object." "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" + "AI_SEARCH_ENABLED=True\n" "GN3_LOCAL_URL=\"" (string-append "http://localhost:" (number->string gn3-port)) @@ -516,6 +517,7 @@ server described by CONFIG, a <genenetwork-configuration> object." #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) @@ -884,6 +886,7 @@ described by CONFIG, a <genenetwork-configuration> object." #~(begin (use-modules (guix build utils)) + (setenv "LC_ALL" "en_US.UTF-8") (invoke #$(file-append tissue "/bin/tissue") "pull" "issues.genenetwork.org")))))))) (ci-jobs-trigger 'webhook))) @@ -1137,6 +1140,20 @@ gn-auth." ";") "proxy_set_header Host $host;"))))))) +(define set-build-directory-permissions-gexp + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) + (append (find-files #$%xapian-directory + #:directories? #t) + (find-files #$%transform-genenetwork-database-export-directory + #:directories? #t)))))) + ;; Port on which webhook is listening (define %webhook-port 9091) ;; Port on which genenetwork2 is listening @@ -1230,18 +1247,7 @@ gn-auth." (xapian-db-path %xapian-directory))) (simple-service 'set-build-directory-permissions activation-service-type - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - - (for-each (lambda (file) - (chown file - (passwd:uid (getpw "laminar")) - (passwd:gid (getpw "laminar")))) - (append (find-files #$%xapian-directory - #:directories? #t) - (find-files #$%transform-genenetwork-database-export-directory - #:directories? #t)))))) + set-build-directory-permissions-gexp) (service tissue-service-type (tissue-configuration (socket diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index 34d70df..6905686 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -30,7 +30,9 @@ #:use-module (gnu system file-systems) #:use-module (gnu system shadow) #:use-module (guix build python-build-system) + #:use-module (guix diagnostics) #:use-module (guix gexp) + #:use-module (guix i18n) #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix records) @@ -83,10 +85,14 @@ (default 8083)) (gn-auth-port genenetwork-configuration-gn-auth-port (default 8084)) + (gn3-alias-server-port genenetwork-gn3-alias-server-port + (default 8000)) (sql-uri genenetwork-configuration-sql-uri (default "mysql://username:password@localhost/database")) (auth-db genenetwork-configuration-auth-db (default "/var/genenetwork/auth.db")) + (llm-db-path genenetwork-configuration-llm-db-path + (default "/var/genenetwork/llm.db")) (xapian-db genenetwork-configuration-xapian-db (default "/var/genenetwork/xapian")) (genotype-files genenetwork-configuration-genotype-files @@ -94,7 +100,7 @@ (sparql-endpoint genenetwork-configuration-sparql-endpoint (default "http://localhost:8081/sparql")) (gn-sourcecode-directory genenetwork-configuration-gn-sourcecode-directory - (default "/var/empty")) + (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) (gn2-secrets genenetwork-configuration-gn2-secrets @@ -102,7 +108,10 @@ (gn3-secrets genenetwork-configuration-gn3-secrets (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets - (default "/etc/genenetwork"))) + (default "/etc/genenetwork")) + (log-level genenetwork-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) (define-record-type* <gn-uploader-configuration> gn-uploader-configuration make-gn-uploader-configuration @@ -119,11 +128,20 @@ (default "/var/genenetwork")) (secrets gn-uploader-configuration-secrets (default "/etc/genenetwork/gn-uploader-secrets.py")) - (auth-server-url gn-uploader-auth-server-url + (auth-server-url gn-uploader-configuration-auth-server-url (default "https://auth.genenetwork.org")) - (gn2-server-url gn-uploader-gn2-server-url + (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) - (log-level gn-uploader-log-level (default "WARNING"))) + (log-level gn-uploader-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) + +(define (sanitize-log-level log-level) + (case log-level + ((fatal error warning info debug trace) log-level) + (else + (leave (G_ "Log level ~a is invalid. It must be one of the following symbols---fatal, error, warn, info, debug or trace.~%") + log-level)))) (define %genenetwork-accounts (list (user-group @@ -146,8 +164,8 @@ (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) (gn3-profile (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t))) + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) (python3-version (python-version (package-version python)))) (with-imported-modules '((guix build utils)) #~(begin @@ -277,7 +295,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path log-level) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -300,8 +318,7 @@ object." ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("SSL_PRIVATE_KEY" ,(string-append gn2-secrets "/gn2-ssl-private-key.pem")) - ("AUTH_SERVER_SSL_PUBLIC_KEY" ,(string-append gn2-secrets "/gn-auth-ssl-public-key.pem")))))) + ("AI_SEARCH_ENABLED" "True"))))) (gn3-conf (computed-file "gn3.conf" (configuration-file-gexp `(("AUTH_DB" ,auth-db) @@ -309,7 +326,10 @@ object." ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) ("SQL_URI" ,sql-uri) - ("XAPIAN_DB_PATH" ,xapian-db))))) + ("XAPIAN_DB_PATH" ,xapian-db) + ("GENOTYPE_FILES" ,genotype-files) + ("REAPER_COMMAND" ,(file-append gn2-profile "/bin/qtlreaper")) + ("LLM_DB_PATH" ,llm-db-path))))) (gn-auth-conf (computed-file "gn-auth.conf" (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) @@ -358,7 +378,10 @@ object." (file-system-mapping (source gn2-secrets) (target source) - (writable? #t))))) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "genenetwork3") (package genenetwork3) @@ -407,9 +430,12 @@ object." (source xapian-db) (target source)) (file-system-mapping - (source auth-db) + (source llm-db-path) (target source) - (writable? #t))))) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "gn-auth") (package gn-auth) @@ -445,7 +471,7 @@ object." reverse proxies for the genenetwork service described by @var{config}, a @code{<genenetwork-configuration>} record." (match-record config <genenetwork-configuration> - (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port) + (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port gn3-alias-server-port) (list (nginx-server-configuration (server-name (list server-name)) (locations @@ -461,7 +487,16 @@ a @code{<genenetwork-configuration>} record." (body (list "rewrite /api3/(.*) /api/$1 break;" (string-append "proxy_pass http://localhost:" (number->string gn3-port) ";") - "proxy_set_header Host $host;")))))) + "proxy_set_header Host $host;"))) + (nginx-location-configuration + (uri "/gn3/") + (body + (list "rewrite /gn3/(.*) /$1 break;" + (string-append "proxy_pass http://localhost:" + (number->string gn3-alias-server-port) + ";") + "proxy_redirect off;" + "proxy_set_header Host $host;")))))) (nginx-server-configuration (server-name (list gn-auth-server-name)) (locations @@ -567,7 +602,9 @@ a @code{<genenetwork-configuration>} record." (file-system-mapping (source gn-uploader-profile) (target source)))) - (extra-cli-arguments (list "--log-level" log-level))))))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (gn-uploader-nginx-server-block config) (match-record config <gn-uploader-configuration> @@ -582,7 +619,7 @@ a @code{<genenetwork-configuration>} record." #$(file-append gn-uploader "/lib/python" (python-version (package-version python)) - "/site-packages/qc_app;"))))) + "/site-packages/uploader;"))))) (nginx-location-configuration (uri "/") (body (list (string-append "proxy_pass http://localhost:" diff --git a/production-deploy.sh b/production-deploy.sh index b4924a7..a744f41 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> # # This file is part of genenetwork-machines. # @@ -27,14 +28,16 @@ container_script=$(guix system container \ --verbosity=3 \ --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/mysql=/var/lib/mysql \ + --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/mysql/database=/var/lib/mysql \ --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ --share=/export2/guix-containers/genenetwork/var/log=/var/log \ --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --expose=/export/data/genenetwork-xapian \ - --share=/export/data/genenetwork-sqlite \ - --expose=/export/data/genenetwork/genotype_files \ + --share=/export2/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld=/run/mysqld \ + --share=/export2/guix-containers/genenetwork/tmp=/tmp \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 399c921..ca4a655 100644 --- a/production.scm +++ b/production.scm @@ -1,5 +1,6 @@ ;;; genenetwork-machines --- Guix configuration for genenetwork machines ;;; Copyright © 2022–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> ;;; ;;; This file is part of genenetwork-machines. ;;; @@ -17,6 +18,9 @@ ;;; along with genenetwork-machines. If not, see ;;; <https://www.gnu.org/licenses/>. +;;; This is the production genenetwork container currently deployed on +;;; tux04. + (use-modules (gnu) (genenetwork services genenetwork) ((gnu packages admin) #:select (shepherd)) @@ -45,32 +49,45 @@ (auto-upgrade? #f))) (service virtuoso-service-type (virtuoso-configuration - (server-port 7892) - (http-server-port 7893))) + (server-port 9892) + (http-server-port 9893) + (dirs-allowed "/var/lib/virtuoso") + (number-of-buffers 4000000) + (maximum-dirty-buffers 3000000) + (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") + (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") - (port 7890))) + (port 9890))) (https-listen (forge-ip-socket (ip "0.0.0.0") - (port 7891))))) + (port 9891))))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) + (service redis-service-type + (redis-configuration + (bind "127.0.0.1") + (port 6379) + (working-directory "/var/lib/redis"))) (service genenetwork-service-type (genenetwork-configuration - (server-name "test1.genenetwork.org") - (gn-auth-server-name "test1-auth.genenetwork.org") - (gn2-port 7894) - (gn3-port 7895) - (gn-auth-port 7896) + (server-name "genenetwork.org") + (gn-auth-server-name "auth.genenetwork.org") + (gn2-port 9894) + (gn3-port 9895) + (gn-auth-port 9896) (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") - (sparql-endpoint "http://localhost:7893/sparql") - (gn3-data-directory "/export/data/genenetwork") + (xapian-db "/var/lib/xapian") + (sparql-endpoint "http://localhost:9893/sparql") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (auth-db "/var/lib/genenetwork-sqlite/auth.db") + (llm-db-path "/var/lib/genenetwork-gnqa/llm.db") + (gn3-alias-server-port 9800) + (log-level 'debug))) %base-services))) @@ -45,6 +45,17 @@ (define slurm (package (inherit guix:slurm) + (name "slurm") + (version "24.05.3") + (source (origin + (inherit (package-source guix:slurm)) + (method url-fetch) + (uri (string-append + "https://download.schedmd.com/slurm/slurm-" + version ".tar.bz2")) + (sha256 + (base32 + "095fck6016kslggd1d9mnwahr66b1fahpmlmvdyqdbmnx49hbd5h")))) (arguments (substitute-keyword-arguments (package-arguments guix:slurm) ((#:configure-flags flags #~'()) @@ -73,9 +84,7 @@ (substitute* (string-append #$output "/etc/slurmrestd.service") ;; Set user and group to run slurmrestd as. (("# User=") "User=slurmrestd") - (("# Group=") "Group=slurmrestd") - ;; Disable listening on Unix socket by default. - ((" unix:[^ ]*") "")))))))) + (("# Group=") "Group=slurmrestd")))))))) (inputs (modify-inputs (package-inputs guix:slurm) (prepend dbus http-parser json-c libjwt diff --git a/uploader-deploy.sh b/uploader-deploy.sh index 90fd7e4..fdbbe0c 100755 --- a/uploader-deploy.sh +++ b/uploader-deploy.sh @@ -49,7 +49,7 @@ container_script=$(guix system container \ --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ --share=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ - --expose=/export/data/genenetwork/genotype_files=/export/data/genenetwork/genotype_files \ + --expose=/export/data/uploader/genotype_files=/export/data/genenetwork/genotype_files \ --expose=/export/data/uploader/genenetwork3 \ --share=/export/data/uploader/gn-uploader \ --share=/var/run/mysqld3307=/run/mysqld \ |