diff options
Diffstat (limited to 'genenetwork')
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 143 |
1 files changed, 112 insertions, 31 deletions
diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index ec3af23..3f4123b 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -21,18 +21,23 @@ (define-module (genenetwork services genenetwork) #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn packages guile) #:select (gn-guile)) + #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) #:use-module ((gnu packages admin) #:select (shadow shepherd)) + #:use-module ((gnu packages version-control) #:select (git-minimal)) #:use-module ((gnu packages python) #:select (python)) #:use-module (gnu services) #:use-module (gnu services web) #:use-module (gnu services mcron) + #:use-module (gnu services shepherd) #:use-module (gnu system file-systems) #:use-module (gnu system shadow) #:use-module (guix build python-build-system) #:use-module (guix diagnostics) #:use-module (guix gexp) #:use-module (guix i18n) + #:use-module (guix least-authority) #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix records) @@ -40,6 +45,7 @@ #:use-module (forge nginx) #:use-module (forge gunicorn) #:use-module (forge socket) + #:use-module (forge utils) #:use-module (srfi srfi-1) #:use-module (ice-9 match) #:export (genenetwork-service-type @@ -109,6 +115,14 @@ (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/export/data/gn-docs")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) (log-level genenetwork-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -160,6 +174,8 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) @@ -193,7 +209,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -202,7 +220,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -259,7 +279,7 @@ (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -279,11 +299,17 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets #:directories? #t))) - (chown #$gn3-secrets - (passwd:uid (getpw "gunicorn-genenetwork3")) - (passwd:gid (getpw "gunicorn-genenetwork3"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-genenetwork3")) + (passwd:gid (getpw "gunicorn-genenetwork3")))) + (cons #$gn3-secrets + (find-files #$(dirname llm-db-path) + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -322,7 +348,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -388,7 +414,7 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) @@ -401,13 +427,14 @@ object." (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (source gn-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -442,7 +469,7 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) @@ -469,8 +496,8 @@ object." (source gn3-data-directory) (target source)) ; Rqtl usese this (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (source gn-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) @@ -539,7 +566,8 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" @@ -575,6 +603,57 @@ a @code{<genenetwork-configuration>} record." #:user "gunicorn-gn-auth") )) +(define (gn-guile-gexp gn-guile-port) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + +(define (gn-guile-shepherd-service config) + (match-record config <genenetwork-configuration> + (gn-doc-git-checkout gn-guile-port) + (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/gn-guile.log"))) + (stop #~(make-kill-destructor))))) + (define genenetwork-service-type (service-type (name 'genenetwork) @@ -588,6 +667,8 @@ a @code{<genenetwork-configuration>} record." genenetwork-gunicorn-apps) (service-extension forge-nginx-service-type genenetwork-nginx-server-blocks) + (service-extension shepherd-root-service-type + (compose list gn-guile-shepherd-service)) (service-extension mcron-service-type genenetwork-mcron-jobs))) (default-value (genenetwork-configuration)))) @@ -619,22 +700,22 @@ a @code{<genenetwork-configuration>} record." (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t))) - (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOAD_FOLDER" ,(string-append data-directory + "/uploads")) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) |