aboutsummaryrefslogtreecommitdiff
path: root/genenetwork/services/genenetwork.scm
diff options
context:
space:
mode:
Diffstat (limited to 'genenetwork/services/genenetwork.scm')
-rw-r--r--genenetwork/services/genenetwork.scm149
1 files changed, 107 insertions, 42 deletions
diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm
index b863bcf..a403f21 100644
--- a/genenetwork/services/genenetwork.scm
+++ b/genenetwork/services/genenetwork.scm
@@ -109,6 +109,8 @@
(default "/var/empty"))
(gn3-data-directory genenetwork-configuration-gn3-data-directory
(default "/var/genenetwork"))
+ (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir
+ (default "/var/genenetwork/sessions/genenetwork2"))
(gn2-secrets genenetwork-configuration-gn2-secrets
(default "/etc/genenetwork"))
(gn3-secrets genenetwork-configuration-gn3-secrets
@@ -119,6 +121,10 @@
(default 8091))
(gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout
(default "/export/data/gn-docs"))
+ (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory
+ (default "/export/data/virtuoso/ttl"))
+ (gn-tmpdir genenetwork-configuration-gn-tmpdir
+ (default "/opt/gn/tmp"))
(log-level genenetwork-configuration-log-level
(default 'warning)
(sanitize sanitize-log-level)))
@@ -142,6 +148,8 @@
(default "https://auth.genenetwork.org"))
(gn2-server-url gn-uploader-configuration-gn2-server-url
(default "https://genenetwork.org"))
+ (sessions-dir gn-uploader-sessions-dir
+ (default "/var/genenetwork/sessions/gn-uploader"))
(log-level gn-uploader-configuration-log-level
(default 'warning)
(sanitize sanitize-log-level)))
@@ -170,6 +178,8 @@
(genenetwork3 (genenetwork-configuration-genenetwork3 config))
(xapian-directory (genenetwork-configuration-xapian-db config))
(sparql-endpoint (genenetwork-configuration-sparql-endpoint config))
+ (virtuoso-ttl-directory
+ (genenetwork-configuration-gn-virtuoso-ttl-directory config))
(xapian-build-directory (string-append xapian-directory "/build"))
(herd (file-append shepherd "/bin/herd"))
(index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork"))
@@ -203,7 +213,9 @@
"is-data-modified"
#$xapian-directory
#$sql-uri
- #$sparql-endpoint))))
+ #$sparql-endpoint
+ "--virtuoso-ttl-directory"
+ #$virtuoso-ttl-directory))))
(dynamic-wind
(const #t)
;; build the index
@@ -212,7 +224,9 @@
"create-xapian-index"
#$xapian-build-directory
#$sql-uri
- #$sparql-endpoint)
+ #$sparql-endpoint
+ "--virtuoso-ttl-directory"
+ #$virtuoso-ttl-directory)
(dynamic-wind
;; stop GN3: Here there is magic!!!
;; The name `gunicorn-genenetwork' is magical. It is not set
@@ -242,7 +256,7 @@
(define (genenetwork-activation config)
(match-record config <genenetwork-configuration>
- (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path)
+ (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
@@ -262,7 +276,11 @@
(passwd:uid (getpw "gunicorn-genenetwork2"))
(passwd:gid (getpw "gunicorn-genenetwork2"))))
(append (list #$gn2-secrets)
+ (find-files #$genotype-files
+ #:directories? #t)
(find-files #$gn2-secrets
+ #:directories? #t)
+ (find-files #$gn2-sessions-dir
#:directories? #t)))
(for-each (lambda (file)
(chown file
@@ -278,7 +296,31 @@
(find-files #$gn2-secrets
#:directories? #f)
(find-files #$gn-auth-secrets
- #:directories? #f)))))))
+ #:directories? #f)))
+ ;; Make sub-directories for various apps under gn-tmpdir and assign
+ ;; appropriate permissions
+ (for-each (match-lambda
+ ((subdir user)
+ (let ((full-path
+ (string-append #$gn-tmpdir "/" subdir)))
+ (unless (file-exists? full-path)
+ (mkdir full-path #o755))
+ (for-each (lambda (file)
+ (chown file
+ (passwd:uid (getpw user))
+ (passwd:gid (getpw user))))
+ (find-files full-path
+ #:directories? #t)))))
+ '(("gn2-tmpdir" "gunicorn-genenetwork2")
+ ("gn3-tmpdir" "gunicorn-genenetwork3")))
+
+ ;; setup correct ownership for gn-docs
+ (for-each (lambda (file)
+ (chown file
+ (passwd:uid (getpw "genenetwork"))
+ (passwd:gid (getpw "genenetwork"))))
+ (find-files #$gn-doc-git-checkout
+ #:directories? #t))))))
(define (configuration-file-gexp alist)
"Return a G-expression that constructs a configuration file of
@@ -309,7 +351,7 @@ G-expressions or numbers."
described by @var{config}, a @code{<genenetwork-configuration>}
object."
(match-record config <genenetwork-configuration>
- (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path log-level)
+ (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level)
;; If we mapped only the mysqld.sock socket file, it would break
;; when the external mysqld server is restarted.
(let* ((database-mapping (file-system-mapping
@@ -333,7 +375,9 @@ object."
("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript"))
("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2"))
("SQL_URI" ,sql-uri)
- ("AI_SEARCH_ENABLED" "True")))))
+ ("AI_SEARCH_ENABLED" "True")
+ ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir)
+ ("MAX_FORM_MEMORY_SIZE" 52428800)))))
(gn3-profile (profile
(content (package->development-manifest genenetwork3))
(allow-collisions? #t)))
@@ -358,9 +402,9 @@ object."
(configuration-file-gexp
`(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py"))
("AUTH_DB" ,auth-db)
- ("SQL_URI" ,sql-uri)
- ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys"))
- ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem")))))))
+ ("SQL_URI" ,sql-uri)))))
+ (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir"))
+ (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir")))
(list (gunicorn-app
(name "genenetwork2")
(package genenetwork2)
@@ -375,7 +419,7 @@ object."
(value gn2-profile))
(environment-variable
(name "TMPDIR")
- (value "/tmp"))
+ (value gn2-tmpdir))
(environment-variable
(name "GN2_SETTINGS")
(value gn2-conf))
@@ -388,13 +432,17 @@ object."
(mappings (list database-mapping
(file-system-mapping
(source genotype-files)
- (target source))
+ (target source)
+ (writable? #t))
(file-system-mapping
(source gn-sourcecode-directory)
(target source))
- (file-system-mapping ; GN2 and GN3 need to share TMPDIR
- (source "/tmp")
- (target "/tmp")
+ (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR
+ (source gn-tmpdir)
+ (target source))
+ (file-system-mapping
+ (source gn2-tmpdir)
+ (target source)
(writable? #t))
(file-system-mapping
(source gn2-conf)
@@ -408,7 +456,11 @@ object."
(writable? #t))
(file-system-mapping
(source gn2-ca-bundle)
- (target source))))
+ (target source))
+ (file-system-mapping
+ (source gn2-sessions-dir)
+ (target source)
+ (writable? #t))))
(extra-cli-arguments
(list "--log-level"
(string-upcase (symbol->string log-level)))))
@@ -429,7 +481,7 @@ object."
(value gn3-conf))
(environment-variable
(name "TMPDIR")
- (value "/tmp"))
+ (value gn3-tmpdir))
(environment-variable
(name "GN3_SECRETS")
(value gn3-secrets))
@@ -455,9 +507,12 @@ object."
(file-system-mapping
(source gn3-data-directory)
(target source)) ; Rqtl usese this
- (file-system-mapping ; GN2 and GN3 need to share TMPDIR
- (source "/tmp")
- (target "/tmp")
+ (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR
+ (source gn-tmpdir)
+ (target source))
+ (file-system-mapping
+ (source gn3-tmpdir)
+ (target source)
(writable? #t))
(file-system-mapping
(source xapian-db)
@@ -526,7 +581,8 @@ a @code{<genenetwork-configuration>} record."
(number->string gn2-port) ";")
"proxy_set_header Host $host;"
"proxy_read_timeout 20m;"
- "proxy_set_header X-Forwarded-Proto $scheme;")))
+ "proxy_set_header X-Forwarded-Proto $scheme;"
+ "client_max_body_size 8050m;")))
(nginx-location-configuration
(uri "/api3/")
(body (list "rewrite /api3/(.*) /api/$1 break;"
@@ -628,14 +684,18 @@ a @code{<genenetwork-configuration>} record."
(define (gn-uploader-activation config)
(match-record config <gn-uploader-configuration>
- (secrets data-directory)
+ (secrets data-directory sessions-dir)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
;; Let service user own their own secrets files.
- (chown #$secrets
- (passwd:uid (getpw "gunicorn-gn-uploader"))
- (passwd:gid (getpw "gunicorn-gn-uploader")))
+ (for-each (lambda (file)
+ (chown file
+ (passwd:uid (getpw "gunicorn-gn-uploader"))
+ (passwd:gid (getpw "gunicorn-gn-uploader"))))
+ (append (list #$secrets)
+ (find-files #$sessions-dir
+ #:directories? #t)))
;; Set owner-only permissions on secrets files.
(for-each (lambda (file)
(chmod file #o600))
@@ -651,25 +711,26 @@ a @code{<genenetwork-configuration>} record."
(define (gn-uploader-gunicorn-app config)
(match-record config <gn-uploader-configuration>
- (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url)
+ (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir)
;; If we mapped only the mysqld.sock socket file, it would break
;; when the external mysqld server is restarted.
- (let ((database-mapping (file-system-mapping
- (source "/run/mysqld")
- (target source)
- (writable? #t)))
- (gn-uploader-conf (computed-file "gn-uploader.conf"
- (configuration-file-gexp
- `(("UPLOADER_SECRETS" ,secrets)
- ("SQL_URI" ,sql-uri)
- ("UPLOAD_FOLDER" ,(string-append data-directory
- "/uploads"))
- ("AUTH_SERVER_URL" ,auth-server-url)
- ("GN2_SERVER_URL" ,gn2-server-url)))))
- (gn-uploader-profile (profile
- (content (package->development-manifest gn-uploader))
- (allow-collisions? #t)))
- (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt")))
+ (let* ((database-mapping (file-system-mapping
+ (source "/run/mysqld")
+ (target source)
+ (writable? #t)))
+ (gn-uploader-conf (computed-file "gn-uploader.conf"
+ (configuration-file-gexp
+ `(("UPLOADER_SECRETS" ,secrets)
+ ("SQL_URI" ,sql-uri)
+ ("UPLOAD_FOLDER" ,(string-append data-directory
+ "/uploads"))
+ ("AUTH_SERVER_URL" ,auth-server-url)
+ ("GN2_SERVER_URL" ,gn2-server-url)
+ ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir)))))
+ (gn-uploader-profile (profile
+ (content (package->development-manifest gn-uploader))
+ (allow-collisions? #t)))
+ (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt")))
(list (gunicorn-app
(name "gn-uploader")
(package gn-uploader)
@@ -706,7 +767,11 @@ a @code{<genenetwork-configuration>} record."
(target source))
(file-system-mapping
(source gn-uploader-ca-bundle)
- (target source))))
+ (target source))
+ (file-system-mapping
+ (source sessions-dir)
+ (target source)
+ (writable? #t))))
(extra-cli-arguments
(list "--log-level"
(string-upcase (symbol->string log-level)))))))))
generated by cgit v1.2.3 (git 2.47.1) at 2025-04-11 00:05:27 +0000