diff options
Diffstat (limited to 'genenetwork/services/genenetwork.scm')
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 365 |
1 files changed, 301 insertions, 64 deletions
diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index 34d70df..4aa35b9 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -21,16 +21,23 @@ (define-module (genenetwork services genenetwork) #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn packages guile) #:select (gn-guile)) + #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) #:use-module ((gnu packages admin) #:select (shadow shepherd)) + #:use-module ((gnu packages version-control) #:select (git-minimal)) #:use-module ((gnu packages python) #:select (python)) #:use-module (gnu services) #:use-module (gnu services web) #:use-module (gnu services mcron) + #:use-module (gnu services shepherd) #:use-module (gnu system file-systems) #:use-module (gnu system shadow) #:use-module (guix build python-build-system) + #:use-module (guix diagnostics) #:use-module (guix gexp) + #:use-module (guix i18n) + #:use-module (guix least-authority) #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix records) @@ -38,6 +45,7 @@ #:use-module (forge nginx) #:use-module (forge gunicorn) #:use-module (forge socket) + #:use-module (forge utils) #:use-module (srfi srfi-1) #:use-module (ice-9 match) #:export (genenetwork-service-type @@ -49,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -83,10 +92,14 @@ (default 8083)) (gn-auth-port genenetwork-configuration-gn-auth-port (default 8084)) + (gn3-alias-server-port genenetwork-gn3-alias-server-port + (default 8000)) (sql-uri genenetwork-configuration-sql-uri (default "mysql://username:password@localhost/database")) (auth-db genenetwork-configuration-auth-db (default "/var/genenetwork/auth.db")) + (llm-db-path genenetwork-configuration-llm-db-path + (default "/var/genenetwork/llm.db")) (xapian-db genenetwork-configuration-xapian-db (default "/var/genenetwork/xapian")) (genotype-files genenetwork-configuration-genotype-files @@ -94,15 +107,30 @@ (sparql-endpoint genenetwork-configuration-sparql-endpoint (default "http://localhost:8081/sparql")) (gn-sourcecode-directory genenetwork-configuration-gn-sourcecode-directory - (default "/var/empty")) + (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) + (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir + (default "/var/genenetwork/sessions/genenetwork2")) (gn2-secrets genenetwork-configuration-gn2-secrets (default "/etc/genenetwork")) (gn3-secrets genenetwork-configuration-gn3-secrets (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets - (default "/etc/genenetwork"))) + (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/export/data/gn-docs")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (log-level genenetwork-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) (define-record-type* <gn-uploader-configuration> gn-uploader-configuration make-gn-uploader-configuration @@ -119,11 +147,24 @@ (default "/var/genenetwork")) (secrets gn-uploader-configuration-secrets (default "/etc/genenetwork/gn-uploader-secrets.py")) - (auth-server-url gn-uploader-auth-server-url + (auth-server-url gn-uploader-configuration-auth-server-url (default "https://auth.genenetwork.org")) - (gn2-server-url gn-uploader-gn2-server-url + (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) - (log-level gn-uploader-log-level (default "WARNING"))) + (sessions-dir gn-uploader-sessions-dir + (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (log-level gn-uploader-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) + +(define (sanitize-log-level log-level) + (case log-level + ((fatal error warning info debug trace) log-level) + (else + (leave (G_ "Log level ~a is invalid. It must be one of the following symbols---fatal, error, warn, info, debug or trace.~%") + log-level)))) (define %genenetwork-accounts (list (user-group @@ -142,12 +183,14 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) (gn3-profile (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t))) + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) (python3-version (python-version (package-version python)))) (with-imported-modules '((guix build utils)) #~(begin @@ -175,7 +218,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -184,7 +229,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -214,7 +261,7 @@ (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -234,11 +281,19 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets + #:directories? #t) + (find-files #$gn2-sessions-dir #:directories? #t))) - (chown #$gn3-secrets - (passwd:uid (getpw "gunicorn-genenetwork3")) - (passwd:gid (getpw "gunicorn-genenetwork3"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-genenetwork3")) + (passwd:gid (getpw "gunicorn-genenetwork3")))) + (cons #$gn3-secrets + (find-files #$(dirname llm-db-path) + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -246,7 +301,31 @@ (find-files #$gn2-secrets #:directories? #f) (find-files #$gn-auth-secrets - #:directories? #f))))))) + #:directories? #f))) + ;; Make sub-directories for various apps under gn-tmpdir and assign + ;; appropriate permissions + (for-each (match-lambda + ((subdir user) + (let ((full-path + (string-append #$gn-tmpdir "/" subdir))) + (unless (file-exists? full-path) + (mkdir full-path #o755)) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw user)) + (passwd:gid (getpw user)))) + (find-files full-path + #:directories? #t))))) + '(("gn2-tmpdir" "gunicorn-genenetwork2") + ("gn3-tmpdir" "gunicorn-genenetwork3"))) + + ;; setup correct ownership for gn-docs + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (find-files #$(dirname gn-doc-git-checkout) + #:directories? #t)))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -277,7 +356,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -287,6 +366,7 @@ object." (gn2-profile (profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) + (gn2-ca-bundle (file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) (gn2-conf (computed-file "gn2.conf" (configuration-file-gexp `(("GN2_SECRETS" ,(string-append gn2-secrets "/gn2-secrets.py")) @@ -295,28 +375,45 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("SSL_PRIVATE_KEY" ,(string-append gn2-secrets "/gn2-ssl-private-key.pem")) - ("AUTH_SERVER_SSL_PUBLIC_KEY" ,(string-append gn2-secrets "/gn-auth-ssl-public-key.pem")))))) + ("AI_SEARCH_ENABLED" "True") + ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir) + ("MAX_FORM_MEMORY_SIZE" 52428800))))) + (gn3-profile (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (gn3-ca-bundle (file-append gn3-profile "/etc/ssl/certs/ca-certificates.crt")) (gn3-conf (computed-file "gn3.conf" (configuration-file-gexp `(("AUTH_DB" ,auth-db) + ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) ("SQL_URI" ,sql-uri) - ("XAPIAN_DB_PATH" ,xapian-db))))) + ("XAPIAN_DB_PATH" ,xapian-db) + ("GENOTYPE_FILES" ,genotype-files) + ("REAPER_COMMAND" ,(file-append gn2-profile "/bin/qtlreaper")) + ("LLM_DB_PATH" ,llm-db-path))))) + (gn-auth-profile (profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (gn-auth-ca-bundle (file-append gn-auth-profile "/etc/ssl/certs/ca-certificates.crt")) (gn-auth-conf (computed-file "gn-auth.conf" (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) ("AUTH_DB" ,auth-db) - ("SQL_URI" ,sql-uri) - ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys")) - ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem"))))))) + ("SQL_URI" ,sql-uri))))) + (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir")) + (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir"))) (list (gunicorn-app (name "genenetwork2") (package genenetwork2) @@ -331,23 +428,30 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn2-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) (environment-variable (name "HOME") - (value "/tmp")))) + (value "/tmp")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn2-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn2-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -358,7 +462,17 @@ object." (file-system-mapping (source gn2-secrets) (target source) - (writable? #t))))) + (writable? #t)) + (file-system-mapping + (source gn2-ca-bundle) + (target source)) + (file-system-mapping + (source gn2-sessions-dir) + (target source) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "genenetwork3") (package genenetwork3) @@ -376,13 +490,16 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn3-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) (environment-variable (name "HOME") - (value "/tmp")))) + (value "/tmp")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn3-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn3-conf) @@ -399,17 +516,26 @@ object." (file-system-mapping (source gn3-data-directory) (target source)) ; Rqtl usese this - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn3-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) (target source)) (file-system-mapping - (source auth-db) + (source llm-db-path) (target source) - (writable? #t))))) + (writable? #t)) + (file-system-mapping + (source gn3-ca-bundle) + (target source)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "gn-auth") (package gn-auth) @@ -417,6 +543,7 @@ object." (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -426,7 +553,10 @@ object." (value "/tmp")) (environment-variable (name "AUTHLIB_INSECURE_TRANSPORT") - (value "true")))) + (value "true")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn-auth-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn-auth-conf) @@ -438,14 +568,20 @@ object." (file-system-mapping (source gn-auth-secrets) (target source) - (writable? #t))))))))) + (writable? #t)) + (file-system-mapping + (source gn-auth-ca-bundle) + (target source)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (genenetwork-nginx-server-blocks config) "Return a list of @code{<nginx-server-configuration>} records specifying reverse proxies for the genenetwork service described by @var{config}, a @code{<genenetwork-configuration>} record." (match-record config <genenetwork-configuration> - (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port) + (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port gn3-alias-server-port) (list (nginx-server-configuration (server-name (list server-name)) (locations @@ -455,13 +591,23 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" (string-append "proxy_pass http://localhost:" (number->string gn3-port) ";") - "proxy_set_header Host $host;")))))) + "proxy_set_header Host $host;"))) + (nginx-location-configuration + (uri "/gn3/") + (body + (list "rewrite /gn3/(.*) /$1 break;" + (string-append "proxy_pass http://localhost:" + (number->string gn3-alias-server-port) + ";") + "proxy_redirect off;" + "proxy_set_header Host $host;")))))) (nginx-server-configuration (server-name (list gn-auth-server-name)) (locations @@ -477,6 +623,69 @@ a @code{<genenetwork-configuration>} record." (build-xapian-index-cron-gexp config)) #:user "root"))) +(define (gn-guile-gexp gn-guile-port gn-guile-pkg) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile-pkg)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (current-repo-path (string-append (pk "CWD" (getcwd)) "/gn-docs"))) + ;; These have to be setup manually here an not in the + ;; `gn-guile-shepherd-service' function, otherwise, they do not take + ;; effect for some reason. + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + +(define (gn-guile-shepherd-service config) + (match-record config <genenetwork-configuration> + (gn-guile gn-doc-git-checkout gn-guile-port) + (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port gn-guile)) + #:name "gn-guile-pola-wrapper" + #:directory (dirname gn-doc-git-checkout) + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source (dirname gn-doc-git-checkout)) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/gn-guile.log"))) + (stop #~(make-kill-destructor))))) + (define genenetwork-service-type (service-type (name 'genenetwork) @@ -490,19 +699,27 @@ a @code{<genenetwork-configuration>} record." genenetwork-gunicorn-apps) (service-extension forge-nginx-service-type genenetwork-nginx-server-blocks) + (service-extension shepherd-root-service-type + (compose list gn-guile-shepherd-service)) (service-extension mcron-service-type genenetwork-mcron-jobs))) (default-value (genenetwork-configuration)))) (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory) + (secrets data-directory sessions-dir sqlite-databases-directory) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Let service user own their own secrets files. - (chown #$secrets - (passwd:uid (getpw "gunicorn-gn-uploader")) - (passwd:gid (getpw "gunicorn-gn-uploader"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader")))) + (append (list #$secrets) + (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -518,24 +735,27 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t)))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOAD_FOLDER" ,(string-append data-directory + "/uploads")) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) @@ -543,6 +763,7 @@ a @code{<genenetwork-configuration>} record." (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -552,7 +773,10 @@ a @code{<genenetwork-configuration>} record." (value "/tmp")) (environment-variable (name "GN_UPLOADER_ENVIRONMENT") - (value gn-uploader-profile)))) + (value gn-uploader-profile)) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn-uploader-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn-uploader-conf) @@ -566,8 +790,21 @@ a @code{<genenetwork-configuration>} record." (writable? #t)) (file-system-mapping (source gn-uploader-profile) - (target source)))) - (extra-cli-arguments (list "--log-level" log-level))))))) + (target source)) + (file-system-mapping + (source gn-uploader-ca-bundle) + (target source)) + (file-system-mapping + (source sessions-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (gn-uploader-nginx-server-block config) (match-record config <gn-uploader-configuration> @@ -582,7 +819,7 @@ a @code{<genenetwork-configuration>} record." #$(file-append gn-uploader "/lib/python" (python-version (package-version python)) - "/site-packages/qc_app;"))))) + "/site-packages/uploader;"))))) (nginx-location-configuration (uri "/") (body (list (string-append "proxy_pass http://localhost:" |