diff options
Diffstat (limited to 'genenetwork/services')
| -rw-r--r-- | genenetwork/services/genecup.scm | 121 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 123 | ||||
| -rw-r--r-- | genenetwork/services/mouse-longevity.scm | 33 |
3 files changed, 254 insertions, 23 deletions
diff --git a/genenetwork/services/genecup.scm b/genenetwork/services/genecup.scm new file mode 100644 index 0000000..6ee2812 --- /dev/null +++ b/genenetwork/services/genecup.scm @@ -0,0 +1,121 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services genecup) + #:use-module (guix) + #:use-module (gnu) + #:use-module (guix git) + #:use-module (guix modules) + #:use-module (guix profiles) + #:use-module (guix records) + #:use-module (srfi srfi-1) + #:use-module (ice-9 match) + #:use-module (forge utils) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gn packages machine-learning) + #:use-module (gnu packages certs) + #:use-module (gnu packages curl) + #:use-module (gn packages python) + #:use-module (gnu packages admin) + #:use-module (gnu packages bioinformatics) + #:use-module (gnu packages compression) + #:use-module ((gnu packages python) #:select (python)) + #:use-module (gnu packages python-xyz) + #:use-module (guix build python-build-system) + #:use-module (gnu packages python-crypto) + #:use-module (gnu packages python-web) + #:use-module (gnu services shepherd) + #:use-module (gnu packages python-science) + #:use-module (gnu packages machine-learning) + #:use-module ((gnu packages admin) #:select (shepherd)) + #:export (genecup-configuration + genecup-configuration? + genecup-configuration-package + genecup-service-type)) + +(define-record-type* <genecup-configuration> + genecup-configuration make-genecup-configuration + genecup-configuration? + (package genecup-configuration-package ; <package> + (default genecup-latest-with-tensorflow-native))) + +(define (genecup-activation config) + (match-record config <genecup-configuration> + (package) + (with-packages + (list python + python-nltk + nss-certs ; Needed for downloading data with nltk.downloader. + curl) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((nltk-data "/var/cache/nltk_data/") + (data-dir "/export/ratspub/tmp")) + (unless (file-exists? nltk-data) + (begin + (mkdir-p nltk-data) + (chdir nltk-data) + (invoke #$(file-append python "/bin/python3") "-m" "nltk.downloader" "-d" nltk-data "punkt"))) + (unless (file-exists? (string-append data-dir "/userspub.sqlite")) + (begin + (install-file #$(file-append package "/userspub.sqlite") data-dir) + (chmod (string-append data-dir "/userspub.sqlite") #o554))))))))) + +(define genecup-shepherd-service + (match-lambda + (($ <genecup-configuration> package) + (with-imported-modules (source-module-closure + '((guix search-paths) + (gnu build shepherd) + (gnu system file-systems))) + (list (shepherd-service + (provision '(genecup)) + (requirement '(networking)) + (modules '((gnu build shepherd) + (gnu system file-systems))) + (start + #~(make-forkexec-constructor + (list #$(file-append package "/server.py")) + #:directory #$package + #:environment-variables + (list + "NLTK_DATA=/var/cache/nltk_data" + (string-append "EDIRECT_PUBMED_MASTER=" + #$(file-append package "/minipubmed")) + (string-append "PERL_LWP_SSL_CA_FILE=" + #$(file-append (profile (content (packages->manifest (list curl nss-certs)))) + "/etc/ssl/certs/ca-certificates.crt"))) + #:log-file "/var/log/genecup.log")) + (stop #~(make-kill-destructor)))))))) + +(define genecup-service-type + (service-type + (name 'genecup) + (extensions + (list + (service-extension activation-service-type + genecup-activation) + (service-extension shepherd-root-service-type + genecup-shepherd-service))) + (default-value (genecup-configuration)) + (description + "Run a GeneCup Webserver."))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index a403f21..5c6b9f0 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -20,7 +20,7 @@ ;;; <https://www.gnu.org/licenses/>. (define-module (genenetwork services genenetwork) - #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) #:use-module ((gn packages guile) #:select (gn-guile)) #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) @@ -57,6 +57,7 @@ genenetwork-configuration-port ; external port genenetwork-configuration-gn2-port ; internal port genenetwork-configuration-gn3-port ; internal port + genenetwork-configuration-gn-guile-port ; aka gn4 internal port (may be external) genenetwork-configuration-auth-db ; RW auth DB genenetwork-configuration-xapian-db ; RO search index, unless you want to regenerate inside VM genenetwork-configuration-genotype-files ; RO genotype files @@ -117,6 +118,8 @@ (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork")) + (gn-guile genenetwork-configuration-gn-guile + (default gn-guile)) (gn-guile-port genenetwork-configuration-gn-guile-port (default 8091)) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout @@ -150,6 +153,12 @@ (default "https://genenetwork.org")) (sessions-dir gn-uploader-sessions-dir (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (gn-tmpdir gn-uploader-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (genotype-files-directory gn-uploader-configuration-genotype-files-directory + (default "/var/genenetwork/genotype-files")) (log-level gn-uploader-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -254,6 +263,32 @@ (chmod file #o644)) (find-files #$xapian-directory))))))))) +(define (samples-count-script-gexp config) + (match-record config <genenetwork-configuration> + (genenetwork2 sql-uri) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (setenv "PYTHONPATH" + (string-append + #$(file-append genenetwork2 + "/lib/python" + (python-version (package-version python)) + "/site-packages") + ":" + #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "/lib/python" + #$(python-version (package-version python)) + "/site-packages")) + + (invoke #$(file-append python "/bin/python3") + "-m" + "gn2.scripts.sample_count" + #$sql-uri))))) + (define (genenetwork-activation config) (match-record config <genenetwork-configuration> (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) @@ -319,7 +354,7 @@ (chown file (passwd:uid (getpw "genenetwork")) (passwd:gid (getpw "genenetwork")))) - (find-files #$gn-doc-git-checkout + (find-files #$(dirname gn-doc-git-checkout) #:directories? #t)))))) (define (configuration-file-gexp alist) @@ -351,7 +386,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level gn-guile-port) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -370,6 +405,8 @@ object." ("GENENETWORK_FILES" ,genotype-files) ("GN3_LOCAL_URL" ,(string-append "http://localhost:" (number->string gn3-port))) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/" )) ("GN_SERVER_URL" ,(string-append "https://" server-name "/api3/")) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) @@ -386,6 +423,8 @@ object." (configuration-file-gexp `(("AUTH_DB" ,auth-db) ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) + ("GN_GUILE_SERVER_URL" ,(string-append "http://localhost:" ; AKA GN4 + (number->string gn-guile-port) "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) @@ -518,7 +557,7 @@ object." (source xapian-db) (target source)) (file-system-mapping - (source llm-db-path) + (source (dirname llm-db-path)) (target source) (writable? #t)) (file-system-mapping @@ -534,6 +573,7 @@ object." (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -552,9 +592,9 @@ object." (source gn-auth-conf) (target source)) (file-system-mapping - (source auth-db) - (target source) - (writable? #t)) + (source (dirname auth-db)) + (target source) + (writable? #t)) (file-system-mapping (source gn-auth-secrets) (target source) @@ -611,24 +651,38 @@ a @code{<genenetwork-configuration>} record." (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron-gexp" (build-xapian-index-cron-gexp config)) - #:user "root"))) + #:user "root") + #~(job '(next-minute-from (next-hour) '(17)) ;17th minute of every hour + #$(program-file "samples-count-script-gexp" + (samples-count-script-gexp config))))) -(define (gn-guile-gexp gn-guile-port) +(define (gn-guile-gexp gn-guile-port gn-guile-pkg bare-repo) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) - (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (let* ((gn-guile-profile #$(profile (content (package->development-manifest gn-guile-pkg)) + (allow-collisions? #t))) + (ssl-cert-dir (string-append gn-guile-profile "/etc/ssl/certs")) + (ssl-cert-file (string-append ssl-cert-dir "/ca-certificates.crt")) + (current-repo-path (string-append (pk "CWD" (getcwd)) "/gn-docs"))) + ;; These have to be setup manually here an not in the + ;; `gn-guile-shepherd-service' function, otherwise, they do not take + ;; effect for some reason. + (setenv "SSL_CERT_DIR" ssl-cert-dir) + (setenv "SSL_CERT_FILE" ssl-cert-file) + (setenv "GUILE_TLS_CERTIFICATE_DIRECTORY" ssl-cert-dir) + (when (file-exists? current-repo-path) (delete-file-recursively current-repo-path)) (setenv "CURRENT_REPO_PATH" current-repo-path) (invoke #$(file-append git-minimal "/bin/git") - "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + "clone" "--depth" "1" #$(string-append "file://" bare-repo))) (invoke #$(file-append gn-guile "/bin/gn-guile") (number->string #$gn-guile-port))))) (define (gn-guile-shepherd-service config) (match-record config <genenetwork-configuration> - (gn-doc-git-checkout gn-guile-port) + (gn-guile gn-doc-git-checkout gn-guile-port) (shepherd-service (documentation "Run gn-guile server.") (provision '(gn-guile)) @@ -644,14 +698,15 @@ a @code{<genenetwork-configuration>} record." #~(make-forkexec-constructor (list #$(least-authority-wrapper (program-file "gn-guile" - (gn-guile-gexp gn-guile-port)) + (gn-guile-gexp gn-guile-port gn-guile gn-doc-git-checkout)) #:name "gn-guile-pola-wrapper" + #:directory (dirname gn-doc-git-checkout) #:preserved-environment-variables (map first gn-guile-settings) #:mappings (list (file-system-mapping - (source gn-doc-git-checkout) - (target source) - (writable? #t))) + (source (dirname gn-doc-git-checkout)) + (target source) + (writable? #t))) #:namespaces (delq 'net %namespaces)) "127.0.0.1" #$(number->string gn-guile-port)) #:user "genenetwork" @@ -684,7 +739,7 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory sessions-dir) + (secrets data-directory sessions-dir sqlite-databases-directory gn-tmpdir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -693,8 +748,10 @@ a @code{<genenetwork-configuration>} record." (chown file (passwd:uid (getpw "gunicorn-gn-uploader")) (passwd:gid (getpw "gunicorn-gn-uploader")))) - (append (list #$secrets) + (append (list #$(dirname secrets)) (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) @@ -707,26 +764,32 @@ a @code{<genenetwork-configuration>} record." (passwd:gid (getpw "gunicorn-gn-uploader")))) (append (list #$data-directory) (find-files #$data-directory + #:directories? #t) + (find-files #$(string-append gn-tmpdir "/gn-uploader-tmpdir") #:directories? #t))))))) (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory gn-tmpdir genotype-files-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping (source "/run/mysqld") (target source) (writable? #t))) + (gn-uploader-tmpdir (string-append gn-tmpdir "/gn-uploader-tmpdir")) (gn-uploader-conf (computed-file "gn-uploader.conf" (configuration-file-gexp `(("UPLOADER_SECRETS" ,secrets) ("SQL_URI" ,sql-uri) ("UPLOAD_FOLDER" ,(string-append data-directory "/uploads")) + ("TEMPORARY_DIRECTORY" ,gn-uploader-tmpdir) ("AUTH_SERVER_URL" ,auth-server-url) ("GN2_SERVER_URL" ,gn2-server-url) - ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir))))) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")) + ("GENOTYPE_FILES_DIRECTORY" ,genotype-files-directory))))) (gn-uploader-profile (profile (content (package->development-manifest gn-uploader)) (allow-collisions? #t))) @@ -738,6 +801,7 @@ a @code{<genenetwork-configuration>} record." (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -756,8 +820,9 @@ a @code{<genenetwork-configuration>} record." (source gn-uploader-conf) (target source)) (file-system-mapping - (source secrets) - (target source)) + (source (dirname secrets)) + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source) @@ -771,7 +836,19 @@ a @code{<genenetwork-configuration>} record." (file-system-mapping (source sessions-dir) (target source) - (writable? #t)))) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-tmpdir) + (target source) + (writable? #t)) + (file-system-mapping + (source genotype-files-directory) + (target source) + (writable? #f)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))))))) diff --git a/genenetwork/services/mouse-longevity.scm b/genenetwork/services/mouse-longevity.scm new file mode 100644 index 0000000..c9b977d --- /dev/null +++ b/genenetwork/services/mouse-longevity.scm @@ -0,0 +1,33 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2024 jgart <jgart@dismail.de> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (genenetwork services mouse-longevity) + #:use-module (gn packages genecup) + #:use-module (gn packages mouse-longevity) + #:use-module (gn services rshiny) + #:use-module (gnu services) + #:export (mouse-longevity-service)) + +(define* (mouse-longevity-service #:optional (package mouse-longevity-app)) + (simple-service 'mouse-longevity + rshiny-service-type + (list + (rshiny-configuration + (package package) + (binary "mouse-longevity-app"))))) |