diff options
| -rw-r--r-- | .guix-channel | 2 | ||||
| -rwxr-xr-x | genenetwork-development-deploy.sh | 2 | ||||
| -rw-r--r-- | genenetwork-development.scm | 36 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 8 | ||||
| -rw-r--r-- | guix/gn-machines/genenetwork.scm | 57 | ||||
| -rw-r--r-- | guix/gn-machines/services/monitoring.scm | 68 | ||||
| -rwxr-xr-x | production-deploy.sh | 4 | ||||
| -rw-r--r-- | production.scm | 4 | ||||
| -rw-r--r-- | public-sparql.scm | 4 | ||||
| -rw-r--r-- | services/README.md | 17 | ||||
| -rw-r--r-- | services/gn-guile.scm | 52 | ||||
| -rw-r--r-- | services/opensmtpd.scm | 21 |
12 files changed, 155 insertions, 120 deletions
diff --git a/.guix-channel b/.guix-channel index f43f5ea..b8f3873 100644 --- a/.guix-channel +++ b/.guix-channel @@ -27,7 +27,7 @@ (channel (name guix-bioinformatics) (url "https://git.genenetwork.org/guix-bioinformatics") - (commit "903465c85c9b2ae28480b236c3364da873ca8f51")) + (commit "9b0955f14ec725990abb1f6af3b9f171e4943f77")) ;; Until https://issues.guix.gnu.org/68797 is resolved, we need to ;; explicitly list guix-past and guix-rust-past-crates—the ;; dependencies of the guix-bioinformatics channel—here. diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index d63dcf0..53c53d9 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -29,7 +29,7 @@ # /etc/genenetwork/conf instead of merely exposing it. container_script=$(guix system container --network \ --verbosity=3 \ - --load-path=. \ + --load-path=./guix/ \ --share=/home/git/public \ --share=/var/guix/daemon-socket=/var/host-guix/daemon-socket \ --share=/export2/guix-containers/genenetwork-development/var/lib/acme=/var/lib/acme \ diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 8e4e1e8..bfd2fc1 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,6 +21,7 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) + (gn-machines services monitoring) ((gn-machines genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) (gn services databases) ((gn packages guile) #:select (gn-guile)) @@ -529,8 +530,11 @@ server described by CONFIG, a <genenetwork-configuration> object." (getenv "GN2_PROFILE") "/etc/ssl/certs/ca-certificates.crt")) (setenv "PYTHONPATH" (string-append - (getenv "GN2_PROFILE") - "/lib/python3.11/site-packages")) + (getenv "GN3_PYTHONPATH") + ":" + (string-append + (getenv "GN2_PROFILE") + "/lib/python3.11/site-packages"))) (setenv "PATH" (string-append (getenv "GN2_PROFILE") "/bin:$PATH")) (setenv "R_LIBS_SITE" (string-append (getenv "GN2_PROFILE") "/site-library")) (setenv "JS_GUIX_PATH" (string-append (getenv "GN2_PROFILE") "/share/genenetwork2/javascript")) @@ -732,6 +736,7 @@ server described by CONFIG, a <genenetwork-configuration> object." (setenv "GIT_SSL_CAINFO" (getenv "SSL_CERT_FILE")) (setenv "CURL_CA_BUNDLE" (getenv "SSL_CERT_FILE")) (setenv "REQUESTS_CA_BUNDLE" (getenv "SSL_CERT_FILE")) + (setenv "SPARQL-ENDPOINT" "http://localhost:9082/sparql/") (setenv "GIT_PAGER" #$(file-append coreutils-minimal "/bin/cat")) (let ((current-repo-path "/home/genenetwork/gn-docs")) (setenv "CURRENT_REPO_PATH" current-repo-path) @@ -1131,12 +1136,6 @@ described by CONFIG, a <genenetwork-configuration> object." (copy-recursively build-directory #$virtuoso-data-dir) ;; Load RDF into virtuoso. (invoke "./pre-inst-env" "./load-rdf.scm" #$connection-settings) - ;; Visualize schema and archive results. - (invoke "./pre-inst-env" "./visualize-schema.scm" #$connection-settings) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "sql.dot" (string-append "-o" (getenv "ARCHIVE") "/sql.svg")) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "rdf.dot" (string-append "-o" (getenv "ARCHIVE") "/rdf.svg")) (delete-file-recursively build-directory))))))) (define transform-genenetwork-database-project @@ -1429,6 +1428,19 @@ gn-auth." ";") "proxy_set_header Host $host;"))))))) +(define (gn-guile-reverse-proxy-server-block) + "Return an <nginx-server-configuration> object to reverse proxy +gn-guile to display RDF pages." + (nginx-server-configuration + (server-name '("rdf.genenetwork.org")) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list (string-append "proxy_pass http://localhost:" + (number->string %gn-guile-port) + ";") + "proxy_set_header Host $host;"))))))) + (define set-build-directory-permissions-gexp (with-imported-modules '((guix build utils)) #~(begin @@ -1453,6 +1465,8 @@ gn-auth." (define %gn-auth-port 9094) ;; Port on which virtuoso's SPARQL endpoint is listening (define %virtuoso-sparql-port 9082) +;; Port on which gn-guile is listening +(define %gn-guile-port 8091) (operating-system (host-name "genenetwork-development") @@ -1577,7 +1591,11 @@ gn-auth." (list 'gn-bioinformatics 'guix-bioinformatics)) (tissue-reverse-proxy-server-block) - (gn-auth-reverse-proxy-server-block))))) + (gn-auth-reverse-proxy-server-block) + (gn-guile-reverse-proxy-server-block))))) + (service guile-sheepdog-service-type + (guile-sheepdog-configuration + (settings-file "/etc/genenetwork/conf/sheepdog.scm"))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index 5c6b9f0..f66dcd9 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -450,7 +450,7 @@ object." (sockets (list (forge-ip-socket (port gn2-port)))) (wsgi-app-module "gn2.wsgi") - (workers 20) + (workers 10) (timeout 1200) (environment-variables (list (environment-variable @@ -509,7 +509,7 @@ object." (sockets (list (forge-ip-socket (port gn3-port)))) (wsgi-app-module "gn3.app:create_app()") - (workers 20) + (workers 10) ;; gunicorn's default 30 second timeout is insufficient ;; for Fahamu AI endpoints and results in worker timeout ;; errors. @@ -572,7 +572,7 @@ object." (sockets (list (forge-ip-socket (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") - (workers 20) + (workers 10) (timeout 1200) (environment-variables (list (environment-variable @@ -800,7 +800,7 @@ a @code{<genenetwork-configuration>} record." (sockets (list (forge-ip-socket (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") - (workers 20) + (workers 10) (timeout 1200) (environment-variables (list (environment-variable diff --git a/guix/gn-machines/genenetwork.scm b/guix/gn-machines/genenetwork.scm index 0e68279..657322e 100644 --- a/guix/gn-machines/genenetwork.scm +++ b/guix/gn-machines/genenetwork.scm @@ -2,11 +2,14 @@ #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader gn-libs) #:prefix gn:) + #:use-module((gn packages guile) + #:select (gn-guile guile-sheepdog) + #:prefix gng:) #:use-module (guix git-download) #:use-module (guix packages)) (define-public genenetwork2 - (let ((commit "c057054b69e673108410894ce87c5059aebb7b68") + (let ((commit "905cacdc33a054b463d3b98c2adcd882dc3573fe") (revision "4")) (package (inherit gn:genenetwork2) @@ -20,14 +23,14 @@ (file-name (string-append name "-" version)) (sha256 (base32 - "0r7yisni908ppmd57vz0j9i5cvbxnj3gpfxqdv3bjhzrrisx704l")))) + "0vjsfihj94g1cxivkvmfnixjpmmn3wsciiiry2zkqjnzb3yryy2x")))) (propagated-inputs (modify-inputs (package-propagated-inputs gn:genenetwork2) (replace "gn-libs" gn-libs) (replace "genenetwork3" genenetwork3)))))) (define-public genenetwork3 - (let ((commit "c16eedfa97f7dfbf0a217d9dee2e210d1c51c7ed") + (let ((commit "34a6b37b312b2e2e92324683ec47046b6ab63fe4") (revision "5")) (package (inherit gn:genenetwork3) @@ -42,13 +45,13 @@ (file-name (git-file-name name version)) (sha256 (base32 - "0f3z98yg8mz2x2kgbghdhjj90qm35zc84v5fjmr8g949gws1k7xv")))) + "1mz71kc4icgcj8mjpnipbyfv3nvyrdmqdzpjy1nd3j2f9zjxypz0")))) (propagated-inputs (modify-inputs (package-propagated-inputs gn:genenetwork3) (replace "gn-libs" gn-libs)))))) (define-public gn-auth - (let ((commit "1f2c00e2657f1b7edcfcc9f9bd1a573625d7fb29") + (let ((commit "bd0f453033327a392afa3194538db5d2be22969e") (revision "1")) (package (inherit gn:gn-auth) @@ -64,13 +67,13 @@ (hash (content-hash (base32 - "04xra524dfm1w048ml8n9mc76020j9ipp3dbmf4cc0jps7vq3j91"))))) + "0iamdg2xbj4f5z0qd8z8jildmq2scr10s547fn2k4rac2xm0yn59"))))) (propagated-inputs (modify-inputs (package-propagated-inputs gn:gn-auth) (replace "gn-libs" gn-libs)))))) (define-public gn-uploader - (let ((commit "a65b120e08979d25b641180310e2859b6b045004") + (let ((commit "096fe9dfe3a582658c6bf9179b6c2662cc78711b") (revision "0")) (package (inherit gn:gn-uploader) @@ -85,13 +88,13 @@ (hash (content-hash (base32 - "1p2ncdxip61zywizixjj9mgy6r2r8d1dqzlfsa586l16c381kihn"))))) + "02ds5vv8x2367r95zv0ddh42cqdlwh2jjbqipfd6ppn1dms3p9yj"))))) (propagated-inputs (modify-inputs (package-propagated-inputs gn:gn-uploader) (replace "gn-libs" gn-libs)))))) (define-public gn-libs - (let ((commit "edfbd94378980e1189fcfb769a207e0519af228b") + (let ((commit "5f977e69b46e0ee99845fb69d504f299dc7429a1") (revision "03")) (package (inherit gn:gn-libs) @@ -105,4 +108,38 @@ (file-name (string-append name "-" version)) (sha256 (base32 - "1ran4hrv4waf1fi8zal1kk2asjp4br7n1dpyds7wxwdia1gnxjnk"))))))) + "0hvixdpmqhhby4xv2sq7wnw15gbbp3vicxxa04rmgq1fd4mj0j49"))))))) + +(define-public gn-guile + (let ((commit "454244b774ece37f04f146c74353ea4ec35d43e1") + (revision "0")) + (package + (inherit gng:gn-guile) + (name "gn-guile") + (version (git-version "4.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://git.genenetwork.org/gn-guile/") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1n1zjpwfksq8bf42d7xr7lnblgsbqc0yszkbid253s943fyxzhfn"))))))) + +(define-public guile-sheepdog + (let ((commit "1426617d58f305a4126bb867202843e8cf7dd4b2") + (revision "0")) + (package + (inherit gng:guile-sheepdog) + (name "guile-sheepdog") + (version (git-version "4.0.0" revision commit)) + (source (origin + (method git-fetch) + (uri (git-reference + (url "https://github.com/BonfaceKilz/guile-sheepdog.git") + (commit commit))) + (file-name (string-append name "-" version)) + (sha256 + (base32 + "1z0xzg11p75s2hk312akxlg2h5278w2abma27dhzjf981g3lcqvr"))))))) diff --git a/guix/gn-machines/services/monitoring.scm b/guix/gn-machines/services/monitoring.scm new file mode 100644 index 0000000..7fa59c9 --- /dev/null +++ b/guix/gn-machines/services/monitoring.scm @@ -0,0 +1,68 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(define-module (gn-machines services monitoring) + #:use-module (gnu) + #:use-module (gnu services shepherd) + #:use-module (gnu services databases) + #:use-module ((gn-machines genenetwork) #:select (guile-sheepdog)) + #:use-module (guix gexp) + #:use-module (guix records) + #:use-module (ice-9 match) + #:export (guile-sheepdog-configuration + guile-sheepdog-configuration? + guile-sheepdog-configuration-settings-file + guile-sheepdog-configuration-package + guile-sheepdog-service-type)) + +(define-record-type* <guile-sheepdog-configuration> + guile-sheepdog-configuration + make-guile-sheepdog-configuration + guile-sheepdog-configuration? + (settings-file guile-sheepdog-configuration-settings-file + (default "/etc/conn.scm")) + (package guile-sheepdog-configuration-package (default guile-sheepdog))) + +(define (guile-sheepdog-gexp config) + (match-record config <guile-sheepdog-configuration> (settings-file package) + (program-file + "guile-sheepdog" + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (invoke #$(file-append package "/bin/guile-sheepdog") #$settings-file)))))) + +(define (guile-sheepdog-shepherd-service config) + (shepherd-service + (documentation "Run Sheepdog") + (provision '(guile-sheepdog)) + (requirement '(networking redis)) + (start #~(make-forkexec-constructor + (list #$(guile-sheepdog-gexp config)) + #:log-file "/var/log/sheepdog.log")) + (stop #~(make-kill-destructor)))) + +(define guile-sheepdog-service-type + (service-type + (name 'guile-sheepdog) + (description "Run sheepdog monitor") + (extensions + (list (service-extension shepherd-root-service-type + (compose list guile-sheepdog-shepherd-service)))) + (default-value (guile-sheepdog-configuration)))) diff --git a/production-deploy.sh b/production-deploy.sh index c8e30fe..8fb3d36 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -3,6 +3,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> # Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> +# Copyright © 2026 Munyoki Kilyungi <me@bonfacemunyoki.com> # # This file is part of genenetwork-machines. # @@ -24,7 +25,8 @@ container_script=$(guix system container \ --network \ - --load-path=. \ + --load-path=./guix/ \ + --load-path=./ \ --verbosity=3 \ --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ diff --git a/production.scm b/production.scm index bfd9e48..6339b66 100644 --- a/production.scm +++ b/production.scm @@ -49,8 +49,8 @@ (server-port 9892) (http-server-port 9893) (dirs-allowed (list "/export/data/virtuoso")) - (number-of-buffers 4000000) - (maximum-dirty-buffers 3000000) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) (service forge-nginx-service-type diff --git a/public-sparql.scm b/public-sparql.scm index edfcd87..87ef843 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -60,9 +60,9 @@ SPARQL endpoint is listening on." (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) - (number-of-buffers 4000000) + (number-of-buffers 680000) + (maximum-dirty-buffers 500000) (dirs-allowed (list "/export/data/virtuoso")) - (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/public-virtuoso.db") (transaction-file "/var/lib/virtuoso/public-virtuoso.trx") (error-log-file "/var/lib/public-virtuoso-errors.log") diff --git a/services/README.md b/services/README.md deleted file mode 100644 index d0d1c01..0000000 --- a/services/README.md +++ /dev/null @@ -1,17 +0,0 @@ -# Services - -This directory contains small and simple services that may be run independently. -Note that composition is not the goal. -For example, databases and web proxies are handled outsite the containers. -Use these services for simple deployment and ad hoc testing. -One nice aspect of small system containers is that you can run these easily on your laptop. - -IMPORTANT: more complex services do not belong in this directory. - -# Examples - -## gn-guile - -`gn-guile` is our next generation service (gn4?). It provides a REST API, at this point, and some portals, such as https://aging.genenetwork.org/. - -See the [gn-guile](./gn-guile.scm) system definition example. diff --git a/services/gn-guile.scm b/services/gn-guile.scm deleted file mode 100644 index 2f675a8..0000000 --- a/services/gn-guile.scm +++ /dev/null @@ -1,52 +0,0 @@ -;; This is an example definition for the gn-guile/GN4 service -;; -;; Run with -;; -;; export runner=$(guix system container gn-guile.scm) -;; -;; as root -;; -;; sudo bash -c $runner -;; echo $runner -;; -;; make a note of pid and -;; -;; sudo bash -c "nsenter -a -t 4050285" -;; -;; now you should be inside the container (note bash should be in the container!) - -(use-modules (gnu) - (guix records) - (forge utils)) - -(define-record-type* <gn-guile-configuration> - gn-guile-configuration make-gn-guile-configuration - gn-guile-configuration? - (gn2-repository gn-guile-configuration-gn2-repository - (default "https://github.com/genenetwork/genenetwork2")) - (gn2-port gn-guile-configuration-gn2-port - (default 8082))) - - -(define gn-guile-service-type - (service-type - (name 'gn-guile) - (description "gn-guile/GN4 webservice") - (extensions '()) - )) - -(operating-system - (host-name "gn-guile") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service gn-guile-service-type - (gn-guile-configuration)) - %base-services))) diff --git a/services/opensmtpd.scm b/services/opensmtpd.scm deleted file mode 100644 index 1b1e58f..0000000 --- a/services/opensmtpd.scm +++ /dev/null @@ -1,21 +0,0 @@ -(use-modules (gnu) - (gnu services mail)) - -(operating-system - (host-name "mail") - (timezone "UTC") - (locale "en_US.utf8") - (bootloader (bootloader-configuration - (bootloader grub-bootloader) - (targets (list "/dev/sdX")))) - (file-systems %base-file-systems) - (users %base-user-accounts) - (packages %base-packages) - - (services (cons - (service opensmtpd-service-type - (opensmtpd-configuration - (config-file %default-opensmtpd-config-file - ; (config-file (local-file "./my-smtpd.conf"))) - ))) - %base-services))) |