diff options
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 105 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 60 | ||||
| -rw-r--r-- | uploader.scm | 76 |
3 files changed, 239 insertions, 2 deletions
diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index cdb7ff4..0382d01 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -1,5 +1,6 @@ ;;; genenetwork-machines --- Guix configuration for genenetwork machines ;;; Copyright © 2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2024 Frederick M. Muriithi <fredmanglis@gmail.com> ;;; ;;; This file is part of genenetwork-machines. ;;; @@ -18,7 +19,7 @@ ;;; <https://www.gnu.org/licenses/>. (define-module (genenetwork services genenetwork) - #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) + #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) #:use-module ((gnu packages admin) #:select (shadow)) #:use-module (gnu services) #:use-module (gnu services web) @@ -48,7 +49,13 @@ genenetwork-configuration-sparql-endpoint genenetwork-configuration-gn3-data-directory genenetwork-configuration-gn2-secrets - genenetwork-configuration-gn3-secrets)) + genenetwork-configuration-gn3-secrets + gn-uploader-service-type + gn-uploader-configuration + gn-uploader-configuration? + gn-uploader-configuration-server-name + gn-uploader-configuration-port + gn-uploader-configuration-secrets)) (define-record-type* <genenetwork-configuration> genenetwork-configuration make-genenetwork-configuration @@ -88,6 +95,22 @@ (gn-auth-secrets genenetwork-configuration-gn-auth-secrets (default "/etc/genenetwork/gn-auth-secrets.py"))) +(define-record-type* <gn-uploader-configuration> + gn-uploader-configuration make-gn-uploader-configuration + gn-uploader-configuration? + (gn-uploader gn-uploader-configuration-gn-uploader + (default gn-uploader)) + (server-name gn-uploader-server-name + (default "upload.genenetwork.org")) + (port gn-uploader-configuration-port + (default 8085)) + (sql-uri gn-uploader-configuration-sql-uri + (default "mysql://username:password@localhost/database")) + (data-directory gn-uploader-configuration-data-directory + (default "/var/genenetwork")) + (secrets gn-uploader-configuration-secrets + (default "/etc/genenetwork/gn-uploader-secrets.py"))) + (define %genenetwork-accounts (list (user-group (name "genenetwork") @@ -334,3 +357,81 @@ a @code{<genenetwork-configuration>} record." (service-extension forge-nginx-service-type genenetwork-nginx-server-blocks))) (default-value (genenetwork-configuration)))) + +(define (gn-uploader-activation config) + (match-record config <gn-uploader-configuration> + (secrets) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + ;; Let service user own their own secrets files. + (chown #$secrets + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader"))) + ;; Set owner-only permissions on secrets files. + (for-each (lambda (file) + (chmod file #o600)) + (list #$secrets)))))) + +(define (gn-uploader-gunicorn-app config) + (match-record config <gn-uploader-configuration> + (gn-uploader sql-uri port data-directory secrets) + ;; If we mapped only the mysqld.sock socket file, it would break + ;; when the external mysqld server is restarted. + (let ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("QCAPP_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri)))))) + (list (gunicorn-app + (name "gn-uploader") + (package gn-uploader) + (sockets (list (forge-ip-socket + (port port)))) + (wsgi-app-module "qc_app:create_app()") + (workers 20) + (environment-variables + (list (environment-variable + (name "QCAPP_CONF") + (value gn-uploader-conf)) + (environment-variable + (name "HOME") + (value "/tmp")))) + (mappings (list database-mapping + (file-system-mapping + (source gn-uploader-conf) + (target source)) + (file-system-mapping + (source secrets) + (target source)) + (file-system-mapping + (source data-directory) + (target source))))))))) + +(define (gn-uploader-nginx-server-block config) + (match-record config <gn-uploader-configuration> + (server-name port) + (list (nginx-server-configuration + (server-name (list server-name)) + (locations + (list (nginx-location-configuration + (uri "/") + (body (list (string-append "proxy_pass http://localhost:" + (number->string port) ";") + "proxy_set_header Host $host;"))))))))) + +(define gn-uploader-service-type + (service-type + (name 'gn-uploader) + (description "GeneNetwork data uploader service.") + (extensions + (list (service-extension activation-service-type + gn-uploader-activation) + (service-extension gunicorn-service-type + gn-uploader-gunicorn-app) + (service-extension forge-nginx-service-type + gn-uploader-nginx-server-block))) + (default-value (genenetwork-configuration)))) diff --git a/uploader-deploy.sh b/uploader-deploy.sh new file mode 100755 index 0000000..77a91e6 --- /dev/null +++ b/uploader-deploy.sh @@ -0,0 +1,60 @@ +#! /bin/bash -e + +# genenetwork-machines --- Guix configuration for genenetwork machines +# Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2022, 2024 Frederick M. Muriithi <fredmanglis@gmail.com> +# +# This file is part of genenetwork-machines. +# +# genenetwork-machines is free software: you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# genenetwork-machines is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with genenetwork-machines. If not, see +# <https://www.gnu.org/licenses/>. + +# Build and install genenetwork production container on tux02. + +SCRIPTARGS=("$@") + +MODULEARGS=() +if [ "${#SCRIPTARGS[@]}" -gt 0 ] +then + echo "===== Auxilliary module load paths =====" + for path in "${SCRIPTARGS[@]}" + do + echo "-L ${path}" + MODULEARGS+=("-L" "${path}") + done + echo "===== END: Auxilliary module load paths =====" +fi + +container_script=$(guix system container \ + "${MODULEARGS[@]}" \ + --network \ + --load-path=. \ + --verbosity=3 \ + --share=/export2/guix-containers/genenetwork/uploader/var/genenetwork=/var/genenetwork \ + --share=/export2/guix-containers/genenetwork/uploader/var/lib/acme=/var/lib/acme \ + --share=/export2/guix-containers/genenetwork/uploader/var/lib/mysql=/var/lib/mysql \ + --share=/export2/guix-containers/genenetwork/uploader/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export2/guix-containers/genenetwork/uploader/var/log=/var/log \ + --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ + --expose=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ + --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ + --expose=/export/data/genenetwork/genotype_files=/export/data/genenetwork/genotype_files \ + --expose=/export/data/uploader/genenetwork3 \ + --expose=/export/data/uploader/gn-uploader \ + --share=/var/run/mysqld3307=/run/mysqld \ + uploader.scm) + +echo "${container_script}" +sudo ln --force --symbolic "${container_script}" /usr/local/bin/genenetwork-uploader-container +sudo ln --force --symbolic /usr/local/bin/genenetwork-uploader-container /var/guix/gcroots diff --git a/uploader.scm b/uploader.scm new file mode 100644 index 0000000..9491a22 --- /dev/null +++ b/uploader.scm @@ -0,0 +1,76 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2022–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2022–2024 Frederick M. Muriithi <fredmanglis@gmail.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +(use-modules (gnu) + (genenetwork services genenetwork) + ((gnu packages admin) #:select (shepherd)) + (gn services databases) + (gn packages genenetwork) + (gnu services databases) + (forge acme) + (forge nginx) + (forge socket)) + +(operating-system + (host-name "genenetwork-uploader") + (timezone "UTC") + (locale "en_US.utf8") + (bootloader (bootloader-configuration + (bootloader grub-bootloader) + (targets (list "/dev/sdX")))) + (file-systems %base-file-systems) + (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages %base-packages) + (services (cons* (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "0.0.0.0") + (port 10890))) + (https-listen (forge-ip-socket + (ip "0.0.0.0") + (port 10891))))) + (service acme-service-type + (acme-configuration + (email "arunisaac@systemreboot.net"))) + (service genenetwork-service-type + (genenetwork-configuration + (genenetwork2 genenetwork2) + (server-name "staging.genenetwork.org") + (gn-auth-server-name "staging-auth.genenetwork.org") + (gn2-port 10894) + (gn3-port 10895) + (gn-auth-port 10896) + (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (auth-db "/export/data/genenetwork-sqlite/auth.db") + (xapian-db "/export/data/genenetwork-xapian") + (genotype-files "/export/data/genenetwork/genotype_files") + (sparql-endpoint "http://localhost:10892/sparql") + (gn3-data-directory "/export/data/uploader/genenetwork3"))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "staging-uploader.genenetwork.org") + (port 10897) + (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (data-directory "/export/data/uploader/gn-uploader"))) + %base-services))) |