diff options
| -rwxr-xr-x | etc/auth.db | bin | 0 -> 14548992 bytes | |||
| -rw-r--r-- | etc/conn.scm | 11 | ||||
| -rw-r--r-- | etc/gn-auth-secrets.py | 8 | ||||
| -rw-r--r-- | etc/gn-uploader-secrets.py | 3 | ||||
| -rw-r--r-- | etc/gn-uploader.db | 0 | ||||
| -rw-r--r-- | etc/gn2-secrets.py | 3 | ||||
| -rw-r--r-- | etc/gn3-secrets.py | 7 | ||||
| -rw-r--r-- | etc/llm.db | 0 | ||||
| -rw-r--r-- | genenetwork-development.scm | 18 | ||||
| -rw-r--r-- | genenetwork-local-container.scm | 662 | ||||
| -rwxr-xr-x | genenetwork-local-container.sh | 650 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 149 | ||||
| -rwxr-xr-x | production-deploy.sh | 6 | ||||
| -rw-r--r-- | production.scm | 10 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 1 | ||||
| -rw-r--r-- | public-sparql.scm | 7 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 26 | ||||
| -rw-r--r-- | uploader.scm | 24 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 1 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
20 files changed, 1505 insertions, 83 deletions
diff --git a/etc/auth.db b/etc/auth.db Binary files differnew file mode 100755 index 0000000..ca05c2d --- /dev/null +++ b/etc/auth.db diff --git a/etc/conn.scm b/etc/conn.scm new file mode 100644 index 0000000..6f448ab --- /dev/null +++ b/etc/conn.scm @@ -0,0 +1,11 @@ +((sql-username . "webqtlout") + (sql-password . "webqtlout") + (sql-database . "db_webqtl_local") + (sql-host . "localhost") + (sql-port . 3306) + (virtuoso-port . 7081) + (virtuoso-username . "dba") + (virtuoso-password . "") + (sparql-scheme . http) + (sparql-host . "localhost") + (sparql-port . 7082)) diff --git a/etc/gn-auth-secrets.py b/etc/gn-auth-secrets.py new file mode 100644 index 0000000..0478bcf --- /dev/null +++ b/etc/gn-auth-secrets.py @@ -0,0 +1,8 @@ +SECRET_KEY = "qQIrgiK29kXZU6v8D09y4uw_sk8I4cqgNZniYUrRoUk" +SMTP_TIMEOUT = 200 +SMTP_HOST = "smtp.XXXX.XXX" +SMTP_PORT = 587 +SMTP_USER = "XXXX" +SMTP_PASSWORD = "XXXX" +EMAIL_ADDRESS = "XXXX@XXXX.com" +EMAIL_DISPLAY_NAME = "GeneNetwork: Local Server" diff --git a/etc/gn-uploader-secrets.py b/etc/gn-uploader-secrets.py new file mode 100644 index 0000000..1c5a030 --- /dev/null +++ b/etc/gn-uploader-secrets.py @@ -0,0 +1,3 @@ +SECRET_KEY="qQIrgiK29kXZU6v8D09y4uw_sk8I4cqgNZniYUrRoUk" +OAUTH2_CLIENT_ID="dae2e4fc-2e32-4e87-88a0-f1e63d84470b" +OAUTH2_CLIENT_SECRET="eNL4WZ3lpX,S>V$Bd*+p" diff --git a/etc/gn-uploader.db b/etc/gn-uploader.db new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/etc/gn-uploader.db diff --git a/etc/gn2-secrets.py b/etc/gn2-secrets.py new file mode 100644 index 0000000..bc3733a --- /dev/null +++ b/etc/gn2-secrets.py @@ -0,0 +1,3 @@ +SECRET_KEY="qQIrgiK29kXZU6v8D09y4uw_sk8I4cqgNZniYUrRoUk" +OAUTH2_CLIENT_ID="0bbfca82-d73f-4bd4-a140-5ae7abb4a64d" +OAUTH2_CLIENT_SECRET="qQIrgiK29kXZU6v8D09y4uw_sk8I4cqgNZniYUrRoUk" diff --git a/etc/gn3-secrets.py b/etc/gn3-secrets.py new file mode 100644 index 0000000..8f945a7 --- /dev/null +++ b/etc/gn3-secrets.py @@ -0,0 +1,7 @@ +SECRET_KEY="random-key" +OAUTH2_CLIENT_ID="0bbfca82-d73f-4bd4-a140-5ae7abb4a64d" +OAUTH2_CLIENT_SECRET="yadabadaboo" +SPARQL_USER="dba" +SPARQL_PASSWORD="dba" +SPARQL_AUTH_URI="http://localhost:7082/sparql-auth/" +FAHAMU_AUTH_TOKEN = "XXXX" diff --git a/etc/llm.db b/etc/llm.db new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/etc/llm.db diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 50d72a5..0b22f77 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -241,7 +241,7 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock") (chdir "genenetwork2") ;; XXXX: FIXME: R/Qtl tests fail because files are generated in ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 @@ -277,7 +277,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" + "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -322,7 +322,7 @@ genenetwork3 source from the latest commit of @var{project}." (system* (string-append gn3-dir "/scripts/index-genenetwork") "is-data-modified" #$%xapian-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" + "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql")))) (setenv "LAMINAR_REASON" "Nightly xapian index rebuild") (invoke #$(file-append laminar "/bin/laminarc") @@ -505,7 +505,7 @@ server described by CONFIG, a <genenetwork-configuration> object." "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock\"\n" "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) @@ -1372,7 +1372,7 @@ gn-auth." (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (server-port 9081) - (dirs-allowed "/var/lib/data") + (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) (service genenetwork-service-type (genenetwork-configuration @@ -1399,8 +1399,12 @@ gn-auth." (hosts (list (tissue-host (name "issues.genenetwork.org") - (user "laminar") - (upstream-repository "https://github.com/genenetwork/gn-gemtext-threads")))))) + (projects (list (tissue-project + (name "issues.genenetwork.org") + (user "laminar") + (base-path "/") + (upstream-repository + "https://github.com/genenetwork/gn-gemtext-threads"))))))))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket diff --git a/genenetwork-local-container.scm b/genenetwork-local-container.scm new file mode 100644 index 0000000..d5848b7 --- /dev/null +++ b/genenetwork-local-container.scm @@ -0,0 +1,662 @@ +;;; genenetwork-machines --- Guix configuration for genenetwork machines +;;; Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +;;; +;;; This file is part of genenetwork-machines. +;;; +;;; genenetwork-machines is free software: you can redistribute it +;;; and/or modify it under the terms of the GNU General Public License +;;; as published by the Free Software Foundation, either version 3 of +;;; the License, or (at your option) any later version. +;;; +;;; genenetwork-machines is distributed in the hope that it will be +;;; useful, but WITHOUT ANY WARRANTY; without even the implied +;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. +;;; See the GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with genenetwork-machines. If not, see +;;; <https://www.gnu.org/licenses/>. + +;;; This is the production genenetwork container currently deployed on +;;; tux04. + +(use-modules (gnu) + (gnu build linux-container) + ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs gn-uploader)) + ((gnu packages admin) #:select (shepherd shadow)) + ((gnu packages certs) #:select (nss-certs)) + ((gnu packages bash) #:select (bash)) + ((gnu packages check) #:select (python-pylint python-hypothesis)) + ((gnu packages python-check) #:select (python-mypy python-mypy-extensions)) + ((gnu packages python-web) #:select (gunicorn python-flask python-flask-cors)) + ((gnu packages version-control) #:select (git-minimal)) + ((gn packages guile) #:select (gn-guile)) + (gn services databases) + (gnu services databases) + (guix modules) + (gnu services shepherd) + (guix search-paths) + (guix least-authority) + (guix packages) + (guix profiles) + (guix records) + (forge socket) + (forge utils) + (srfi srfi-1) + (ice-9 match)) + +(define-record-type* <genenetwork-configuration> + genenetwork-configuration make-genenetwork-configuration + genenetwork-configuration? + (gn2-port genenetwork-configuration-gn2-port + (default 8082)) + (gn3-port genenetwork-configuration-gn3-port + (default 8083)) + (gn-auth-port genenetwork-configuration-gn-auth-port + (default 8084)) + (gn2-secrets genenetwork-configuration-gn2-secrets + (default "/etc/genenetwork/conf/gn2/secrets.py")) + (gn3-secrets genenetwork-configuration-gn3-secrets + (default "/etc/genenetwork/conf/gn3/secrets.py")) + (gn-auth-secrets genenetwork-configuration-gn-auth-secrets + (default "/etc/genenetwork/conf/gn-auth/secrets.py")) + (genotype-files genenetwork-configuration-genotype-files + (default "/var/genenetwork/genotype-files")) + (sparql-endpoint genenetwork-configuration-sparql-endpoint + (default "http://localhost:7082/sparql")) + (data-directory genenetwork-data-directory + (default "/var/genenetwork")) + (xapian-db-path genenetwork-xapian-db-path + (default "/var/lib/xapian")) + (auth-db-path genenetwork-auth-db-path + (default "/var/genenetwork/auth.db")) + (llm-db-path genenetwork-llm-db-path + (default "/var/lib/genenetwork-sqlite/llm.db")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/var/lib/gn-docs")) + (sql-uri genenetwork-configuration-sql-uri + (default + "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock&charset=utf8")) + (gn-uploader-data-directory genenetwork-configuration-gn-uploader-data-directory + (default "/var/lib/gn-uploader/data")) + (gn-uploader-port genenetwork-configuration-gn-uploader-port + (default 8085)) + (gn-uploader-secrets genenetwork-configuration-gn-uploader-configuration-secrets + (default "/etc/genenetwork/conf/gn-uploader/secrets.py")) + (gn-uploader-sessions-dir genenetwork-configuration-gn-uploader-sessions-dir + (default "/var/lib/gn-uploader/sessions"))) + + +(define (genenetwork2-gexp config) + "Return a G-expression that runs the latest genenetwork2 development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (sql-uri gn2-port gn2-secrets gn3-port gn-auth-port genotype-files) + (with-packages + (list coreutils git-minimal gunicorn nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 ftw) + (ice-9 match)) + + ;; Override the genenetwork3 used by genenetwork2. + (setenv "GN3_PYTHONPATH" "/genenetwork3") + (setenv "GN2_PROFILE" #$(profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN2_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (unless (file-exists? "/etc/genenetwork/conf/gn2.conf") + (with-output-to-file "/etc/genenetwork/conf/gn2.conf" + (lambda () + (display #$(string-append + "GN2_SECRETS=\"" gn2-secrets "\"\n" + "AI_SEARCH_ENABLED=True\n" + "TEST_FEATURE_SWITCH=True\n" + "GN3_LOCAL_URL=\"http://localhost:" (number->string gn3-port) "\"\n" + "GN_SERVER_URL=\"http://localhost:" (number->string gn3-port) "/api/\"\n" + "AUTH_SERVER_URL=\"http://localhost:" (number->string gn-auth-port) "\"\n" + "SQL_URI=\"" sql-uri "\"\n" + "SSL_PRIVATE_KEY=\"/etc/genenetwork/conf/gn2/private.pem\"\n" + "AUTH_SERVER_SSL_PUBLIC_KEY=\"/etc/genenetwork/conf/gn-auth/clients-public-keys/gn-auth.pem\"\n")) + ;; We actually set here twice so that we can refer to it + ;; when running things manually. + (display "GN2_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest genenetwork2)) + (allow-collisions? #t)) + "\"\n"))))) + (setenv "GN2_SETTINGS" "/etc/genenetwork/conf/gn2.conf") + ;; Start genenetwork2. + (with-directory-excursion "/genenetwork2" + (invoke #$(file-append bash "/bin/sh") + "bin/genenetwork2" "gn2/default_settings.py" "-gunicorn-dev"))))))) + +(define (genenetwork3-gexp config) + "Return a G-expression that runs the latest genenetwork3 development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (sql-uri gn3-port gn3-secrets gn-auth-port sparql-endpoint data-directory xapian-db-path auth-db-path llm-db-path) + (with-manifest (package->development-manifest genenetwork3) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + ;; Configure genenetwork3. + (setenv "RSCRIPT" #$(file-append + (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t)) + "/bin/Rscript")) + (unless (file-exists? "/etc/genenetwork/conf/gn3.conf") + (with-output-to-file "/etc/genenetwork/conf/gn3.conf" + (lambda () + (display #$(string-append + "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" + "DATA_DIR=\"" data-directory "\"\n" + "AUTH_SERVER_URL=\"http://localhost:8084/\"\n" + "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" + "AUTH_DB=\"" auth-db-path "\"\n" + "LLM_DB_PATH=\"" llm-db-path "\"\n")) + (display "GN3_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t)) + "\"\n")) + (display "R_SCRIPT=\"") + (display #$(file-append + (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t)) + "/bin/Rscript\"\n"))))) + (setenv "GN3_PROFILE" #$(profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN3_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN3_CONF" "/etc/genenetwork/conf/gn3.conf") + (setenv "GN3_SECRETS" #$gn3-secrets) + (setenv "HOME" "/tmp") + (setenv "SQL_URI" #$sql-uri) + + (setenv "FLASK_ENV" "development") + (setenv "FLASK_DEBUG" "1") + ;; Run genenetwork3. + (with-directory-excursion "/genenetwork3" + (invoke #$(file-append python-flask "/bin/flask") + "run" + #$(string-append "--port=" (number->string gn3-port))))))))) + +(define (gn-auth-gexp config) + "Return a G-expression that runs the latest gn-auth development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (sql-uri gn-auth-port auth-db-path gn-auth-secrets) + (with-manifest (package->development-manifest gn-auth) + (with-packages (list git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + ;; Configure gn-auth. + (unless (file-exists? "/etc/genenetwork/conf/gn-auth.conf") + (with-output-to-file "/etc/genenetwork/conf/gn-auth.conf" + (lambda () + (display #$(string-append + "LOGLEVEL=\"DEBUG\"\n" + "SQL_URI=\"" sql-uri "\"\n" + "AUTH_DB=\"" auth-db-path "\"\n" + "GN_AUTH_SECRETS=\"" gn-auth-secrets "\"\n")) + (display "GN_AUTH_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t)) + "\"\n"))))) + (setenv "GN_AUTH_PROFILE" #$(profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (setenv "REQUESTS_CA_BUNDLE" (string-append + (getenv "GN_AUTH_PROFILE") + "/etc/ssl/certs/ca-certificates.crt")) + (setenv "GN_AUTH_CONF" "/etc/genenetwork/conf/gn-auth.conf") + (setenv "HOME" "/tmp") + (setenv "AUTHLIB_INSECURE_TRANSPORT" "true") + ;; Run gn-auth. + (with-directory-excursion "/gn-auth" + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn-auth-port)) + "--workers" "8" + "gn_auth.wsgi:app")))))))) + +(define (gn-uploader-gexp config) + (match-record + config <genenetwork-configuration> + (sql-uri gn-uploader-port gn-uploader-data-directory + gn-uploader-secrets gn-auth-port + gn2-port gn-uploader-sessions-dir) + (with-manifest + (package->development-manifest gn-uploader) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 ftw)) + (unless (file-exists? "/etc/genenetwork/conf/gn-uploader.conf") + (with-output-to-file "/etc/genenetwork/conf/gn-uploader.conf" + (lambda () + (display #$(string-append + "UPLOADER_SECRETS=\"" gn-uploader-secrets "\"\n" + "SQL_URI=\"" sql-uri "\"\n" + "ASYNCHRONOUS_JOBS_SQLITE_DB=\"/var/genenetwork/gn-uploader.db\"\n" + "UPLOAD_FOLDER=\"" gn-uploader-data-directory "\"\n" + "AUTH_SERVER_URL=\"http://localhost:" (number->string gn-auth-port) "\"\n" + "GN2_SERVER_URL=\"http://localhost:" (number->string gn2-port) "\"\n" + "SESSION_FILESYSTEM_CACHE_PATH=\"" gn-uploader-sessions-dir "\"\n")) + (display "GN_UPLOADER_PROFILE=\"") + (display #$(file-append (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t)) + "\"\n"))))) + ;; Run gn-uploader + (with-directory-excursion "/gn-uploader" + ;; Configure Uploader + (setenv "UPLOADER_CONF" "/etc/genenetwork/conf/gn-uploader.conf") + (setenv "FLASK_APP" "scripts/qcapp_wsgi.py") + (setenv "FLASK_ENV" "development") + (setenv "FLASK_DEBUG" "1") + (setenv "ASYNCHRONOUS_JOBS_SQLITE_DB" "/var/genenetwork/gn-uploader.db") + (setenv "GN_UPLOADER_ENVIRONMENT" + #$(profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (invoke #$(file-append (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t)) + "/bin/flask") + "run" + #$(string-append "--port=" (number->string gn-uploader-port))))))))) + +(define (genenetwork-activation config) + (match-record + config <genenetwork-configuration> + (gn2-secrets + gn3-secrets gn3-port gn-auth-port + auth-db-path gn-auth-secrets gn-doc-git-checkout + gn-uploader-data-directory + gn-uploader-secrets + gn-uploader-sessions-dir) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 ftw)) + ;; Set ownership of files. + (for-each (lambda (file) + (when (eq? (stat:type (stat file)) 'directory) + (chmod file #o755)) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (append + '("/etc/genenetwork/conf" "/genenetwork2/flask_session" + "/gn-uploader/flask_session") + (find-files #$(dirname gn-auth-secrets) + #:directories? #t) + (find-files #$(dirname gn2-secrets) + #:directories? #t) + (find-files #$(dirname gn3-secrets) + #:directories? #t) + (find-files #$(dirname gn-uploader-secrets) + #:directories? #t) + (find-files #$(dirname auth-db-path) + #:directories? #t) + (find-files #$(dirname gn-doc-git-checkout) + #:directories? #t) + (find-files #$(dirname gn-uploader-data-directory) + #:directories? #t) + (find-files #$(dirname gn-uploader-sessions-dir) + #:directories? #t))) + ;; Prevent other users from reading secret files. + (for-each (lambda (file) + (chmod file #o600)) + (append + (find-files #$gn-auth-secrets + #:directories? #f) + (find-files #$gn2-secrets + #:directories? #f) + (find-files #$gn3-secrets + #:directories? #f) + (find-files #$gn-uploader-secrets + #:directories? #f))))))) + +(define (gn-guile-gexp gn-guile-port) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + + +(define (genenetwork-shepherd-services config) + "Return shepherd services to run the genenetwork development server +described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (gn2-port + gn3-port gn-auth-port genotype-files + data-directory xapian-db-path gn2-secrets + auth-db-path gn-auth-secrets + llm-db-path gn-doc-git-checkout + gn-guile-port + gn-uploader-port + gn-uploader-data-directory + gn-uploader-sessions-dir) + (list (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/gn-guile.log"))) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run GeneNetwork 2 development server.") + (provision '(genenetwork2)) + ;; FIXME: The genenetwork2 service should depend on redis. + (requirement '(networking genenetwork3)) + (modules '((guix search-paths) + (ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn2-manifest (packages->manifest (list genenetwork2))) + (gn2-profile (profile + (content gn2-manifest) + (allow-collisions? #t))) + (gn2-settings + `(("SERVER_PORT" ,(number->string gn2-port)) + ("GENENETWORK_FILES" ,genotype-files) + ("HOME" "/tmp") + ("LC_ALL" "en_US.UTF-8") + ("NO_REDIS" "no-redis") + ("RUST_BACKTRACE" "1")))) + (with-imported-modules (source-module-closure '((guix search-paths))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork2" + (genenetwork2-gexp config)) + #:name "genenetwork2-pola-wrapper" + #:preserved-environment-variables + (append '("REQUESTS_CA_BUNDLE") + (map first gn2-settings) + (map search-path-specification-variable + (manifest-search-paths gn2-manifest))) + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source genotype-files) + (target source)) + (file-system-mapping + (source "/genenetwork2") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source "/etc/genenetwork/conf") + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn2-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (append + '("REQUESTS_CA_BUNDLE=" + #$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) + (map (match-lambda + ((spec . value) + (string-append (search-path-specification-variable spec) + "=" + value))) + (evaluate-search-paths + (map sexp->search-path-specification + '#$(map search-path-specification->sexp + (manifest-search-paths gn2-manifest))) + (list #$gn2-profile))) + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn2-settings)) + #:log-file "/var/log/genenetwork2.log")))) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run GeneNetwork 3 development server.") + (provision '(genenetwork3)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork3" + (genenetwork3-gexp config)) + #:name "genenetwork3-pola-wrapper" + #:mappings (list (file-system-mapping + (source "/genenetwork3") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source data-directory) + (target source)) + (file-system-mapping + (source xapian-db-path) + (target source)) + (file-system-mapping + (source "/etc/genenetwork/conf") + (target source) + (writable? #t)) + (file-system-mapping + (source auth-db-path) + (target source) + (writable? #t)) + (file-system-mapping + (source llm-db-path) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn3-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/genenetwork3.log")) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run gn-uploader") + (provision '(gn-uploader)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-uploader" + (gn-uploader-gexp config)) + #:name "gn-uploader-pola-wrapper" + #:mappings (list (file-system-mapping + (source "/gn-uploader") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source "/var/genenetwork/gn-uploader.db") + (target source) + (writable? #t)) + (file-system-mapping + (source "/etc/genenetwork/conf") + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-data-directory) + (target source) + (writable? #t)) + (file-system-mapping + (source gn-uploader-sessions-dir) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-uploader-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/gn-uploader.log")) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run gn-auth development server.") + (provision '(gn-auth)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-auth" + (gn-auth-gexp config)) + #:name "gn-auth-pola-wrapper" + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source "/gn-auth") + (target source) + (writable? #t)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source data-directory) + (target source)) + (file-system-mapping + (source "/var/genenetwork") + (target source) + (writable? #t)) + (file-system-mapping + (source "/etc/genenetwork/conf") + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-auth-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/gn-auth.log")) + (stop #~(make-kill-destructor)))))) + +(define %genenetwork-accounts + (list (user-group + (name "genenetwork") + (system? #t)) + (user-account + (name "genenetwork") + (group "genenetwork") + (system? #t) + (comment "GeneNetwork user") + (home-directory "/var/empty") + (shell (file-append shadow "/sbin/nologin"))))) + + +(define genenetwork-service-type + (service-type + (name 'genenetwork) + (description "Run GeneNetwork development servers and CI.") + (extensions + (list (service-extension account-service-type + (const %genenetwork-accounts)) + (service-extension activation-service-type + genenetwork-activation) + (service-extension shepherd-root-service-type + genenetwork-shepherd-services))) + (default-value (genenetwork-configuration)))) + + +(operating-system + (host-name "genenetwork-work-container") + (timezone "UTC") + (locale "en_US.utf8") + (bootloader (bootloader-configuration + (bootloader grub-bootloader) + (targets (list "/dev/sdX")))) + (file-systems %base-file-systems) + (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) + (packages (cons* genenetwork2 python-hypothesis gn-auth + python-mypy python-mypy-extensions python-pylint + %base-packages)) + (services (cons* (service virtuoso-service-type + (virtuoso-configuration + (server-port 7081) + (http-server-port 7082) + (dirs-allowed (list "/var/lib/data")) + (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") + (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) + (service redis-service-type + (redis-configuration + (bind "127.0.0.1") + (port 6379) + (working-directory "/var/lib/redis"))) + (service genenetwork-service-type) + %base-services))) diff --git a/genenetwork-local-container.sh b/genenetwork-local-container.sh new file mode 100755 index 0000000..7632d8a --- /dev/null +++ b/genenetwork-local-container.sh @@ -0,0 +1,650 @@ +#! /bin/sh -e + +# genenetwork-machines --- Guix configuration for genenetwork machines +# Copyright © 2025 Munyoki Kilyungi <me@bonfacemunyoki.com> +# +# This file is part of genenetwork-machines. +# +# genenetwork-machines is free software: you can redistribute it +# and/or modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation, either version 3 of +# the License, or (at your option) any later version. +# +# genenetwork-machines is distributed in the hope that it will be +# useful, but WITHOUT ANY WARRANTY; without even the implied warranty +# of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with genenetwork-machines. If not, see +# <https://www.gnu.org/licenses/>. + +# Build and install genenetwork container on your local machine. + +set -euo pipefail + + +BASE_DIR="${HOME:-/home/$USER}/genenetwork" +SYSTEM_DIRECTORIES=( + "$BASE_DIR" + "$BASE_DIR/var/log" + "$BASE_DIR/var/genenetwork" + "$BASE_DIR/etc/genenetwork/conf" + "$BASE_DIR/etc/genenetwork" + "$BASE_DIR/var/lib/redis" + "$BASE_DIR/var/lib/virtuoso" + "$BASE_DIR/var/lib/data" + "$BASE_DIR/var/lib/gn-uploader/data" + "$BASE_DIR/var/lib/gn-uploader/sessions" + "$BASE_DIR/var/lib/xapian" + "$BASE_DIR/var/genenetwork/genotype-files/genotype/json" + "$BASE_DIR/var/lib/genenetwork-sqlite" + "$BASE_DIR/var/lib/genenetwork-gnqa" + "/tmp/local-container" +) + +GN_PROJECTS=( + "genenetwork2:https://github.com/genenetwork/genenetwork2.git" + "genenetwork3:https://github.com/genenetwork/genenetwork3.git" + "gn-transform-databases:https://git.genenetwork.org/gn-transform-databases/" + "gn-auth:https://git.genenetwork.org/gn-auth" + "gn-docs:https://git.genenetwork.org/gn-docs" + "gn-uploader:https://git.genenetwork.org/gn-uploader" +) + +# File mappings: source -> destination +declare -A FILE_MAPPINGS=( + ["etc/auth.db"]="$BASE_DIR/var/genenetwork/auth.db" + ["etc/gn-uploader.db"]="$BASE_DIR/var/genenetwork/gn-uploader.db" + ["etc/llm.db"]="$BASE_DIR/var/lib/genenetwork-sqlite/llm.db" + ["etc/gn2-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn2/secrets.py" + ["etc/gn3-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn3/secrets.py" + ["etc/gn-auth-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn-auth/secrets.py" + ["etc/gn-uploader-secrets.py"]="$BASE_DIR/etc/genenetwork/conf/gn-uploader/secrets.py" +) + +CONTAINER_SCM="genenetwork-local-container.scm" +CONTAINER_BIN="/usr/local/bin/genenetwork-local-container" +GC_ROOT="/var/guix/gcroots/genenetwork-local-container" + +log() { + local level="$1" + shift + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $level: $*" >&2 +} + +# Check dependencies +for cmd in git sudo diff cp grep; do + if ! command -v "$cmd" &>/dev/null; then + log "ERROR" "Required command '$cmd' not found" + exit 1 + fi +done + +if ! command -v guix &>/dev/null; then + echo "Please install Guix to proceed. Follow the instructions at:" + echo "https://issues.genenetwork.org/topics/octopus/set-up-guix-for-new-users" + echo "After installing Guix, try running this script again." + exit 1 +fi + +# Check for gn-bioinformatics channel +guix describe | grep gn-bioinformatics\ + &> /dev/null && log "INFO" "guix guix-informatics $(guix describe | grep gn-bioinformatics | cut -d ' ' -f 4)" || \ + (clear && guix describe && + echo "" + log "ERROR" "Please make sure your current profile has gn-bioinformatics" && + echo "" && + echo "Refer to:" && + echo " https://issues.genenetwork.org/topics/guix/guix-profiles" && + echo "to help you set your profile" && echo "" && + exit 1) + +# Validate HOME is set +if [ -z "${HOME:-}" ]; then + log "ERROR" "HOME environment variable is not set" + exit 1 +fi + +init_container() { + log "INFO" "Creating system directories..." + for dir in "${SYSTEM_DIRECTORIES[@]}"; do + # Check if directory exists and is accessible + if [ -d "$dir" ]; then + if [ -w "$dir" ]; then + log "DEBUG" "Directory exists and is writable: $dir" + else + log "WARNING" "Directory exists but is not writable: $dir. Making this writable" + sudo chown -R "$USER" "$dir" + fi + continue + fi + + # Attempt to create directory + log "INFO" "Creating directory: $dir" + if [ -w "$(dirname "$dir")" ]; then + # Parent directory is writable, try without sudo + if ! mkdir -p "$dir"; then + log "ERROR" "Failed to create directory without sudo: $dir" + exit 1 + fi + else + # Parent directory requires root, use sudo + if ! sudo mkdir -p "$dir"; then + log "ERROR" "Failed to create directory with sudo: $dir" + exit 1 + fi + # Set ownership to current user if created with sudo + if ! sudo chown -R "$USER" "$dir"; then + log "WARNING" "Failed to set ownership for: $dir" + fi + fi + done + + # Check and copy configuration files + log "INFO" "Checking and copying configuration files..." + for src in "${!FILE_MAPPINGS[@]}"; do + dest="${FILE_MAPPINGS[$src]}" + log "INFO" "Processing $src -> $dest" + + # Check if source file exists + if [ ! -f "$src" ]; then + log "ERROR" "Source file does not exist: $src" + exit 1 + fi + + # Check if destination file exists + if [ ! -f "$dest" ]; then + log "INFO" "Destination file does not exist, copying $src to $dest" + mkdir -p "$(dirname "$dest")" + if ! cp "$src" "$dest"; then + log "ERROR" "Failed to copy $src to $dest" + exit 1 + fi + continue + fi + + # Compare files using diff + log "INFO" "Comparing $src with $dest" + if diff_output=$(diff -u "$dest" "$src" 2>&1); then + log "INFO" "Files $src and $dest are identical" + else + log "INFO" "Differences found between $src and $dest:" + echo "$diff_output" >&2 + log "INFO" "Copying $src to $dest" + if ! cp "$src" "$dest"; then + log "ERROR" "Failed to copy $src to $dest" + exit 1 + fi + fi + done + + is_git_repository() { + local dir="$1" + # Check for standard repository + if [ -d "$dir/.git" ]; then + if [ -f "$dir/.git/HEAD" ] && [ -d "$dir/.git/refs" ]; then + log "DEBUG" "Detected standard Git repository: $dir" + return 0 + else + log "ERROR" "Directory $dir/.git exists but is not a valid Git repository" + return 1 + fi + # Check for bare repository + elif [ -f "$dir/HEAD" ] && [ -d "$dir/refs" ] && [ -d "$dir/objects" ]; then + log "DEBUG" "Detected bare Git repository: $dir" + return 0 + else + log "ERROR" "$dir exists but is not a Git repository (neither standard nor bare)" + return 1 + fi + } + # Clone GeneNetwork projects + log "INFO" "Cloning GeneNetwork projects..." + for project_entry in "${GN_PROJECTS[@]}"; do + IFS=':' read -r project repo_url <<< "$project_entry" + dir="$BASE_DIR/$project" + if [ ! -d "$dir" ]; then + log "INFO" "Cloning $project from $repo_url to $dir" + if ! git clone "$repo_url" "$dir"; then + log "ERROR" "Failed to clone $project" + exit 1 + fi + else + log "DEBUG" "Directory exists, skipping clone: $dir" + if ! is_git_repository "$dir"; then + log "ERROR" "$dir exists but is not a Git repository" + exit 1 + fi + fi + done + + FLASK_SESSIONS=( + "gn-uploader:$BASE_DIR/gn-uploader/flask_session" + "genenetwork2:$BASE_DIR/genenetwork2/flask_session" + ) + for flask_session in "${FLASK_SESSIONS[@]}"; do + IFS=':' read -r project session_dir <<< "$flask_session" + if [ ! -d "$session_dir" ]; then + log "INFO" "Creating FLASK_SESSION directory: $session_dir" + if ! mkdir -p "$session_dir"; then + log "ERROR" "Failed to create FLASK_SESSION directory: $session_dir" + exit 1 + fi + else + log "DEBUG" "FLASK_SESSION directory already exists: $session_dir" + fi + done + + # Verify container SCM file exists + if [ ! -f "$CONTAINER_SCM" ]; then + log "ERROR" "Container SCM file not found: $CONTAINER_SCM" + exit 1 + fi + + # Create Guix system container + log "INFO" "Creating Guix system container..." + SHARE_OPTS=( + "--share=$BASE_DIR/var/log=/var/log" + "--share=$BASE_DIR/var/genenetwork=/var/genenetwork" + "--share=$BASE_DIR/etc/genenetwork/conf=/etc/genenetwork/conf" + "--share=$BASE_DIR/etc/genenetwork=/etc/genenetwork" + "--share=$BASE_DIR/var/lib/redis=/var/lib/redis" + "--share=$BASE_DIR/var/lib/virtuoso=/var/lib/virtuoso" + "--share=$BASE_DIR/var/lib/data=/var/lib/data" + "--share=$BASE_DIR/var/lib/gn-uploader/data=/var/lib/gn-uploader/data" + "--share=$BASE_DIR/var/lib/gn-uploader/sessions=/var/lib/gn-uploader/sessions" + "--share=$BASE_DIR/genenetwork2=/genenetwork2" + "--share=$BASE_DIR/genenetwork3=/genenetwork3" + "--share=$BASE_DIR/gn-uploader=/gn-uploader" + "--share=$BASE_DIR/gn-auth=/gn-auth" + "--share=$BASE_DIR/var/lib/xapian=/var/lib/xapian" + "--share=$BASE_DIR/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite" + "--share=$BASE_DIR/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa" + "--share=/tmp/local-container=/tmp" + "--share=$BASE_DIR/gn-docs=/var/lib/gn-docs" + "--share=/run/mysqld=/run/mysqld" + ) + + container_script=$(guix system container \ + --network --fallback \ + --load-path=. \ + --verbosity=3 \ + "${SHARE_OPTS[@]}" \ + "$CONTAINER_SCM") + + log $container_script + + # Create symbolic links + log "INFO" "Creating symbolic links..." + if ! sudo ln -sf "$container_script" "$CONTAINER_BIN"; then + log "ERROR" "Failed to create symbolic link: $CONTAINER_BIN" + exit 1 + fi + + if ! sudo ln -sf "$container_script" "$GC_ROOT"; then + log "ERROR" "Failed to create GC root link: $GC_ROOT" + exit 1 + fi + + log "INFO" "Setup completed successfully!" + log "INFO" "Container script: $container_script" + log "INFO" "Run with: $CONTAINER_BIN" + log "INFO" "Email: test@development.user" + log "INFO" "Password: testpasswd" +} + +if [ "$1" = "--init-container" ]; then + init_container + exit 0 +fi + +init_sql() { + # Configuration + URL="https://files.genenetwork.org/database/db_webqtl_s-2025-02-18.sql.xz" + DOWNLOAD_DIR="/tmp" + FILE_NAME=$(basename "$URL") + EXTRACTED_FILE="${FILE_NAME%.xz}" + DB_USER="webqtlout" + DB_PASSWORD="webqtlout" + DB_HOST="localhost" + DB_NAME="db_webqtl_local" + MYSQL_ROOT_USER="" # Change to your MySQL admin user if different + MYSQL_ROOT_PASSWORD="" # Set this or leave empty to prompt + + # Check for required tools + for cmd in wget xz mysql; do + if ! command -v "$cmd" &> /dev/null; then + log ERROR "Required command '$cmd' not found" + exit 1 + fi + done + + # Extract the .xz file + if [ ! -f "/tmp/db_webqtl_s-2025-02-18.sql" ]; then + # Download the file + log INFO "Downloading $URL to $DOWNLOAD_DIR/$FILE_NAME" + if ! wget -O "$DOWNLOAD_DIR/$FILE_NAME" "$URL"; then + log ERROR "Failed to download $URL" + exit 1 + fi + + log INFO "Extracting $DOWNLOAD_DIR/$FILE_NAME" + if ! xz -d "$DOWNLOAD_DIR/$FILE_NAME"; then + log ERROR "Failed to extract $DOWNLOAD_DIR/$FILE_NAME" + rm -f "$DOWNLOAD_DIR/$FILE_NAME" + exit 1 + fi + fi + + # Prepare MySQL user and password credentials + if [ -z "$MYSQL_ROOT_USER" ]; then + log INFO "MySQL root user not set, prompting for input" + read -s -p "Enter MySQL user: " MYSQL_ROOT_USER + fi + if [ -z "$MYSQL_ROOT_PASSWORD" ]; then + log INFO "MySQL root password not set, prompting for input" + read -s -p "Enter MySQL root password: " MYSQL_ROOT_PASSWORD + fi + + # Check if DB user exists, create if not + log INFO "Checking if MySQL user $DB_USER exists" + USER_EXISTS=$(mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '$DB_USER' AND host = 'localhost') AS user_exists;" 2>/dev/null | grep -o '[0-1]$') + if [ "$USER_EXISTS" = "0" ]; then + log INFO "Creating MySQL user $DB_USER" + if ! mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "CREATE USER '$DB_USER'@'localhost' IDENTIFIED BY '$DB_PASSWORD'; GRANT ALL PRIVILEGES ON $DB_NAME.* TO '$DB_USER'@'localhost'; FLUSH PRIVILEGES;" 2>/dev/null; then + log ERROR "Failed to create MySQL user $DB_USER" + rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" + exit 1 + fi + else + log INFO "User $DB_USER already exists, ensuring privileges" + if ! mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "GRANT ALL PRIVILEGES ON $DB_NAME.* TO '$DB_USER'@'localhost'; FLUSH PRIVILEGES;" 2>/dev/null; then + log ERROR "Failed to update privileges for $DB_USER" + rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" + exit 1 + fi + fi + + # Create database if it doesn't exist + log INFO "Ensuring database $DB_NAME exists" + if ! mysql -h "$DB_HOST" -u "$MYSQL_ROOT_USER" -p"$MYSQL_ROOT_PASSWORD" -e "CREATE DATABASE IF NOT EXISTS $DB_NAME;" 2>/dev/null; then + log ERROR "Failed to create or verify database $DB_NAME" + rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" + exit 1 + fi + + # Install the SQL dump into the database + log INFO "Importing $DOWNLOAD_DIR/$EXTRACTED_FILE into $DB_NAME" + if ! mysql -h "$DB_HOST" -u "$DB_USER" -p"$DB_PASSWORD" "$DB_NAME" < "$DOWNLOAD_DIR/$EXTRACTED_FILE"; then + log ERROR "Failed to import $DOWNLOAD_DIR/$EXTRACTED_FILE into $DB_NAME" + rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" + exit 1 + fi + + # Clean up + log INFO "Removing $DOWNLOAD_DIR/$EXTRACTED_FILE" + rm -f "$DOWNLOAD_DIR/$EXTRACTED_FILE" + + log INFO "Database import completed successfully" +} + +if [ "$1" = "--init-sql" ]; then + init_sql + exit 0 +fi + +init_rdf() { + # Check for required tools + for cmd in guix; do + if ! command -v "$cmd" &> /dev/null; then + log ERROR "Required command '$cmd' not found" + exit 1 + fi + done + + log INFO "Please make sure your container is running for this to work" + log INFO "This takes some time to run" + # Check if directory exists and is accessible + if [ -d "$BASE_DIR/var/lib/data" ]; then + if [ -w "$BASE_DIR/var/lib/data" ]; then + log "DEBUG" "Directory exists and is writable: $BASE_DIR/var/lib/data" + else + log "WARNING" "Directory exists but is not writable: $BASE_DIR/var/lib/data. Making this writable" + sudo chown -R "$USER" "$BASE_DIR/var/lib/data" + fi + continue + fi + curr_dir="$PWD" + cd "$BASE_DIR/gn-transform-databases" + if ls $BASE_DIR/var/lib/data/*ttl >/dev/null 2>&1; then + log INFO "Removing all the ttl files and generating them again" + rm $BASE_DIR/var/lib/data/*ttl + fi + + log INFO "Generating the ttl files" + guix shell -m "manifest.scm" -- guile "generate-ttl-files.scm" \ + --settings "$curr_dir/etc/conn.scm" --output "$BASE_DIR/var/lib/data" + log INFO "Loading ttl files" + guix shell guile-dbi -m "manifest.scm" -- guile load-rdf.scm \ + "$curr_dir/etc/conn.scm" + cd $curr_dir +} + +if [ "$1" = "--init-rdf" ]; then + init_rdf + exit 0 +fi + +init_xapian() { + # Check for required tools + log INFO "Please make sure your container is running for this to work" + log INFO "This takes some time to run" + # Check if directory exists and is accessible + if [ -d "$BASE_DIR/var/lib/xapian" ]; then + if [ -w "$BASE_DIR/var/lib/xapian" ]; then + log "DEBUG" "Directory exists and is writable: $BASE_DIR/var/lib/xapian" + else + log "WARNING" "Directory exists but is not writable: $BASE_DIR/var/lib/xapian. Making this writable" + sudo chown -R "$USER" "$BASE_DIR/var/lib/xapian" + fi + fi + # Check if build directory exists + if [ ! -f "$BASE_DIR/var/lib/xapian/build" ]; then + rm -rf "$BASE_DIR/var/lib/xapian/build" + fi + guix shell python-wrapper genenetwork3 \ + --share="$BASE_DIR/var/lib/xapian" -- \ + "$BASE_DIR/genenetwork3/scripts/index-genenetwork" create-xapian-index \ + "$BASE_DIR/var/lib/xapian/build" \ + "mysql://webqtlout:webqtlout@localhost/db_webqtl_local?unix_socket=/run/mysqld/mysqld.sock&charset=utf8" \ + "http://localhost:7082/sparql" + mv $BASE_DIR/var/lib/xapian/build/* $BASE_DIR/var/lib/xapian/ \ + && rmdir $BASE_DIR/var/lib/xapian/build/ + log INFO "Please restart the container to set the correct permissions" +} + +if [ "$1" = "--init-xapian" ]; then + init_xapian + exit 0 +fi + + +# Function to handle yes/no prompts +prompt_yes_no() { + local prompt="$1" + while true; do + read -p "$prompt (yes/no): " response + case "$response" in + [Yy][Ee][Ss]|[Yy] ) return 0 ;; + [Nn][Oo]|[Nn] ) return 1 ;; + * ) echo "Please answer 'yes' or 'no'." ;; + esac + done +} + +# Main tutorial function +run_beginner_setup_tutorial() { + clear + # Step 1: Initialize Container + echo "Step 1: Setting Up the Container" + echo "--------------------------------" + echo "Before we begin, make sure you are using the correct guix-profile that has:" + echo "" + echo " guix-bioinformatics" + echo "" + echo "Otherwise, I will terminate early. If you don't know how to do that, read:" + echo "" + echo " https://issues.genenetwork.org/topics/guix/guix-profiles" + echo "" + echo "This step prepares the container environment where GeneNetwork runs." + echo "It ensures all necessary files, folders, and configurations are in place." + echo "" + echo "This step is similar to running:" + echo "" + echo "./genenetwork-local-container.sh --init-container" + echo "" + echo "NOTE: If you have any of the following projects:" + echo " genenetwork2" + echo " genenetwork3" + echo " gn-uploader" + echo "" + echo "Move/copy them to $BASE_DIR. This is where'll you'll be hacking" + echo "on them from. Because of how we mount things inside the container," + echo "make sure the names appear as above." + echo "" + echo "Now, some things to note. AI search won't work. To get this working, if you want so," + echo "Get into the container, and modify FAHAMU_AUTH_TOKEN in:" + echo " /etc/genenetwork/conf/gn3/secrets.py" + echo "" + echo "Similarly, e-mail functionality ala resetting passwords or confirming a new user" + echo "won't work unless you modify:" + echo " SMTP_HOST = 'XXXX'" + echo " SMTP_USER = 'XXXX'" + echo " SMTP_PASSWORD = 'XXXX'" + echo " SMTP_PORT = 'XXXX'" + echo " EMAIL_ADDRESS = 'XXXX'" + echo " EMAIL_DISPLAY_NAME = 'XXXX'" + echo "inside:" + echo " /etc/genenetwork/conf/gn-auth/secrets.py" + echo "" + if prompt_yes_no "Would you like to set up the container now?"; then + init_container + clear + echo "Container setup complete!" + echo "" + echo "Right now you have a working container. You need to set-up sql, upload rdf data" + echo "and build a xapian index to have a fully functioning system akin to production and" + echo "our test environment." + echo "" + else + clear + echo "Skipping container setup. Moving to the next step." + fi + echo "" + + # Step 2: Initialize SQL Database + echo "Step 2: Setting Up the SQL Database" + echo "-----------------------------------" + echo "Please make sure you have a working MariaDB installation on your Linux system." + echo "" + echo "This step creates and configures the database for storing GeneNetwork data." + echo "" + echo "It sets up tables and initial data needed for Genenetwork to run" + echo "" + echo "This step is similar to running:" + echo "./genenetwork-local-container.sh --init-sql" + echo "" + echo "Make sure you have a running mysql instance for this to work" + echo "" + if prompt_yes_no "Would you like to set up the database now?"; then + init_sql + clear + echo "Database setup complete!" + else + clear + echo "Skipping database setup. Moving to the next step: Importing RDF" + fi + echo "" + + # Step 3: Initialize RDF + echo "" + echo "Step 3: Importing RDF into your local set-up" + echo "---------------------------------------------" + echo "This step adds RDF (Resource Description Framework) data." + echo "" + echo "RDF helps organize and link data for GeneNetwork." + echo "" + echo "IMPORTANT. This step requires you to have a running container instance." + echo " Otherwise, it will fail." + echo "To run your container, in a different shell, run:" + echo "sudo /usr/local/bin/genenetwork-local-container" + echo "" + echo "This step is similar to running:" + echo "./genenetwork-local-container.sh --init-rdf" + echo "" + if prompt_yes_no "Would you like to set up RDF now?"; then + init_rdf + clear + echo "RDF setup complete!" + else + clear + echo "Skipping RDF setup." + fi + echo "" + + # Step 4: Initialize XAPIAN + clear + echo "Step 3: Setting up XAPIAN search locally" + echo "---------------------------------------------" + echo "This indexes all your data from GeneNetwork and indexes it using XAPIAN" + echo "" + echo "Read more here:" + echo "https://issues.genenetwork.org/topics/xapian/xapian-search" + echo "" + echo "IMPORTANT. This step requires you to have a running container instance." + echo " Otherwise, it will fail." + echo " To run your container, in a different shell, run:" + echo " sudo /usr/local/bin/genenetwork-local-container" + echo "" + echo "This step is similar to running:" + echo "./genenetwork-local-container.sh --init-xapian" + echo "" + if prompt_yes_no "Would you like to build the xapian index now?"; then + init_xapian + clear + echo "XAPIAN setup complete!" + else + clear + echo "Skipping RDF setup." + fi + echo "" + + # Completion Message + echo "=============================================================" + echo "Congratulations! You've completed the GeneNetwork Beginner Setup Tutorial!" + echo "=============================================================" + echo "You may have skipped some steps. To complete them, run the tutorial again." + echo "Your environment is ready to the extent you chose to set it up." + echo "If you run your container, you can access the different web-services at:" + echo "gn-guile : localhost:8091" + echo "genenetwork2 : localhost:8082" + echo "genenetwork3 : localhost:8083" + echo "gn-uploader : localhost:8085" + echo "sparql web ui: localhost:7082/sparql" + echo "" + echo "To access virtuoso back-end service, run:" + echo "guix shell virtuoso-ose -- isql 7081" + echo "" + echo "To login in gn-uploader/genenetwork2, use the following credentials:" + echo " Email: test@development.user" + echo " Password: testpasswd" + exit 0 +} + +# Check for --beginner-set-up flag or run tutorial interactively +if [ "$1" = "--beginner-set-up" ]; then + # Interactive mode: run the tutorial + run_beginner_setup_tutorial + exit 0 +fi diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index b863bcf..a403f21 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -109,6 +109,8 @@ (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) + (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir + (default "/var/genenetwork/sessions/genenetwork2")) (gn2-secrets genenetwork-configuration-gn2-secrets (default "/etc/genenetwork")) (gn3-secrets genenetwork-configuration-gn3-secrets @@ -119,6 +121,10 @@ (default 8091)) (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout (default "/export/data/gn-docs")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) (log-level genenetwork-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -142,6 +148,8 @@ (default "https://auth.genenetwork.org")) (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) + (sessions-dir gn-uploader-sessions-dir + (default "/var/genenetwork/sessions/gn-uploader")) (log-level gn-uploader-configuration-log-level (default 'warning) (sanitize sanitize-log-level))) @@ -170,6 +178,8 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) @@ -203,7 +213,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -212,7 +224,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -242,7 +256,7 @@ (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -262,7 +276,11 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets + #:directories? #t) + (find-files #$gn2-sessions-dir #:directories? #t))) (for-each (lambda (file) (chown file @@ -278,7 +296,31 @@ (find-files #$gn2-secrets #:directories? #f) (find-files #$gn-auth-secrets - #:directories? #f))))))) + #:directories? #f))) + ;; Make sub-directories for various apps under gn-tmpdir and assign + ;; appropriate permissions + (for-each (match-lambda + ((subdir user) + (let ((full-path + (string-append #$gn-tmpdir "/" subdir))) + (unless (file-exists? full-path) + (mkdir full-path #o755)) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw user)) + (passwd:gid (getpw user)))) + (find-files full-path + #:directories? #t))))) + '(("gn2-tmpdir" "gunicorn-genenetwork2") + ("gn3-tmpdir" "gunicorn-genenetwork3"))) + + ;; setup correct ownership for gn-docs + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (find-files #$gn-doc-git-checkout + #:directories? #t)))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -309,7 +351,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path log-level) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -333,7 +375,9 @@ object." ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("AI_SEARCH_ENABLED" "True"))))) + ("AI_SEARCH_ENABLED" "True") + ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir) + ("MAX_FORM_MEMORY_SIZE" 52428800))))) (gn3-profile (profile (content (package->development-manifest genenetwork3)) (allow-collisions? #t))) @@ -358,9 +402,9 @@ object." (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) ("AUTH_DB" ,auth-db) - ("SQL_URI" ,sql-uri) - ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys")) - ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem"))))))) + ("SQL_URI" ,sql-uri))))) + (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir")) + (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir"))) (list (gunicorn-app (name "genenetwork2") (package genenetwork2) @@ -375,7 +419,7 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn2-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) @@ -388,13 +432,17 @@ object." (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn2-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -408,7 +456,11 @@ object." (writable? #t)) (file-system-mapping (source gn2-ca-bundle) - (target source)))) + (target source)) + (file-system-mapping + (source gn2-sessions-dir) + (target source) + (writable? #t)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))) @@ -429,7 +481,7 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn3-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) @@ -455,9 +507,12 @@ object." (file-system-mapping (source gn3-data-directory) (target source)) ; Rqtl usese this - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn3-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) @@ -526,7 +581,8 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" @@ -628,14 +684,18 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory) + (secrets data-directory sessions-dir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Let service user own their own secrets files. - (chown #$secrets - (passwd:uid (getpw "gunicorn-gn-uploader")) - (passwd:gid (getpw "gunicorn-gn-uploader"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader")))) + (append (list #$secrets) + (find-files #$sessions-dir + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -651,25 +711,26 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t))) - (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOAD_FOLDER" ,(string-append data-directory + "/uploads")) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) @@ -706,7 +767,11 @@ a @code{<genenetwork-configuration>} record." (target source)) (file-system-mapping (source gn-uploader-ca-bundle) - (target source)))) + (target source)) + (file-system-mapping + (source sessions-dir) + (target source) + (writable? #t)))) (extra-cli-arguments (list "--log-level" (string-upcase (symbol->string log-level))))))))) diff --git a/production-deploy.sh b/production-deploy.sh index bd95814..a88fcb8 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -29,7 +29,6 @@ container_script=$(guix system container \ --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ --share=/export2/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ - --share=/export/mysql/database=/var/lib/mysql \ --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ --share=/export2/guix-containers/genenetwork/var/log=/var/log \ --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ @@ -38,7 +37,10 @@ container_script=$(guix system container \ --share=/export2/guix-containers/genenetwork/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld=/run/mysqld \ --share=/export/data/gn-docs/ \ - --share=/export2/guix-containers/genenetwork/tmp=/tmp \ + --share=/export2/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export2/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export2/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 933320a..9e629f0 100644 --- a/production.scm +++ b/production.scm @@ -44,14 +44,11 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service mysql-service-type - (mysql-configuration - (auto-upgrade? #f))) - (service virtuoso-service-type + (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port 9892) (http-server-port 9893) - (dirs-allowed "/var/lib/virtuoso") + (dirs-allowed (list "/export/data/virtuoso")) (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") @@ -80,7 +77,7 @@ (gn3-port 9895) (gn-auth-port 9896) (sql-uri - "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:9893/sparql") (gn3-data-directory "/var/genenetwork/data/genenetwork3") @@ -90,5 +87,6 @@ (auth-db "/var/lib/genenetwork-sqlite/auth.db") (llm-db-path "/var/lib/genenetwork-gnqa/llm.db") (gn3-alias-server-port 9800) + (gn-tmpdir "/opt/gn/tmp") (log-level 'debug))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index bee9abf..4ecacc6 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -26,6 +26,7 @@ container_script=$(guix system container \ --share=/export2/guix-containers/public-sparql/tmp=/tmp \ --share=/export2/guix-containers/public-sparql/var/log=/var/log \ --share=/export2/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index f6efb15..4603cec 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -20,6 +20,7 @@ (use-modules (gnu) (gn services databases) (gnu services web) + ((gnu packages admin) #:select (shepherd)) (forge nginx) (forge socket)) @@ -50,13 +51,17 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) (number-of-buffers 4000000) - (dirs-allowed "/var/lib/virtuoso") + (dirs-allowed (list "/export/data/virtuoso")) (maximum-dirty-buffers 3000000) (database-file "/var/lib/virtuoso/public-virtuoso.db") (transaction-file "/var/lib/virtuoso/public-virtuoso.trx"))) diff --git a/uploader-deploy.sh b/uploader-deploy.sh index fdbbe0c..415790b 100755 --- a/uploader-deploy.sh +++ b/uploader-deploy.sh @@ -41,18 +41,22 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/uploader/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/uploader/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ - --share=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ - --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ - --expose=/export/data/uploader/genotype_files=/export/data/genenetwork/genotype_files \ - --expose=/export/data/uploader/genenetwork3 \ - --share=/export/data/uploader/gn-uploader \ + --share=/export/guix-containers/uploader/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/uploader/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/uploader/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/uploader/var/log=/var/log \ + --share=/export/guix-containers/uploader/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-xapian=/var/lib/xapian \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld3307=/run/mysqld \ + --share=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/uploader/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/uploader/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/data/uploader/gn-uploader \ uploader.scm) echo "${container_script}" diff --git a/uploader.scm b/uploader.scm index 62ab35f..5064426 100644 --- a/uploader.scm +++ b/uploader.scm @@ -42,11 +42,7 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service virtuoso-service-type - (virtuoso-configuration - (server-port 10892) - (http-server-port 10893))) - (service forge-nginx-service-type + (services (cons* (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") @@ -64,24 +60,26 @@ (gn2-port 10894) (gn3-port 10895) (gn-auth-port 10896) - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") - (auth-db "/export/data/genenetwork-sqlite/auth.db") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (auth-db "/var/lib/genenetwork-sqlite/auth.db") + (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:10893/sparql") - (gn3-data-directory "/export/data/uploader/genenetwork3") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (llm-db-path "/var/lib/genenetwork-gnqa/llm.db"))) (service gn-uploader-service-type (gn-uploader-configuration (gn-uploader gn-uploader) (server-name "staging-uploader.genenetwork.org") (port 10897) (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (data-directory "/export/data/uploader/gn-uploader") - (log-level "DEBUG") + (log-level 'debug) (auth-server-url "https://staging-auth.genenetwork.org/") (gn2-server-url "https://staging.genenetwork.org"))) %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0414a65..0dd2509 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -25,6 +25,7 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ virtuoso.scm) echo $container_script diff --git a/virtuoso.scm b/virtuoso.scm index edcd575..3272f41 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed "/var/lib/virtuoso"))) + (dirs-allowed (list "/export/data/virtuoso")))) %base-services))) |