diff options
| -rwxr-xr-x | genenetwork-development-deploy.sh | 2 | ||||
| -rw-r--r-- | genenetwork-development.scm | 379 | ||||
| -rw-r--r-- | genenetwork/services/genenetwork.scm | 346 | ||||
| -rwxr-xr-x | production-deploy.sh | 26 | ||||
| -rw-r--r-- | production.scm | 67 | ||||
| -rwxr-xr-x | public-sparql-deploy.sh | 8 | ||||
| -rw-r--r-- | public-sparql.scm | 35 | ||||
| -rw-r--r-- | slurm.scm | 15 | ||||
| -rwxr-xr-x | uploader-deploy.sh | 26 | ||||
| -rw-r--r-- | uploader.scm | 24 | ||||
| -rwxr-xr-x | virtuoso-deploy.sh | 1 | ||||
| -rw-r--r-- | virtuoso.scm | 2 |
12 files changed, 683 insertions, 248 deletions
diff --git a/genenetwork-development-deploy.sh b/genenetwork-development-deploy.sh index d39bb7f..b251033 100755 --- a/genenetwork-development-deploy.sh +++ b/genenetwork-development-deploy.sh @@ -42,8 +42,8 @@ container_script=$(guix system container --network \ --expose=/export/data/genenetwork \ --share=/export/data/genenetwork-xapian \ --share=/export/data/genenetwork-sqlite \ - --share=/export/genenetwork-database-dump \ --share=/var/run/mysqld=/run/mysqld \ + --share=/export/data/gn-docs/ \ genenetwork-development.scm) echo $container_script diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 5fafc12..f841255 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,8 +21,9 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) + ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) (gn services databases) + ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) ((gnu packages base) #:select (gnu-make tar)) @@ -64,6 +65,7 @@ (guix packages) (guix profiles) (guix records) + (guix search-paths) (guix store) (guix utils) (forge acme) @@ -120,7 +122,9 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "https://git.genenetwork.org/gn-auth")) + (default "/home/git/public/gn-auth")) + (gn-libs-repository genenetwork-configuration-gn-libs-repository + (default "/home/git/public/gn-libs")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -144,7 +148,11 @@ be imported into G-expressions." (auth-db-path genenetwork-auth-db-path (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path - (default "/export/data/genenetwork-sqlite/llm.db"))) + (default "/export/data/genenetwork-sqlite/llm.db")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/export/data/gn-docs"))) ;;; @@ -182,11 +190,12 @@ described by CONFIG, a <genenetwork-configuration> object. TEST-COMMAND is a list of strings specifying the command to be executed." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn3-port genotype-files) + (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) (with-packages (list bash coreutils git-minimal nss-certs) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (srfi srfi-26)) (define (hline) "Print a horizontal line 50 '=' characters long." @@ -199,9 +208,19 @@ executed." (invoke "git" "log" "--max-count" "1") (hline)) + (define (call-with-temporary-directory proc) + (let ((tmp-dir (mkdtemp "/tmp/gn.XXXXXX"))) + (dynamic-wind + (const #t) + (cut proc tmp-dir) + (cut delete-file-recursively tmp-dir)))) + (invoke "git" "clone" "--depth" "1" #$gn3-repository) (with-directory-excursion "genenetwork3" (show-head-commit)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository) + (with-directory-excursion "gn-libs" + (show-head-commit)) (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) @@ -222,9 +241,15 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl") + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") (chdir "genenetwork2") - (apply invoke '#$test-command)))))) + ;; XXXX: FIXME: R/Qtl tests fail because files are generated in + ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 + ;; so tests will fail because of permission issues. + (call-with-temporary-directory + (lambda (tmp-dir) + (setenv "TMPDIR" tmp-dir) + (apply invoke '#$test-command)))))))) (define %xapian-directory "/export/data/genenetwork-xapian") @@ -252,7 +277,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -297,7 +322,7 @@ genenetwork3 source from the latest commit of @var{project}." (system* (string-append gn3-dir "/scripts/index-genenetwork") "is-data-modified" #$%xapian-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql")))) (setenv "LAMINAR_REASON" "Nightly xapian index rebuild") (invoke #$(file-append laminar "/bin/laminarc") @@ -307,7 +332,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port) (list (forge-project (name "genenetwork2") (repository gn2-repository) @@ -380,6 +405,20 @@ genenetwork3 source from the latest commit of @var{project}." (trigger? #f)))) (ci-jobs-trigger 'webhook)) (forge-project + (name "gn-libs") + (repository gn-libs-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-libs") + (run (guix-channel-job-gexp + (list (channel + (name 'gn-libs) + (url (forge-project-repository this-forge-project)) + (branch "main"))) + #:variables (list (variable-specification + (module '(gn-libs)) + (name 'gn-libs))) + #:guix-daemon-uri %guix-daemon-uri)))))) + (forge-project (name "gn-auth") (repository gn-auth-repository) (ci-jobs (list (forge-laminar-job @@ -414,8 +453,7 @@ genenetwork3 source from the latest commit of @var{project}." #:variables (list (variable-specification (module '(gn-auth)) (name 'gn-auth-all-tests))) - #:guix-daemon-uri %guix-daemon-uri))))) - (ci-jobs-trigger 'webhook))))) + #:guix-daemon-uri %guix-daemon-uri))))))))) (define (genenetwork2-cd-gexp config) "Return a G-expression that runs the latest genenetwork2 development @@ -453,31 +491,21 @@ server described by CONFIG, a <genenetwork-configuration> object." (string-append (getcwd) "/genenetwork3")) ;; Set other environment variables required by ;; genenetwork2. - (setenv "SERVER_PORT" #$(number->string gn2-port)) (setenv "GN2_PROFILE" #$(profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) - (setenv "GN_SERVER_URL" "https://cd.genenetwork.org/api3/") - (setenv "GN3_LOCAL_URL" - #$(string-append "http://localhost:" - (number->string gn3-port))) - (setenv "GENENETWORK_FILES" #$genotype-files) - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl") - (setenv "HOME" "/tmp") - (setenv "NO_REDIS" "no-redis") - (setenv "RUST_BACKTRACE" "1") - (setenv "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" "GN2_SECRETS=\"" gn2-secrets "/gn2-secrets.py\"\n" + "AI_SEARCH_ENABLED=True\n" "GN3_LOCAL_URL=\"" (string-append "http://localhost:" (number->string gn3-port)) "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) @@ -516,11 +544,17 @@ server described by CONFIG, a <genenetwork-configuration> object." #$(mixed-text-file "gn3.conf" "SPARQL_ENDPOINT=\"" sparql-endpoint "\"\n" "DATA_DIR=\"" data-directory "\"\n" + "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" "XAPIAN_DB_PATH=\"" xapian-db-path "\"\n" "AUTH_DB=\"" auth-db-path "\"\n" "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) + (setenv "RSCRIPT" #$(file-append + (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t)) + "/bin/Rscript")) ;; Run genenetwork3. (with-directory-excursion "genenetwork3" (show-head-commit) @@ -575,40 +609,135 @@ server described by CONFIG, a <genenetwork-configuration> object." "--workers" "8" "gn_auth.wsgi:app")))))))) +(define (gn-guile-gexp gn-guile-port) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) (list (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/cd/gn-guile.log"))) + (stop #~(make-kill-destructor))) + (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) ;; FIXME: The genenetwork2 service should depend on redis. (requirement '(networking genenetwork3)) - (start #~(make-forkexec-constructor - (list #$(least-authority-wrapper - (program-file "genenetwork2" - (genenetwork2-cd-gexp config)) - #:name "genenetwork2-pola-wrapper" - ;; If we mapped only the mysqld.sock - ;; socket file, it would break when the - ;; external mysqld server is restarted. - #:mappings (list (file-system-mapping - (source genotype-files) - (target source)) - (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t)) - (file-system-mapping - (source gn2-secrets) - (target source) - (writable? #t))) - #:namespaces (delq 'net %namespaces)) - "127.0.0.1" #$(number->string gn2-port)) - #:user "genenetwork" - #:group "genenetwork" - #:log-file "/var/log/cd/genenetwork2.log")) + (modules '((guix search-paths) + (ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn2-manifest (packages->manifest (list genenetwork2))) + (gn2-profile (profile + (content gn2-manifest) + (allow-collisions? #t))) + (gn2-settings + `(("SERVER_PORT" ,(number->string gn2-port)) + ("GENENETWORK_FILES" ,genotype-files) + ("HOME" "/tmp") + ("LC_ALL" "en_US.UTF-8") + ("NO_REDIS" "no-redis") + ("RUST_BACKTRACE" "1")))) + (with-imported-modules (source-module-closure '((guix search-paths))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork2" + (genenetwork2-cd-gexp config)) + #:name "genenetwork2-pola-wrapper" + #:preserved-environment-variables + (append '("REQUESTS_CA_BUNDLE") + (map first gn2-settings) + (map search-path-specification-variable + (manifest-search-paths gn2-manifest))) + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source genotype-files) + (target source)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source gn2-secrets) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn2-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (append + '("REQUESTS_CA_BUNDLE=" + #$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) + (map (match-lambda + ((spec . value) + (string-append (search-path-specification-variable spec) + "=" + value))) + (evaluate-search-paths + (map sexp->search-path-specification + '#$(map search-path-specification->sexp + (manifest-search-paths gn2-manifest))) + (list #$gn2-profile))) + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn2-settings)) + #:log-file "/var/log/cd/genenetwork2.log")))) (stop #~(make-kill-destructor))) (shepherd-service (documentation "Run GeneNetwork 3 development server.") @@ -626,6 +755,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) @@ -711,6 +849,8 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) + (find-files "/export/data/gn-docs" + #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets @@ -744,10 +884,6 @@ described by CONFIG, a <genenetwork-configuration> object." ;;; transform-genenetwork-database ;;; -;; Path to genenetwork database dump export directory that has lots of -;; free space -(define %transform-genenetwork-database-export-directory - "/export/genenetwork-database-dump") ;; Unreleased version of ccwl that is required by ;; transform-genenetwork-database for its graphql library. @@ -804,57 +940,54 @@ described by CONFIG, a <genenetwork-configuration> object." (description "run64 is a SRFI-64 test runner for Scheme.") (license license:gpl3+))) -(define (transform-genenetwork-database project) +;; Connection settings for Virtuoso and MySQL used to load data into Virtuoso +(define %connection-settings + "/etc/genenetwork/conf/gn-transform-database/conn.scm") + +;; Path to where the data directory from which virtuoso loads all the files +(define %virtuoso-data-dir "/var/lib/data") + +(define (transform-genenetwork-database-gexp connection-settings virtuoso-data-dir repository) (with-imported-modules '((guix build utils)) (with-packages (list ccwl git-minimal gnu-make guile-3.0 guile-dbd-mysql guile-dbi guile-hashing guile-libyaml guile-sparql - guile-zlib nss-certs virtuoso-ose) - #~(begin - (use-modules (guix build utils) - (srfi srfi-26) - (ice-9 threads)) - - (invoke "git" "clone" - "--depth" "1" - #$(forge-project-repository project) - ".") - (invoke "make" "-j" (number->string (current-processor-count))) - (let ((connection-settings-file #$(string-append %transform-genenetwork-database-export-directory - "/conn.scm")) - (dump-directory #$(string-append %transform-genenetwork-database-export-directory - "/dump"))) - (when (file-exists? dump-directory) - (delete-file-recursively dump-directory)) - (mkdir-p dump-directory) - ;; Dump data to RDF. - (invoke "./pre-inst-env" "./dump.scm" - connection-settings-file - dump-directory) - ;; Validate dumped RDF, sending the error output to - ;; oblivion because we don't want to print out potentially - ;; sensitive data. - (with-error-to-file "/dev/null" - (cut invoke - #$(file-append raptor2 "/bin/rapper") - "--input" "turtle" - "--count" - (string-append dump-directory "/dump.ttl"))) - ;; Load RDF into virtuoso. - (invoke "./pre-inst-env" "./load-rdf.scm" - connection-settings-file - (string-append dump-directory "/dump.ttl")) - ;; Visualize schema and archive results. - (invoke "./pre-inst-env" "./visualize-schema.scm" - connection-settings-file) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "sql.dot" (string-append "-o" (getenv "ARCHIVE") "/sql.svg")) - (invoke #$(file-append graphviz "/bin/dot") - "-Tsvg" "rdf.dot" (string-append "-o" (getenv "ARCHIVE") "/rdf.svg"))))))) + guile-zlib nss-certs virtuoso-ose raptor2) + #~(begin + (use-modules (guix build utils) + (srfi srfi-26) + (ice-9 threads)) + (setenv "LC_ALL" "en_US.UTF-8") + (let ((build-directory (string-append #$virtuoso-data-dir + "/build"))) + ;; Only run this job if the build directory does not + ;; exists. This ensures that no other process is + ;; running this. + (unless (file-exists? build-directory) + (invoke "git" "clone" "--depth" "1" #$repository ".") + (invoke "make" "-j" (number->string (current-processor-count))) + (invoke "./generate-ttl-files.scm" "--settings" + #$connection-settings "--output" build-directory) + ;; First clear all the files in our virtuoso directory + (for-each (lambda (file) + (unless (string-suffix? "build" (dirname file)) + (delete-file file))) + (find-files #$virtuoso-data-dir ".ttl")) + ;; Move data into the container's virtuoso data directory + (copy-recursively build-directory #$virtuoso-data-dir) + ;; Load RDF into virtuoso. + (invoke "./pre-inst-env" "./load-rdf.scm" #$connection-settings) + ;; Visualize schema and archive results. + (invoke "./pre-inst-env" "./visualize-schema.scm" #$connection-settings) + (invoke #$(file-append graphviz "/bin/dot") + "-Tsvg" "sql.dot" (string-append "-o" (getenv "ARCHIVE") "/sql.svg")) + (invoke #$(file-append graphviz "/bin/dot") + "-Tsvg" "rdf.dot" (string-append "-o" (getenv "ARCHIVE") "/rdf.svg")) + (delete-file-recursively build-directory))))))) (define transform-genenetwork-database-project (forge-project (name "transform-genenetwork-database") - (repository "/home/git/public/gn-transform-databases/") + (repository "/home/git/public/gn-transform-databases") (ci-jobs (list (forge-laminar-job (name "transform-genenetwork-database-tests") (run (guix-channel-job-gexp @@ -866,7 +999,10 @@ described by CONFIG, a <genenetwork-configuration> object." #:guix-daemon-uri %guix-daemon-uri))) (forge-laminar-job (name "transform-genenetwork-database") - (run (transform-genenetwork-database this-forge-project))))))) + (run (transform-genenetwork-database-gexp + %connection-settings + %virtuoso-data-dir + "https://git.genenetwork.org/gn-transform-databases"))))))) ;;; @@ -884,6 +1020,7 @@ described by CONFIG, a <genenetwork-configuration> object." #~(begin (use-modules (guix build utils)) + (setenv "LC_ALL" "en_US.UTF-8") (invoke #$(file-append tissue "/bin/tissue") "pull" "issues.genenetwork.org")))))))) (ci-jobs-trigger 'webhook))) @@ -1137,6 +1274,20 @@ gn-auth." ";") "proxy_set_header Host $host;"))))))) +(define set-build-directory-permissions-gexp + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "laminar")) + (passwd:gid (getpw "laminar")))) + (append (find-files #$%xapian-directory + #:directories? #t) + (find-files #$%virtuoso-data-dir + #:directories? #t)))))) + ;; Port on which webhook is listening (define %webhook-port 9091) ;; Port on which genenetwork2 is listening @@ -1194,6 +1345,15 @@ gn-auth." (jobs (list #~(job '(next-hour) #$(program-file "build-xapian-index-cron" build-xapian-index-cron-gexp) + #:user "laminar") + ;; Run cron once a week at midnight on Sunday morning + ;; Verify using: https://crontab.guru/#0_0_*_*_0 + #~(job "0 0 * * 0" + #$(program-file "update-virtuoso" + (transform-genenetwork-database-gexp + %connection-settings + %virtuoso-data-dir + "https://git.genenetwork.org/gn-transform-databases")) #:user "laminar"))))) (simple-service 'install-laminar-template activation-service-type @@ -1212,7 +1372,7 @@ gn-auth." (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (server-port 9081) - (dirs-allowed "/var/lib/data") + (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) (service genenetwork-service-type (genenetwork-configuration @@ -1230,18 +1390,7 @@ gn-auth." (xapian-db-path %xapian-directory))) (simple-service 'set-build-directory-permissions activation-service-type - (with-imported-modules '((guix build utils)) - #~(begin - (use-modules (guix build utils)) - - (for-each (lambda (file) - (chown file - (passwd:uid (getpw "laminar")) - (passwd:gid (getpw "laminar")))) - (append (find-files #$%xapian-directory - #:directories? #t) - (find-files #$%transform-genenetwork-database-export-directory - #:directories? #t)))))) + set-build-directory-permissions-gexp) (service tissue-service-type (tissue-configuration (socket @@ -1250,8 +1399,12 @@ gn-auth." (hosts (list (tissue-host (name "issues.genenetwork.org") - (user "laminar") - (upstream-repository "https://github.com/genenetwork/gn-gemtext-threads")))))) + (projects (list (tissue-project + (name "issues.genenetwork.org") + (user "laminar") + (base-path "/") + (upstream-repository + "https://github.com/genenetwork/gn-gemtext-threads"))))))))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket diff --git a/genenetwork/services/genenetwork.scm b/genenetwork/services/genenetwork.scm index 34d70df..80b6c3c 100644 --- a/genenetwork/services/genenetwork.scm +++ b/genenetwork/services/genenetwork.scm @@ -21,16 +21,23 @@ (define-module (genenetwork services genenetwork) #:use-module ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-uploader)) + #:use-module ((gn packages guile) #:select (gn-guile)) + #:use-module (gnu build linux-container) #:use-module ((gnu packages web) #:select (nginx)) #:use-module ((gnu packages admin) #:select (shadow shepherd)) + #:use-module ((gnu packages version-control) #:select (git-minimal)) #:use-module ((gnu packages python) #:select (python)) #:use-module (gnu services) #:use-module (gnu services web) #:use-module (gnu services mcron) + #:use-module (gnu services shepherd) #:use-module (gnu system file-systems) #:use-module (gnu system shadow) #:use-module (guix build python-build-system) + #:use-module (guix diagnostics) #:use-module (guix gexp) + #:use-module (guix i18n) + #:use-module (guix least-authority) #:use-module (guix packages) #:use-module (guix profiles) #:use-module (guix records) @@ -38,6 +45,7 @@ #:use-module (forge nginx) #:use-module (forge gunicorn) #:use-module (forge socket) + #:use-module (forge utils) #:use-module (srfi srfi-1) #:use-module (ice-9 match) #:export (genenetwork-service-type @@ -83,10 +91,14 @@ (default 8083)) (gn-auth-port genenetwork-configuration-gn-auth-port (default 8084)) + (gn3-alias-server-port genenetwork-gn3-alias-server-port + (default 8000)) (sql-uri genenetwork-configuration-sql-uri (default "mysql://username:password@localhost/database")) (auth-db genenetwork-configuration-auth-db (default "/var/genenetwork/auth.db")) + (llm-db-path genenetwork-configuration-llm-db-path + (default "/var/genenetwork/llm.db")) (xapian-db genenetwork-configuration-xapian-db (default "/var/genenetwork/xapian")) (genotype-files genenetwork-configuration-genotype-files @@ -94,15 +106,28 @@ (sparql-endpoint genenetwork-configuration-sparql-endpoint (default "http://localhost:8081/sparql")) (gn-sourcecode-directory genenetwork-configuration-gn-sourcecode-directory - (default "/var/empty")) + (default "/var/empty")) (gn3-data-directory genenetwork-configuration-gn3-data-directory (default "/var/genenetwork")) + (gn2-sessions-dir genenetwork-configuration-gn2-sessions-dir + (default "/var/genenetwork/sessions/genenetwork2")) (gn2-secrets genenetwork-configuration-gn2-secrets (default "/etc/genenetwork")) (gn3-secrets genenetwork-configuration-gn3-secrets (default "/etc/genenetwork/gn3-secrets.py")) (gn-auth-secrets genenetwork-configuration-gn-auth-secrets - (default "/etc/genenetwork"))) + (default "/etc/genenetwork")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/export/data/gn-docs")) + (gn-virtuoso-ttl-directory genenetwork-configuration-gn-virtuoso-ttl-directory + (default "/export/data/virtuoso/ttl")) + (gn-tmpdir genenetwork-configuration-gn-tmpdir + (default "/opt/gn/tmp")) + (log-level genenetwork-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) (define-record-type* <gn-uploader-configuration> gn-uploader-configuration make-gn-uploader-configuration @@ -119,11 +144,24 @@ (default "/var/genenetwork")) (secrets gn-uploader-configuration-secrets (default "/etc/genenetwork/gn-uploader-secrets.py")) - (auth-server-url gn-uploader-auth-server-url + (auth-server-url gn-uploader-configuration-auth-server-url (default "https://auth.genenetwork.org")) - (gn2-server-url gn-uploader-gn2-server-url + (gn2-server-url gn-uploader-configuration-gn2-server-url (default "https://genenetwork.org")) - (log-level gn-uploader-log-level (default "WARNING"))) + (sessions-dir gn-uploader-sessions-dir + (default "/var/genenetwork/sessions/gn-uploader")) + (sqlite-databases-directory gn-uploader-sqlite-databases-directory + (default "/var/genenetwork/sqlite/gn-uploader")) + (log-level gn-uploader-configuration-log-level + (default 'warning) + (sanitize sanitize-log-level))) + +(define (sanitize-log-level log-level) + (case log-level + ((fatal error warning info debug trace) log-level) + (else + (leave (G_ "Log level ~a is invalid. It must be one of the following symbols---fatal, error, warn, info, debug or trace.~%") + log-level)))) (define %genenetwork-accounts (list (user-group @@ -142,12 +180,14 @@ (genenetwork3 (genenetwork-configuration-genenetwork3 config)) (xapian-directory (genenetwork-configuration-xapian-db config)) (sparql-endpoint (genenetwork-configuration-sparql-endpoint config)) + (virtuoso-ttl-directory + (genenetwork-configuration-gn-virtuoso-ttl-directory config)) (xapian-build-directory (string-append xapian-directory "/build")) (herd (file-append shepherd "/bin/herd")) (index-genenetwork (file-append genenetwork3 "/bin/index-genenetwork")) (gn3-profile (profile - (content (package->development-manifest genenetwork3)) - (allow-collisions? #t))) + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) (python3-version (python-version (package-version python)))) (with-imported-modules '((guix build utils)) #~(begin @@ -175,7 +215,9 @@ "is-data-modified" #$xapian-directory #$sql-uri - #$sparql-endpoint)))) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory)))) (dynamic-wind (const #t) ;; build the index @@ -184,7 +226,9 @@ "create-xapian-index" #$xapian-build-directory #$sql-uri - #$sparql-endpoint) + #$sparql-endpoint + "--virtuoso-ttl-directory" + #$virtuoso-ttl-directory) (dynamic-wind ;; stop GN3: Here there is magic!!! ;; The name `gunicorn-genenetwork' is magical. It is not set @@ -214,7 +258,7 @@ (define (genenetwork-activation config) (match-record config <genenetwork-configuration> - (gn2-secrets gn3-secrets gn-auth-secrets auth-db) + (gn2-secrets gn3-secrets gn-auth-secrets auth-db llm-db-path genotype-files gn-tmpdir gn-doc-git-checkout gn2-sessions-dir) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -234,11 +278,19 @@ (passwd:uid (getpw "gunicorn-genenetwork2")) (passwd:gid (getpw "gunicorn-genenetwork2")))) (append (list #$gn2-secrets) + (find-files #$genotype-files + #:directories? #t) (find-files #$gn2-secrets + #:directories? #t) + (find-files #$gn2-sessions-dir #:directories? #t))) - (chown #$gn3-secrets - (passwd:uid (getpw "gunicorn-genenetwork3")) - (passwd:gid (getpw "gunicorn-genenetwork3"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-genenetwork3")) + (passwd:gid (getpw "gunicorn-genenetwork3")))) + (cons #$gn3-secrets + (find-files #$(dirname llm-db-path) + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -246,7 +298,31 @@ (find-files #$gn2-secrets #:directories? #f) (find-files #$gn-auth-secrets - #:directories? #f))))))) + #:directories? #f))) + ;; Make sub-directories for various apps under gn-tmpdir and assign + ;; appropriate permissions + (for-each (match-lambda + ((subdir user) + (let ((full-path + (string-append #$gn-tmpdir "/" subdir))) + (unless (file-exists? full-path) + (mkdir full-path #o755)) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw user)) + (passwd:gid (getpw user)))) + (find-files full-path + #:directories? #t))))) + '(("gn2-tmpdir" "gunicorn-genenetwork2") + ("gn3-tmpdir" "gunicorn-genenetwork3"))) + + ;; setup correct ownership for gn-docs + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "genenetwork")) + (passwd:gid (getpw "genenetwork")))) + (find-files #$gn-doc-git-checkout + #:directories? #t)))))) (define (configuration-file-gexp alist) "Return a G-expression that constructs a configuration file of @@ -277,7 +353,7 @@ G-expressions or numbers." described by @var{config}, a @code{<genenetwork-configuration>} object." (match-record config <genenetwork-configuration> - (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets) + (genenetwork2 genenetwork3 gn-auth server-name gn-auth-server-name gn2-port gn3-port gn-auth-port sql-uri auth-db xapian-db genotype-files gn2-sessions-dir sparql-endpoint gn-sourcecode-directory gn3-data-directory gn2-secrets gn3-secrets gn-auth-secrets llm-db-path gn-tmpdir log-level) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. (let* ((database-mapping (file-system-mapping @@ -287,6 +363,7 @@ object." (gn2-profile (profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) + (gn2-ca-bundle (file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) (gn2-conf (computed-file "gn2.conf" (configuration-file-gexp `(("GN2_SECRETS" ,(string-append gn2-secrets "/gn2-secrets.py")) @@ -300,23 +377,36 @@ object." ("JS_GUIX_PATH" ,(file-append gn2-profile "/share/genenetwork2/javascript")) ("PLINK_COMMAND" ,(file-append gn2-profile "/bin/plink2")) ("SQL_URI" ,sql-uri) - ("SSL_PRIVATE_KEY" ,(string-append gn2-secrets "/gn2-ssl-private-key.pem")) - ("AUTH_SERVER_SSL_PUBLIC_KEY" ,(string-append gn2-secrets "/gn-auth-ssl-public-key.pem")))))) + ("AI_SEARCH_ENABLED" "True") + ("SESSION_FILESYSTEM_CACHE_PATH" ,gn2-sessions-dir) + ("MAX_FORM_MEMORY_SIZE" 52428800))))) + (gn3-profile (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t))) + (gn3-ca-bundle (file-append gn3-profile "/etc/ssl/certs/ca-certificates.crt")) (gn3-conf (computed-file "gn3.conf" (configuration-file-gexp `(("AUTH_DB" ,auth-db) + ("AUTH_SERVER_URL" ,(string-append "https://" gn-auth-server-name "/")) ("DATA_DIR" ,gn3-data-directory) ("SOURCE_DIR" ,gn-sourcecode-directory) ("SPARQL_ENDPOINT" ,sparql-endpoint) ("SQL_URI" ,sql-uri) - ("XAPIAN_DB_PATH" ,xapian-db))))) + ("XAPIAN_DB_PATH" ,xapian-db) + ("GENOTYPE_FILES" ,genotype-files) + ("REAPER_COMMAND" ,(file-append gn2-profile "/bin/qtlreaper")) + ("LLM_DB_PATH" ,llm-db-path))))) + (gn-auth-profile (profile + (content (package->development-manifest gn-auth)) + (allow-collisions? #t))) + (gn-auth-ca-bundle (file-append gn-auth-profile "/etc/ssl/certs/ca-certificates.crt")) (gn-auth-conf (computed-file "gn-auth.conf" (configuration-file-gexp `(("GN_AUTH_SECRETS" ,(string-append gn-auth-secrets "/gn-auth-secrets.py")) ("AUTH_DB" ,auth-db) - ("SQL_URI" ,sql-uri) - ("CLIENTS_SSL_PUBLIC_KEYS_DIR" ,(string-append gn-auth-secrets "/clients-public-keys")) - ("SSL_PRIVATE_KEY" ,(string-append gn-auth-secrets "/gn-auth-ssl-private-key.pem"))))))) + ("SQL_URI" ,sql-uri))))) + (gn2-tmpdir (string-append gn-tmpdir "/gn2-tmpdir")) + (gn3-tmpdir (string-append gn-tmpdir "/gn3-tmpdir"))) (list (gunicorn-app (name "genenetwork2") (package genenetwork2) @@ -331,23 +421,30 @@ object." (value gn2-profile)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn2-tmpdir)) (environment-variable (name "GN2_SETTINGS") (value gn2-conf)) (environment-variable (name "HOME") - (value "/tmp")))) + (value "/tmp")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn2-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source genotype-files) - (target source)) + (target source) + (writable? #t)) (file-system-mapping (source gn-sourcecode-directory) (target source)) - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn2-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source gn2-conf) @@ -358,7 +455,17 @@ object." (file-system-mapping (source gn2-secrets) (target source) - (writable? #t))))) + (writable? #t)) + (file-system-mapping + (source gn2-ca-bundle) + (target source)) + (file-system-mapping + (source gn2-sessions-dir) + (target source) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "genenetwork3") (package genenetwork3) @@ -376,13 +483,16 @@ object." (value gn3-conf)) (environment-variable (name "TMPDIR") - (value "/tmp")) + (value gn3-tmpdir)) (environment-variable (name "GN3_SECRETS") (value gn3-secrets)) (environment-variable (name "HOME") - (value "/tmp")))) + (value "/tmp")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn3-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn3-conf) @@ -399,17 +509,26 @@ object." (file-system-mapping (source gn3-data-directory) (target source)) ; Rqtl usese this - (file-system-mapping ; GN2 and GN3 need to share TMPDIR - (source "/tmp") - (target "/tmp") + (file-system-mapping ; GN2 and GN3 need to communicate via TMPDIR + (source gn-tmpdir) + (target source)) + (file-system-mapping + (source gn3-tmpdir) + (target source) (writable? #t)) (file-system-mapping (source xapian-db) (target source)) (file-system-mapping - (source auth-db) + (source llm-db-path) (target source) - (writable? #t))))) + (writable? #t)) + (file-system-mapping + (source gn3-ca-bundle) + (target source)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))) (gunicorn-app (name "gn-auth") (package gn-auth) @@ -417,6 +536,7 @@ object." (port gn-auth-port)))) (wsgi-app-module "gn_auth:create_app()") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "GN_AUTH_CONF") @@ -426,7 +546,10 @@ object." (value "/tmp")) (environment-variable (name "AUTHLIB_INSECURE_TRANSPORT") - (value "true")))) + (value "true")) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn-auth-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn-auth-conf) @@ -438,14 +561,20 @@ object." (file-system-mapping (source gn-auth-secrets) (target source) - (writable? #t))))))))) + (writable? #t)) + (file-system-mapping + (source gn-auth-ca-bundle) + (target source)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (genenetwork-nginx-server-blocks config) "Return a list of @code{<nginx-server-configuration>} records specifying reverse proxies for the genenetwork service described by @var{config}, a @code{<genenetwork-configuration>} record." (match-record config <genenetwork-configuration> - (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port) + (server-name gn-auth-server-name gn2-port gn3-port gn-auth-port gn3-alias-server-port) (list (nginx-server-configuration (server-name (list server-name)) (locations @@ -455,13 +584,23 @@ a @code{<genenetwork-configuration>} record." (number->string gn2-port) ";") "proxy_set_header Host $host;" "proxy_read_timeout 20m;" - "proxy_set_header X-Forwarded-Proto $scheme;"))) + "proxy_set_header X-Forwarded-Proto $scheme;" + "client_max_body_size 8050m;"))) (nginx-location-configuration (uri "/api3/") (body (list "rewrite /api3/(.*) /api/$1 break;" (string-append "proxy_pass http://localhost:" (number->string gn3-port) ";") - "proxy_set_header Host $host;")))))) + "proxy_set_header Host $host;"))) + (nginx-location-configuration + (uri "/gn3/") + (body + (list "rewrite /gn3/(.*) /$1 break;" + (string-append "proxy_pass http://localhost:" + (number->string gn3-alias-server-port) + ";") + "proxy_redirect off;" + "proxy_set_header Host $host;")))))) (nginx-server-configuration (server-name (list gn-auth-server-name)) (locations @@ -477,6 +616,57 @@ a @code{<genenetwork-configuration>} record." (build-xapian-index-cron-gexp config)) #:user "root"))) +(define (gn-guile-gexp gn-guile-port) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + +(define (gn-guile-shepherd-service config) + (match-record config <genenetwork-configuration> + (gn-doc-git-checkout gn-guile-port) + (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/gn-guile.log"))) + (stop #~(make-kill-destructor))))) + (define genenetwork-service-type (service-type (name 'genenetwork) @@ -490,19 +680,27 @@ a @code{<genenetwork-configuration>} record." genenetwork-gunicorn-apps) (service-extension forge-nginx-service-type genenetwork-nginx-server-blocks) + (service-extension shepherd-root-service-type + (compose list gn-guile-shepherd-service)) (service-extension mcron-service-type genenetwork-mcron-jobs))) (default-value (genenetwork-configuration)))) (define (gn-uploader-activation config) (match-record config <gn-uploader-configuration> - (secrets data-directory) + (secrets data-directory sessions-dir sqlite-databases-directory) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Let service user own their own secrets files. - (chown #$secrets - (passwd:uid (getpw "gunicorn-gn-uploader")) - (passwd:gid (getpw "gunicorn-gn-uploader"))) + (for-each (lambda (file) + (chown file + (passwd:uid (getpw "gunicorn-gn-uploader")) + (passwd:gid (getpw "gunicorn-gn-uploader")))) + (append (list #$secrets) + (find-files #$sessions-dir + #:directories? #t) + (find-files #$sqlite-databases-directory + #:directories? #t))) ;; Set owner-only permissions on secrets files. (for-each (lambda (file) (chmod file #o600)) @@ -518,24 +716,27 @@ a @code{<genenetwork-configuration>} record." (define (gn-uploader-gunicorn-app config) (match-record config <gn-uploader-configuration> - (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url) + (gn-uploader sql-uri port data-directory secrets log-level auth-server-url gn2-server-url sessions-dir sqlite-databases-directory) ;; If we mapped only the mysqld.sock socket file, it would break ;; when the external mysqld server is restarted. - (let ((database-mapping (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t))) - (gn-uploader-conf (computed-file "gn-uploader.conf" - (configuration-file-gexp - `(("UPLOADER_SECRETS" ,secrets) - ("SQL_URI" ,sql-uri) - ("UPLOAD_FOLDER" ,(string-append data-directory - "/uploads")) - ("AUTH_SERVER_URL" ,auth-server-url) - ("GN2_SERVER_URL" ,gn2-server-url))))) - (gn-uploader-profile (profile - (content (package->development-manifest gn-uploader)) - (allow-collisions? #t)))) + (let* ((database-mapping (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t))) + (gn-uploader-conf (computed-file "gn-uploader.conf" + (configuration-file-gexp + `(("UPLOADER_SECRETS" ,secrets) + ("SQL_URI" ,sql-uri) + ("UPLOAD_FOLDER" ,(string-append data-directory + "/uploads")) + ("AUTH_SERVER_URL" ,auth-server-url) + ("GN2_SERVER_URL" ,gn2-server-url) + ("SESSION_FILESYSTEM_CACHE_PATH" ,sessions-dir) + ("ASYNCHRONOUS_JOBS_SQLITE_DB" ,(string-append sqlite-databases-directory "/background-jobs.db")))))) + (gn-uploader-profile (profile + (content (package->development-manifest gn-uploader)) + (allow-collisions? #t))) + (gn-uploader-ca-bundle (file-append gn-uploader-profile "/etc/ssl/certs/ca-certificates.crt"))) (list (gunicorn-app (name "gn-uploader") (package gn-uploader) @@ -543,6 +744,7 @@ a @code{<genenetwork-configuration>} record." (port port)))) (wsgi-app-module "scripts.qcapp_wsgi:app") (workers 20) + (timeout 1200) (environment-variables (list (environment-variable (name "UPLOADER_CONF") @@ -552,7 +754,10 @@ a @code{<genenetwork-configuration>} record." (value "/tmp")) (environment-variable (name "GN_UPLOADER_ENVIRONMENT") - (value gn-uploader-profile)))) + (value gn-uploader-profile)) + (environment-variable + (name "REQUESTS_CA_BUNDLE") + (value gn-uploader-ca-bundle)))) (mappings (list database-mapping (file-system-mapping (source gn-uploader-conf) @@ -566,8 +771,21 @@ a @code{<genenetwork-configuration>} record." (writable? #t)) (file-system-mapping (source gn-uploader-profile) - (target source)))) - (extra-cli-arguments (list "--log-level" log-level))))))) + (target source)) + (file-system-mapping + (source gn-uploader-ca-bundle) + (target source)) + (file-system-mapping + (source sessions-dir) + (target source) + (writable? #t)) + (file-system-mapping + (source sqlite-databases-directory) + (target source) + (writable? #t)))) + (extra-cli-arguments + (list "--log-level" + (string-upcase (symbol->string log-level))))))))) (define (gn-uploader-nginx-server-block config) (match-record config <gn-uploader-configuration> @@ -582,7 +800,7 @@ a @code{<genenetwork-configuration>} record." #$(file-append gn-uploader "/lib/python" (python-version (package-version python)) - "/site-packages/qc_app;"))))) + "/site-packages/uploader;"))))) (nginx-location-configuration (uri "/") (body (list (string-append "proxy_pass http://localhost:" diff --git a/production-deploy.sh b/production-deploy.sh index b4924a7..7cd1cc7 100755 --- a/production-deploy.sh +++ b/production-deploy.sh @@ -2,6 +2,7 @@ # genenetwork-machines --- Guix configuration for genenetwork machines # Copyright © 2022, 2024 Arun Isaac <arunisaac@systemreboot.net> +# Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> # # This file is part of genenetwork-machines. # @@ -25,16 +26,23 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ - --expose=/export/data/genenetwork-xapian \ - --share=/export/data/genenetwork-sqlite \ - --expose=/export/data/genenetwork/genotype_files \ + --share=/export/guix-containers/genenetwork/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/genenetwork/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/genenetwork/var/log=/var/log \ + --share=/export/guix-containers/genenetwork/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/genenetwork/var/lib/xapian=/var/lib/xapian \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-auth=/var/lib/genenetwork/sqlite/gn-auth \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/genenetwork3=/var/lib/genenetwork/sqlite/genenetwork3 \ --share=/var/run/mysqld=/run/mysqld \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs.git=/var/lib/gn-docs.git \ + --share=/export/guix-containers/genenetwork/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/genenetwork/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/genenetwork/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/uploader=/var/lib/genenetwork/uploader \ + --share=/export/guix-containers/genenetwork/var/lib/genenetwork/sqlite/gn-uploader=/var/lib/genenetwork/sqlite/gn-uploader \ production.scm) echo $container_script diff --git a/production.scm b/production.scm index 399c921..ffa75da 100644 --- a/production.scm +++ b/production.scm @@ -1,5 +1,6 @@ ;;; genenetwork-machines --- Guix configuration for genenetwork machines ;;; Copyright © 2022–2024 Arun Isaac <arunisaac@systemreboot.net> +;;; Copyright © 2024 Frederick Muriuki Muriithi <fredmanglis@protonmail.com> ;;; ;;; This file is part of genenetwork-machines. ;;; @@ -17,6 +18,9 @@ ;;; along with genenetwork-machines. If not, see ;;; <https://www.gnu.org/licenses/>. +;;; This is the production genenetwork container currently deployed on +;;; tux04. + (use-modules (gnu) (genenetwork services genenetwork) ((gnu packages admin) #:select (shepherd)) @@ -40,37 +44,62 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service mysql-service-type - (mysql-configuration - (auto-upgrade? #f))) - (service virtuoso-service-type + (services (cons* (service virtuoso-service-type (virtuoso-configuration - (server-port 7892) - (http-server-port 7893))) + (server-port 9892) + (http-server-port 9893) + (dirs-allowed (list "/export/data/virtuoso")) + (number-of-buffers 4000000) + (maximum-dirty-buffers 3000000) + (database-file "/var/lib/virtuoso/genenetwork-virtuoso.db") + (transaction-file "/var/lib/virtuoso/genenetwork-virtuoso.trx"))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") - (port 7890))) + (port 9890))) (https-listen (forge-ip-socket (ip "0.0.0.0") - (port 7891))))) + (port 9891))))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) + (service redis-service-type + (redis-configuration + (bind "127.0.0.1") + (port 6379) + (working-directory "/var/lib/redis"))) (service genenetwork-service-type (genenetwork-configuration - (server-name "test1.genenetwork.org") - (gn-auth-server-name "test1-auth.genenetwork.org") - (gn2-port 7894) - (gn3-port 7895) - (gn-auth-port 7896) - (sql-uri "mysql://webqtlout:webqtlout@localhost/db_webqtl") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") - (sparql-endpoint "http://localhost:7893/sparql") - (gn3-data-directory "/export/data/genenetwork") + (server-name "genenetwork.org") + (gn-auth-server-name "auth.genenetwork.org") + (gn2-port 9894) + (gn3-port 9895) + (gn-auth-port 9896) + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (xapian-db "/var/lib/xapian") + (sparql-endpoint "http://localhost:9893/sparql") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (auth-db "/var/lib/genenetwork/sqlite/gn-auth/auth.db") + (llm-db-path "/var/lib/genenetwork/sqlite/genenetwork3/llm.db") + (gn3-alias-server-port 9800) + (gn-tmpdir "/opt/gn/tmp") + (gn-doc-git-checkout "/var/lib/gn-docs.git") + (log-level 'debug))) + (service gn-uploader-service-type + (gn-uploader-configuration + (server-name "uploader.genenetwork.org") + (port 9897) + (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (data-directory "/var/lib/genenetwork/uploader/data") + (auth-server-url "https://auth.genenetwork.org/") + (gn2-server-url "https://genenetwork.org") + (sqlite-databases-directory "/var/lib/genenetwork/sqlite/gn-uploader") + (log-level 'debug))) %base-services))) diff --git a/public-sparql-deploy.sh b/public-sparql-deploy.sh index bc4c23a..bd8b938 100755 --- a/public-sparql-deploy.sh +++ b/public-sparql-deploy.sh @@ -22,9 +22,11 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ - --share=/export2/guix-containers/public-sparql/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export/data/genenetwork-virtuoso=/var/lib/data \ + --share=/export/guix-containers/public-sparql/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/public-sparql/tmp=/tmp \ + --share=/export/guix-containers/public-sparql/var/log=/var/log \ + --share=/export/guix-containers/public-sparql/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ public-sparql.scm) echo $container_script diff --git a/public-sparql.scm b/public-sparql.scm index 9781b5d..4603cec 100644 --- a/public-sparql.scm +++ b/public-sparql.scm @@ -19,15 +19,17 @@ (use-modules (gnu) (gn services databases) - (gnu services web)) + (gnu services web) + ((gnu packages admin) #:select (shepherd)) + (forge nginx) + (forge socket)) -(define (virtuoso-reverse-proxy-server-block listen sparql-port) +(define (virtuoso-reverse-proxy-server-block sparql-port) "Return an <nginx-server-configuration> object listening on LISTEN to reverse proxy the Virtuoso server. SPARQL-PORT is the port virtuoso's SPARQL endpoint is listening on." (nginx-server-configuration (server-name '("sparql.genenetwork.org")) - (listen (list listen)) (locations (list (nginx-location-configuration (uri "/") @@ -35,9 +37,10 @@ SPARQL endpoint is listening on." (number->string sparql-port) ";") "proxy_set_header Host $host;"))))))) -(define %reverse-proxy-port 8990) +(define %reverse-http-proxy-port 8990) (define %virtuoso-port 8981) (define %sparql-port 8982) +(define %reverse-https-proxy-port 8993) (operating-system (host-name "sparql") @@ -48,18 +51,28 @@ SPARQL endpoint is listening on." (targets (list "/dev/sdX")))) (file-systems %base-file-systems) (users %base-user-accounts) + (sudoers-file + (mixed-text-file "sudoers" + "@include " %sudoers-specification + "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) (services (cons* (service virtuoso-service-type (virtuoso-configuration (server-port %virtuoso-port) (http-server-port %sparql-port) (number-of-buffers 4000000) - (dirs-allowed "/var/lib/data") - (maximum-dirty-buffers 3000000))) - (service nginx-service-type - (nginx-configuration + (dirs-allowed (list "/export/data/virtuoso")) + (maximum-dirty-buffers 3000000) + (database-file "/var/lib/virtuoso/public-virtuoso.db") + (transaction-file "/var/lib/virtuoso/public-virtuoso.trx"))) + (service forge-nginx-service-type + (forge-nginx-configuration + (http-listen (forge-ip-socket + (ip "0.0.0.0") + (port %reverse-http-proxy-port))) + (https-listen (forge-ip-socket + (ip "0.0.0.0") + (port %reverse-https-proxy-port))) (server-blocks - (list (virtuoso-reverse-proxy-server-block - (number->string %reverse-proxy-port) - %sparql-port))))) + (list (virtuoso-reverse-proxy-server-block %sparql-port))))) %base-services))) @@ -45,6 +45,17 @@ (define slurm (package (inherit guix:slurm) + (name "slurm") + (version "24.05.3") + (source (origin + (inherit (package-source guix:slurm)) + (method url-fetch) + (uri (string-append + "https://download.schedmd.com/slurm/slurm-" + version ".tar.bz2")) + (sha256 + (base32 + "095fck6016kslggd1d9mnwahr66b1fahpmlmvdyqdbmnx49hbd5h")))) (arguments (substitute-keyword-arguments (package-arguments guix:slurm) ((#:configure-flags flags #~'()) @@ -73,9 +84,7 @@ (substitute* (string-append #$output "/etc/slurmrestd.service") ;; Set user and group to run slurmrestd as. (("# User=") "User=slurmrestd") - (("# Group=") "Group=slurmrestd") - ;; Disable listening on Unix socket by default. - ((" unix:[^ ]*") "")))))))) + (("# Group=") "Group=slurmrestd")))))))) (inputs (modify-inputs (package-inputs guix:slurm) (prepend dbus http-parser json-c libjwt diff --git a/uploader-deploy.sh b/uploader-deploy.sh index 90fd7e4..415790b 100755 --- a/uploader-deploy.sh +++ b/uploader-deploy.sh @@ -41,18 +41,22 @@ container_script=$(guix system container \ --network \ --load-path=. \ --verbosity=3 \ - --share=/export2/guix-containers/genenetwork/uploader/var/genenetwork=/var/genenetwork \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/acme=/var/lib/acme \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/mysql=/var/lib/mysql \ - --share=/export2/guix-containers/genenetwork/uploader/var/lib/virtuoso=/var/lib/virtuoso \ - --share=/export2/guix-containers/genenetwork/uploader/var/log=/var/log \ - --share=/export2/guix-containers/genenetwork/uploader/etc/genenetwork=/etc/genenetwork \ - --share=/export/data/uploader/genenetwork-xapian=/export/data/genenetwork-xapian \ - --share=/export/data/uploader/genenetwork-sqlite=/export/data/genenetwork-sqlite \ - --expose=/export/data/genenetwork/genotype_files=/export/data/genenetwork/genotype_files \ - --expose=/export/data/uploader/genenetwork3 \ - --share=/export/data/uploader/gn-uploader \ + --share=/export/guix-containers/uploader/var/genenetwork=/var/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/acme=/var/lib/acme \ + --share=/export/guix-containers/uploader/var/lib/redis=/var/lib/redis \ + --share=/export/guix-containers/uploader/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export/guix-containers/uploader/var/log=/var/log \ + --share=/export/guix-containers/uploader/etc/genenetwork=/etc/genenetwork \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-xapian=/var/lib/xapian \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-sqlite=/var/lib/genenetwork-sqlite \ + --share=/export/guix-containers/uploader/var/lib/genenetwork-gnqa=/var/lib/genenetwork-gnqa \ --share=/var/run/mysqld3307=/run/mysqld \ + --share=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/tmp=/opt/gn/tmp \ + --expose=/export/guix-containers/uploader/data/virtuoso=/export/data/virtuoso/ \ + --share=/export/guix-containers/uploader/var/lib/gn-docs=/export/data/gn-docs \ + --share=/export/guix-containers/uploader/var/genenetwork/sessions=/var/genenetwork/sessions \ + --share=/export/data/uploader/gn-uploader \ uploader.scm) echo "${container_script}" diff --git a/uploader.scm b/uploader.scm index 62ab35f..5064426 100644 --- a/uploader.scm +++ b/uploader.scm @@ -42,11 +42,7 @@ "@include " %sudoers-specification "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (packages %base-packages) - (services (cons* (service virtuoso-service-type - (virtuoso-configuration - (server-port 10892) - (http-server-port 10893))) - (service forge-nginx-service-type + (services (cons* (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket (ip "0.0.0.0") @@ -64,24 +60,26 @@ (gn2-port 10894) (gn3-port 10895) (gn-auth-port 10896) - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") - (auth-db "/export/data/genenetwork-sqlite/auth.db") - (xapian-db "/export/data/genenetwork-xapian") - (genotype-files "/export/data/genenetwork/genotype_files") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") + (auth-db "/var/lib/genenetwork-sqlite/auth.db") + (xapian-db "/var/lib/xapian") (sparql-endpoint "http://localhost:10893/sparql") - (gn3-data-directory "/export/data/uploader/genenetwork3") + (gn3-data-directory "/var/genenetwork/data/genenetwork3") (gn2-secrets "/etc/genenetwork/genenetwork2") (gn3-secrets "/etc/genenetwork/genenetwork3/gn3-secrets.py") - (gn-auth-secrets "/etc/genenetwork/gn-auth"))) + (gn-auth-secrets "/etc/genenetwork/gn-auth") + (llm-db-path "/var/lib/genenetwork-gnqa/llm.db"))) (service gn-uploader-service-type (gn-uploader-configuration (gn-uploader gn-uploader) (server-name "staging-uploader.genenetwork.org") (port 10897) (secrets "/etc/genenetwork/gn-uploader/gn-uploader-secrets.py") - (sql-uri "mysql://webqtlout:webqtlout@127.0.0.1:3307/db_webqtl") + (sql-uri + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock&charset=utf8") (data-directory "/export/data/uploader/gn-uploader") - (log-level "DEBUG") + (log-level 'debug) (auth-server-url "https://staging-auth.genenetwork.org/") (gn2-server-url "https://staging.genenetwork.org"))) %base-services))) diff --git a/virtuoso-deploy.sh b/virtuoso-deploy.sh index 0414a65..0dd2509 100755 --- a/virtuoso-deploy.sh +++ b/virtuoso-deploy.sh @@ -25,6 +25,7 @@ container_script=$(guix system container \ --network \ --verbosity=3 \ --share=/export2/guix-containers/virtuoso/var/lib/virtuoso=/var/lib/virtuoso \ + --share=/export2/guix-containers/genenetwork/data/virtuoso=/export/data/virtuoso \ virtuoso.scm) echo $container_script diff --git a/virtuoso.scm b/virtuoso.scm index edcd575..3272f41 100644 --- a/virtuoso.scm +++ b/virtuoso.scm @@ -34,5 +34,5 @@ (virtuoso-configuration (server-port 8891) (http-server-port 8892) - (dirs-allowed "/var/lib/virtuoso"))) + (dirs-allowed (list "/export/data/virtuoso")))) %base-services))) |