diff options
Diffstat (limited to 'genenetwork-development.scm')
| -rw-r--r-- | genenetwork-development.scm | 254 |
1 files changed, 198 insertions, 56 deletions
diff --git a/genenetwork-development.scm b/genenetwork-development.scm index 5894cef..f841255 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -21,8 +21,9 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) - ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth)) + ((gn packages genenetwork) #:select (genenetwork2 genenetwork3 gn-auth gn-libs)) (gn services databases) + ((gn packages guile) #:select (gn-guile)) (gnu build linux-container) ((gnu packages admin) #:select (shepherd shadow)) ((gnu packages base) #:select (gnu-make tar)) @@ -64,6 +65,7 @@ (guix packages) (guix profiles) (guix records) + (guix search-paths) (guix store) (guix utils) (forge acme) @@ -120,7 +122,9 @@ be imported into G-expressions." (gn3-repository genenetwork-configuration-gn3-repository (default "https://github.com/genenetwork/genenetwork3")) (gn-auth-repository genenetwork-configuration-gn-auth-repository - (default "https://git.genenetwork.org/gn-auth")) + (default "/home/git/public/gn-auth")) + (gn-libs-repository genenetwork-configuration-gn-libs-repository + (default "/home/git/public/gn-libs")) (gn2-port genenetwork-configuration-gn2-port (default 8082)) (gn3-port genenetwork-configuration-gn3-port @@ -144,7 +148,11 @@ be imported into G-expressions." (auth-db-path genenetwork-auth-db-path (default "/export/data/genenetwork-sqlite/auth.db")) (llm-db-path genenetwork-llm-db-path - (default "/export/data/genenetwork-sqlite/llm.db"))) + (default "/export/data/genenetwork-sqlite/llm.db")) + (gn-guile-port genenetwork-configuration-gn-guile-port + (default 8091)) + (gn-doc-git-checkout genenetwork-configuration-gn-doc-git-checkout + (default "/export/data/gn-docs"))) ;;; @@ -182,11 +190,12 @@ described by CONFIG, a <genenetwork-configuration> object. TEST-COMMAND is a list of strings specifying the command to be executed." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn3-port genotype-files) + (gn2-repository gn3-repository gn-libs-repository gn3-port genotype-files) (with-imported-modules '((guix build utils)) (with-packages (list bash coreutils git-minimal nss-certs) #~(begin - (use-modules (guix build utils)) + (use-modules (guix build utils) + (srfi srfi-26)) (define (hline) "Print a horizontal line 50 '=' characters long." @@ -199,9 +208,19 @@ executed." (invoke "git" "log" "--max-count" "1") (hline)) + (define (call-with-temporary-directory proc) + (let ((tmp-dir (mkdtemp "/tmp/gn.XXXXXX"))) + (dynamic-wind + (const #t) + (cut proc tmp-dir) + (cut delete-file-recursively tmp-dir)))) + (invoke "git" "clone" "--depth" "1" #$gn3-repository) (with-directory-excursion "genenetwork3" (show-head-commit)) + (invoke "git" "clone" "--depth" "1" #$gn-libs-repository) + (with-directory-excursion "gn-libs" + (show-head-commit)) (invoke "git" "clone" "--depth" "1" #$gn2-repository) (with-directory-excursion "genenetwork2" (show-head-commit)) @@ -222,9 +241,15 @@ executed." (setenv "GN3_LOCAL_URL" (string-append "http://localhost:" (number->string #$gn3-port))) (setenv "GENENETWORK_FILES" #$genotype-files) (setenv "HOME" "/tmp") - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl") + (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock") (chdir "genenetwork2") - (apply invoke '#$test-command)))))) + ;; XXXX: FIXME: R/Qtl tests fail because files are generated in + ;; the "/tmp" directory. Currently, "/tmp" is mapped by gn2/gn3 + ;; so tests will fail because of permission issues. + (call-with-temporary-directory + (lambda (tmp-dir) + (setenv "TMPDIR" tmp-dir) + (apply invoke '#$test-command)))))))) (define %xapian-directory "/export/data/genenetwork-xapian") @@ -252,7 +277,7 @@ genenetwork3 source from the latest commit of @var{project}." (setenv "PYTHONPATH" (getcwd)) (invoke "./scripts/index-genenetwork" "create-xapian-index" xapian-build-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql") ;; Stop genenetwork3, replace old xapian index and ;; start genenetwork3. @@ -297,7 +322,7 @@ genenetwork3 source from the latest commit of @var{project}." (system* (string-append gn3-dir "/scripts/index-genenetwork") "is-data-modified" #$%xapian-directory - "mysql://webqtlout:webqtlout@localhost/db_webqtl" + "mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock" "http://localhost:9082/sparql")))) (setenv "LAMINAR_REASON" "Nightly xapian index rebuild") (invoke #$(file-append laminar "/bin/laminarc") @@ -307,7 +332,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn-auth-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn-libs-repository gn2-port) (list (forge-project (name "genenetwork2") (repository gn2-repository) @@ -380,6 +405,20 @@ genenetwork3 source from the latest commit of @var{project}." (trigger? #f)))) (ci-jobs-trigger 'webhook)) (forge-project + (name "gn-libs") + (repository gn-libs-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-libs") + (run (guix-channel-job-gexp + (list (channel + (name 'gn-libs) + (url (forge-project-repository this-forge-project)) + (branch "main"))) + #:variables (list (variable-specification + (module '(gn-libs)) + (name 'gn-libs))) + #:guix-daemon-uri %guix-daemon-uri)))))) + (forge-project (name "gn-auth") (repository gn-auth-repository) (ci-jobs (list (forge-laminar-job @@ -414,8 +453,7 @@ genenetwork3 source from the latest commit of @var{project}." #:variables (list (variable-specification (module '(gn-auth)) (name 'gn-auth-all-tests))) - #:guix-daemon-uri %guix-daemon-uri))))) - (ci-jobs-trigger 'webhook))))) + #:guix-daemon-uri %guix-daemon-uri))))))))) (define (genenetwork2-cd-gexp config) "Return a G-expression that runs the latest genenetwork2 development @@ -453,20 +491,9 @@ server described by CONFIG, a <genenetwork-configuration> object." (string-append (getcwd) "/genenetwork3")) ;; Set other environment variables required by ;; genenetwork2. - (setenv "SERVER_PORT" #$(number->string gn2-port)) (setenv "GN2_PROFILE" #$(profile (content (package->development-manifest genenetwork2)) (allow-collisions? #t))) - (setenv "GN_SERVER_URL" "https://cd.genenetwork.org/api3/") - (setenv "GN3_LOCAL_URL" - #$(string-append "http://localhost:" - (number->string gn3-port))) - (setenv "GENENETWORK_FILES" #$genotype-files) - (setenv "SQL_URI" "mysql://webqtlout:webqtlout@localhost/db_webqtl") - (setenv "HOME" "/tmp") - (setenv "NO_REDIS" "no-redis") - (setenv "RUST_BACKTRACE" "1") - (setenv "GN2_SETTINGS" #$(mixed-text-file "gn2.conf" @@ -478,7 +505,7 @@ server described by CONFIG, a <genenetwork-configuration> object." "\"\n" "GN_SERVER_URL=\"https://cd.genenetwork.org/api3/\"\n" "AUTH_SERVER_URL=\"https://auth-cd.genenetwork.org/\"\n" - "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl\"\n" + "SQL_URI=\"mysql://webqtlout:webqtlout@localhost/db_webqtl?unix_socket=/run/mysqld/mysqld.sock\"\n" "SSL_PRIVATE_KEY=\"" gn2-secrets "/gn2-ssl-private-key.pem\"\n" "AUTH_SERVER_SSL_PUBLIC_KEY=\"" gn2-secrets "/gn-auth-ssl-public-key.pem\"\n")) @@ -523,6 +550,11 @@ server described by CONFIG, a <genenetwork-configuration> object." "LLM_DB_PATH=\"" llm-db-path "\"\n")) (setenv "HOME" "/tmp") (setenv "GN3_SECRETS" #$gn3-secrets) + (setenv "RSCRIPT" #$(file-append + (profile + (content (package->development-manifest genenetwork3)) + (allow-collisions? #t)) + "/bin/Rscript")) ;; Run genenetwork3. (with-directory-excursion "genenetwork3" (show-head-commit) @@ -577,40 +609,135 @@ server described by CONFIG, a <genenetwork-configuration> object." "--workers" "8" "gn_auth.wsgi:app")))))))) +(define (gn-guile-gexp gn-guile-port) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + (let ((current-repo-path (string-append (getcwd) "/gn-docs"))) + (when (file-exists? current-repo-path) + (delete-file-recursively current-repo-path)) + (setenv "CURRENT_REPO_PATH" current-repo-path) + (invoke #$(file-append git-minimal "/bin/git") + "clone" "--depth" "1" (getenv "CGIT_REPO_PATH"))) + (invoke #$(file-append gn-guile "/bin/gn-guile") + (number->string #$gn-guile-port))))) + (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path gn2-secrets auth-db-path gn-auth-secrets llm-db-path gn-doc-git-checkout gn-guile-port) (list (shepherd-service + (documentation "Run gn-guile server.") + (provision '(gn-guile)) + (requirement '(networking)) + (modules '((ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn-guile-settings + `(("CGIT_REPO_PATH" ,gn-doc-git-checkout) + ("LC_ALL" "en_US.UTF-8") + ("GIT_COMMITTER_NAME" "genenetwork") + ("GIT_COMMITTER_EMAIL" "no-reply@git.genenetwork.org")))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-guile" + (gn-guile-gexp gn-guile-port)) + #:name "gn-guile-pola-wrapper" + #:preserved-environment-variables + (map first gn-guile-settings) + #:mappings (list (file-system-mapping + (source gn-doc-git-checkout) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-guile-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn-guile-settings) + #:log-file "/var/log/cd/gn-guile.log"))) + (stop #~(make-kill-destructor))) + (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) ;; FIXME: The genenetwork2 service should depend on redis. (requirement '(networking genenetwork3)) - (start #~(make-forkexec-constructor - (list #$(least-authority-wrapper - (program-file "genenetwork2" - (genenetwork2-cd-gexp config)) - #:name "genenetwork2-pola-wrapper" - ;; If we mapped only the mysqld.sock - ;; socket file, it would break when the - ;; external mysqld server is restarted. - #:mappings (list (file-system-mapping - (source genotype-files) - (target source)) - (file-system-mapping - (source "/run/mysqld") - (target source) - (writable? #t)) - (file-system-mapping - (source gn2-secrets) - (target source) - (writable? #t))) - #:namespaces (delq 'net %namespaces)) - "127.0.0.1" #$(number->string gn2-port)) - #:user "genenetwork" - #:group "genenetwork" - #:log-file "/var/log/cd/genenetwork2.log")) + (modules '((guix search-paths) + (ice-9 match) + (srfi srfi-1))) + (start + (let* ((gn2-manifest (packages->manifest (list genenetwork2))) + (gn2-profile (profile + (content gn2-manifest) + (allow-collisions? #t))) + (gn2-settings + `(("SERVER_PORT" ,(number->string gn2-port)) + ("GENENETWORK_FILES" ,genotype-files) + ("HOME" "/tmp") + ("LC_ALL" "en_US.UTF-8") + ("NO_REDIS" "no-redis") + ("RUST_BACKTRACE" "1")))) + (with-imported-modules (source-module-closure '((guix search-paths))) + #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "genenetwork2" + (genenetwork2-cd-gexp config)) + #:name "genenetwork2-pola-wrapper" + #:preserved-environment-variables + (append '("REQUESTS_CA_BUNDLE") + (map first gn2-settings) + (map search-path-specification-variable + (manifest-search-paths gn2-manifest))) + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source genotype-files) + (target source)) + (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) + (file-system-mapping + (source gn2-secrets) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn2-port)) + #:user "genenetwork" + #:group "genenetwork" + #:environment-variables + (append + '("REQUESTS_CA_BUNDLE=" + #$(file-append gn2-profile "/etc/ssl/certs/ca-certificates.crt")) + (map (match-lambda + ((spec . value) + (string-append (search-path-specification-variable spec) + "=" + value))) + (evaluate-search-paths + (map sexp->search-path-specification + '#$(map search-path-specification->sexp + (manifest-search-paths gn2-manifest))) + (list #$gn2-profile))) + (map (match-lambda + ((spec value) + (string-append spec "=" value))) + '#$gn2-settings)) + #:log-file "/var/log/cd/genenetwork2.log")))) (stop #~(make-kill-destructor))) (shepherd-service (documentation "Run GeneNetwork 3 development server.") @@ -628,6 +755,15 @@ described by CONFIG, a <genenetwork-configuration> object." (source "/run/mysqld") (target source) (writable? #t)) + ;; XXXX: FIXME: R/Qtl generates + ;; files in "/tmp" and + ;; "/tmp/gn2". These files are + ;; accessed by gn3 for R/Qtl + ;; mapping + (file-system-mapping + (source "/tmp") + (target source) + (writable? #t)) (file-system-mapping (source data-directory) (target source)) @@ -713,6 +849,8 @@ described by CONFIG, a <genenetwork-configuration> object." (cons* #$gn3-secrets (append (find-files #$gn2-secrets #:directories? #t) + (find-files "/export/data/gn-docs" + #:directories? #t) (find-files #$(dirname auth-db-path) #:directories? #t) (find-files #$gn-auth-secrets @@ -831,7 +969,8 @@ described by CONFIG, a <genenetwork-configuration> object." #$connection-settings "--output" build-directory) ;; First clear all the files in our virtuoso directory (for-each (lambda (file) - (delete-file file)) + (unless (string-suffix? "build" (dirname file)) + (delete-file file))) (find-files #$virtuoso-data-dir ".ttl")) ;; Move data into the container's virtuoso data directory (copy-recursively build-directory #$virtuoso-data-dir) @@ -848,7 +987,7 @@ described by CONFIG, a <genenetwork-configuration> object." (define transform-genenetwork-database-project (forge-project (name "transform-genenetwork-database") - (repository "https://git.genenetwork.org/gn-transform-databases") + (repository "/home/git/public/gn-transform-databases") (ci-jobs (list (forge-laminar-job (name "transform-genenetwork-database-tests") (run (guix-channel-job-gexp @@ -863,8 +1002,7 @@ described by CONFIG, a <genenetwork-configuration> object." (run (transform-genenetwork-database-gexp %connection-settings %virtuoso-data-dir - "https://git.genenetwork.org/gn-transform-databases"))))) - (ci-jobs-trigger 'webhook))) + "https://git.genenetwork.org/gn-transform-databases"))))))) ;;; @@ -1234,7 +1372,7 @@ gn-auth." (number-of-buffers 4000000) (maximum-dirty-buffers 3000000) (server-port 9081) - (dirs-allowed "/var/lib/data") + (dirs-allowed (list "/var/lib/data")) (http-server-port %virtuoso-sparql-port))) (service genenetwork-service-type (genenetwork-configuration @@ -1261,8 +1399,12 @@ gn-auth." (hosts (list (tissue-host (name "issues.genenetwork.org") - (user "laminar") - (upstream-repository "https://github.com/genenetwork/gn-gemtext-threads")))))) + (projects (list (tissue-project + (name "issues.genenetwork.org") + (user "laminar") + (base-path "/") + (upstream-repository + "https://github.com/genenetwork/gn-gemtext-threads"))))))))) (service forge-nginx-service-type (forge-nginx-configuration (http-listen (forge-ip-socket |