diff options
| -rw-r--r-- | genenetwork-development.scm | 197 | ||||
| -rw-r--r-- | genenetwork/development-helper.scm | 37 |
2 files changed, 206 insertions, 28 deletions
diff --git a/genenetwork-development.scm b/genenetwork-development.scm index e6eee8a..d6f9dbc 100644 --- a/genenetwork-development.scm +++ b/genenetwork-development.scm @@ -19,6 +19,7 @@ ;;; <https://www.gnu.org/licenses/>. (use-modules (gnu) + (gn-auth) ((gn packages genenetwork) #:select (genenetwork2 genenetwork3)) ((gn packages quality-control) #:select (sbcl-qc)) ((gn packages python) #:select (python-mypy-0.981)) @@ -116,6 +117,8 @@ be imported into G-expressions." (default 8082)) (gn3-port genenetwork-configuration-gn3-port (default 8083)) + (gn-auth-port genenetwork-configuration-gn-auth-port + (default 8084)) (genotype-files genenetwork-configuration-genotype-files (default "/var/genenetwork/genotype-files")) (sparql-endpoint genenetwork-configuration-sparql-endpoint @@ -223,7 +226,7 @@ command to be executed." (genenetwork3-tests (list "pytest" "-k" "unit_test") (package->development-manifest genenetwork3))) -(define (genenetwork3-auth-migrations-genenetwork config) +(define (gn-auth-migrations config) "Return a G-expression that runs the migrations for the auth(entic|oris)ation database. This is the actual migration run by the genenetwork user." @@ -244,29 +247,29 @@ the genenetwork user." #$(mixed-text-file "yoyo.ini" "[DEFAULT]\n" - "sources = genenetwork3/migrations/auth/\n" + "sources = gn-auth/migrations/auth/\n" "database = sqlite:///" auth-db-path "\n" "migration_table = _yoyo_migration\n" "batch_mode = on\n" "verbosity = 2")))))) -(define (genenetwork3-auth-migrations-laminar config) +(define (gn-auth-migrations-laminar config) "Return a G-expression that runs the migrations for the auth(entic|oris)ation database. This is the wrapper script run by the laminar user." (match-record config <genenetwork-configuration> - (gn3-repository) + (gn-auth-repository) (with-packages (list git-minimal nss-certs) (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) ;; Clone the latest genenetwork3 repository. - (invoke "git" "clone" "--depth" "1" #$gn3-repository) + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) ;; Run the actual migrations as the genenetwork user. (invoke #$sudo "--user=genenetwork" - #$(program-file "genenetwork3-auth-migrations" - (genenetwork3-auth-migrations-genenetwork config)))))))) + #$(program-file "gn-auth-migrations" + (gn-auth-migrations config)))))))) (define genenetwork3-pylint (with-imported-modules (source-module-closure '((genenetwork development-helper)) @@ -285,6 +288,19 @@ laminar user." (manifest-cons python-mypy-0.981 (package->development-manifest genenetwork3)))) +(define gn-auth-pylint + (with-imported-modules (source-module-closure '((genenetwork development-helper)) + #:select? import-module?) + #~(lambda (gn-auth-checkout) + ((@@ (genenetwork development-helper) + genenetwork-lint-gexp) + gn-auth-checkout + #$(profile + (content (manifest-cons* python-pylint shellcheck + (package->development-manifest gn-auth))) + (allow-collisions? #t)) + (list "main.py" "setup.py" "wsgi.py" "tests" "gn_auth" "scripts"))))) + (define %xapian-directory "/export/data/genenetwork-xapian") @@ -326,7 +342,7 @@ genenetwork3 source from the latest commit of @var{project}." "Return forge projects for genenetwork described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-repository gn3-repository gn2-port) + (gn2-repository gn3-repository gn-auth-repository gn2-port) (list (forge-project (name "genenetwork2") (repository gn2-repository) @@ -368,14 +384,18 @@ genenetwork3 source from the latest commit of @var{project}." (branch "main")) %default-channels) #:guix-daemon-uri %guix-daemon-uri)) - ;; If unit tests pass, trigger the auth migrations. - (after (with-imported-modules '((guix build utils)) + ;; If tests run successfully, redeploy + ;; genenetwork3 and trigger genenetwork2 tests. + (after (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) (when (string=? (getenv "RESULT") "success") + (invoke #$sudo + #$(file-append shepherd "/bin/herd") + "restart" "genenetwork3") (invoke #$(file-append laminar "/bin/laminarc") - "queue" "genenetwork3-auth-migrations")))))) + "queue" "genenetwork2")))))) (forge-laminar-job (name "genenetwork3-pylint") (run (derivation-job-gexp @@ -391,10 +411,50 @@ genenetwork3 source from the latest commit of @var{project}." genenetwork3-mypy #:guix-daemon-uri %guix-daemon-uri))) (forge-laminar-job - (name "genenetwork3-auth-migrations") - (run (genenetwork3-auth-migrations-laminar config)) - ;; If migrations run successfully, redeploy - ;; genenetwork3 and trigger genenetwork2 tests. + (name "genenetwork3-build-xapian-index") + (run (build-xapian-index-gexp this-forge-project)) + (trigger? #f)))) + (ci-jobs-trigger 'webhook)) + (forge-project + (name "gn-auth") + (repository gn-auth-repository) + (ci-jobs (list (forge-laminar-job + (name "gn-auth") + (run (guix-channel-job-gexp + (cons (channel + (name 'gn-auth) + (url (forge-project-repository this-forge-project)) + (branch "main")) + %default-channels) + #:guix-daemon-uri %guix-daemon-uri)) + ;; If unit tests pass, trigger the auth migrations. + (after (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils)) + + (when (string=? (getenv "RESULT") "success") + (invoke #$(file-append laminar "/bin/laminarc") + "queue" "gn-auth-migrations")))))) + (forge-laminar-job + (name "gn-auth-pylint") + (run (derivation-job-gexp + this-forge-project + this-forge-laminar-job + gn-auth-pylint + #:guix-daemon-uri %guix-daemon-uri))) + (forge-laminar-job + (name "gn-auth-mypy") + (run (derivation-job-gexp + this-forge-project + this-forge-laminar-job + (genenetwork3-tests (list "mypy" ".") + (manifest-cons python-mypy + (package->development-manifest gn-auth))) + #:guix-daemon-uri %guix-daemon-uri))) + (forge-laminar-job + (name "gn-auth-migrations") + (run (gn-auth-migrations-laminar config)) + ;; If migrations run successfully, redeploy gn-auth (after (with-imported-modules '((guix build utils)) #~(begin (use-modules (guix build utils)) @@ -402,14 +462,8 @@ genenetwork3 source from the latest commit of @var{project}." (when (string=? (getenv "RESULT") "success") (invoke #$sudo #$(file-append shepherd "/bin/herd") - "restart" "genenetwork3") - (invoke #$(file-append laminar "/bin/laminarc") - "queue" "genenetwork2"))))) - (trigger? #f)) - (forge-laminar-job - (name "genenetwork3-build-xapian-index") - (run (build-xapian-index-gexp this-forge-project)) - (trigger? #f)))) + "restart" "gn-auth"))))) + (trigger? #f)))) (ci-jobs-trigger 'webhook))))) (define (genenetwork2-cd-gexp config) @@ -508,11 +562,48 @@ server described by CONFIG, a <genenetwork-configuration> object." "-b" #$(string-append "localhost:" (number->string gn3-port)) "gn3.app:create_app()")))))))) +(define (gn-auth-cd-gexp config) + "Return a G-expression that runs the latest gn-auth development +server described by CONFIG, a <genenetwork-configuration> object." + (match-record config <genenetwork-configuration> + (gn-auth-repository gn-auth-port auth-db-path) + (with-manifest (package->development-manifest gn-auth) + (with-packages (list git-minimal nss-certs) + (with-imported-modules '((guix build utils)) + #~(begin + (use-modules (guix build utils) + (ice-9 match)) + + (define (hline) + "Print a horizontal line 50 '=' characters long." + (display (make-string 50 #\=)) + (newline) + (force-output)) + + (define (show-head-commit) + (hline) + (invoke "git" "log" "--max-count" "1") + (hline)) + + ;; Clone the latest gn-auth repository. + (invoke "git" "clone" "--depth" "1" #$gn-auth-repository) + ;; Configure gn-auth. + (setenv "GN_AUTH_CONF" + #$(mixed-text-file "gn-auth.conf" + "AUTH_DB=\"" auth-db-path "\"\n")) + (setenv "HOME" "/tmp") + ;; Run gn-auth. + (with-directory-excursion "gn-auth" + (show-head-commit) + (invoke #$(file-append gunicorn "/bin/gunicorn") + "-b" #$(string-append "localhost:" (number->string gn-auth-port)) + "gn_auth:create_app()")))))))) + (define (genenetwork-shepherd-services config) "Return shepherd services to run the genenetwork development server described by CONFIG, a <genenetwork-configuration> object." (match-record config <genenetwork-configuration> - (gn2-port gn3-port genotype-files data-directory xapian-db-path auth-db-path) + (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path auth-db-path) (list (shepherd-service (documentation "Run GeneNetwork 2 development server.") (provision '(genenetwork2)) @@ -570,6 +661,35 @@ described by CONFIG, a <genenetwork-configuration> object." #:user "genenetwork" #:group "genenetwork" #:log-file "/var/log/cd/genenetwork3.log")) + (stop #~(make-kill-destructor))) + (shepherd-service + (documentation "Run gn-auth development server.") + (provision '(gn-auth)) + (requirement '(networking)) + (start #~(make-forkexec-constructor + (list #$(least-authority-wrapper + (program-file "gn-auth" + (gn-auth-cd-gexp config)) + #:name "gn-auth-pola-wrapper" + ;; If we mapped only the mysqld.sock + ;; socket file, it would break when the + ;; external mysqld server is restarted. + #:mappings (list (file-system-mapping + (source "/run/mysqld") + (target source) + (writable? #t)) + (file-system-mapping + (source data-directory) + (target source)) + (file-system-mapping + (source auth-db-path) + (target source) + (writable? #t))) + #:namespaces (delq 'net %namespaces)) + "127.0.0.1" #$(number->string gn-auth-port)) + #:user "genenetwork" + #:group "genenetwork" + #:log-file "/var/log/cd/gn-auth.log")) (stop #~(make-kill-destructor)))))) (define %genenetwork-accounts @@ -983,12 +1103,28 @@ tissue." (body (list (string-append "proxy_pass http://localhost:" (number->string %tissue-port) ";") "proxy_set_header Host $host;"))))))) +(define (gn-auth-reverse-proxy-server-block) + "Return an <nginx-server-configuration> object to reverse proxy +gn-auth." + (nginx-server-configuration + (server-name '("auth.genenetwork.org")) + (root "/var/lib/gn-auth/auth.genenetwork.org/website") + (locations + (list (nginx-location-configuration + (uri "/") + (body (list (string-append "proxy_pass http://localhost:" + (number->string %gn-auth-port) + ";") + "proxy_set_header Host $host;"))))))) + ;; Port on which webhook is listening (define %webhook-port 9091) ;; Port on which genenetwork2 is listening (define %genenetwork2-port 9092) ;; Port on which genenetwork3 is listening (define %genenetwork3-port 9093) +;; Port on which gn-auth is listening +(define %gn-auth-port 9094) ;; Port on which virtuoso's SPARQL endpoint is listening (define %virtuoso-sparql-port 9082) @@ -1011,12 +1147,15 @@ tissue." (file-append shepherd "/bin/herd") " restart genenetwork2, " (file-append shepherd "/bin/herd") " start genenetwork3, " (file-append shepherd "/bin/herd") " stop genenetwork3, " - (file-append shepherd "/bin/herd") " restart genenetwork3\n" + (file-append shepherd "/bin/herd") " restart genenetwork3," + (file-append shepherd "/bin/herd") " start gn-auth, " + (file-append shepherd "/bin/herd") " stop gn-auth, " + (file-append shepherd "/bin/herd") " restart gn-auth\n" ;; Permit the laminar user to run auth db ;; migrations as the genenetwork user. "\nlaminar ALL = (genenetwork) NOPASSWD: " - (program-file "genenetwork3-auth-migrations" - (genenetwork3-auth-migrations-genenetwork (genenetwork-configuration))) + (program-file "gn-auth-migrations" + (gn-auth-migrations (genenetwork-configuration))) ;; Permit the acme user to restart nginx. "\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n")) (services (cons* (service forge-service-type @@ -1067,6 +1206,7 @@ tissue." (genenetwork-configuration (gn2-port %genenetwork2-port) (gn3-port %genenetwork3-port) + (gn-auth-port %gn-auth-port) (genotype-files "/export/data/genenetwork/genotype_files") (sparql-endpoint (string-append "http://localhost:" (number->string %virtuoso-sparql-port) @@ -1111,7 +1251,8 @@ tissue." (laminar-reverse-proxy-server-block "localhost:9089" %webhook-port (list 'gn-bioinformatics)) - (tissue-reverse-proxy-server-block))))) + (tissue-reverse-proxy-server-block) + (gn-auth-reverse-proxy-server-block))))) (service acme-service-type (acme-configuration (email "arunisaac@systemreboot.net"))) diff --git a/genenetwork/development-helper.scm b/genenetwork/development-helper.scm index 81bfc3b..627a80a 100644 --- a/genenetwork/development-helper.scm +++ b/genenetwork/development-helper.scm @@ -101,3 +101,40 @@ with genenetwork3 dependencies." (find-files "." shell-script?)) (invoke "pylint" "main.py" "setup.py" "wsgi.py" "setup_commands" "tests" "gn3" "scripts" "sheepdog")) (mkdir-p #$output))))) + +(define (genenetwork-lint-gexp source-repo-checkout profile files-and-modules) + "Return a G-expression that runs GeneNetwork3 lint tests in PROFILE +with SOURCE-REPO-CHECKOUT as the current directory. SOURCE-REPO-CHECKOUT +is a checkout of the genenetwork3 source code. PROFILE is a profile +with genenetwork3 dependencies." + (with-imported-modules '((guix build utils)) + (with-profile profile + #~(begin + (use-modules (rnrs exceptions) + (srfi srfi-26) + (ice-9 rdelim) + (guix build utils)) + + (define (shell-script? filename stat-obj) + (and (eq? (stat:type stat-obj) 'regular) + (call-with-input-file filename + (lambda (port) + (let ((first-line (read-line port))) + (and (not (eof-object? first-line)) + (> (string-length first-line) 2) + (string=? (string-take first-line 2) "#!") + (or (string-contains first-line "/bin/sh") + (string-contains first-line "/bin/bash")))))))) + + (chdir #$source-repo-checkout) + (guard (condition ((invoke-error? condition) + (format (current-error-port) + "`~a~{ ~a~}' failed with exit status ~a~%" + (invoke-error-program condition) + (invoke-error-arguments condition) + (invoke-error-exit-status condition)) + (exit #f))) + (for-each (cut invoke "shellcheck" <>) + (find-files "." shell-script?)) + (invoke "pylint" #$@files-and-modules)) + (mkdir-p #$output))))) |