From 64d81df4ce47a8977b37baf1a68ca983c2dd99a1 Mon Sep 17 00:00:00 2001
From: Frederick Muriuki Muriithi
Date: Mon, 14 Aug 2023 11:43:17 +0300
Subject: Add CI/CD definitions for gn-auth
---
genenetwork-development.scm | 197 +++++++++++++++++++++++++++++++------
genenetwork/development-helper.scm | 37 +++++++
2 files changed, 206 insertions(+), 28 deletions(-)
diff --git a/genenetwork-development.scm b/genenetwork-development.scm
index e6eee8a..d6f9dbc 100644
--- a/genenetwork-development.scm
+++ b/genenetwork-development.scm
@@ -19,6 +19,7 @@
;;; .
(use-modules (gnu)
+ (gn-auth)
((gn packages genenetwork) #:select (genenetwork2 genenetwork3))
((gn packages quality-control) #:select (sbcl-qc))
((gn packages python) #:select (python-mypy-0.981))
@@ -116,6 +117,8 @@ be imported into G-expressions."
(default 8082))
(gn3-port genenetwork-configuration-gn3-port
(default 8083))
+ (gn-auth-port genenetwork-configuration-gn-auth-port
+ (default 8084))
(genotype-files genenetwork-configuration-genotype-files
(default "/var/genenetwork/genotype-files"))
(sparql-endpoint genenetwork-configuration-sparql-endpoint
@@ -223,7 +226,7 @@ command to be executed."
(genenetwork3-tests (list "pytest" "-k" "unit_test")
(package->development-manifest genenetwork3)))
-(define (genenetwork3-auth-migrations-genenetwork config)
+(define (gn-auth-migrations config)
"Return a G-expression that runs the migrations for the
auth(entic|oris)ation database. This is the actual migration run by
the genenetwork user."
@@ -244,29 +247,29 @@ the genenetwork user."
#$(mixed-text-file
"yoyo.ini"
"[DEFAULT]\n"
- "sources = genenetwork3/migrations/auth/\n"
+ "sources = gn-auth/migrations/auth/\n"
"database = sqlite:///" auth-db-path "\n"
"migration_table = _yoyo_migration\n"
"batch_mode = on\n"
"verbosity = 2"))))))
-(define (genenetwork3-auth-migrations-laminar config)
+(define (gn-auth-migrations-laminar config)
"Return a G-expression that runs the migrations for the
auth(entic|oris)ation database. This is the wrapper script run by the
laminar user."
(match-record config
- (gn3-repository)
+ (gn-auth-repository)
(with-packages (list git-minimal nss-certs)
(with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
;; Clone the latest genenetwork3 repository.
- (invoke "git" "clone" "--depth" "1" #$gn3-repository)
+ (invoke "git" "clone" "--depth" "1" #$gn-auth-repository)
;; Run the actual migrations as the genenetwork user.
(invoke #$sudo "--user=genenetwork"
- #$(program-file "genenetwork3-auth-migrations"
- (genenetwork3-auth-migrations-genenetwork config))))))))
+ #$(program-file "gn-auth-migrations"
+ (gn-auth-migrations config))))))))
(define genenetwork3-pylint
(with-imported-modules (source-module-closure '((genenetwork development-helper))
@@ -285,6 +288,19 @@ laminar user."
(manifest-cons python-mypy-0.981
(package->development-manifest genenetwork3))))
+(define gn-auth-pylint
+ (with-imported-modules (source-module-closure '((genenetwork development-helper))
+ #:select? import-module?)
+ #~(lambda (gn-auth-checkout)
+ ((@@ (genenetwork development-helper)
+ genenetwork-lint-gexp)
+ gn-auth-checkout
+ #$(profile
+ (content (manifest-cons* python-pylint shellcheck
+ (package->development-manifest gn-auth)))
+ (allow-collisions? #t))
+ (list "main.py" "setup.py" "wsgi.py" "tests" "gn_auth" "scripts")))))
+
(define %xapian-directory
"/export/data/genenetwork-xapian")
@@ -326,7 +342,7 @@ genenetwork3 source from the latest commit of @var{project}."
"Return forge projects for genenetwork described by CONFIG, a
object."
(match-record config
- (gn2-repository gn3-repository gn2-port)
+ (gn2-repository gn3-repository gn-auth-repository gn2-port)
(list (forge-project
(name "genenetwork2")
(repository gn2-repository)
@@ -368,14 +384,18 @@ genenetwork3 source from the latest commit of @var{project}."
(branch "main"))
%default-channels)
#:guix-daemon-uri %guix-daemon-uri))
- ;; If unit tests pass, trigger the auth migrations.
- (after (with-imported-modules '((guix build utils))
+ ;; If tests run successfully, redeploy
+ ;; genenetwork3 and trigger genenetwork2 tests.
+ (after (with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
(when (string=? (getenv "RESULT") "success")
+ (invoke #$sudo
+ #$(file-append shepherd "/bin/herd")
+ "restart" "genenetwork3")
(invoke #$(file-append laminar "/bin/laminarc")
- "queue" "genenetwork3-auth-migrations"))))))
+ "queue" "genenetwork2"))))))
(forge-laminar-job
(name "genenetwork3-pylint")
(run (derivation-job-gexp
@@ -391,10 +411,50 @@ genenetwork3 source from the latest commit of @var{project}."
genenetwork3-mypy
#:guix-daemon-uri %guix-daemon-uri)))
(forge-laminar-job
- (name "genenetwork3-auth-migrations")
- (run (genenetwork3-auth-migrations-laminar config))
- ;; If migrations run successfully, redeploy
- ;; genenetwork3 and trigger genenetwork2 tests.
+ (name "genenetwork3-build-xapian-index")
+ (run (build-xapian-index-gexp this-forge-project))
+ (trigger? #f))))
+ (ci-jobs-trigger 'webhook))
+ (forge-project
+ (name "gn-auth")
+ (repository gn-auth-repository)
+ (ci-jobs (list (forge-laminar-job
+ (name "gn-auth")
+ (run (guix-channel-job-gexp
+ (cons (channel
+ (name 'gn-auth)
+ (url (forge-project-repository this-forge-project))
+ (branch "main"))
+ %default-channels)
+ #:guix-daemon-uri %guix-daemon-uri))
+ ;; If unit tests pass, trigger the auth migrations.
+ (after (with-imported-modules '((guix build utils))
+ #~(begin
+ (use-modules (guix build utils))
+
+ (when (string=? (getenv "RESULT") "success")
+ (invoke #$(file-append laminar "/bin/laminarc")
+ "queue" "gn-auth-migrations"))))))
+ (forge-laminar-job
+ (name "gn-auth-pylint")
+ (run (derivation-job-gexp
+ this-forge-project
+ this-forge-laminar-job
+ gn-auth-pylint
+ #:guix-daemon-uri %guix-daemon-uri)))
+ (forge-laminar-job
+ (name "gn-auth-mypy")
+ (run (derivation-job-gexp
+ this-forge-project
+ this-forge-laminar-job
+ (genenetwork3-tests (list "mypy" ".")
+ (manifest-cons python-mypy
+ (package->development-manifest gn-auth)))
+ #:guix-daemon-uri %guix-daemon-uri)))
+ (forge-laminar-job
+ (name "gn-auth-migrations")
+ (run (gn-auth-migrations-laminar config))
+ ;; If migrations run successfully, redeploy gn-auth
(after (with-imported-modules '((guix build utils))
#~(begin
(use-modules (guix build utils))
@@ -402,14 +462,8 @@ genenetwork3 source from the latest commit of @var{project}."
(when (string=? (getenv "RESULT") "success")
(invoke #$sudo
#$(file-append shepherd "/bin/herd")
- "restart" "genenetwork3")
- (invoke #$(file-append laminar "/bin/laminarc")
- "queue" "genenetwork2")))))
- (trigger? #f))
- (forge-laminar-job
- (name "genenetwork3-build-xapian-index")
- (run (build-xapian-index-gexp this-forge-project))
- (trigger? #f))))
+ "restart" "gn-auth")))))
+ (trigger? #f))))
(ci-jobs-trigger 'webhook)))))
(define (genenetwork2-cd-gexp config)
@@ -508,11 +562,48 @@ server described by CONFIG, a object."
"-b" #$(string-append "localhost:" (number->string gn3-port))
"gn3.app:create_app()"))))))))
+(define (gn-auth-cd-gexp config)
+ "Return a G-expression that runs the latest gn-auth development
+server described by CONFIG, a object."
+ (match-record config
+ (gn-auth-repository gn-auth-port auth-db-path)
+ (with-manifest (package->development-manifest gn-auth)
+ (with-packages (list git-minimal nss-certs)
+ (with-imported-modules '((guix build utils))
+ #~(begin
+ (use-modules (guix build utils)
+ (ice-9 match))
+
+ (define (hline)
+ "Print a horizontal line 50 '=' characters long."
+ (display (make-string 50 #\=))
+ (newline)
+ (force-output))
+
+ (define (show-head-commit)
+ (hline)
+ (invoke "git" "log" "--max-count" "1")
+ (hline))
+
+ ;; Clone the latest gn-auth repository.
+ (invoke "git" "clone" "--depth" "1" #$gn-auth-repository)
+ ;; Configure gn-auth.
+ (setenv "GN_AUTH_CONF"
+ #$(mixed-text-file "gn-auth.conf"
+ "AUTH_DB=\"" auth-db-path "\"\n"))
+ (setenv "HOME" "/tmp")
+ ;; Run gn-auth.
+ (with-directory-excursion "gn-auth"
+ (show-head-commit)
+ (invoke #$(file-append gunicorn "/bin/gunicorn")
+ "-b" #$(string-append "localhost:" (number->string gn-auth-port))
+ "gn_auth:create_app()"))))))))
+
(define (genenetwork-shepherd-services config)
"Return shepherd services to run the genenetwork development server
described by CONFIG, a object."
(match-record config
- (gn2-port gn3-port genotype-files data-directory xapian-db-path auth-db-path)
+ (gn2-port gn3-port gn-auth-port genotype-files data-directory xapian-db-path auth-db-path)
(list (shepherd-service
(documentation "Run GeneNetwork 2 development server.")
(provision '(genenetwork2))
@@ -570,6 +661,35 @@ described by CONFIG, a object."
#:user "genenetwork"
#:group "genenetwork"
#:log-file "/var/log/cd/genenetwork3.log"))
+ (stop #~(make-kill-destructor)))
+ (shepherd-service
+ (documentation "Run gn-auth development server.")
+ (provision '(gn-auth))
+ (requirement '(networking))
+ (start #~(make-forkexec-constructor
+ (list #$(least-authority-wrapper
+ (program-file "gn-auth"
+ (gn-auth-cd-gexp config))
+ #:name "gn-auth-pola-wrapper"
+ ;; If we mapped only the mysqld.sock
+ ;; socket file, it would break when the
+ ;; external mysqld server is restarted.
+ #:mappings (list (file-system-mapping
+ (source "/run/mysqld")
+ (target source)
+ (writable? #t))
+ (file-system-mapping
+ (source data-directory)
+ (target source))
+ (file-system-mapping
+ (source auth-db-path)
+ (target source)
+ (writable? #t)))
+ #:namespaces (delq 'net %namespaces))
+ "127.0.0.1" #$(number->string gn-auth-port))
+ #:user "genenetwork"
+ #:group "genenetwork"
+ #:log-file "/var/log/cd/gn-auth.log"))
(stop #~(make-kill-destructor))))))
(define %genenetwork-accounts
@@ -983,12 +1103,28 @@ tissue."
(body (list (string-append "proxy_pass http://localhost:" (number->string %tissue-port) ";")
"proxy_set_header Host $host;")))))))
+(define (gn-auth-reverse-proxy-server-block)
+ "Return an object to reverse proxy
+gn-auth."
+ (nginx-server-configuration
+ (server-name '("auth.genenetwork.org"))
+ (root "/var/lib/gn-auth/auth.genenetwork.org/website")
+ (locations
+ (list (nginx-location-configuration
+ (uri "/")
+ (body (list (string-append "proxy_pass http://localhost:"
+ (number->string %gn-auth-port)
+ ";")
+ "proxy_set_header Host $host;")))))))
+
;; Port on which webhook is listening
(define %webhook-port 9091)
;; Port on which genenetwork2 is listening
(define %genenetwork2-port 9092)
;; Port on which genenetwork3 is listening
(define %genenetwork3-port 9093)
+;; Port on which gn-auth is listening
+(define %gn-auth-port 9094)
;; Port on which virtuoso's SPARQL endpoint is listening
(define %virtuoso-sparql-port 9082)
@@ -1011,12 +1147,15 @@ tissue."
(file-append shepherd "/bin/herd") " restart genenetwork2, "
(file-append shepherd "/bin/herd") " start genenetwork3, "
(file-append shepherd "/bin/herd") " stop genenetwork3, "
- (file-append shepherd "/bin/herd") " restart genenetwork3\n"
+ (file-append shepherd "/bin/herd") " restart genenetwork3,"
+ (file-append shepherd "/bin/herd") " start gn-auth, "
+ (file-append shepherd "/bin/herd") " stop gn-auth, "
+ (file-append shepherd "/bin/herd") " restart gn-auth\n"
;; Permit the laminar user to run auth db
;; migrations as the genenetwork user.
"\nlaminar ALL = (genenetwork) NOPASSWD: "
- (program-file "genenetwork3-auth-migrations"
- (genenetwork3-auth-migrations-genenetwork (genenetwork-configuration)))
+ (program-file "gn-auth-migrations"
+ (gn-auth-migrations (genenetwork-configuration)))
;; Permit the acme user to restart nginx.
"\nacme ALL = NOPASSWD: " (file-append shepherd "/bin/herd") " restart nginx\n"))
(services (cons* (service forge-service-type
@@ -1067,6 +1206,7 @@ tissue."
(genenetwork-configuration
(gn2-port %genenetwork2-port)
(gn3-port %genenetwork3-port)
+ (gn-auth-port %gn-auth-port)
(genotype-files "/export/data/genenetwork/genotype_files")
(sparql-endpoint (string-append "http://localhost:"
(number->string %virtuoso-sparql-port)
@@ -1111,7 +1251,8 @@ tissue."
(laminar-reverse-proxy-server-block
"localhost:9089" %webhook-port
(list 'gn-bioinformatics))
- (tissue-reverse-proxy-server-block)))))
+ (tissue-reverse-proxy-server-block)
+ (gn-auth-reverse-proxy-server-block)))))
(service acme-service-type
(acme-configuration
(email "arunisaac@systemreboot.net")))
diff --git a/genenetwork/development-helper.scm b/genenetwork/development-helper.scm
index 81bfc3b..627a80a 100644
--- a/genenetwork/development-helper.scm
+++ b/genenetwork/development-helper.scm
@@ -101,3 +101,40 @@ with genenetwork3 dependencies."
(find-files "." shell-script?))
(invoke "pylint" "main.py" "setup.py" "wsgi.py" "setup_commands" "tests" "gn3" "scripts" "sheepdog"))
(mkdir-p #$output)))))
+
+(define (genenetwork-lint-gexp source-repo-checkout profile files-and-modules)
+ "Return a G-expression that runs GeneNetwork3 lint tests in PROFILE
+with SOURCE-REPO-CHECKOUT as the current directory. SOURCE-REPO-CHECKOUT
+is a checkout of the genenetwork3 source code. PROFILE is a profile
+with genenetwork3 dependencies."
+ (with-imported-modules '((guix build utils))
+ (with-profile profile
+ #~(begin
+ (use-modules (rnrs exceptions)
+ (srfi srfi-26)
+ (ice-9 rdelim)
+ (guix build utils))
+
+ (define (shell-script? filename stat-obj)
+ (and (eq? (stat:type stat-obj) 'regular)
+ (call-with-input-file filename
+ (lambda (port)
+ (let ((first-line (read-line port)))
+ (and (not (eof-object? first-line))
+ (> (string-length first-line) 2)
+ (string=? (string-take first-line 2) "#!")
+ (or (string-contains first-line "/bin/sh")
+ (string-contains first-line "/bin/bash"))))))))
+
+ (chdir #$source-repo-checkout)
+ (guard (condition ((invoke-error? condition)
+ (format (current-error-port)
+ "`~a~{ ~a~}' failed with exit status ~a~%"
+ (invoke-error-program condition)
+ (invoke-error-arguments condition)
+ (invoke-error-exit-status condition))
+ (exit #f)))
+ (for-each (cut invoke "shellcheck" <>)
+ (find-files "." shell-script?))
+ (invoke "pylint" #$@files-and-modules))
+ (mkdir-p #$output)))))
--
cgit v1.2.3