aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMunyoki Kilyungi2024-06-17 19:24:02 +0300
committerMunyoki Kilyungi2024-06-19 19:18:17 +0300
commit1a6c7982d40357374fe79b9ca916f85a55988512 (patch)
tree55d21bceab4a9e97b195f02fc703979c1a6a8cb4
parented922a94b072453a6c32d3ccbe8cfa3090132829 (diff)
downloadgn-machines-1a6c7982d40357374fe79b9ca916f85a55988512.tar.gz
Move xapian re-indexing checks into the indexing script.
Let the python xapian indexing script check the checksums, instead of using a G-expression. This approach is simpler and more understandable; it is also more versatile and allows easier modification. * genenetwork-development.scm (build-xapian-index-cron-gexp): Use the indexing script to compare checksums. If the checksums changed, queue the genenetwork3-build-xapian-index laminar job. Co-authored-by: John Nduli <rookie101@jnduli.co.ke>
-rw-r--r--genenetwork-development.scm86
1 files changed, 38 insertions, 48 deletions
diff --git a/genenetwork-development.scm b/genenetwork-development.scm
index f0db2b7..a015154 100644
--- a/genenetwork-development.scm
+++ b/genenetwork-development.scm
@@ -1,6 +1,8 @@
;;; genenetwork-machines --- Guix configuration for genenetwork machines
;;; Copyright © 2022–2024 Arun Isaac <arunisaac@systemreboot.net>
;;; Copyright © 2022–2024 Frederick Muriuki Muriithi <fredmanglis@gmail.com>
+;;; Copyright © 2024 Munyoki Kilyungi <me@bonfacemunyoki.com>
+;;; Copyright © 2024 John Nduli <rookie101@jnduli.co.ke>
;;;
;;; This file is part of genenetwork-machines.
;;;
@@ -248,8 +250,10 @@ genenetwork3 source from the latest commit of @var{project}."
(lambda ()
;; Build xapian index.
(setenv "PYTHONPATH" (getcwd))
- (invoke "./scripts/index-genenetwork" xapian-build-directory
- "mysql://webqtlout:webqtlout@localhost/db_webqtl")
+ (invoke "./scripts/index-genenetwork" "create-xapian-index"
+ xapian-build-directory
+ "mysql://webqtlout:webqtlout@localhost/db_webqtl"
+ "http://localhost:9082/sparql")
;; Stop genenetwork3, replace old xapian index and
;; start genenetwork3.
(dynamic-wind
@@ -264,54 +268,40 @@ genenetwork3 source from the latest commit of @var{project}."
;; G-expression that is configured in cron to trigger a rebuild of the
;; xapian index
(define build-xapian-index-cron-gexp
- (with-extensions (list guile-dsv guile-lib guile-xapian)
- (with-imported-modules '((guix build utils))
+ (with-imported-modules '((guix build utils))
+ (with-manifest
+ (manifest-cons* git-minimal nss-certs
+ (package->development-manifest genenetwork3))
#~(begin
(use-modules (guix build utils)
- (dsv)
- (xapian wrap)
- (xapian xapian)
- (ice-9 match)
- (ice-9 popen)
- (srfi srfi-26)
- (srfi srfi-71)
- (rnrs io ports))
-
- ;; guile-dbi does not support guile 3. So, we shell out to
- ;; mysql instead.
- (define (mysql-table-checksums user password database tables)
- "Return list of checksums for @var{tables} in MySQL
-@var{database}. Authenticate with @var{user} and @var{password}."
- (let ((from to pids (pipeline `((#$(file-append mariadb "/bin/mysql")
- "--batch"
- ,(string-append "--user=" user)
- ,(string-append "--password=" password)
- ,database)))))
- (put-string to (string-append "CHECKSUM TABLES "
- (string-join tables ",")
- ";"))
- (close to)
- (match (dsv->scm from #\tab)
- ((header (tables checksums) ...)
- (close from)
- checksums))))
-
- (call-with-database #$%xapian-directory
- (lambda (db)
- (let ((tables (string-split (Database-get-metadata db "tables")
- #\space))
- (checksums (map string->number
- (string-split (Database-get-metadata db "checksums")
- #\space))))
- ;; Trigger xapian index rebuild when table checksums
- ;; in the index do not match with the SQL database and
- ;; there is no running build job.
- (unless (or (equal? (mysql-table-checksums "webqtlout" "webqtlout" "db_webqtl" tables)
- checksums)
- (file-exists? (string-append #$%xapian-directory "/build")))
- (setenv "LAMINAR_REASON" "Nightly xapian index rebuild")
- (invoke #$(file-append laminar "/bin/laminarc")
- "queue" "genenetwork3-build-xapian-index")))))))))
+ (srfi srfi-26))
+
+ (define (call-with-temporary-directory proc)
+ (let ((tmp-dir (mkdtemp "/tmp/xapian-index-cron.XXXXXX")))
+ (dynamic-wind
+ (const #t)
+ (cut proc tmp-dir)
+ (cut delete-file-recursively tmp-dir))))
+
+ (call-with-temporary-directory
+ (lambda (gn3-dir)
+ (invoke "git" "clone" "--depth" "1"
+ "https://github.com/genenetwork/genenetwork3.git"
+ gn3-dir)
+ (setenv "PYTHONPATH" gn3-dir)
+ ;; Only trigger a laminar build when the data is modified.
+ (when
+ (and
+ (not (file-exists? (string-append #$%xapian-directory "/build")))
+ (zero? (status:exit-val
+ (system* (string-append gn3-dir "/scripts/index-genenetwork")
+ "is-data-modified"
+ #$%xapian-directory
+ "mysql://webqtlout:webqtlout@localhost/db_webqtl"
+ "http://localhost:9082/sparql"))))
+ (setenv "LAMINAR_REASON" "Nightly xapian index rebuild")
+ (invoke #$(file-append laminar "/bin/laminarc")
+ "queue" "genenetwork3-build-xapian-index"))))))))
(define (genenetwork-projects config)
"Return forge projects for genenetwork described by CONFIG, a