From 1a6c7982d40357374fe79b9ca916f85a55988512 Mon Sep 17 00:00:00 2001
From: Munyoki Kilyungi
Date: Mon, 17 Jun 2024 19:24:02 +0300
Subject: Move xapian re-indexing checks into the indexing script.

Let the python xapian indexing script check the checksums, instead of using a
G-expression.  This approach is simpler and more understandable; it is also
more versatile and allows easier modification.

* genenetwork-development.scm (build-xapian-index-cron-gexp): Use the indexing
script to compare checksums.  If the checksums changed, queue the
genenetwork3-build-xapian-index laminar job.

Co-authored-by: John Nduli <rookie101@jnduli.co.ke>
---
 genenetwork-development.scm | 86 ++++++++++++++++++++-------------------------
 1 file changed, 38 insertions(+), 48 deletions(-)

diff --git a/genenetwork-development.scm b/genenetwork-development.scm
index f0db2b7..a015154 100644
--- a/genenetwork-development.scm
+++ b/genenetwork-development.scm
@@ -1,6 +1,8 @@
 ;;; genenetwork-machines --- Guix configuration for genenetwork machines
 ;;; Copyright © 2022–2024 Arun Isaac <arunisaac@systemreboot.net>
 ;;; Copyright © 2022–2024 Frederick Muriuki Muriithi <fredmanglis@gmail.com>
+;;; Copyright © 2024 Munyoki Kilyungi <me@bonfacemunyoki.com>
+;;; Copyright © 2024 John Nduli <rookie101@jnduli.co.ke>
 ;;;
 ;;; This file is part of genenetwork-machines.
 ;;;
@@ -248,8 +250,10 @@ genenetwork3 source from the latest commit of @var{project}."
               (lambda ()
                 ;; Build xapian index.
                 (setenv "PYTHONPATH" (getcwd))
-                (invoke "./scripts/index-genenetwork" xapian-build-directory
-			"mysql://webqtlout:webqtlout@localhost/db_webqtl")
+                (invoke "./scripts/index-genenetwork" "create-xapian-index"
+                        xapian-build-directory
+			"mysql://webqtlout:webqtlout@localhost/db_webqtl"
+                        "http://localhost:9082/sparql")
                 ;; Stop genenetwork3, replace old xapian index and
                 ;; start genenetwork3.
                 (dynamic-wind
@@ -264,54 +268,40 @@ genenetwork3 source from the latest commit of @var{project}."
 ;; G-expression that is configured in cron to trigger a rebuild of the
 ;; xapian index
 (define build-xapian-index-cron-gexp
-  (with-extensions (list guile-dsv guile-lib guile-xapian)
-    (with-imported-modules '((guix build utils))
+  (with-imported-modules '((guix build utils))
+    (with-manifest
+        (manifest-cons* git-minimal nss-certs
+                        (package->development-manifest genenetwork3))
       #~(begin
           (use-modules (guix build utils)
-                       (dsv)
-                       (xapian wrap)
-                       (xapian xapian)
-                       (ice-9 match)
-                       (ice-9 popen)
-                       (srfi srfi-26)
-                       (srfi srfi-71)
-                       (rnrs io ports))
-
-          ;; guile-dbi does not support guile 3. So, we shell out to
-          ;; mysql instead.
-          (define (mysql-table-checksums user password database tables)
-            "Return list of checksums for @var{tables} in MySQL
-@var{database}. Authenticate with @var{user} and @var{password}."
-            (let ((from to pids (pipeline `((#$(file-append mariadb "/bin/mysql")
-                                               "--batch"
-                                               ,(string-append "--user=" user)
-                                               ,(string-append "--password=" password)
-                                               ,database)))))
-              (put-string to (string-append "CHECKSUM TABLES "
-                                            (string-join tables ",")
-                                            ";"))
-              (close to)
-              (match (dsv->scm from #\tab)
-                ((header (tables checksums) ...)
-                 (close from)
-                 checksums))))
-
-          (call-with-database #$%xapian-directory
-            (lambda (db)
-              (let ((tables (string-split (Database-get-metadata db "tables")
-                                          #\space))
-                    (checksums (map string->number
-                                    (string-split (Database-get-metadata db "checksums")
-                                                  #\space))))
-                ;; Trigger xapian index rebuild when table checksums
-                ;; in the index do not match with the SQL database and
-                ;; there is no running build job.
-                (unless (or (equal? (mysql-table-checksums "webqtlout" "webqtlout" "db_webqtl" tables)
-                                    checksums)
-                            (file-exists? (string-append #$%xapian-directory "/build")))
-                  (setenv "LAMINAR_REASON" "Nightly xapian index rebuild")
-                  (invoke #$(file-append laminar "/bin/laminarc")
-                          "queue" "genenetwork3-build-xapian-index")))))))))
+                       (srfi srfi-26))
+
+          (define (call-with-temporary-directory proc)
+            (let ((tmp-dir (mkdtemp "/tmp/xapian-index-cron.XXXXXX")))
+              (dynamic-wind
+                (const #t)
+                (cut proc tmp-dir)
+                (cut delete-file-recursively tmp-dir))))
+
+          (call-with-temporary-directory
+           (lambda (gn3-dir)
+             (invoke "git" "clone" "--depth" "1"
+                     "https://github.com/genenetwork/genenetwork3.git"
+                     gn3-dir)
+             (setenv "PYTHONPATH" gn3-dir)
+             ;; Only trigger a laminar build when the data is modified.
+             (when
+                 (and
+                  (not (file-exists? (string-append #$%xapian-directory "/build")))
+                  (zero? (status:exit-val
+                          (system* (string-append gn3-dir "/scripts/index-genenetwork")
+                                   "is-data-modified"
+                                   #$%xapian-directory
+                                   "mysql://webqtlout:webqtlout@localhost/db_webqtl"
+                                   "http://localhost:9082/sparql"))))
+               (setenv "LAMINAR_REASON" "Nightly xapian index rebuild")
+               (invoke #$(file-append laminar "/bin/laminarc")
+                       "queue" "genenetwork3-build-xapian-index"))))))))
 
 (define (genenetwork-projects config)
   "Return forge projects for genenetwork described by CONFIG, a
-- 
cgit v1.2.3