about summary refs log tree commit diff
path: root/scripts
diff options
context:
space:
mode:
authorAlexander_Kabui <alexanderkabui@proton.me>2026-10-14 07:28:47 +0300
committerAlexander_Kabui <alexanderkabui@proton.me>2026-10-14 07:28:47 +0300
commitd607e8faa7692f28f7e6cf3d0517a69a44557a27 (patch)
tree5643ec45ed1f1d1296bf0732f775e9fd93afd882 /scripts
parentb0c027a40c1e64d5c3a6b7e5dd79d1f3c4cb35eb (diff)
downloadgenenetwork4-d607e8faa7692f28f7e6cf3d0517a69a44557a27.tar.gz
indexer: add versioning-based DELTA mode
* src/indexer/delta.rs: change detection over MariaDB system versioning
  (ROW_START/ROW_END): coverage marker, versioning probe, purge check,
  per-table candidate queries and identity-term fan-out.
* src/indexer/run.rs (run_delta): reuse the streaming crawl and shard diff,
  scoped to the affected datasets.
* src/indexer/sidecar.rs: manifest v3 with versioning_marker/versioning_flavour.
* src/bin/gn-indexer.rs: add the delta mode.
* scripts/gn-indexer-run.sh, scripts/gn-indexer-lag.sh: flock wrapper and lag SLA.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/gn-indexer-lag.sh114
-rwxr-xr-xscripts/gn-indexer-run.sh98
2 files changed, 212 insertions, 0 deletions
diff --git a/scripts/gn-indexer-lag.sh b/scripts/gn-indexer-lag.sh
new file mode 100755
index 0000000..64ec655
--- /dev/null
+++ b/scripts/gn-indexer-lag.sh
@@ -0,0 +1,114 @@
+#!/bin/sh
+# gn-indexer-lag — the lag SLA check, on the DB clock (no host clock).
+#
+# Reads `versioning_marker` from <index>.sidecar.json, then one mysql
+# round-trip with the two design §3 quantities:
+#   lag_seconds   = TIMESTAMPDIFF(SECOND, marker, NOW(6))
+#   newest_change = MAX(ROW_START) over the seven versioned tables
+#
+# Alert (exit 1 + ERROR) only when lag_seconds > N
+# (GN4_INDEXER_LAG_SLA_MINUTES, default 120 = worst-case 99 + tick 10 +
+# slack 10), or when `newest_change > marker` has PERSISTED past N
+# (a single fresh gap is ordinary live write load and never pages).
+#
+# usage: gn-indexer-lag <index-path> [--db DB]
+#   * mysql client: $GN4_INDEXER_LAG_MYSQL (else mysql -u USER -h HOST)
+#   * user/host:    GN4_INDEXER_LAG_USER (webqtlout) / _HOST (127.0.0.1)
+#   * database:     --db DB or GN4_INDEXER_LAG_DB
+#   * table prefix: GN4_INDEXER_DELTA_TABLE_PREFIX (test seam, direct)
+#   * state file:   $GN4_INDEXER_LAG_STATE else <index>.lag.state
+set -u
+
+usage() {
+    echo "usage: gn-indexer-lag <index-path> [--db DB]" >&2
+    exit 2
+}
+
+[ "$#" -ge 1 ] || usage
+index=$1
+shift
+db=${GN4_INDEXER_LAG_DB:-}
+while [ "$#" -ge 1 ]; do
+    case "$1" in
+        --db) db=${2:-} ; shift 2 ;;
+        *) usage ;;
+    esac
+done
+
+sla_minutes=${GN4_INDEXER_LAG_SLA_MINUTES:-120}
+sla_seconds=$(( sla_minutes * 60 ))
+prefix=${GN4_INDEXER_DELTA_TABLE_PREFIX:-}
+state=${GN4_INDEXER_LAG_STATE:-${index}.lag.state}
+sidecar=${index}.sidecar.json
+
+log() {
+    echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) gn-indexer-lag: $*"
+}
+
+mysql_cmd=${GN4_INDEXER_LAG_MYSQL:-"mysql -u ${GN4_INDEXER_LAG_USER:-webqtlout} -h ${GN4_INDEXER_LAG_HOST:-127.0.0.1} --batch --skip-column-names"}
+if [ -n "$db" ]; then
+    mysql_q() { $mysql_cmd "$db" -e "$1"; }
+else
+    mysql_q() { $mysql_cmd -e "$1"; }
+fi
+
+[ -f "$sidecar" ] || {
+    log "ERROR: sidecar not found: $sidecar"
+    exit 1
+}
+
+marker=$(sed -n 's/.*"versioning_marker"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$sidecar" | head -n 1)
+if [ -z "$marker" ]; then
+    log "INFO: no versioning marker in $sidecar (versioning absent or not yet adopted); lag SLA not applicable"
+    exit 0
+fi
+
+union=
+for t in PublishXRef PublishFreeze ProbeSetXRef ProbeSetFreeze ProbeSet InbredSet Species; do
+    part="SELECT MAX(ROW_START) AS ROW_START FROM ${prefix}${t} FOR SYSTEM_TIME ALL"
+    if [ -z "$union" ]; then
+        union=$part
+    else
+        union="$union UNION ALL $part"
+    fi
+done
+
+newest_change=$(mysql_q "SELECT MAX(ROW_START) FROM ( $union ) u" 2>/dev/null) || {
+    log "ERROR: lag query failed (mysql client or privileges?)"
+    exit 1
+}
+lag_seconds=$(mysql_q "SELECT TIMESTAMPDIFF(SECOND, '$marker', NOW(6))" 2>/dev/null) || {
+    log "ERROR: lag arithmetic query failed"
+    exit 1
+}
+
+log "marker=$marker newest_change=${newest_change:-NULL} lag_seconds=$lag_seconds sla_seconds=$sla_seconds"
+
+if [ "$lag_seconds" -gt "$sla_seconds" ]; then
+    log "ERROR: lag ${lag_seconds}s > SLA ${sla_seconds}s"
+    exit 1
+fi
+
+# Freshness gap: versioned data newer than the marker. Page only when it
+# persists past the SLA — track the first observation in the state file.
+if [ -n "$newest_change" ] && [ "$newest_change" \> "$marker" ]; then
+    now=$(date -u +%s)
+    first_seen=
+    [ -f "$state" ] && first_seen=$(cat "$state" 2>/dev/null || true)
+    if [ -z "$first_seen" ]; then
+        printf '%s\n' "$now" >"$state"
+        log "INFO: freshness gap (newest_change=$newest_change > marker=$marker) first observed; not paging yet"
+        exit 0
+    fi
+    gap_seconds=$(( now - first_seen ))
+    if [ "$gap_seconds" -gt "$sla_seconds" ]; then
+        log "ERROR: freshness gap persisted ${gap_seconds}s > SLA ${sla_seconds}s (newest_change=$newest_change > marker=$marker)"
+        exit 1
+    fi
+    log "INFO: freshness gap persisted ${gap_seconds}s < SLA ${sla_seconds}s"
+    exit 0
+fi
+
+rm -f "$state"
+log "INFO: within SLA"
+exit 0
diff --git a/scripts/gn-indexer-run.sh b/scripts/gn-indexer-run.sh
new file mode 100755
index 0000000..8a33c45
--- /dev/null
+++ b/scripts/gn-indexer-run.sh
@@ -0,0 +1,98 @@
+#!/bin/sh
+# gn-indexer-run — the cron wrapper: acquire the per-index single-instance
+# flock, record the hold start, then hand off to the indexer (which does
+# all the work and decides nothing about schedule or mutex).
+#
+# SUPERSEDES DESIGN-CRON §1's crontab-level `flock`: a single flock inside
+# this wrapper is the authoritative mutex. Do NOT also wrap the crontab
+# line in flock — two locks would defeat the held-since bookkeeping (and
+# the wrapper's own skip log).
+#
+# usage: gn-indexer-run <delta|sweep|full> --index PATH [indexer args…]
+#   * `--index PATH` (or GN4_INDEXER_INDEX) is REQUIRED: the wrapper needs
+#     it both for the lock path and to pass it through to the indexer.
+#   * `sweep` maps to the indexer's `incremental` (the verification sweep).
+#   * lock file:      $GN4_INDEXER_LOCK   else <index>.lock
+#   * hold-start file: $GN4_INDEXER_HELD  else <index>.lock.held
+#   * hung-holder ERROR threshold: GN4_INDEXER_WORST_CASE_MINUTES (99)
+#
+# The kernel releases the flock when the holder exits — including kill -9 —
+# so there is no pidfile, no operator unlock, and no queue.
+#
+# crontab (install is the operator's job):
+#   */10 * * * *  /srv/gn4/bin/gn-indexer-run delta --index /srv/gn4/index
+#   0 */3 * * *   /srv/gn4/bin/gn-indexer-run sweep --index /srv/gn4/index
+set -u
+
+log() {
+    echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) gn-indexer-run[$mode_arg]: $*"
+}
+
+usage() {
+    echo "usage: gn-indexer-run <delta|sweep|full> --index PATH [indexer args…]" >&2
+    exit 2
+}
+
+[ "$#" -ge 1 ] || usage
+mode=$1
+shift
+case "$mode" in
+    delta) mode_arg=delta ;;
+    sweep) mode_arg=incremental ;;
+    full) mode_arg=full ;;
+    *) usage ;;
+esac
+
+# Resolve --index from the remaining args (or GN4_INDEXER_INDEX).
+index=${GN4_INDEXER_INDEX:-}
+prev=
+for arg in "$@"; do
+    if [ "$prev" = "--index" ]; then
+        index=$arg
+    fi
+    prev=$arg
+done
+[ -n "$index" ] || usage
+
+bin=${GN4_INDEXER_BIN:-gn-indexer}
+lock=${GN4_INDEXER_LOCK:-${index}.lock}
+held=${GN4_INDEXER_HELD:-${index}.lock.held}
+worst_case=${GN4_INDEXER_WORST_CASE_MINUTES:-99}
+
+# Hold the lock on fd 9 for the wrapper's lifetime (inherited by the child).
+exec 9>"$lock" || {
+    log "ERROR: cannot open lock file $lock"
+    exit 1
+}
+if ! flock -n 9; then
+    # Skip-and-log: a busy lock is expected while a long sweep runs; rc 0
+    # for cron. Only a holder past the worst-case threshold is an ERROR.
+    if [ -f "$held" ]; then
+        since=$(cat "$held" 2>/dev/null || echo 0)
+        now=$(date -u +%s)
+        age_min=$(( (now - since) / 60 ))
+        if [ "$age_min" -ge "$worst_case" ]; then
+            log "ERROR: skip: lock held ${age_min}min >= worst-case ${worst_case}min (hung holder?); held-since=$since"
+        else
+            log "INFO: skip: lock busy (held since $since, ${age_min}min)"
+        fi
+    else
+        log "INFO: skip: lock busy"
+    fi
+    exit 0
+fi
+
+# We hold the lock: record the hold start so a later skip can state the age.
+now=$(date -u +%s)
+printf '%s\n' "$now" >"$held"
+trap 'rm -f "$held"' EXIT HUP INT TERM
+
+# The indexer does the work; propagate its exit code (a failed run is not
+# a skip). fd 9 is closed for the child so the WRAPPER is the sole lock
+# holder: a kill -9 of the wrapper releases the flock immediately (E11).
+"$bin" "$mode_arg" "$@" 9>&-
+rc=$?
+if [ "$rc" -ne 0 ]; then
+    log "ERROR: run failed rc=$rc"
+fi
+exit "$rc"