diff options
| author | Alexander_Kabui <alexanderkabui@proton.me> | 2026-10-14 07:28:47 +0300 |
|---|---|---|
| committer | Alexander_Kabui <alexanderkabui@proton.me> | 2026-10-14 07:28:47 +0300 |
| commit | d607e8faa7692f28f7e6cf3d0517a69a44557a27 (patch) | |
| tree | 5643ec45ed1f1d1296bf0732f775e9fd93afd882 /scripts | |
| parent | b0c027a40c1e64d5c3a6b7e5dd79d1f3c4cb35eb (diff) | |
| download | genenetwork4-d607e8faa7692f28f7e6cf3d0517a69a44557a27.tar.gz | |
indexer: add versioning-based DELTA mode
* src/indexer/delta.rs: change detection over MariaDB system versioning (ROW_START/ROW_END): coverage marker, versioning probe, purge check, per-table candidate queries and identity-term fan-out. * src/indexer/run.rs (run_delta): reuse the streaming crawl and shard diff, scoped to the affected datasets. * src/indexer/sidecar.rs: manifest v3 with versioning_marker/versioning_flavour. * src/bin/gn-indexer.rs: add the delta mode. * scripts/gn-indexer-run.sh, scripts/gn-indexer-lag.sh: flock wrapper and lag SLA.
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gn-indexer-lag.sh | 114 | ||||
| -rwxr-xr-x | scripts/gn-indexer-run.sh | 98 |
2 files changed, 212 insertions, 0 deletions
diff --git a/scripts/gn-indexer-lag.sh b/scripts/gn-indexer-lag.sh new file mode 100755 index 0000000..64ec655 --- /dev/null +++ b/scripts/gn-indexer-lag.sh @@ -0,0 +1,114 @@ +#!/bin/sh +# gn-indexer-lag — the lag SLA check, on the DB clock (no host clock). +# +# Reads `versioning_marker` from <index>.sidecar.json, then one mysql +# round-trip with the two design §3 quantities: +# lag_seconds = TIMESTAMPDIFF(SECOND, marker, NOW(6)) +# newest_change = MAX(ROW_START) over the seven versioned tables +# +# Alert (exit 1 + ERROR) only when lag_seconds > N +# (GN4_INDEXER_LAG_SLA_MINUTES, default 120 = worst-case 99 + tick 10 + +# slack 10), or when `newest_change > marker` has PERSISTED past N +# (a single fresh gap is ordinary live write load and never pages). +# +# usage: gn-indexer-lag <index-path> [--db DB] +# * mysql client: $GN4_INDEXER_LAG_MYSQL (else mysql -u USER -h HOST) +# * user/host: GN4_INDEXER_LAG_USER (webqtlout) / _HOST (127.0.0.1) +# * database: --db DB or GN4_INDEXER_LAG_DB +# * table prefix: GN4_INDEXER_DELTA_TABLE_PREFIX (test seam, direct) +# * state file: $GN4_INDEXER_LAG_STATE else <index>.lag.state +set -u + +usage() { + echo "usage: gn-indexer-lag <index-path> [--db DB]" >&2 + exit 2 +} + +[ "$#" -ge 1 ] || usage +index=$1 +shift +db=${GN4_INDEXER_LAG_DB:-} +while [ "$#" -ge 1 ]; do + case "$1" in + --db) db=${2:-} ; shift 2 ;; + *) usage ;; + esac +done + +sla_minutes=${GN4_INDEXER_LAG_SLA_MINUTES:-120} +sla_seconds=$(( sla_minutes * 60 )) +prefix=${GN4_INDEXER_DELTA_TABLE_PREFIX:-} +state=${GN4_INDEXER_LAG_STATE:-${index}.lag.state} +sidecar=${index}.sidecar.json + +log() { + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) gn-indexer-lag: $*" +} + +mysql_cmd=${GN4_INDEXER_LAG_MYSQL:-"mysql -u ${GN4_INDEXER_LAG_USER:-webqtlout} -h ${GN4_INDEXER_LAG_HOST:-127.0.0.1} --batch --skip-column-names"} +if [ -n "$db" ]; then + mysql_q() { $mysql_cmd "$db" -e "$1"; } +else + mysql_q() { $mysql_cmd -e "$1"; } +fi + +[ -f "$sidecar" ] || { + log "ERROR: sidecar not found: $sidecar" + exit 1 +} + +marker=$(sed -n 's/.*"versioning_marker"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$sidecar" | head -n 1) +if [ -z "$marker" ]; then + log "INFO: no versioning marker in $sidecar (versioning absent or not yet adopted); lag SLA not applicable" + exit 0 +fi + +union= +for t in PublishXRef PublishFreeze ProbeSetXRef ProbeSetFreeze ProbeSet InbredSet Species; do + part="SELECT MAX(ROW_START) AS ROW_START FROM ${prefix}${t} FOR SYSTEM_TIME ALL" + if [ -z "$union" ]; then + union=$part + else + union="$union UNION ALL $part" + fi +done + +newest_change=$(mysql_q "SELECT MAX(ROW_START) FROM ( $union ) u" 2>/dev/null) || { + log "ERROR: lag query failed (mysql client or privileges?)" + exit 1 +} +lag_seconds=$(mysql_q "SELECT TIMESTAMPDIFF(SECOND, '$marker', NOW(6))" 2>/dev/null) || { + log "ERROR: lag arithmetic query failed" + exit 1 +} + +log "marker=$marker newest_change=${newest_change:-NULL} lag_seconds=$lag_seconds sla_seconds=$sla_seconds" + +if [ "$lag_seconds" -gt "$sla_seconds" ]; then + log "ERROR: lag ${lag_seconds}s > SLA ${sla_seconds}s" + exit 1 +fi + +# Freshness gap: versioned data newer than the marker. Page only when it +# persists past the SLA — track the first observation in the state file. +if [ -n "$newest_change" ] && [ "$newest_change" \> "$marker" ]; then + now=$(date -u +%s) + first_seen= + [ -f "$state" ] && first_seen=$(cat "$state" 2>/dev/null || true) + if [ -z "$first_seen" ]; then + printf '%s\n' "$now" >"$state" + log "INFO: freshness gap (newest_change=$newest_change > marker=$marker) first observed; not paging yet" + exit 0 + fi + gap_seconds=$(( now - first_seen )) + if [ "$gap_seconds" -gt "$sla_seconds" ]; then + log "ERROR: freshness gap persisted ${gap_seconds}s > SLA ${sla_seconds}s (newest_change=$newest_change > marker=$marker)" + exit 1 + fi + log "INFO: freshness gap persisted ${gap_seconds}s < SLA ${sla_seconds}s" + exit 0 +fi + +rm -f "$state" +log "INFO: within SLA" +exit 0 diff --git a/scripts/gn-indexer-run.sh b/scripts/gn-indexer-run.sh new file mode 100755 index 0000000..8a33c45 --- /dev/null +++ b/scripts/gn-indexer-run.sh @@ -0,0 +1,98 @@ +#!/bin/sh +# gn-indexer-run — the cron wrapper: acquire the per-index single-instance +# flock, record the hold start, then hand off to the indexer (which does +# all the work and decides nothing about schedule or mutex). +# +# SUPERSEDES DESIGN-CRON §1's crontab-level `flock`: a single flock inside +# this wrapper is the authoritative mutex. Do NOT also wrap the crontab +# line in flock — two locks would defeat the held-since bookkeeping (and +# the wrapper's own skip log). +# +# usage: gn-indexer-run <delta|sweep|full> --index PATH [indexer args…] +# * `--index PATH` (or GN4_INDEXER_INDEX) is REQUIRED: the wrapper needs +# it both for the lock path and to pass it through to the indexer. +# * `sweep` maps to the indexer's `incremental` (the verification sweep). +# * lock file: $GN4_INDEXER_LOCK else <index>.lock +# * hold-start file: $GN4_INDEXER_HELD else <index>.lock.held +# * hung-holder ERROR threshold: GN4_INDEXER_WORST_CASE_MINUTES (99) +# +# The kernel releases the flock when the holder exits — including kill -9 — +# so there is no pidfile, no operator unlock, and no queue. +# +# crontab (install is the operator's job): +# */10 * * * * /srv/gn4/bin/gn-indexer-run delta --index /srv/gn4/index +# 0 */3 * * * /srv/gn4/bin/gn-indexer-run sweep --index /srv/gn4/index +set -u + +log() { + echo "$(date -u +%Y-%m-%dT%H:%M:%SZ) gn-indexer-run[$mode_arg]: $*" +} + +usage() { + echo "usage: gn-indexer-run <delta|sweep|full> --index PATH [indexer args…]" >&2 + exit 2 +} + +[ "$#" -ge 1 ] || usage +mode=$1 +shift +case "$mode" in + delta) mode_arg=delta ;; + sweep) mode_arg=incremental ;; + full) mode_arg=full ;; + *) usage ;; +esac + +# Resolve --index from the remaining args (or GN4_INDEXER_INDEX). +index=${GN4_INDEXER_INDEX:-} +prev= +for arg in "$@"; do + if [ "$prev" = "--index" ]; then + index=$arg + fi + prev=$arg +done +[ -n "$index" ] || usage + +bin=${GN4_INDEXER_BIN:-gn-indexer} +lock=${GN4_INDEXER_LOCK:-${index}.lock} +held=${GN4_INDEXER_HELD:-${index}.lock.held} +worst_case=${GN4_INDEXER_WORST_CASE_MINUTES:-99} + +# Hold the lock on fd 9 for the wrapper's lifetime (inherited by the child). +exec 9>"$lock" || { + log "ERROR: cannot open lock file $lock" + exit 1 +} +if ! flock -n 9; then + # Skip-and-log: a busy lock is expected while a long sweep runs; rc 0 + # for cron. Only a holder past the worst-case threshold is an ERROR. + if [ -f "$held" ]; then + since=$(cat "$held" 2>/dev/null || echo 0) + now=$(date -u +%s) + age_min=$(( (now - since) / 60 )) + if [ "$age_min" -ge "$worst_case" ]; then + log "ERROR: skip: lock held ${age_min}min >= worst-case ${worst_case}min (hung holder?); held-since=$since" + else + log "INFO: skip: lock busy (held since $since, ${age_min}min)" + fi + else + log "INFO: skip: lock busy" + fi + exit 0 +fi + +# We hold the lock: record the hold start so a later skip can state the age. +now=$(date -u +%s) +printf '%s\n' "$now" >"$held" +trap 'rm -f "$held"' EXIT HUP INT TERM + +# The indexer does the work; propagate its exit code (a failed run is not +# a skip). fd 9 is closed for the child so the WRAPPER is the sole lock +# holder: a kill -9 of the wrapper releases the flock immediately (E11). +"$bin" "$mode_arg" "$@" 9>&- +rc=$? +if [ "$rc" -ne 0 ]; then + log "ERROR: run failed rc=$rc" +fi +exit "$rc" |
