#!/usr/bin/env python3 """monitor-gn4 -- is this genenetwork4 instance alive, endpoint by endpoint? The point of this tool is not "does the server answer" -- /version does that -- but "does every endpoint work, and when one does not, which *other* thing is missing". Most of the interesting endpoints call out: MariaDB, the Xapian index, gemma-wrapper and gemma and R, R/qtl and R/qtl2, Julia and its depot, ravanan and ccwl and slurmrestd, a python plotter, Wikidata over the network. Each check therefore names what it depends on, and a failure prints the server's own diagnosis (the reply's `code` and `detail`) rather than just the status code. ./bin/monitor-gn4 # localhost:3000, core+read+search ./bin/monitor-gn4 --url https://example.org/gn4/ # somewhere else ./bin/monitor-gn4 --tier all --token-file ~/.gn4-tokens/pjotrp1.token ./bin/monitor-gn4 --json > report.json Tiers, cheapest first, so a monitor run is not a load test: core no auth, no database: /version, /mapping-methods read the MariaDB-backed surface (needs a token) search the search engine (Xapian index, or gn3 if SEARCH_ENGINE=forward) external checks that need the outside world (Wikidata), off by default: a normal run must be fast and its failures GN4's own, not the net's compute every /compute/* engine -- minutes, and it will spawn processes plots the plotters, chained off the compute results they draw Exit status is 0 when nothing failed, 1 when anything failed. SKIPped checks (no token, discovery found nothing to use) do not fail a run: they are printed with a reason and counted separately, because a monitor that fails when it has nothing to test trains people to ignore it. Performance tests belong here too, later, and share the same discovery and reporting; they are not implemented yet. """ import argparse import json import os import ssl import sys import time import urllib.error import urllib.parse import urllib.request DEFAULT_URL = "http://127.0.0.1:3000" DEFAULT_TOKEN_FILE = os.path.expanduser( "~/tmp/ai-root/factory/.gn4-tokens/pjotrp1.token") # Per-check timeouts in seconds. The default is deliberately short: a liveness # monitor that waits a minute per endpoint is not one. Engine calls get what # they actually need, measured on this box. TIMEOUTS = { "/version": 10, "/search": 30, "/genes": 60, "/compute/correlation": 300, "/compute/bnw": 120, "/compute/gemma": 600, "/compute/rqtl": 300, "/compute/qtl2": 300, "/compute/fitqtl": 300, "/compute/bulklmm": 300, "/compute/gemma/ravanan": 1200, } OK, FAIL, SKIP, BUSY = "ok", "FAIL", "skip", "busy" class Skip(Exception): """A check that cannot be exercised here, and why. Not a failure: an endpoint fenced by design (a hit only exists above a LOD threshold) has nothing to answer on a fixture that has no hit, and calling that red would train a reader to ignore red. """ class Reply: """One HTTP exchange, reduced to what the report needs.""" def __init__(self, status, body, ms, error=None): self.status = status self.body = body self.ms = ms self.error = error @property def json(self): if isinstance(self.body, (dict, list)): return self.body return None def fields(self): """The envelope's code/detail, which is where the diagnosis lives.""" data = self.json if not isinstance(data, dict): return None, None return data.get("code"), (data.get("detail") or data.get("title")) class Client: def __init__(self, base, token=None, insecure=False, verbose=False): self.base = base.rstrip("/") + "/" self.token = token self.verbose = verbose self.context = None if insecure: self.context = ssl.create_default_context() self.context.check_hostname = False self.context.verify_mode = ssl.CERT_NONE def call(self, method, path, body=None, timeout=15): url = urllib.parse.urljoin(self.base, path.lstrip("/")) headers = {"accept": "application/json, image/*"} data = None if body is not None: data = json.dumps(body).encode() headers["content-type"] = "application/json" if self.token: headers["authorization"] = "Bearer " + self.token request = urllib.request.Request(url, data=data, headers=headers, method=method) started = time.time() try: with urllib.request.urlopen(request, timeout=timeout, context=self.context) as response: raw = response.read() status = response.status ctype = response.headers.get("content-type", "") except urllib.error.HTTPError as error: raw = error.read() status = error.code ctype = error.headers.get("content-type", "") if error.headers else "" except Exception as error: # noqa: BLE001 -- reported, not raised ms = int((time.time() - started) * 1000) kind = "timeout" if "timed out" in str(error) else type(error).__name__ return Reply(0, None, ms, "%s: %s" % (kind, error)) ms = int((time.time() - started) * 1000) if self.verbose: print(" %-6s %-60s %s %5d ms" % (method, path[:60], status, ms), file=sys.stderr) if "json" in ctype: try: return Reply(status, json.loads(raw or b"null"), ms) except ValueError: return Reply(status, raw[:200].decode("utf-8", "replace"), ms) return Reply(status, {"_content_type": ctype, "_bytes": len(raw)}, ms) def needs_token(path): """Paths that are expected to refuse an anonymous caller. Not the same as "we only send the token there": the token goes on every request, because /mapping-methods answers 401 invalid_token without one and a public endpoint ignoring the header costs nothing. This predicate decides only whether a *missing* token makes a check SKIP instead of FAIL. """ return not path.lstrip("/").startswith(("version", "openapi")) class Check: """One endpoint, what it needs, and how to judge its answer.""" def __init__(self, name, tier, method, path, depends, body=None, expect=200, timeout=None, judge=None, hint=None, caches=False): self.name = name self.tier = tier self.method = method self.path = path self.depends = depends self.body = body self.expect = expect self.timeout = timeout or TIMEOUTS.get(_root(path), 20) self.judge = judge self.hint = hint # True for an engine whose answer is cached on its inputs: the check can # then be run cold and warm (see --cold). self.caches = caches def body_for(self, client): """The body to post: a dict, or built from earlier replies. A callable is how a check exercises a flow (values + genotypes -> plot) instead of inventing a payload, and it is where a fixture that cannot support the check says so with `Skip`. """ if callable(self.body): return self.body(client) return self.body def run(self, client, cold=False): try: body = self.body_for(client) except Skip as reason: return Result(self, None, SKIP, str(reason)) cold_ms = None if cold and self.caches: # A cold run means an input the cache has never seen. Perturbing one # value does that without touching the service's caches: every # engine here keys on the content it is given (gemma-wrapper's # content hash, ravanan's interned inputs, rqtl/qtl2/bulklmm's own # hashes), so a changed number is a cache miss, and the identical # body afterwards is the warm hit. unseen = perturbed(body) if unseen is None: return Result(self, None, SKIP, "cannot force a cache miss for this payload") cold_reply = client.call(self.method, self.path, unseen, self.timeout) cold_verdict, cold_why = self._judge(cold_reply) cold_ms = cold_reply.ms if cold_verdict != OK: return Result(self, cold_reply, cold_verdict, "cold run: " + cold_why, cold_ms) reply = client.call(self.method, self.path, body, self.timeout) verdict, why = self._judge(reply) if cold_ms is not None and verdict == OK: why = "%s; cold %s / warm %s" % ( why or "", _seconds(cold_ms), _seconds(reply.ms)) return Result(self, reply, verdict, why, cold_ms) def _judge(self, reply): if reply.error: hint = self.hint or "" return FAIL, (reply.error + ((" -- " + hint) if hint else "")) if reply.status == 409: code, detail = reply.fields() if code == "compute_busy": # Another caller holds the engine's slot. The endpoint works; # retrying later would test it. Counting this as a failure # would make a shared box red whenever anyone else is working. return BUSY, "another job holds the slot: %s" % str(detail)[:120] if reply.status != self.expect: code, detail = reply.fields() why = "http %s" % reply.status if code: why += " " + code if detail: why += ": " + str(detail)[:180] if reply.status in (401, 403): why += " -- credential: scope, registry or host" if reply.status in (502, 503) and self.hint: why += " -- " + self.hint return FAIL, why if self.judge: try: return self.judge(reply) except Exception as error: # noqa: BLE001 return FAIL, "unreadable answer: %s" % error return OK, "" def _root(path): """The leading path segments a timeout table is keyed on.""" parts = [p for p in path.split("?")[0].split("/") if p] for n in (4, 3, 2, 1): candidate = "/" + "/".join(parts[:n]) if candidate in TIMEOUTS: return candidate return "/" + "/".join(parts[:2]) class LocalCheck: """A check that reads the local box rather than an endpoint.""" def __init__(self, name, tier, depends): self.name = name self.tier = tier self.depends = depends self.method = "-" self.path = "-" class Result: def __init__(self, check, reply, verdict, why, cold_ms=None): self.check = check self.reply = reply self.verdict = verdict self.why = why # Set when --cold ran the check twice: the first, cache-missing run. self.cold_ms = cold_ms @property def name(self): return self.check.name def has_rows(reply): data = reply.json rows = data.get("data") if isinstance(data, dict) else data if isinstance(rows, list) and rows: return OK, "%d rows" % len(rows) if isinstance(rows, dict) and rows: return OK, "%d keys" % len(rows) return FAIL, "answered, but no rows" def has_permutation(reply): """A qtl2 reply carrying the permutation null: n, thresholds, and values.""" data = reply.json or {} permutation = ((data.get("meta") or {}).get("permutation") or {}) if permutation.get("n") and permutation.get("significant") is not None: return OK, "%d permutations, sig LOD %.2f" % ( permutation["n"], permutation["significant"]) return FAIL, "no meta.permutation in the reply" def collection_ok(reply): """An empty list is the right answer for a user with no baskets yet.""" data = reply.json rows = data.get("data") if isinstance(data, dict) else data if isinstance(rows, list): return OK, "%d rows" % len(rows) return FAIL, "not a list" def is_image(reply): ctype = (reply.json or {}).get("_content_type", "") if isinstance( reply.json, dict) else "" if ctype.startswith("image/") or "svg" in ctype: return OK, ctype.split(";")[0] return FAIL, "not an image: %s" % (ctype or "no content type") def version_is_reachable(reply): data = reply.json if not isinstance(data, dict): return FAIL, "not an envelope" data = data.get("data") or {} database = data.get("database") if database != "reachable": return FAIL, "database: %s (MariaDB)" % database return OK, "%s db:%s" % (data.get("git"), database) class Discovery: """Find a group, a trait and its samples, so checks use real names.""" def __init__(self, client): self.client = client self.group = self.publish = self.expression = None self.trait = self.samples = None self.genotype_version = self.symbol = None self.attribute = self.pmid = self.probeset = self.marker = None self.plot_trait = self.plot_locus = None self.control_trait = None self.notes = [] def run(self): groups = self.client.call("GET", "/groups").json names = [g.get("name") for g in _rows(groups) if g.get("name")] if not names: self.notes.append("no groups") return self self.group = "BXD" if "BXD" in names else names[0] datasets = _rows(self.client.call( "GET", "/groups/%s/datasets" % self.group).json) for dataset in datasets: name = dataset.get("name") if not name: continue kind = (dataset.get("type") or dataset.get("category") or dataset.get("tissue") or "").lower() if "publish" in kind and not self.publish: self.publish = name elif not self.expression: self.expression = name # The group's datasets listing is the expression side; published traits # are reached as /traits/Publish, not through /datasets. self.publish = self.publish or "%sPublish" % self.group self.expression = self.expression or "HC_M2_0606_P" traits = _rows(self.client.call( "GET", "/traits/%s" % self.publish).json) for trait in traits: tid = trait.get("id") or trait.get("trait_id") or trait.get("name") if tid: self.trait = str(tid) break # The genotype plot is fenced: /peak-genotypes only answers inside a # confirmed hit (LOD >= 3, i.e. LRS >= 13.8), so a plot check needs a # trait that has one or it earns an honest 422. Trait 10001 sits at # LOD 2.93 and is exactly the near-miss that made this look broken. for trait in traits: lrs = trait.get("lrs", trait.get("LRS")) tid = trait.get("id") or trait.get("trait_id") or trait.get("name") if not tid or not isinstance(lrs, (int, float)) or lrs < 14: continue locus = trait.get("locus") or trait.get("Locus") if not locus: continue self.plot_trait = str(tid) self.plot_locus = str(locus) break if self.trait: # A second published trait, for the partial axis: it needs something # to hold constant. for trait in traits: candidate = trait.get("id") or trait.get("trait_id") if candidate is not None and str(candidate) != self.trait: self.control_trait = str(candidate) break values = self.client.call( "GET", "/traits/%s/%s/values" % (self.publish, self.trait)).json samples = _rows(values) named = [(s.get("name"), s.get("value")) for s in samples if s.get("name") and isinstance(s.get("value"), (int, float))] if named: self.samples = [{"name": n, "value": v} for n, v in named] attributes = self.client.call( "GET", "/groups/%s/case-attributes" % self.group).json data = attributes.get("data") if isinstance(attributes, dict) else None if isinstance(data, dict): names = data.get("names") or [] if names: self.attribute = str(names[0]) for pmid in ("21464291", "15637629", "11746377"): reply = self.client.call("GET", "/publications/%s/traits" % pmid) if reply.status == 200: self.pmid = pmid break genotypes = _rows(self.client.call( "GET", "/groups/%s/genotypes" % self.group).json, "versions", "genotypes") for version in genotypes: name = version.get("version") or version.get("name") if name: self.genotype_version = str(name) break if self.genotype_version: markers = self.client.call( "GET", "/groups/%s/genotypes/%s/markers" % (self.group, self.genotype_version)).json data = _rows(markers, "markers") if data: first = data[0] if isinstance(first, dict): self.marker = first.get("name") elif isinstance(first, list) and first: # /markers answers positional rows: ["rsm10000000001"] self.marker = str(first[0]) else: self.marker = str(first) probesets = _rows(self.client.call( "GET", "/datasets/%s/probesets" % self.expression).json) for probeset in probesets: if not self.probeset and probeset.get("name"): self.probeset = str(probeset["name"]) if not self.symbol and probeset.get("symbol"): self.symbol = str(probeset["symbol"]) if self.probeset and self.symbol: break return self def _rows(payload, *keys): """The list inside an envelope, wherever that endpoint puts it. Most answers are `{data: [...]}`, but /groups/{g}/genotypes nests the versions one level down, so a fixed `data` unwrap silently found nothing and the fitqtl fixture went in with no markers. """ if isinstance(payload, dict): payload = payload.get("data") if isinstance(payload, list): return payload if isinstance(payload, dict): for key in keys or ("rows", "items", "versions", "markers", "genotypes"): candidate = payload.get(key) if isinstance(candidate, list): return candidate return [] def process_row(url): """The listening process behind URL, when /proc shows it. Returns (verdict, text). A container with its own PID namespace sees the host's listening sockets in /proc/net/tcp but no owning process, so this says that instead of pretending the service has no pid. """ port = urllib.parse.urlsplit(url).port or (443 if url.startswith("https") else 80) if urllib.parse.urlsplit(url).hostname not in ("127.0.0.1", "localhost", "::1"): return SKIP, "remote host: its processes are not ours to read" inodes = {} for table in ("/proc/net/tcp", "/proc/net/tcp6"): try: lines = open(table, encoding="utf-8").read().splitlines()[1:] except OSError: continue for line in lines: fields = line.split() if len(fields) > 9 and fields[3] == "0A": if int(fields[1].split(":")[1], 16) == port: inodes[fields[9]] = True if not inodes: return FAIL, "nothing listens on :%d" % port import glob for descriptor in glob.glob("/proc/[0-9]*/fd/*"): try: target = os.readlink(descriptor) except OSError: continue if target.startswith("socket:[") and target[8:-1] in inodes: pid = descriptor.split("/")[2] try: status = open("/proc/%s/status" % pid, encoding="utf-8").read() except OSError: continue rss = "" for line in status.splitlines(): if line.startswith("VmRSS:"): rss = " rss %s" % line.split()[1] + " kB" return OK, "pid %s%s" % (pid, rss) return SKIP, ("listening, but no owning process in this PID namespace " "(the service runs outside it)") def build_checks(found, tiers): """The check list: every endpoint, with what it depends on.""" checks = [] add = checks.append if "core" in tiers: add(Check("version", "core", "GET", "/version", "the process itself", judge=version_is_reachable)) add(Check("mapping-methods", "core", "GET", "/mapping-methods", "static table", judge=has_rows)) if "read" in tiers: g = found.group d, x, t = found.publish, found.expression, found.trait add(Check("species", "read", "GET", "/species", "MariaDB Species", judge=has_rows)) add(Check("groups", "read", "GET", "/groups", "MariaDB InbredSet", judge=has_rows)) add(Check("group", "read", "GET", "/groups/%s" % g, "MariaDB", judge=has_rows)) add(Check("group-strains", "read", "GET", "/groups/%s/strains" % g, "MariaDB Strain", judge=has_rows)) add(Check("group-datasets", "read", "GET", "/groups/%s/datasets" % g, "MariaDB *Freeze", judge=has_rows)) add(Check("group-traits", "read", "GET", "/groups/%s/traits" % g, "MariaDB PublishXRef", judge=has_rows)) add(Check("group-case-attributes", "read", "GET", "/groups/%s/case-attributes" % g, "MariaDB CaseAttribute", judge=has_rows)) add(Check("group-parents-f1s", "read", "GET", "/groups/%s/parents-and-f1s" % g, "parental metadata", judge=has_rows)) add(Check("group-phewas", "read", "GET", "/groups/%s/phewas?chr=1&mb=100" % g, "MariaDB PublishXRef", judge=has_rows)) add(Check("dataset", "read", "GET", "/datasets/%s" % x, "MariaDB ProbeSetFreeze", judge=has_rows)) add(Check("dataset-probesets", "read", "GET", "/datasets/%s/probesets" % x, "MariaDB ProbeSet", judge=has_rows)) if t: add(Check("trait", "read", "GET", "/traits/%s/%s" % (d, t), "MariaDB PublishXRef", judge=has_rows)) add(Check("trait-values", "read", "GET", "/traits/%s/%s/values" % (d, t), "MariaDB PublishData", judge=has_rows)) add(Check("group-trait-values", "read", "GET", "/groups/%s/traits/%s/values" % (g, t), "MariaDB PublishData", judge=has_rows)) if found.attribute: add(Check("trait-case-attribute", "read", "GET", "/groups/%s/traits/%s/case-attributes/%s" % (g, t, urllib.parse.quote(found.attribute)), "MariaDB CaseAttributeXRefNew", expect=200)) add(Check("genotypes", "read", "GET", "/groups/%s/genotypes" % g, "genotype files on disk", judge=has_rows)) if found.genotype_version: v = found.genotype_version add(Check("genotype-version", "read", "GET", "/groups/%s/genotypes/%s" % (g, v), "genotype files", judge=has_rows)) add(Check("genotype-markers", "read", "GET", "/groups/%s/genotypes/%s/markers" % (g, v), "genotype files", judge=has_rows)) if found.probeset: add(Check("probeset", "read", "GET", "/probesets/%s" % urllib.parse.quote(found.probeset), "MariaDB ProbeSet", judge=has_rows)) if found.symbol: add(Check("gene-traits", "read", "GET", "/genes/%s/traits" % found.symbol, "MariaDB", judge=has_rows)) if found.symbol and "external" in tiers: add(Check("gene-aliases", "external", "GET", "/genes/%s/aliases" % found.symbol, "Wikidata over the network (cached in LMDB)", judge=has_rows, hint="Wikidata or its cache (WIKIDATA_CACHE_DIR); " "off by default, use --tier external")) if found.pmid: add(Check("publication-traits", "read", "GET", "/publications/%s/traits" % found.pmid, "MariaDB PublicationXRef", expect=200)) add(Check("baskets", "read", "GET", "/baskets", "basket SQLite", judge=collection_ok)) add(Check("matrices", "read", "GET", "/matrices", "basket SQLite", judge=collection_ok)) add(Check("whoami", "read", "GET", "/tokens/who", "token registry", judge=has_rows, hint="localhost only: 403 not_local is correct remotely")) if "search" in tiers: add(Check("search-phenotype", "search", "GET", "/search?q=alcohol%20AND%20dependence&type=phenotype&limit=2", "Xapian index, or gn3 when SEARCH_ENGINE=forward", judge=has_rows, hint="SEARCH_ENGINE=forward proxies gn3 on 127.0.0.1:5003")) add(Check("search-gene", "search", "GET", "/search?q=insulin&type=gene&limit=2", "Xapian index (gene documents)", judge=has_rows, hint="SEARCH_ENGINE=forward proxies gn3 on 127.0.0.1:5003")) if "read" in tiers or "compute" in tiers: if found.samples: body = {"dataset": found.publish, "trait_name": found.trait, "target_dataset": found.publish, "method": "pearson", "top_n": 5} add(Check("compute-correlation", "compute", "POST", "/compute/correlation", "correlation engine + MariaDB", body=body, judge=has_rows)) if "compute" in tiers: g = found.group samples = found.samples if samples: if found.control_trait: # The matrix: two published traits, correlated with each other. add(Check("compute-correlation-matrix", "compute", "POST", "/compute/correlation/matrix", "the N x N matrix (pure Rust)", body={"traits": [ {"dataset": found.publish, "trait_name": found.trait}, {"dataset": found.publish, "trait_name": found.control_trait}], "method": "pearson"}, judge=has_matrix, hint="no external tool: pure Rust + MariaDB")) if found.control_trait: # PCA over the matrix: the reaction is over the r matrix the # previous check already exercised, so what is tested here is # the eigendecomposition and the reply's shape. add(Check("compute-correlation-pca", "compute", "POST", "/compute/correlation/matrix", "PCA of the matrix (Jacobi, pure Rust)", body={"traits": [ {"dataset": found.publish, "trait_name": found.trait}, {"dataset": found.publish, "trait_name": found.control_trait}], "method": "pearson", "pca": True}, judge=has_pca, hint="no external tool: pure Rust + MariaDB")) if found.control_trait: # Clustered ordering: average linkage on 1 - r, for the heatmap. add(Check("compute-correlation-heatmap", "compute", "POST", "/compute/correlation/matrix", "clustered order + tree (pure Rust)", body={"traits": [ {"dataset": found.publish, "trait_name": found.trait}, {"dataset": found.publish, "trait_name": found.control_trait}], "method": "pearson", "cluster": True}, judge=has_clustering, hint="no external tool: pure Rust + MariaDB")) add(Check("compute-gemma", "compute", "POST", "/compute/gemma", "gemma-wrapper + gemma + R (R_LIBS_SITE)", body={"group": g, "lmm": 9, "samples": samples}, judge=has_rows, caches=True, hint="gemma-wrapper, GEMMA_WRAPPER_CMD/GEMMA_DIR")) if found.control_trait: # The partial axis: same endpoint, a second trait held constant. # Two traits out of a publish set give a low n, but the path # (fetch, complete cases, least squares, t on n-k-2) is what is # being checked, not the biology. add(Check("compute-correlation-partial", "compute", "POST", "/compute/correlation", "partial correlation, one control (pure Rust)", body={"dataset": found.publish, "trait_name": found.trait, "target_dataset": found.publish, "type": "partial", "control_traits": [ {"dataset": found.publish, "trait_name": found.control_trait}], "top_n": 5}, judge=has_rows, hint="no external tool: pure Rust + MariaDB")) add(Check("compute-rqtl", "compute", "POST", "/compute/rqtl", "R + r-qtl", body={"group": g, "method": "hk", "trait_name": found.trait, "samples": samples}, judge=has_rows, caches=True, hint="Rscript and r-qtl")) add(Check("compute-qtl2", "compute", "POST", "/compute/qtl2", "R + r-qtl2 + QTL2_SCRIPT_PATH", body={"group": g, "samples": samples}, judge=has_rows, caches=True, hint="r-qtl2 and scripts/qtl2_scan.R")) # Permutations are the one qtl2 option whose failure is not a # missing binary but a broken R path, so they get their own check. # Ten keeps it cheap; the reply must carry meta.permutation. add(Check("compute-qtl2-perm", "compute", "POST", "/compute/qtl2", "R + qtl2 + scan1perm (meta.permutation)", body={"group": g, "trait_name": found.trait, "samples": samples, "nperm": 10, "perm_seed": 1}, judge=has_permutation, hint="scan1perm threshold code, or --nperm support")) add(Check("compute-fitqtl", "compute", "POST", "/compute/fitqtl", "R + r-qtl", body={"group": g, "method": "hk", "markers": [found.marker] if found.marker else [], "trait_name": found.trait, "samples": samples}, judge=has_rows, caches=True, hint="Rscript and r-qtl")) add(Check("compute-bulklmm", "compute", "POST", "/compute/bulklmm", "Julia + its depot (BULKLMM_DEPOT/ENV)", body={"group": g, "samples": samples}, judge=has_rows, caches=True, hint="julia, BULKLMM_ENV and BULKLMM_DEPOT")) add(Check("compute-ravanan", "compute", "POST", "/compute/gemma/ravanan", "ravanan + ccwl + slurmrestd", body={"group": g, "samples": samples}, judge=has_rows, caches=True, hint="RAVANAN_CMD, ccwl on PATH, slurmrestd :6820")) add(Check("compute-bnw", "compute", "POST", "/compute/bnw", "python + the gn-bnw driver (BNW_CMD)", body={"values": bnw_matrix()}, judge=has_rows, hint="BNW_CMD, BNW_BIN_DIR, python3")) add(Check("bnw-examples", "compute", "GET", "/compute/bnw/examples", "the BNW example matrices on disk", judge=has_rows)) if "plots" in tiers: if found.control_trait: # The scree plot of the matrix PCA: two steps, like the genotype # plot below -- ask the matrix for `pca: true`, then post that # reply to the plotter. add(Check("plot-pca", "plots", "POST", "/plot/pca", "matrix PCA -> scree svg (PCA_PLOTTER)", body=pca_plot_body(found), expect=200, hint="PCA_PLOTTER and python3", judge=has_svg)) if found.control_trait: add(Check("plot-heatmap", "plots", "POST", "/plot/heatmap", "clustered matrix -> heatmap svg (HEATMAP_PLOTTER)", body=heatmap_plot_body(found), expect=200, hint="HEATMAP_PLOTTER and python3", judge=has_svg)) if found.samples: # The Manhattan plot: a GEMMA run's reply, posted back as-is. add(Check("plot-gemma", "plots", "POST", "/plot/gemma", "GEMMA result -> Manhattan svg (GEMMA_PLOTTER)", body=gemma_plot_body(found), expect=200, hint="GEMMA_PLOTTER and python3", judge=has_svg)) add(Check("plot-bnw", "plots", "POST", "/plot/bnw", "BNW result -> network svg (BNW_PLOTTER)", body=bnw_plot_body(), expect=200, hint="BNW_PLOTTER and python3", judge=has_svg)) add(Check("plot-genotypes", "plots", "POST", "/plot/genotypes", "peak-genotypes + values -> svg (GENOTYPE_PLOTTER)", body=genotype_plot_body(found), expect=200, hint="GENOTYPE_PLOTTER and python3")) return checks def _seconds(ms): return "%.1f s" % (ms / 1000.0) if ms >= 1000 else "%d ms" % ms def perturbed(body): """The same request with one number changed, so the cache cannot answer it. Returns None when the payload has nothing to nudge -- then a cold/warm pair would be two warm runs and saying so is better than pretending. """ if not isinstance(body, dict): return None for key in ("samples", "values", "trait_values"): rows = body.get(key) if not isinstance(rows, list) or not rows: continue copy = json.loads(json.dumps(body)) row = copy[key][0] if isinstance(row, dict) and isinstance(row.get("value"), (int, float)): # A thousandth of a standard-ish unit is enough to change every # content hash and nowhere near enough to change the biology. row["value"] = row["value"] + 0.001 * (1 + time.time() % 1) return copy return None def has_clustering(reply): """A clustered order that is a permutation, with n-1 merges.""" data = (reply.json or {}).get("data") or {} heatmap = data.get("heatmap") or {} order = heatmap.get("order") or [] merges = heatmap.get("merges") or [] traits = data.get("traits") or [] if len(order) != len(traits): return FAIL, "order lists %d, traits %d" % (len(order), len(traits)) if sorted(order) != list(range(len(traits))): return FAIL, "order is not a permutation: %s" % order if len(merges) != max(len(traits) - 1, 0): return FAIL, "%d merges for %d traits" % (len(merges), len(traits)) return OK, "%d traits, max distance %.3f" % (len(traits), heatmap.get("max_height") or 0.0) def has_pca(reply): """A PCA block whose eigenvalues sum to the trait count.""" data = (reply.json or {}).get("data") or {} pca = data.get("pca") or {} components = pca.get("components") or [] if not components: return FAIL, "no components (traits=%d, dropped=%d)" % ( len(data.get("traits") or []), len(pca.get("dropped") or [])) kept = len(data.get("traits") or []) - len(pca.get("dropped") or []) trace = sum(c.get("eigenvalue", 0.0) for c in components) ratios = sum(c.get("explained_variance_ratio", 0.0) for c in components) if abs(trace - kept) > 1e-6 * max(1, kept): return FAIL, "eigenvalues sum to %.6f, expected %d" % (trace, kept) if abs(ratios - 1.0) > 1e-9: return FAIL, "explained variance ratios sum to %.9f" % ratios return OK, "%d component(s) over %d trait(s), trace %.6f" % ( len(components), kept, trace) def has_matrix(reply): """A matrix answer: square, symmetric, unit diagonal, n alongside.""" data = (reply.json or {}).get("data") or {} r, n = data.get("r"), data.get("n") if not isinstance(r, list) or not isinstance(n, list) or not r: return FAIL, "no r matrix in the answer" size = len(r) if any(not isinstance(row, list) or len(row) != size for row in r): return FAIL, "r is not square" for row in range(size): if r[row][row] != 1.0: return FAIL, "diagonal r[%d][%d] is %r, not 1.0" % (row, row, r[row][row]) for column in range(size): if r[row][column] != r[column][row]: return FAIL, "r is not symmetric at [%d][%d]" % (row, column) measured = sum(1 for row in r for cell in row if cell is not None) - size return OK, "%dx%d, %d cell(s) measured, n=%s" % ( size, size, measured, n[0][0] if n and n[0] else "?") def has_svg(reply): """An SVG came back, not a JSON envelope. The client hands a binary reply back as a dict of metadata (it keeps the bytes out of the report), so the content type is the thing to judge. """ body = reply.body if isinstance(body, str) and "= 3 to plot a hit for") calls = client.call( "GET", "/groups/%s/traits/%s/peak-genotypes?marker=%s" % (found.group, found.plot_trait, found.plot_locus), timeout=120) if calls.status != 200: code, detail = calls.fields() raise Skip("trait %s at %s is not a confirmed hit: %s %s" % (found.plot_trait, found.plot_locus, code or calls.status, str(detail)[:90])) animals = (calls.json or {}).get("data", {}).get("animals") or [] values = client.call( "GET", "/groups/%s/traits/%s/values" % (found.group, found.plot_trait)) return {"marker": found.plot_locus, "genotypes": animals, "values": (values.json or {}).get("data") or []} return build def bnw_matrix(): """A tiny deterministic network: four variables, five samples, two edges. The shape is the API's own (`BnwVariable`): one entry per variable with its per-sample values -- not a CSV-style matrix. V2 tracks V1, V3 inverts it, V4 is noise, so a working BNW reports V1-V2 and V1-V3 with the right signs. """ samples = ["S1", "S2", "S3", "S4", "S5"] def variable(name, values): return {"name": name, "samples": [{"name": s, "value": v} for s, v in zip(samples, values)]} return [variable("V1", [1.0, 2.0, 3.0, 4.0, 5.0]), variable("V2", [2.1, 3.9, 6.2, 7.8, 10.1]), variable("V3", [5.0, 4.0, 3.0, 2.0, 1.0]), variable("V4", [0.4, 5.1, 1.3, 4.8, 2.2])] def report(results, quiet_skips=True): width = max([len(r.name) for r in results] + [12]) print("%-*s %-8s %-7s %7s %s" % (width, "check", "tier", "verdict", "ms", "depends on / why")) print("-" * (width + 60)) for result in results: if result.verdict == SKIP and quiet_skips: continue note = result.why or result.check.depends print("%-*s %-8s %-7s %7d %s" % ( width, result.name, result.check.tier, result.verdict, result.reply.ms, note)) counted = {"ok": 0, "FAIL": 0, "skip": 0, "busy": 0} for result in results: counted[result.verdict] += 1 print("-" * (width + 60)) print("%d ok, %d failed, %d skipped, %d busy" % ( counted[OK], counted[FAIL], counted[SKIP], counted[BUSY])) for result in results: if result.verdict == SKIP: print(" skipped %-*s %s" % (width, result.name, result.why)) return counted[FAIL] def main(argv): parser = argparse.ArgumentParser( prog="monitor-gn4", description="Check every genenetwork4 endpoint, and name what each needs.") parser.add_argument("--url", default=os.environ.get("GN4_URL", DEFAULT_URL), help="base URL, remote or localhost (default %(default)s)") parser.add_argument("--token-file", default=os.environ.get("GN4_TOKEN_FILE", DEFAULT_TOKEN_FILE), help="file holding a short gn4ai_ token (or a JWT)") parser.add_argument("--token", help="the token itself, instead of a file") parser.add_argument("--tier", action="append", default=None, choices=["core", "read", "search", "compute", "plots", "external", "all"], help="tiers to run (repeatable; default core,read,search)") parser.add_argument("--only", help="comma-separated check names to run") parser.add_argument("--filter", help="only checks whose name, what they depend on, or " "their hint mentions this text (case-insensitive); " "--filter julia picks compute-bulklmm") parser.add_argument("--json", action="store_true", help="machine-readable report") parser.add_argument("--verbose", action="store_true", help="log every request") parser.add_argument("--cold", action="store_true", help="run each cached engine twice: once with an input " "the cache cannot have seen (that is the run a " "fresh deploy does), then again as the warm hit. " "Reported as 'cold X / warm Y'. Slow by nature: " "ravanan's cold run is minutes.") parser.add_argument("--insecure", action="store_true", help="do not verify TLS (self-signed test hosts)") parser.add_argument("--no-discovery", action="store_true", help="skip discovery and use BXD/BXDPublish fallbacks") args = parser.parse_args(argv) tiers = set(args.tier or ["core", "read", "search"]) if "all" in tiers or args.only or args.filter: # --only names checks directly, so the tier gate must not hide them tiers = {"core", "read", "search", "compute", "plots", "external"} token = args.token if not token and args.token_file and os.path.exists(args.token_file): token = open(args.token_file, encoding="utf-8").read().strip() if not token and tiers - {"core"}: print("warning: no token (--token-file %s); read, search and compute " "checks will be skipped" % args.token_file, file=sys.stderr) client = Client(args.url, token, args.insecure, args.verbose) found = Discovery(Client(args.url, token, args.insecure, False)) if not args.no_discovery: found.run() if not found.group: found.group = "BXD" found.publish = found.expression = "BXDPublish" checks = build_checks(found, tiers) if args.only: wanted = {name.strip() for name in args.only.split(",")} checks = [c for c in checks if c.name in wanted] if args.filter: # A substring over the three strings a human would match on: the check # name, the sentence describing what it needs, and the hint that says # what to fix. "--filter julia" therefore finds compute-bulklmm (whose # name does not contain it) as well as anything else Julia-backed. needle = args.filter.strip().lower() checks = [ c for c in checks if needle in c.name.lower() or needle in (c.depends or "").lower() or needle in (c.hint or "").lower() ] if not checks: print(f"no check matches --filter {args.filter!r}", file=sys.stderr) print("(the compute checks are only built once discovery has found a " "group with samples -- drop --no-discovery, or the filter has " "nothing to search)", file=sys.stderr) return 2 results = [] if args.cold: cold_names = [c.name for c in checks if c.caches] if not cold_names: print("--cold: no cached engine among the selected checks", file=sys.stderr) else: print("--cold: running %d cached check(s) twice: %s" % (len(cold_names), ", ".join(cold_names)), file=sys.stderr) if "core" in tiers: verdict, text = process_row(args.url) results.append(Result(LocalCheck("listener", "core", "the listening process, from /proc"), Reply(0, None, 0), verdict, text)) for check in checks: if not token and needs_token(check.path): results.append(Result(check, Reply(0, None, 0), SKIP, "no token")) continue results.append(check.run(client, cold=args.cold)) if args.json: print(json.dumps({ "url": args.url, "tiers": sorted(tiers), "found": {"group": found.group, "publish": found.publish, "expression": found.expression, "trait": found.trait, "samples": len(found.samples or [])}, "checks": [{"name": r.name, "tier": r.check.tier, "method": r.check.method, "path": r.check.path, "depends": r.check.depends, "verdict": r.verdict, "status": r.reply.status, "ms": r.reply.ms, "why": r.why} for r in results], }, indent=2)) failed = sum(1 for r in results if r.verdict == FAIL) else: failed = report(results) return 1 if failed else 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))