#!/usr/bin/env python3 """Run GEMMA over every genotype panel this deployment holds, one at a time. Why: "GEMMA works" has only ever been verified on the panels somebody happened to map. This walks them all, records what happened and how long it took, and can be asked for progress while it runs. scripts/gemma-sweep.py --start --base-url http://127.0.0.1:3001 scripts/gemma-sweep.py --report Driven through the API, so what it proves is that a *caller* can map these panels -- server-side resolution, panel subsetting, the bimbam pair, the kinship step, the LOCO scan -- rather than that the files parse. Three kinds of panel, and what a run of each means: * offered by a group (`/groups/{g}/genotypes` with `gemma: true`): the group's own trait values are used, aligned onto the panel by name; * on disk but not listed by its group (`--orphans` file): the panel's `.geno` header supplies the individual order, so the group's trait values can still be aligned by name, and the request names the bimbam files; * no group at all: nothing to phenotype them with, so a *synthetic* phenotype is derived from the genotypes themselves (allele counts over the first markers). That tests the pipeline, not biology, and the result is labelled `phenotype: synthetic`. State is one JSON object per line, appended as each run finishes -- so it survives a restart, is greppable, and turns into a CSV for plotting (`--csv`). `--report` prints where we are, including the run in flight. """ import argparse import csv import json import math import os import random import re import sys import threading import time import urllib.error import urllib.request GENOTYPE_DIR = "/home/wrk/tmp/ai-root/factory/gn4-data/genotype-files" def slug(text): """A name safe both as a state key and as the run's `?id=` -- which the API restricts to 1-64 ASCII letters, digits, '-' and '_'.""" cleaned = re.sub(r"[^A-Za-z0-9_-]+", "-", text).strip("-") return cleaned[:64] def api(base_url, token, path, body=None, timeout=7200): """One request; returns (status, parsed JSON or raw text).""" url = base_url.rstrip("/") + path data = None headers = {"Authorization": f"Bearer {token}"} if body is not None: data = json.dumps(body).encode() headers["Content-Type"] = "application/json" request = urllib.request.Request(url, data=data, headers=headers, method="POST" if body is not None else "GET") try: with urllib.request.urlopen(request, timeout=timeout) as response: raw = response.read() status = response.status except urllib.error.HTTPError as error: raw = error.read() status = error.code except Exception as error: # connection refused, timeout, ... return 0, {"detail": str(error)} try: return status, json.loads(raw) except Exception: return status, {"detail": raw[:500].decode(errors="replace")} def bimbam_counts(stem): """(individuals, markers) as the *bimbam* pair reports them. The endpoint's `markers` comes from the `.geno` file and can be wildly wrong for GEMMA (AIL: 20 against 515 165), so count the files instead. """ geno = os.path.join(GENOTYPE_DIR, "genotype/bimbam", f"{stem}_geno.txt") snps = os.path.join(GENOTYPE_DIR, "genotype/bimbam", f"{stem}_snps.txt") individuals = 0 try: with open(geno, errors="replace") as handle: for line in handle: line = line.strip() if not line or line.startswith(("#", "@")): continue fields = [field for field in re.split(r"[,\s]+", line) if field] individuals = max(0, len(fields) - 3) break except OSError: pass markers = 0 try: with open(snps, errors="replace") as handle: markers = sum(1 for line in handle if line.strip()) except OSError: pass return individuals, markers def geno_panel(stem): """The .geno header's individual names, in order -- the bimbam column order.""" path = os.path.join(GENOTYPE_DIR, "genotype", f"{stem}.geno") try: with open(path, errors="replace") as handle: for line in handle: if line.lower().startswith("chr\t"): fields = line.rstrip("\n").split("\t") start = 4 if fields[3] == "Mb" else 3 return fields[start:] except OSError: pass return [] def synthetic_phenotype(stem, individuals, markers=200): """A deterministic trait derived from the genotypes: allele counts over the first `markers` markers. Nothing to do with biology -- this is a smoke test for panels no group phenotypes.""" geno = os.path.join(GENOTYPE_DIR, "genotype/bimbam", f"{stem}_geno.txt") totals = [0.0] * individuals try: with open(geno, errors="replace") as handle: seen = 0 for line in handle: line = line.strip() if not line or line.startswith(("#", "@")): continue fields = [field for field in re.split(r"[,\s]+", line) if field][3:] if len(fields) != individuals: continue for index, value in enumerate(fields): try: number = float(value) except ValueError: continue if 0.0 <= number <= 2.0: totals[index] += number seen += 1 if seen >= markers: break except OSError: return None # A placeholder panel (one marker, or markers that are all NA) gives a # constant phenotype, and GEMMA then fails on "no variance" -- an # artefact of the synthetic trait, not of the panel. Fall back to a # seeded series so the panel still gets a real scan. if len({round(value, 6) for value in totals}) < 2: rng = random.Random(abs(hash(stem)) % (2 ** 32)) return [round(rng.gauss(0.0, 1.0), 6) for _ in range(individuals)], True return totals, False def discover(base_url, token): """Every panel to run, cheapest first.""" plans = [] status, groups = api(base_url, token, "/groups", timeout=120) if status != 200: sys.exit(f"cannot list groups: {status} {groups}") for group in groups["data"]: name = group["name"] status, listing = api(base_url, token, f"/groups/{name}/genotypes", timeout=120) if status != 200: continue for version in listing["data"].get("versions", []): if not version.get("gemma"): continue individuals, markers = bimbam_counts(version["version"]) plans.append({ "kind": "group", "slug": slug(f"{name}.{version['version']}"), "group": name, "version": version["version"], "individuals": individuals, "markers": markers, "phenotype": "group trait values, aligned by name", }) offered = {plan["version"] for plan in plans} group_names = {group["name"] for group in groups["data"]} bimbam = os.path.join(GENOTYPE_DIR, "genotype/bimbam") ad_hoc = re.compile(r"_(mm8|mm9|old|orig|fixed|wrong_Mb|wrong_order|no_Mb|unsure|" r"before_polish.*|array|classic)$") for name in sorted(os.listdir(bimbam)): if not name.endswith("_geno.txt"): continue stem = name[:-len("_geno.txt")] if stem in offered or not os.path.exists(os.path.join(bimbam, f"{stem}_snps.txt")): continue if ad_hoc.search(stem): continue # a variant whose own name says it is wrong base = stem.split(".")[0] if re.search(r"\.\d+$", stem) and base in group_names: continue # a superseded version of a live group group = next((candidate for candidate in sorted(group_names, key=len, reverse=True) if stem == candidate or stem.startswith(candidate + ".") or stem.startswith(candidate + "-") or stem.startswith(candidate + "_")), None) individuals, markers = bimbam_counts(stem) plans.append({ "kind": "path", "slug": slug(stem), "group": group, "version": stem, "individuals": individuals, "markers": markers, "phenotype": ("group trait values, aligned through the .geno header" if group else "synthetic (allele counts over 200 markers)"), }) seen = set() unique = [] for plan in plans: if plan["slug"] in seen: continue seen.add(plan["slug"]) unique.append(plan) unique.sort(key=lambda plan: plan["individuals"] * max(plan["markers"], 1)) return unique def load_state(path): done = {} if os.path.exists(path): for line in open(path, errors="replace"): line = line.strip() if not line: continue try: record = json.loads(line) except ValueError: continue done[record["slug"]] = record return done def trait_values_for(base_url, token, group): """First trait of the group with enough values, as [{name, value}].""" status, listing = api(base_url, token, f"/groups/{group}/traits?limit=40", timeout=120) if status != 200: return None for trait in listing["data"]: if (trait.get("nsamples") or 0) < 20: continue status, values = api(base_url, token, f"/groups/{group}/traits/{trait['id']}/values", timeout=120) if status != 200: continue present = [row for row in values["data"] if row.get("value") is not None] if len(present) >= 20: return trait["id"], values["data"] return None def peaks_from(payload, threshold, limit=20): """The markers whose -log10(p) clears `threshold`, strongest first. GEMMA's rows are positional (`meta.columns`), and the score is its p-value column -- `p_lrt` for -lmm 9, whichever `p_*` the lmm produced otherwise. Everything a peak is worth quoting travels with it: where, how strong, and the effect (`additive`, the GeneNetwork convention). """ columns = [name.lower() for name in payload["meta"]["columns"]] score_at = next((index for index, name in enumerate(columns) if name.startswith("p_")), len(columns) - 1) rs_at = columns.index("rs") if "rs" in columns else 1 chr_at = columns.index("chr") if "chr" in columns else 0 pos_at = columns.index("ps") if "ps" in columns else 2 effect_at = columns.index("additive") if "additive" in columns else None scored = [] for row in payload["data"]: try: p = float(row[score_at]) except (TypeError, ValueError): continue if p <= 0: continue peak = {"rs": row[rs_at], "chr": row[chr_at], "mb": round(float(row[pos_at]) / 1e6, 2), "score": round(-math.log10(p), 2), "p": p} if effect_at is not None and isinstance(row[effect_at], (int, float)): peak["additive"] = round(float(row[effect_at]), 4) scored.append(peak) scored.sort(key=lambda peak: -peak["score"]) # The best marker always travels with the panel, whatever the threshold: # "no peak" and "the best it managed was -log10 p 3.1" are different # answers, and only the second tells you the panel did something real. kept = scored[:1] for peak in scored[1:]: if peak["score"] < threshold or len(kept) >= limit: break if peak["rs"] not in {other["rs"] for other in kept}: kept.append(peak) return kept def run_rqtl(base_url, token, plan, body, timeout): """The same panel through `/compute/rqtl`, for panels GEMMA cannot do.""" request = {"trait_name": body.get("trait_name", plan["version"])} if plan["kind"] == "group": request["group"] = plan["group"] request["genotype_version"] = plan["version"] request["include_parents_f1"] = body.get("include_parents_f1", False) if body.get("samples"): request["samples"] = body["samples"] else: # A synthetic run had no names to send: take the panel's own, # in its own order, and pair them with the positional values. status, listing = api(base_url, token, f"/groups/{plan['group']}/genotypes/{plan['version']}", timeout=120) names = listing["data"].get("individuals", []) if status == 200 else [] values = body.get("trait_values") or [] request["samples"] = [ {"name": name, "value": values[index] if index < len(values) else None} for index, name in enumerate(names) ] else: geno = os.path.join(GENOTYPE_DIR, "genotype", f"{plan['version']}.geno") if not os.path.exists(geno): return {"status": "skipped", "error": "no .geno file for R/qtl", "seconds": 0} panel = geno_panel(plan["version"]) or [f"individual-{index + 1}" for index in range(plan["individuals"])] values = body.get("trait_values") or [] request["genotype_file"] = geno request["samples"] = [{"name": name, "value": values[index] if index < len(values) else None} for index, name in enumerate(panel)] run_id = f"rqtl-{plan['slug']}-{int(time.time()) % 100000}" started = time.time() status, payload = api(base_url, token, f"/compute/rqtl?id={run_id}", request, timeout=timeout) seconds = round(time.time() - started, 1) if status == 200: markers = payload["data"]["markers"] best = max(markers, key=lambda row: row[3]) if markers else None return {"status": "ok", "seconds": seconds, "rows": len(markers), "peak": ({"rs": best[0], "chr": best[1], "mb": round(float(best[2]), 2), "score": round(float(best[3]), 2)} if best else None)} detail = payload.get("detail", "") if isinstance(payload, dict) else "" lines = [line.strip() for line in detail.splitlines() if line.strip()] complaints = [line for line in lines if re.search(r"error|fail|cannot|not enough|" r"fewer|singular", line, re.I)] return {"status": "failed", "seconds": seconds, "error": str(payload.get("code", "")) if isinstance(payload, dict) else "", "detail": " / ".join((complaints or lines)[-2:])[:300]} def run_one(base_url, token, state_path, plan, timeout, peak_threshold=4.0, rqtl_fallback=True): """Run one panel: synthesise the request, poll status, record the outcome.""" slug = plan["slug"] record = dict(plan, started=time.strftime("%Y-%m-%dT%H:%M:%S"), status="running") with open(state_path, "a") as handle: handle.write(json.dumps({**record, "state": "started"}) + "\n") body = {"trait_name": plan["version"]} phenotype = plan["phenotype"] if plan["kind"] == "group": body["group"] = plan["group"] body["genotype_version"] = plan["version"] else: body["genotype_file"] = os.path.join(GENOTYPE_DIR, "genotype/bimbam", f"{plan['version']}_geno.txt") body["snps_file"] = os.path.join(GENOTYPE_DIR, "genotype/bimbam", f"{plan['version']}_snps.txt") synthetic = None if plan["group"]: found = trait_values_for(base_url, token, plan["group"]) if found: trait_id, values = found body["trait_name"] = f"{plan['group']}::{trait_id}" if plan["kind"] == "group": body["samples"] = values else: panel = geno_panel(plan["version"]) by_name = {row["name"]: row["value"] for row in values} order = panel or [row["name"] for row in values] body["trait_values"] = [by_name.get(name) for name in order] phenotype = f"{body['trait_name']} ({phenotype})" else: # No trait of this group has enough values to fit: fall back to # the synthetic phenotype rather than skipping, so every panel # still gets a definitive answer about whether GEMMA runs on it. synthetic = True if not plan["group"] or synthetic: synthetic_result = synthetic_phenotype(plan["version"], plan["individuals"]) if synthetic_result is None: record.update(status="skipped", error="cannot read the genotype file", seconds=0, finished=time.strftime("%Y-%m-%dT%H:%M:%S")) with open(state_path, "a") as handle: handle.write(json.dumps(record) + "\n") return record totals, random_fallback = synthetic_result body["trait_values"] = totals body["include_parents_f1"] = True # positional: the full bimbam panel phenotype = ("synthetic (random: the genotypes give no variation)" if random_fallback else "synthetic (allele counts over 200 markers)") record["phenotype"] = phenotype record["request"] = {key: value for key, value in body.items() if key not in ("samples", "trait_values")} run_id = f"sweep-{slug}-{int(time.time()) % 100000}" answer = {} def post(): answer["response"] = api(base_url, token, f"/compute/gemma?id={run_id}", body, timeout=timeout) worker = threading.Thread(target=post, daemon=True) started = time.time() worker.start() last_peek = 0 while worker.is_alive(): time.sleep(5) status, lines = api(base_url, token, f"/compute/gemma/status/{run_id}?peek={last_peek}", timeout=60) if status == 200: last_peek = lines.get("pointer", last_peek) lines_read = [line for line in lines.get("lines", []) if line.strip()] if lines_read: record["last_line"] = lines_read[-1][:160] worker.join() seconds = round(time.time() - started, 1) # Somebody else's large job may hold the single GEMMA slot: wait for it # rather than recording a failure that says nothing about the panel. for _ in range(12): status, payload = answer.get("response", (0, {})) if status != 409: break print(f" busy (someone else's run); waiting 60s before retrying {slug}", flush=True) time.sleep(60) run_id = f"sweep-{slug}-{int(time.time()) % 100000}" worker = threading.Thread(target=post, daemon=True) started = time.time() worker.start() worker.join() seconds = round(time.time() - started, 1) status, payload = answer.get("response", (0, {"detail": "no response"})) record["seconds"] = seconds record["finished"] = time.strftime("%Y-%m-%dT%H:%M:%S") record["http_status"] = status if status == 200: record["status"] = "ok" record["rows"] = payload["meta"]["num_rows"] peaks = peaks_from(payload, peak_threshold) record["peaks"] = peaks record["n_peaks"] = len(peaks) if peaks: record["peak"] = peaks[0] else: record["status"] = "failed" record["error"] = str(payload.get("code", "")) if isinstance(payload, dict) else "" detail = payload.get("detail", "") if isinstance(payload, dict) else "" # The reason is usually in GEMMA's own last lines, not the first 400 # characters (which are the wrapper's banner), so keep the lines that # look like a complaint and, failing that, the tail. lines = [line.strip() for line in detail.splitlines() if line.strip()] complaints = [line for line in lines if re.search(r"error|fail|cannot|not enough|fewer|no snps|singular", line, re.I)] record["detail"] = " / ".join((complaints or lines)[-3:])[:400] record["detail_tail"] = detail[-600:] if rqtl_fallback and record.get("error") not in ("compute_busy",): print(f" GEMMA failed; trying R/qtl on the same panel", flush=True) record["rqtl"] = run_rqtl(base_url, token, plan, body, timeout) with open(state_path, "a") as handle: handle.write(json.dumps(record) + "\n") return record def report(state_path, csv_path, peaks_csv_path=None, threshold=4.0): records = load_state(state_path) finished = [record for record in records.values() if record.get("status") != "running" or record.get("seconds")] # the last line per slug wins: keep only finished states latest = {} for line in open(state_path, errors="replace") if os.path.exists(state_path) else []: line = line.strip() if not line: continue try: record = json.loads(line) except ValueError: continue latest[record["slug"]] = record done = {slug: record for slug, record in latest.items() if record.get("seconds") is not None} running = [record for record in latest.values() if record.get("seconds") is None] ok = [record for record in done.values() if record["status"] == "ok"] failed = [record for record in done.values() if record["status"] == "failed"] skipped = [record for record in done.values() if record["status"] == "skipped"] total_seconds = sum(record["seconds"] for record in done.values()) print(f"finished {len(done)} ({len(ok)} ok, {len(failed)} failed, {len(skipped)} skipped)" f" in {total_seconds/60:.1f} min of GEMMA time") if running: for record in running: print(f"in flight: {record['slug']} (since {record.get('started')}," f" {record['individuals']} individuals x {record['markers']} markers)") if failed: print("\nfailures:") for record in sorted(failed, key=lambda r: r["slug"]): print(f" {record['slug']:36} {record.get('error','')[:24]:24}" f" {record.get('detail','')[:120]}") fell_back = [record for record in done.values() if record.get("rqtl")] if fell_back: ok = [record for record in fell_back if record["rqtl"]["status"] == "ok"] gemma_ok = len([record for record in done.values() if record.get("status") == "ok"]) total = len(done) print(f"\nscore: GEMMA mapped {gemma_ok}/{total}; of the {len(fell_back)} it " f"could not, R/qtl mapped {len(ok)} (so " f"{gemma_ok + len(ok)}/{total} panels map on at least one engine)") for record in sorted(fell_back, key=lambda r: r["slug"])[:15]: outcome = record["rqtl"] peak = outcome.get("peak") summary = (f"{peak['rs']} chr{peak['chr']} LOD {peak['score']}" if peak else outcome.get("error", "")) print(f" {record['slug']:34} rqtl {outcome['status']:7}" f" {outcome['seconds']:7.1f}s {summary[:90]}") with_peaks = [record for record in done.values() if record.get("peaks")] print(f"panels with a peak at -log10(p) >= {threshold}: {len(with_peaks)}") for record in sorted(with_peaks, key=lambda r: -r["peak"]["score"])[:10]: peak = record["peak"] print(f" {record['slug']:34} {peak['rs']:16} chr{peak['chr']} {peak['mb']:9.2f} Mb" f" -log10p {peak['score']:6.2f}" f" additive {peak.get('additive', '')}") if done: slowest = sorted(done.values(), key=lambda r: -r["seconds"])[:5] print("\nslowest so far:") for record in slowest: print(f" {record['slug']:40} {record['seconds']:8.1f}s" f" {record['individuals']} x {record['markers']}") if csv_path: with open(csv_path, "w", newline="") as handle: writer = csv.writer(handle) writer.writerow(["slug", "kind", "group", "version", "individuals", "markers", "phenotype", "status", "seconds", "rows", "n_peaks", "peak_marker", "peak_chr", "peak_mb", "peak_score", "peak_additive", "rqtl_status", "rqtl_seconds", "rqtl_rows", "rqtl_peak", "started", "finished"]) for record in sorted(done.values(), key=lambda r: r["slug"]): peak = record.get("peak") or {} writer.writerow([record["slug"], record["kind"], record.get("group", ""), record["version"], record["individuals"], record["markers"], record.get("phenotype", ""), record["status"], record["seconds"], record.get("rows", ""), record.get("n_peaks", 0), peak.get("rs", ""), peak.get("chr", ""), peak.get("mb", ""), peak.get("score", ""), peak.get("additive", ""), (record.get("rqtl") or {}).get("status", ""), (record.get("rqtl") or {}).get("seconds", ""), (record.get("rqtl") or {}).get("rows", ""), ((record.get("rqtl") or {}).get("peak") or {}).get("rs", ""), record.get("started", ""), record.get("finished", "")]) print(f"\nwrote {csv_path} ({len(done)} rows)") if peaks_csv_path: with open(peaks_csv_path, "w", newline="") as handle: writer = csv.writer(handle) writer.writerow(["slug", "group", "version", "marker", "chr", "mb", "neglog10p", "p", "additive", "individuals", "markers"]) for record in sorted(done.values(), key=lambda r: r["slug"]): for peak in record.get("peaks", []): writer.writerow([record["slug"], record.get("group", ""), record["version"], peak["rs"], peak["chr"], peak["mb"], peak["score"], peak["p"], peak.get("additive", ""), record["individuals"], record["markers"]]) print(f"wrote {peaks_csv_path}") def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) parser.add_argument("--base-url", default="http://127.0.0.1:3001") parser.add_argument("--token-file", default="/tmp/gn4-token.txt") parser.add_argument("--state", default="/tmp/gemma-sweep.jsonl") parser.add_argument("--csv", default="/tmp/gemma-sweep.csv") parser.add_argument("--peaks-csv", default="/tmp/gemma-sweep-peaks.csv") parser.add_argument("--plan", action="store_true", help="list what would run") parser.add_argument("--start", action="store_true", help="run the sweep") parser.add_argument("--report", action="store_true", help="where are we") parser.add_argument("--retry-failed", action="store_true") parser.add_argument("--limit", type=int, default=0) parser.add_argument("--only", default=None, help="substring of the slug") parser.add_argument("--timeout", type=int, default=5400, help="seconds per run") parser.add_argument("--min-free-gb", type=int, default=40) parser.add_argument("--no-rqtl-fallback", action="store_true", help="do not retry panels GEMMA cannot do with R/qtl") parser.add_argument("--peak-threshold", type=float, default=4.0, help="report markers whose -log10(p) clears this") args = parser.parse_args() token = open(args.token_file).read().strip() if args.report: report(args.state, args.csv, args.peaks_csv, args.peak_threshold) return plans = discover(args.base_url, token) done = load_state(args.state) todo = [plan for plan in plans if plan["slug"] not in done or (args.retry_failed and done[plan["slug"]].get("status") == "failed")] if args.only: todo = [plan for plan in todo if args.only in plan["slug"]] if args.limit: todo = todo[:args.limit] if args.plan or not args.start: print(f"{len(plans)} panels, {len(todo)} to run") for plan in todo: print(f" {plan['slug']:40} {plan['kind']:6} {plan['individuals']:6} x" f" {plan['markers']:7} {plan['phenotype']}") return for number, plan in enumerate(todo, 1): free_gb = os.statvfs("/tmp").f_bavail * os.statvfs("/tmp").f_frsize / 1e9 if free_gb < args.min_free_gb: sys.exit(f"stopping: only {free_gb:.0f} GB free on /tmp") print(f"[{number}/{len(todo)}] {plan['slug']} ({plan['individuals']} x" f" {plan['markers']}) ...", flush=True) record = run_one(args.base_url, token, args.state, plan, args.timeout, args.peak_threshold, not args.no_rqtl_fallback) print(f" {record['status']} in {record.get('seconds', 0)}s" f" {record.get('error', '')} {record.get('detail', '')[:120]}", flush=True) if __name__ == "__main__": main()