#!/usr/bin/env python3 """bnw --- headless Bayesian network learning for GeneNetwork The web application (PHP) builds a handful of files in /var/lib/genenet/bnw and then runs three C programs over them. This program does the same computation without the web application: one matrix in, edges out, a private working directory per invocation, and nothing that needs a session key. bnw learn --input matrix.tsv [options] # learn a network bnw version The matrix is variables × samples: a header line naming the variables, then one line per sample. That is exactly what BNW's own example datasets carry (sourcecodes/example_datasets/), and what GeneNetwork4 can build from a set of traits or probes. Output (--format tsv, the default) goes to stdout: # bnw variables=8 samples=200 maxparents=4 k=1 tiers=3 threshold=0.5 fromtoposteriorin_structure Geno1Trait111 ... Every candidate edge is reported, with its posterior, and `in_structure` says whether it survived the threshold. A posterior is not a p-value: it is the share of the k-best networks that contain the edge, so it answers "how often did the search put this arrow there", not "how surprising is it". Exit codes: 0 ok, 2 bad input (the message names the problem), 3 the computation failed (the message is the C program's own), 77 the BNW binaries are not installed -- the conventional "skip" for a test suite. """ import argparse import json import os import shutil import subprocess import sys import tempfile from pathlib import Path VERSION = "0.1.0" # BNW's own defaults, from sourcecodes/create_tiers_gom_part1.php: the web # application writes these into parent.txt, k.txt and thr.txt # unless the user changes them in its settings page. DEFAULT_MAXPARENTS = 4 DEFAULT_K = 1 DEFAULT_TIERS = 3 DEFAULT_THRESHOLD = 0.5 # Where the C programs live, relative to this file: ../var_lib_genenet_bnw, # built by var_lib_genenet_bnw/build.sh. --bin-dir overrides it. HERE = Path(__file__).resolve().parent DEFAULT_BIN_DIR = HERE.parent / "var_lib_genenet_bnw" class InputError(Exception): """Bad input: exit 2, with a message the caller can act on.""" class ComputeError(Exception): """The C programs failed: exit 3, carrying their own message.""" class MissingBinaries(ComputeError): """The BNW binaries are not installed: exit 77, the conventional 'skip'.""" # --------------------------------------------------------------------------- # Input def read_matrix(path, min_variables=2): """Read the variables × samples matrix, and check it before any C runs. A malformed matrix used to travel all the way into the C programs and come back as a segmentation fault or an empty result; every check here is one that failed silently before. """ try: text = Path(path).read_text() except OSError as error: raise InputError(f"cannot read {path}: {error.strerror}") from error lines = [line.rstrip("\r") for line in text.splitlines()] lines = [line for line in lines if line.strip()] if not lines: raise InputError(f"{path} is empty") header = [cell.strip() for cell in lines[0].split("\t")] if len(header) == 1: # A comma-separated header with tab-separated rows is a common mixup. header = [cell.strip() for cell in lines[0].split(",")] if len(header) > 1: raise InputError( "the header is comma-separated but the rows are not: BNW's " "matrix is tab-separated throughout (header and data)") if len(header) < min_variables: raise InputError( f"the header names {len(header)} variable(s); a network needs at " f"least {min_variables}") seen = {} for index, name in enumerate(header): if not name: raise InputError(f"empty variable name in column {index + 1}") if name in seen: raise InputError( f"variable {name!r} appears twice (columns " f"{seen[name] + 1} and {index + 1}); names must be unique — " "rename one copy") seen[name] = index rows = [] for number, line in enumerate(lines[1:], start=2): cells = line.split("\t") if len(cells) != len(header): raise InputError( f"row {number} has {len(cells)} field(s) but the header has " f"{len(header)}") row = [] for name, cell in zip(header, cells): cell = cell.strip() try: row.append(float(cell)) except ValueError: raise InputError( f"row {number}, variable {name!r}: {cell!r} is not a " "number (BNW's continuous model wants numeric values)") from None if row[-1] != row[-1] or row[-1] in (float("inf"), float("-inf")): raise InputError( f"row {number}, variable {name!r}: {cell!r} is not finite") rows.append(row) if len(rows) < 2: raise InputError( f"only {len(rows)} sample row(s): a network cannot be learned from " "fewer than 2 samples") return header, rows def write_input(directory, header, rows): """Write continuous_input.txt: names first, then one line per sample. network_score reads the names with fscanf("%s") and then the numbers, so the file has to be exactly this shape -- tabs in the header, tabs in the data, no trailing blank line. """ path = Path(directory) / "continuous_input.txt" with path.open("w") as handle: handle.write("\t".join(header) + "\n") # The type line: one integer per variable, 1 = continuous, n = n-level # discrete. network_score reads it before the data (and counts rows as # lines - 3, so without it every value shifts by one). This driver is # continuous-only for now, so it is all 1s; the web application writes # the level counts because its upload path accepts discrete columns. handle.write("\t".join("1" for _ in header) + "\n") for row in rows: handle.write("\t".join(repr(value) for value in row) + "\n") return path def write_constraints(directory, banned, white): """Empty ban/white files are what the web application writes by default.""" ban_path = Path(directory) / "ban.txt" white_path = Path(directory) / "white.txt" for path, edges, what in ((ban_path, banned, "ban"), (white_path, white, "white")): if edges is None: path.write_text("") continue lines = [] for edge in edges: parts = edge.replace("->", ",").split(",") if len(parts) != 2 or not all(part.strip() for part in parts): raise InputError(f"--{what} {edge!r} is not 'from,to'") lines.append(f"{parts[0].strip()},{parts[1].strip()}") path.write_text("\n".join(lines) + ("\n" if lines else "")) return ban_path, white_path # --------------------------------------------------------------------------- # Computation def binaries(bin_dir): """Locate the four programs the pipeline needs, or say what is missing.""" bin_dir = Path(bin_dir) wanted = { "network_score": bin_dir / "network_score", "structure": bin_dir / "structure", "get_kbest_parents": bin_dir / "k-best" / "src" / "get_kbest_parents", "get_kbest_nets": bin_dir / "k-best" / "src" / "get_kbest_nets", } # Build output and checkout layout differ: build.sh puts network_score and # structure at the top and the k-best binaries under k-best/src. missing = [name for name, path in wanted.items() if not os.access(path, os.X_OK)] if missing: raise MissingBinaries( "missing BNW binaries: " + ", ".join(missing) + f"\n looked in {bin_dir}" + "\n build them with var_lib_genenet_bnw/build.sh (needs gcc, g++ and libRmath)") return wanted def run(command, what, cwd): """Run one stage, and fail with *its* message rather than a traceback.""" completed = subprocess.run( command, cwd=str(cwd), capture_output=True, text=True, check=False) if completed.returncode != 0: message = (completed.stderr or completed.stdout or "").strip() raise ComputeError( f"{what} failed (exit {completed.returncode})" + (f": {message}" if message else "")) return completed def learn(args): header, rows = read_matrix(args.input) if args.maxparents < 0: raise InputError("--maxparents must be >= 0") if args.k < 1: raise InputError("--k must be >= 1") if not 0.0 <= args.threshold <= 1.0: raise InputError("--threshold is an edge posterior in [0, 1]") workdir = Path(args.workdir) if args.workdir else Path(tempfile.mkdtemp(prefix="bnw-")) workdir.mkdir(parents=True, exist_ok=True) bin_dir = Path(args.bin_dir) if args.bin_dir else DEFAULT_BIN_DIR programs = binaries(bin_dir) try: input_path = write_input(workdir, header, rows) ban_path, white_path = write_constraints(workdir, args.ban, args.white) # Stage 1: local scores. argv[5] is a directory prefix -- the program # writes one binary file of doubles per node into it (0, 1, 2, ...). run([str(programs["network_score"]), str(input_path), str(ban_path), str(white_path), str(args.maxparents), str(workdir)], "network_score", workdir) # Stage 2: the k-best search, which writes postProbEachEdge.txt. run([str(programs["get_kbest_parents"]), str(len(header)), str(workdir), str(args.k)], "get_kbest_parents", workdir) run([str(programs["get_kbest_nets"]), str(len(header)), str(workdir), str(args.k)], "get_kbest_nets", workdir) # Stage 3: threshold the posteriors into an adjacency. structure_tmp = workdir / "structure_input_temp.txt" structure_out = workdir / "structure_input.txt" # `structure' exits with whatever its uninitialised return value # happens to be -- the web application ignores that and reads the file, # so do the same, and only fail if the file is missing or empty. subprocess.run([str(programs["structure"]), str(input_path), str(workdir / "postProbEachEdge.txt"), str(structure_tmp), str(structure_out), str(args.threshold)], cwd=str(workdir), capture_output=True, text=True, check=False) if not structure_out.exists() or structure_out.stat().st_size == 0: raise ComputeError( f"structure wrote no {structure_out.name}; the network could not " "be thresholded (is the posterior table empty?)") edges = read_edges(workdir / "postProbEachEdge.txt", header) structure = read_structure(structure_out, header) for edge in edges: edge["in_structure"] = (edge["from"], edge["to"]) in structure if args.svg: write_svg(args.svg, header, edges, args.threshold) emit(edges, header, len(rows), args) finally: if args.keep: print(f"work directory kept: {workdir}", file=sys.stderr) elif not args.workdir: shutil.rmtree(workdir, ignore_errors=True) def read_edges(path, header): """postProbEachEdge.txt: lines 'j -> i posterior', node numbers.""" edges = [] try: lines = Path(path).read_text().splitlines() except OSError as error: raise ComputeError(f"{path}: {error.strerror}") from error for line in lines: line = line.strip() if not line: continue try: arrow, posterior = line.split("\t") source, target = arrow.split("->") from_index, to_index = int(source), int(target) except ValueError: raise ComputeError( f"unexpected line in {path.name}: {line!r}") from None if not (0 <= from_index < len(header) and 0 <= to_index < len(header)): raise ComputeError( f"{path.name} names node {from_index}->{to_index}, outside " f"the {len(header)} variables") edges.append({ "from": header[from_index], "to": header[to_index], "posterior": float(posterior), }) edges = [edge for edge in edges if edge["from"] != edge["to"]] edges.sort(key=lambda edge: (-edge["posterior"], edge["from"], edge["to"])) return edges def read_structure(path, header): """structure_input.txt: a header of names, then a 0/1 adjacency matrix.""" try: lines = [line.rstrip("\r") for line in Path(path).read_text().splitlines() if line.strip()] except OSError as error: raise ComputeError(f"{path}: {error.strerror}") from error if len(lines) != len(header) + 1: raise ComputeError( f"{path.name} has {len(lines)} line(s); expected {len(header) + 1}") present = set() for row_index, line in enumerate(lines[1:]): cells = [cell for cell in line.split("\t") if cell != ""] if len(cells) != len(header): raise ComputeError( f"{path.name} row {row_index + 1} has {len(cells)} column(s); " f"expected {len(header)}") for col_index, cell in enumerate(cells): if cell.strip() == "1": present.add((header[row_index], header[col_index])) return present # --------------------------------------------------------------------------- # Output def emit(edges, header, samples, args): parameters = { "maxparents": args.maxparents, "k": args.k, "tiers": args.tiers, "threshold": args.threshold, } header_line = ( f"# bnw {VERSION} variables={len(header)} samples={samples} " f"maxparents={args.maxparents} k={args.k} tiers={args.tiers} " f"threshold={args.threshold}") if args.format == "tsv": out = [header_line, "from\tto\tposterior\tin_structure"] for edge in edges: out.append(f"{edge['from']}\t{edge['to']}\t" f"{edge['posterior']:.10g}\t{1 if edge['in_structure'] else 0}") print("\n".join(out)) else: json.dump({ "version": VERSION, "parameters": parameters, "variables": header, "samples": samples, "edges": [ {"from": edge["from"], "to": edge["to"], "posterior": edge["posterior"], "in_structure": edge["in_structure"]} for edge in edges ], "structure": sorted( [edge["from"], edge["to"]] for edge in edges if edge["in_structure"]), }, sys.stdout, indent=1) print() def write_svg(path, header, edges, threshold): """Optional: a dot drawing of the thresholded network, if graphviz exists.""" dot = shutil.which("dot") if not dot: raise ComputeError("--svg needs graphviz's `dot` on PATH") lines = ["digraph bnw {", ' rankdir=LR;', ' node [shape=ellipse, fontsize=10];'] for name in header: lines.append(f' "{name}";') for edge in edges: if edge["in_structure"]: weight = max(0.2, min(1.0, edge["posterior"])) lines.append( f' "{edge["from"]}" -> "{edge["to"]}" ' f'[label="{edge["posterior"]:.2f}", weight="{weight:.2f}"];') lines.append("}") source = "\n".join(lines) + "\n" completed = subprocess.run([dot, "-Tsvg", "-o", str(path)], input=source, text=True, capture_output=True, check=False) if completed.returncode != 0: raise ComputeError(f"dot failed: {(completed.stderr or '').strip()}") # --------------------------------------------------------------------------- def main(argv): parser = argparse.ArgumentParser( prog="bnw", description="Headless Bayesian network learning for GeneNetwork.") sub = parser.add_subparsers(dest="command", required=True) learn_parser = sub.add_parser( "learn", help="learn a network from a variables x samples matrix") learn_parser.add_argument("--input", required=True, help="matrix: header of variable names, then one row per sample") learn_parser.add_argument("--maxparents", type=int, default=DEFAULT_MAXPARENTS, help=f"maximum parents per node (default {DEFAULT_MAXPARENTS})") learn_parser.add_argument("--k", type=int, default=DEFAULT_K, help=f"networks to average over (default {DEFAULT_K})") learn_parser.add_argument("--tiers", type=int, default=DEFAULT_TIERS, help=f"variable ordering tiers (default {DEFAULT_TIERS})") learn_parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, help=f"edge-posterior threshold (default {DEFAULT_THRESHOLD})") learn_parser.add_argument("--ban", action="append", metavar="FROM,TO", help="banned edge; repeatable") learn_parser.add_argument("--white", action="append", metavar="FROM,TO", help="forced edge; repeatable") learn_parser.add_argument("--format", choices=("tsv", "json"), default="tsv") learn_parser.add_argument("--svg", metavar="PATH", help="also write an SVG of the thresholded network (needs graphviz)") learn_parser.add_argument("--workdir", metavar="DIR", help="work here instead of a fresh temporary directory") learn_parser.add_argument("--keep", action="store_true", help="keep the work directory (it is removed otherwise)") learn_parser.add_argument("--bin-dir", metavar="DIR", help="where the BNW C binaries are " "(default: ../var_lib_genenet_bnw beside this script)") sub.add_parser("version", help="print the version and exit") args = parser.parse_args(argv) if args.command == "version": print(f"bnw {VERSION}") return 0 try: learn(args) except InputError as error: print(f"bnw: {error}", file=sys.stderr) return 2 except MissingBinaries as error: print(f"bnw: {error}", file=sys.stderr) return 77 except ComputeError as error: print(f"bnw: {error}", file=sys.stderr) return 3 return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))