diff options
Diffstat (limited to 'bin')
| -rwxr-xr-x | bin/bnw | 444 |
1 files changed, 444 insertions, 0 deletions
diff --git a/bin/bnw b/bin/bnw new file mode 100755 index 00000000..563e1462 --- /dev/null +++ b/bin/bnw @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +"""bnw --- headless Bayesian network learning for GeneNetwork + +The web application (PHP) builds a handful of files in /var/lib/genenet/bnw +and then runs three C programs over them. This program does the same +computation without the web application: one matrix in, edges out, a private +working directory per invocation, and nothing that needs a session key. + + bnw learn --input matrix.tsv [options] # learn a network + bnw version + +The matrix is variables × samples: a header line naming the variables, then +one line per sample. That is exactly what BNW's own example datasets carry +(sourcecodes/example_datasets/), and what GeneNetwork4 can build from a set of +traits or probes. + +Output (--format tsv, the default) goes to stdout: + + # bnw <version> variables=8 samples=200 maxparents=4 k=1 tiers=3 threshold=0.5 + from<TAB>to<TAB>posterior<TAB>in_structure + Geno1<TAB>Trait1<TAB>1<TAB>1 + ... + +Every candidate edge is reported, with its posterior, and `in_structure` says +whether it survived the threshold. A posterior is not a p-value: it is the +share of the k-best networks that contain the edge, so it answers "how often +did the search put this arrow there", not "how surprising is it". + +Exit codes: 0 ok, 2 bad input (the message names the problem), 3 the +computation failed (the message is the C program's own), 77 the BNW binaries +are not installed -- the conventional "skip" for a test suite. +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +VERSION = "0.1.0" + +# BNW's own defaults, from sourcecodes/create_tiers_gom_part1.php: the web +# application writes these into <key>parent.txt, <key>k.txt and <key>thr.txt +# unless the user changes them in its settings page. +DEFAULT_MAXPARENTS = 4 +DEFAULT_K = 1 +DEFAULT_TIERS = 3 +DEFAULT_THRESHOLD = 0.5 + +# Where the C programs live, relative to this file: ../var_lib_genenet_bnw, +# built by var_lib_genenet_bnw/build.sh. --bin-dir overrides it. +HERE = Path(__file__).resolve().parent +DEFAULT_BIN_DIR = HERE.parent / "var_lib_genenet_bnw" + + +class InputError(Exception): + """Bad input: exit 2, with a message the caller can act on.""" + + +class ComputeError(Exception): + """The C programs failed: exit 3, carrying their own message.""" + + +class MissingBinaries(ComputeError): + """The BNW binaries are not installed: exit 77, the conventional 'skip'.""" + + +# --------------------------------------------------------------------------- +# Input + + +def read_matrix(path, min_variables=2): + """Read the variables × samples matrix, and check it before any C runs. + + A malformed matrix used to travel all the way into the C programs and come + back as a segmentation fault or an empty result; every check here is one + that failed silently before. + """ + try: + text = Path(path).read_text() + except OSError as error: + raise InputError(f"cannot read {path}: {error.strerror}") from error + lines = [line.rstrip("\r") for line in text.splitlines()] + lines = [line for line in lines if line.strip()] + if not lines: + raise InputError(f"{path} is empty") + header = [cell.strip() for cell in lines[0].split("\t")] + if len(header) == 1: + # A comma-separated header with tab-separated rows is a common mixup. + header = [cell.strip() for cell in lines[0].split(",")] + if len(header) > 1: + raise InputError( + "the header is comma-separated but the rows are not: BNW's " + "matrix is tab-separated throughout (header and data)") + if len(header) < min_variables: + raise InputError( + f"the header names {len(header)} variable(s); a network needs at " + f"least {min_variables}") + seen = {} + for index, name in enumerate(header): + if not name: + raise InputError(f"empty variable name in column {index + 1}") + if name in seen: + raise InputError( + f"variable {name!r} appears twice (columns " + f"{seen[name] + 1} and {index + 1}); names must be unique — " + "rename one copy") + seen[name] = index + + rows = [] + for number, line in enumerate(lines[1:], start=2): + cells = line.split("\t") + if len(cells) != len(header): + raise InputError( + f"row {number} has {len(cells)} field(s) but the header has " + f"{len(header)}") + row = [] + for name, cell in zip(header, cells): + cell = cell.strip() + try: + row.append(float(cell)) + except ValueError: + raise InputError( + f"row {number}, variable {name!r}: {cell!r} is not a " + "number (BNW's continuous model wants numeric values)") from None + if row[-1] != row[-1] or row[-1] in (float("inf"), float("-inf")): + raise InputError( + f"row {number}, variable {name!r}: {cell!r} is not finite") + rows.append(row) + if len(rows) < 2: + raise InputError( + f"only {len(rows)} sample row(s): a network cannot be learned from " + "fewer than 2 samples") + return header, rows + + +def write_input(directory, header, rows): + """Write continuous_input.txt: names first, then one line per sample. + + network_score reads the names with fscanf("%s") and then the numbers, so + the file has to be exactly this shape -- tabs in the header, tabs in the + data, no trailing blank line. + """ + path = Path(directory) / "continuous_input.txt" + with path.open("w") as handle: + handle.write("\t".join(header) + "\n") + # The type line: one integer per variable, 1 = continuous, n = n-level + # discrete. network_score reads it before the data (and counts rows as + # lines - 3, so without it every value shifts by one). This driver is + # continuous-only for now, so it is all 1s; the web application writes + # the level counts because its upload path accepts discrete columns. + handle.write("\t".join("1" for _ in header) + "\n") + for row in rows: + handle.write("\t".join(repr(value) for value in row) + "\n") + return path + + +def write_constraints(directory, banned, white): + """Empty ban/white files are what the web application writes by default.""" + ban_path = Path(directory) / "ban.txt" + white_path = Path(directory) / "white.txt" + for path, edges, what in ((ban_path, banned, "ban"), (white_path, white, "white")): + if edges is None: + path.write_text("") + continue + lines = [] + for edge in edges: + parts = edge.replace("->", ",").split(",") + if len(parts) != 2 or not all(part.strip() for part in parts): + raise InputError(f"--{what} {edge!r} is not 'from,to'") + lines.append(f"{parts[0].strip()},{parts[1].strip()}") + path.write_text("\n".join(lines) + ("\n" if lines else "")) + return ban_path, white_path + + +# --------------------------------------------------------------------------- +# Computation + + +def binaries(bin_dir): + """Locate the four programs the pipeline needs, or say what is missing.""" + bin_dir = Path(bin_dir) + wanted = { + "network_score": bin_dir / "network_score", + "structure": bin_dir / "structure", + "get_kbest_parents": bin_dir / "k-best" / "src" / "get_kbest_parents", + "get_kbest_nets": bin_dir / "k-best" / "src" / "get_kbest_nets", + } + # Build output and checkout layout differ: build.sh puts network_score and + # structure at the top and the k-best binaries under k-best/src. + missing = [name for name, path in wanted.items() if not os.access(path, os.X_OK)] + if missing: + raise MissingBinaries( + "missing BNW binaries: " + ", ".join(missing) + + f"\n looked in {bin_dir}" + + "\n build them with var_lib_genenet_bnw/build.sh (needs gcc, g++ and libRmath)") + return wanted + + +def run(command, what, cwd): + """Run one stage, and fail with *its* message rather than a traceback.""" + completed = subprocess.run( + command, cwd=str(cwd), capture_output=True, text=True, check=False) + if completed.returncode != 0: + message = (completed.stderr or completed.stdout or "").strip() + raise ComputeError( + f"{what} failed (exit {completed.returncode})" + + (f": {message}" if message else "")) + return completed + + +def learn(args): + header, rows = read_matrix(args.input) + if args.maxparents < 0: + raise InputError("--maxparents must be >= 0") + if args.k < 1: + raise InputError("--k must be >= 1") + if not 0.0 <= args.threshold <= 1.0: + raise InputError("--threshold is an edge posterior in [0, 1]") + + workdir = Path(args.workdir) if args.workdir else Path(tempfile.mkdtemp(prefix="bnw-")) + workdir.mkdir(parents=True, exist_ok=True) + bin_dir = Path(args.bin_dir) if args.bin_dir else DEFAULT_BIN_DIR + programs = binaries(bin_dir) + try: + input_path = write_input(workdir, header, rows) + ban_path, white_path = write_constraints(workdir, args.ban, args.white) + + # Stage 1: local scores. argv[5] is a directory prefix -- the program + # writes one binary file of doubles per node into it (0, 1, 2, ...). + run([str(programs["network_score"]), str(input_path), str(ban_path), + str(white_path), str(args.maxparents), str(workdir)], + "network_score", workdir) + + # Stage 2: the k-best search, which writes postProbEachEdge.txt. + run([str(programs["get_kbest_parents"]), str(len(header)), str(workdir), + str(args.k)], "get_kbest_parents", workdir) + run([str(programs["get_kbest_nets"]), str(len(header)), str(workdir), + str(args.k)], "get_kbest_nets", workdir) + + # Stage 3: threshold the posteriors into an adjacency. + structure_tmp = workdir / "structure_input_temp.txt" + structure_out = workdir / "structure_input.txt" + # `structure' exits with whatever its uninitialised return value + # happens to be -- the web application ignores that and reads the file, + # so do the same, and only fail if the file is missing or empty. + subprocess.run([str(programs["structure"]), str(input_path), + str(workdir / "postProbEachEdge.txt"), str(structure_tmp), + str(structure_out), str(args.threshold)], + cwd=str(workdir), capture_output=True, text=True, check=False) + if not structure_out.exists() or structure_out.stat().st_size == 0: + raise ComputeError( + f"structure wrote no {structure_out.name}; the network could not " + "be thresholded (is the posterior table empty?)") + + edges = read_edges(workdir / "postProbEachEdge.txt", header) + structure = read_structure(structure_out, header) + for edge in edges: + edge["in_structure"] = (edge["from"], edge["to"]) in structure + if args.svg: + write_svg(args.svg, header, edges, args.threshold) + emit(edges, header, len(rows), args) + finally: + if args.keep: + print(f"work directory kept: {workdir}", file=sys.stderr) + elif not args.workdir: + shutil.rmtree(workdir, ignore_errors=True) + + +def read_edges(path, header): + """postProbEachEdge.txt: lines 'j -> i <TAB> posterior', node numbers.""" + edges = [] + try: + lines = Path(path).read_text().splitlines() + except OSError as error: + raise ComputeError(f"{path}: {error.strerror}") from error + for line in lines: + line = line.strip() + if not line: + continue + try: + arrow, posterior = line.split("\t") + source, target = arrow.split("->") + from_index, to_index = int(source), int(target) + except ValueError: + raise ComputeError( + f"unexpected line in {path.name}: {line!r}") from None + if not (0 <= from_index < len(header) and 0 <= to_index < len(header)): + raise ComputeError( + f"{path.name} names node {from_index}->{to_index}, outside " + f"the {len(header)} variables") + edges.append({ + "from": header[from_index], + "to": header[to_index], + "posterior": float(posterior), + }) + edges = [edge for edge in edges if edge["from"] != edge["to"]] + edges.sort(key=lambda edge: (-edge["posterior"], edge["from"], edge["to"])) + return edges + + +def read_structure(path, header): + """structure_input.txt: a header of names, then a 0/1 adjacency matrix.""" + try: + lines = [line.rstrip("\r") for line in Path(path).read_text().splitlines() if line.strip()] + except OSError as error: + raise ComputeError(f"{path}: {error.strerror}") from error + if len(lines) != len(header) + 1: + raise ComputeError( + f"{path.name} has {len(lines)} line(s); expected {len(header) + 1}") + present = set() + for row_index, line in enumerate(lines[1:]): + cells = [cell for cell in line.split("\t") if cell != ""] + if len(cells) != len(header): + raise ComputeError( + f"{path.name} row {row_index + 1} has {len(cells)} column(s); " + f"expected {len(header)}") + for col_index, cell in enumerate(cells): + if cell.strip() == "1": + present.add((header[row_index], header[col_index])) + return present + + +# --------------------------------------------------------------------------- +# Output + + +def emit(edges, header, samples, args): + parameters = { + "maxparents": args.maxparents, + "k": args.k, + "tiers": args.tiers, + "threshold": args.threshold, + } + header_line = ( + f"# bnw {VERSION} variables={len(header)} samples={samples} " + f"maxparents={args.maxparents} k={args.k} tiers={args.tiers} " + f"threshold={args.threshold}") + if args.format == "tsv": + out = [header_line, "from\tto\tposterior\tin_structure"] + for edge in edges: + out.append(f"{edge['from']}\t{edge['to']}\t" + f"{edge['posterior']:.10g}\t{1 if edge['in_structure'] else 0}") + print("\n".join(out)) + else: + json.dump({ + "version": VERSION, + "parameters": parameters, + "variables": header, + "samples": samples, + "edges": [ + {"from": edge["from"], "to": edge["to"], + "posterior": edge["posterior"], + "in_structure": edge["in_structure"]} + for edge in edges + ], + "structure": sorted( + [edge["from"], edge["to"]] for edge in edges if edge["in_structure"]), + }, sys.stdout, indent=1) + print() + + +def write_svg(path, header, edges, threshold): + """Optional: a dot drawing of the thresholded network, if graphviz exists.""" + dot = shutil.which("dot") + if not dot: + raise ComputeError("--svg needs graphviz's `dot` on PATH") + lines = ["digraph bnw {", ' rankdir=LR;', ' node [shape=ellipse, fontsize=10];'] + for name in header: + lines.append(f' "{name}";') + for edge in edges: + if edge["in_structure"]: + weight = max(0.2, min(1.0, edge["posterior"])) + lines.append( + f' "{edge["from"]}" -> "{edge["to"]}" ' + f'[label="{edge["posterior"]:.2f}", weight="{weight:.2f}"];') + lines.append("}") + source = "\n".join(lines) + "\n" + completed = subprocess.run([dot, "-Tsvg", "-o", str(path)], + input=source, text=True, capture_output=True, check=False) + if completed.returncode != 0: + raise ComputeError(f"dot failed: {(completed.stderr or '').strip()}") + + +# --------------------------------------------------------------------------- + + +def main(argv): + parser = argparse.ArgumentParser( + prog="bnw", description="Headless Bayesian network learning for GeneNetwork.") + sub = parser.add_subparsers(dest="command", required=True) + + learn_parser = sub.add_parser( + "learn", help="learn a network from a variables x samples matrix") + learn_parser.add_argument("--input", required=True, + help="matrix: header of variable names, then one row per sample") + learn_parser.add_argument("--maxparents", type=int, default=DEFAULT_MAXPARENTS, + help=f"maximum parents per node (default {DEFAULT_MAXPARENTS})") + learn_parser.add_argument("--k", type=int, default=DEFAULT_K, + help=f"networks to average over (default {DEFAULT_K})") + learn_parser.add_argument("--tiers", type=int, default=DEFAULT_TIERS, + help=f"variable ordering tiers (default {DEFAULT_TIERS})") + learn_parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD, + help=f"edge-posterior threshold (default {DEFAULT_THRESHOLD})") + learn_parser.add_argument("--ban", action="append", metavar="FROM,TO", + help="banned edge; repeatable") + learn_parser.add_argument("--white", action="append", metavar="FROM,TO", + help="forced edge; repeatable") + learn_parser.add_argument("--format", choices=("tsv", "json"), default="tsv") + learn_parser.add_argument("--svg", metavar="PATH", + help="also write an SVG of the thresholded network (needs graphviz)") + learn_parser.add_argument("--workdir", metavar="DIR", + help="work here instead of a fresh temporary directory") + learn_parser.add_argument("--keep", action="store_true", + help="keep the work directory (it is removed otherwise)") + learn_parser.add_argument("--bin-dir", metavar="DIR", + help="where the BNW C binaries are " + "(default: ../var_lib_genenet_bnw beside this script)") + + sub.add_parser("version", help="print the version and exit") + + args = parser.parse_args(argv) + if args.command == "version": + print(f"bnw {VERSION}") + return 0 + try: + learn(args) + except InputError as error: + print(f"bnw: {error}", file=sys.stderr) + return 2 + except MissingBinaries as error: + print(f"bnw: {error}", file=sys.stderr) + return 77 + except ComputeError as error: + print(f"bnw: {error}", file=sys.stderr) + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) |
