about summary refs log tree commit diff
path: root/bin/bnw
diff options
context:
space:
mode:
Diffstat (limited to 'bin/bnw')
-rwxr-xr-xbin/bnw444
1 files changed, 444 insertions, 0 deletions
diff --git a/bin/bnw b/bin/bnw
new file mode 100755
index 00000000..563e1462
--- /dev/null
+++ b/bin/bnw
@@ -0,0 +1,444 @@
+#!/usr/bin/env python3
+"""bnw --- headless Bayesian network learning for GeneNetwork
+
+The web application (PHP) builds a handful of files in /var/lib/genenet/bnw
+and then runs three C programs over them.  This program does the same
+computation without the web application: one matrix in, edges out, a private
+working directory per invocation, and nothing that needs a session key.
+
+    bnw learn --input matrix.tsv [options]        # learn a network
+    bnw version
+
+The matrix is variables × samples: a header line naming the variables, then
+one line per sample.  That is exactly what BNW's own example datasets carry
+(sourcecodes/example_datasets/), and what GeneNetwork4 can build from a set of
+traits or probes.
+
+Output (--format tsv, the default) goes to stdout:
+
+    # bnw <version> variables=8 samples=200 maxparents=4 k=1 tiers=3 threshold=0.5
+    from<TAB>to<TAB>posterior<TAB>in_structure
+    Geno1<TAB>Trait1<TAB>1<TAB>1
+    ...
+
+Every candidate edge is reported, with its posterior, and `in_structure` says
+whether it survived the threshold.  A posterior is not a p-value: it is the
+share of the k-best networks that contain the edge, so it answers "how often
+did the search put this arrow there", not "how surprising is it".
+
+Exit codes: 0 ok, 2 bad input (the message names the problem), 3 the
+computation failed (the message is the C program's own), 77 the BNW binaries
+are not installed -- the conventional "skip" for a test suite.
+"""
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+VERSION = "0.1.0"
+
+# BNW's own defaults, from sourcecodes/create_tiers_gom_part1.php: the web
+# application writes these into <key>parent.txt, <key>k.txt and <key>thr.txt
+# unless the user changes them in its settings page.
+DEFAULT_MAXPARENTS = 4
+DEFAULT_K = 1
+DEFAULT_TIERS = 3
+DEFAULT_THRESHOLD = 0.5
+
+# Where the C programs live, relative to this file: ../var_lib_genenet_bnw,
+# built by var_lib_genenet_bnw/build.sh.  --bin-dir overrides it.
+HERE = Path(__file__).resolve().parent
+DEFAULT_BIN_DIR = HERE.parent / "var_lib_genenet_bnw"
+
+
+class InputError(Exception):
+    """Bad input: exit 2, with a message the caller can act on."""
+
+
+class ComputeError(Exception):
+    """The C programs failed: exit 3, carrying their own message."""
+
+
+class MissingBinaries(ComputeError):
+    """The BNW binaries are not installed: exit 77, the conventional 'skip'."""
+
+
+# ---------------------------------------------------------------------------
+# Input
+
+
+def read_matrix(path, min_variables=2):
+    """Read the variables × samples matrix, and check it before any C runs.
+
+    A malformed matrix used to travel all the way into the C programs and come
+    back as a segmentation fault or an empty result; every check here is one
+    that failed silently before.
+    """
+    try:
+        text = Path(path).read_text()
+    except OSError as error:
+        raise InputError(f"cannot read {path}: {error.strerror}") from error
+    lines = [line.rstrip("\r") for line in text.splitlines()]
+    lines = [line for line in lines if line.strip()]
+    if not lines:
+        raise InputError(f"{path} is empty")
+    header = [cell.strip() for cell in lines[0].split("\t")]
+    if len(header) == 1:
+        # A comma-separated header with tab-separated rows is a common mixup.
+        header = [cell.strip() for cell in lines[0].split(",")]
+        if len(header) > 1:
+            raise InputError(
+                "the header is comma-separated but the rows are not: BNW's "
+                "matrix is tab-separated throughout (header and data)")
+    if len(header) < min_variables:
+        raise InputError(
+            f"the header names {len(header)} variable(s); a network needs at "
+            f"least {min_variables}")
+    seen = {}
+    for index, name in enumerate(header):
+        if not name:
+            raise InputError(f"empty variable name in column {index + 1}")
+        if name in seen:
+            raise InputError(
+                f"variable {name!r} appears twice (columns "
+                f"{seen[name] + 1} and {index + 1}); names must be unique — "
+                "rename one copy")
+        seen[name] = index
+
+    rows = []
+    for number, line in enumerate(lines[1:], start=2):
+        cells = line.split("\t")
+        if len(cells) != len(header):
+            raise InputError(
+                f"row {number} has {len(cells)} field(s) but the header has "
+                f"{len(header)}")
+        row = []
+        for name, cell in zip(header, cells):
+            cell = cell.strip()
+            try:
+                row.append(float(cell))
+            except ValueError:
+                raise InputError(
+                    f"row {number}, variable {name!r}: {cell!r} is not a "
+                    "number (BNW's continuous model wants numeric values)") from None
+            if row[-1] != row[-1] or row[-1] in (float("inf"), float("-inf")):
+                raise InputError(
+                    f"row {number}, variable {name!r}: {cell!r} is not finite")
+        rows.append(row)
+    if len(rows) < 2:
+        raise InputError(
+            f"only {len(rows)} sample row(s): a network cannot be learned from "
+            "fewer than 2 samples")
+    return header, rows
+
+
+def write_input(directory, header, rows):
+    """Write continuous_input.txt: names first, then one line per sample.
+
+    network_score reads the names with fscanf("%s") and then the numbers, so
+    the file has to be exactly this shape -- tabs in the header, tabs in the
+    data, no trailing blank line.
+    """
+    path = Path(directory) / "continuous_input.txt"
+    with path.open("w") as handle:
+        handle.write("\t".join(header) + "\n")
+        # The type line: one integer per variable, 1 = continuous, n = n-level
+        # discrete.  network_score reads it before the data (and counts rows as
+        # lines - 3, so without it every value shifts by one).  This driver is
+        # continuous-only for now, so it is all 1s; the web application writes
+        # the level counts because its upload path accepts discrete columns.
+        handle.write("\t".join("1" for _ in header) + "\n")
+        for row in rows:
+            handle.write("\t".join(repr(value) for value in row) + "\n")
+    return path
+
+
+def write_constraints(directory, banned, white):
+    """Empty ban/white files are what the web application writes by default."""
+    ban_path = Path(directory) / "ban.txt"
+    white_path = Path(directory) / "white.txt"
+    for path, edges, what in ((ban_path, banned, "ban"), (white_path, white, "white")):
+        if edges is None:
+            path.write_text("")
+            continue
+        lines = []
+        for edge in edges:
+            parts = edge.replace("->", ",").split(",")
+            if len(parts) != 2 or not all(part.strip() for part in parts):
+                raise InputError(f"--{what} {edge!r} is not 'from,to'")
+            lines.append(f"{parts[0].strip()},{parts[1].strip()}")
+        path.write_text("\n".join(lines) + ("\n" if lines else ""))
+    return ban_path, white_path
+
+
+# ---------------------------------------------------------------------------
+# Computation
+
+
+def binaries(bin_dir):
+    """Locate the four programs the pipeline needs, or say what is missing."""
+    bin_dir = Path(bin_dir)
+    wanted = {
+        "network_score": bin_dir / "network_score",
+        "structure": bin_dir / "structure",
+        "get_kbest_parents": bin_dir / "k-best" / "src" / "get_kbest_parents",
+        "get_kbest_nets": bin_dir / "k-best" / "src" / "get_kbest_nets",
+    }
+    # Build output and checkout layout differ: build.sh puts network_score and
+    # structure at the top and the k-best binaries under k-best/src.
+    missing = [name for name, path in wanted.items() if not os.access(path, os.X_OK)]
+    if missing:
+        raise MissingBinaries(
+            "missing BNW binaries: " + ", ".join(missing) +
+            f"\n  looked in {bin_dir}" +
+            "\n  build them with var_lib_genenet_bnw/build.sh (needs gcc, g++ and libRmath)")
+    return wanted
+
+
+def run(command, what, cwd):
+    """Run one stage, and fail with *its* message rather than a traceback."""
+    completed = subprocess.run(
+        command, cwd=str(cwd), capture_output=True, text=True, check=False)
+    if completed.returncode != 0:
+        message = (completed.stderr or completed.stdout or "").strip()
+        raise ComputeError(
+            f"{what} failed (exit {completed.returncode})" +
+            (f": {message}" if message else ""))
+    return completed
+
+
+def learn(args):
+    header, rows = read_matrix(args.input)
+    if args.maxparents < 0:
+        raise InputError("--maxparents must be >= 0")
+    if args.k < 1:
+        raise InputError("--k must be >= 1")
+    if not 0.0 <= args.threshold <= 1.0:
+        raise InputError("--threshold is an edge posterior in [0, 1]")
+
+    workdir = Path(args.workdir) if args.workdir else Path(tempfile.mkdtemp(prefix="bnw-"))
+    workdir.mkdir(parents=True, exist_ok=True)
+    bin_dir = Path(args.bin_dir) if args.bin_dir else DEFAULT_BIN_DIR
+    programs = binaries(bin_dir)
+    try:
+        input_path = write_input(workdir, header, rows)
+        ban_path, white_path = write_constraints(workdir, args.ban, args.white)
+
+        # Stage 1: local scores.  argv[5] is a directory prefix -- the program
+        # writes one binary file of doubles per node into it (0, 1, 2, ...).
+        run([str(programs["network_score"]), str(input_path), str(ban_path),
+             str(white_path), str(args.maxparents), str(workdir)],
+            "network_score", workdir)
+
+        # Stage 2: the k-best search, which writes postProbEachEdge.txt.
+        run([str(programs["get_kbest_parents"]), str(len(header)), str(workdir),
+             str(args.k)], "get_kbest_parents", workdir)
+        run([str(programs["get_kbest_nets"]), str(len(header)), str(workdir),
+             str(args.k)], "get_kbest_nets", workdir)
+
+        # Stage 3: threshold the posteriors into an adjacency.
+        structure_tmp = workdir / "structure_input_temp.txt"
+        structure_out = workdir / "structure_input.txt"
+        # `structure' exits with whatever its uninitialised return value
+        # happens to be -- the web application ignores that and reads the file,
+        # so do the same, and only fail if the file is missing or empty.
+        subprocess.run([str(programs["structure"]), str(input_path),
+                        str(workdir / "postProbEachEdge.txt"), str(structure_tmp),
+                        str(structure_out), str(args.threshold)],
+                       cwd=str(workdir), capture_output=True, text=True, check=False)
+        if not structure_out.exists() or structure_out.stat().st_size == 0:
+            raise ComputeError(
+                f"structure wrote no {structure_out.name}; the network could not "
+                "be thresholded (is the posterior table empty?)")
+
+        edges = read_edges(workdir / "postProbEachEdge.txt", header)
+        structure = read_structure(structure_out, header)
+        for edge in edges:
+            edge["in_structure"] = (edge["from"], edge["to"]) in structure
+        if args.svg:
+            write_svg(args.svg, header, edges, args.threshold)
+        emit(edges, header, len(rows), args)
+    finally:
+        if args.keep:
+            print(f"work directory kept: {workdir}", file=sys.stderr)
+        elif not args.workdir:
+            shutil.rmtree(workdir, ignore_errors=True)
+
+
+def read_edges(path, header):
+    """postProbEachEdge.txt: lines 'j -> i <TAB> posterior', node numbers."""
+    edges = []
+    try:
+        lines = Path(path).read_text().splitlines()
+    except OSError as error:
+        raise ComputeError(f"{path}: {error.strerror}") from error
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            arrow, posterior = line.split("\t")
+            source, target = arrow.split("->")
+            from_index, to_index = int(source), int(target)
+        except ValueError:
+            raise ComputeError(
+                f"unexpected line in {path.name}: {line!r}") from None
+        if not (0 <= from_index < len(header) and 0 <= to_index < len(header)):
+            raise ComputeError(
+                f"{path.name} names node {from_index}->{to_index}, outside "
+                f"the {len(header)} variables")
+        edges.append({
+            "from": header[from_index],
+            "to": header[to_index],
+            "posterior": float(posterior),
+        })
+    edges = [edge for edge in edges if edge["from"] != edge["to"]]
+    edges.sort(key=lambda edge: (-edge["posterior"], edge["from"], edge["to"]))
+    return edges
+
+
+def read_structure(path, header):
+    """structure_input.txt: a header of names, then a 0/1 adjacency matrix."""
+    try:
+        lines = [line.rstrip("\r") for line in Path(path).read_text().splitlines() if line.strip()]
+    except OSError as error:
+        raise ComputeError(f"{path}: {error.strerror}") from error
+    if len(lines) != len(header) + 1:
+        raise ComputeError(
+            f"{path.name} has {len(lines)} line(s); expected {len(header) + 1}")
+    present = set()
+    for row_index, line in enumerate(lines[1:]):
+        cells = [cell for cell in line.split("\t") if cell != ""]
+        if len(cells) != len(header):
+            raise ComputeError(
+                f"{path.name} row {row_index + 1} has {len(cells)} column(s); "
+                f"expected {len(header)}")
+        for col_index, cell in enumerate(cells):
+            if cell.strip() == "1":
+                present.add((header[row_index], header[col_index]))
+    return present
+
+
+# ---------------------------------------------------------------------------
+# Output
+
+
+def emit(edges, header, samples, args):
+    parameters = {
+        "maxparents": args.maxparents,
+        "k": args.k,
+        "tiers": args.tiers,
+        "threshold": args.threshold,
+    }
+    header_line = (
+        f"# bnw {VERSION} variables={len(header)} samples={samples} "
+        f"maxparents={args.maxparents} k={args.k} tiers={args.tiers} "
+        f"threshold={args.threshold}")
+    if args.format == "tsv":
+        out = [header_line, "from\tto\tposterior\tin_structure"]
+        for edge in edges:
+            out.append(f"{edge['from']}\t{edge['to']}\t"
+                       f"{edge['posterior']:.10g}\t{1 if edge['in_structure'] else 0}")
+        print("\n".join(out))
+    else:
+        json.dump({
+            "version": VERSION,
+            "parameters": parameters,
+            "variables": header,
+            "samples": samples,
+            "edges": [
+                {"from": edge["from"], "to": edge["to"],
+                 "posterior": edge["posterior"],
+                 "in_structure": edge["in_structure"]}
+                for edge in edges
+            ],
+            "structure": sorted(
+                [edge["from"], edge["to"]] for edge in edges if edge["in_structure"]),
+        }, sys.stdout, indent=1)
+        print()
+
+
+def write_svg(path, header, edges, threshold):
+    """Optional: a dot drawing of the thresholded network, if graphviz exists."""
+    dot = shutil.which("dot")
+    if not dot:
+        raise ComputeError("--svg needs graphviz's `dot` on PATH")
+    lines = ["digraph bnw {", '  rankdir=LR;', '  node [shape=ellipse, fontsize=10];']
+    for name in header:
+        lines.append(f'  "{name}";')
+    for edge in edges:
+        if edge["in_structure"]:
+            weight = max(0.2, min(1.0, edge["posterior"]))
+            lines.append(
+                f'  "{edge["from"]}" -> "{edge["to"]}" '
+                f'[label="{edge["posterior"]:.2f}", weight="{weight:.2f}"];')
+    lines.append("}")
+    source = "\n".join(lines) + "\n"
+    completed = subprocess.run([dot, "-Tsvg", "-o", str(path)],
+                               input=source, text=True, capture_output=True, check=False)
+    if completed.returncode != 0:
+        raise ComputeError(f"dot failed: {(completed.stderr or '').strip()}")
+
+
+# ---------------------------------------------------------------------------
+
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog="bnw", description="Headless Bayesian network learning for GeneNetwork.")
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    learn_parser = sub.add_parser(
+        "learn", help="learn a network from a variables x samples matrix")
+    learn_parser.add_argument("--input", required=True,
+                              help="matrix: header of variable names, then one row per sample")
+    learn_parser.add_argument("--maxparents", type=int, default=DEFAULT_MAXPARENTS,
+                              help=f"maximum parents per node (default {DEFAULT_MAXPARENTS})")
+    learn_parser.add_argument("--k", type=int, default=DEFAULT_K,
+                              help=f"networks to average over (default {DEFAULT_K})")
+    learn_parser.add_argument("--tiers", type=int, default=DEFAULT_TIERS,
+                              help=f"variable ordering tiers (default {DEFAULT_TIERS})")
+    learn_parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD,
+                              help=f"edge-posterior threshold (default {DEFAULT_THRESHOLD})")
+    learn_parser.add_argument("--ban", action="append", metavar="FROM,TO",
+                              help="banned edge; repeatable")
+    learn_parser.add_argument("--white", action="append", metavar="FROM,TO",
+                              help="forced edge; repeatable")
+    learn_parser.add_argument("--format", choices=("tsv", "json"), default="tsv")
+    learn_parser.add_argument("--svg", metavar="PATH",
+                              help="also write an SVG of the thresholded network (needs graphviz)")
+    learn_parser.add_argument("--workdir", metavar="DIR",
+                              help="work here instead of a fresh temporary directory")
+    learn_parser.add_argument("--keep", action="store_true",
+                              help="keep the work directory (it is removed otherwise)")
+    learn_parser.add_argument("--bin-dir", metavar="DIR",
+                              help="where the BNW C binaries are "
+                                   "(default: ../var_lib_genenet_bnw beside this script)")
+
+    sub.add_parser("version", help="print the version and exit")
+
+    args = parser.parse_args(argv)
+    if args.command == "version":
+        print(f"bnw {VERSION}")
+        return 0
+    try:
+        learn(args)
+    except InputError as error:
+        print(f"bnw: {error}", file=sys.stderr)
+        return 2
+    except MissingBinaries as error:
+        print(f"bnw: {error}", file=sys.stderr)
+        return 77
+    except ComputeError as error:
+        print(f"bnw: {error}", file=sys.stderr)
+        return 3
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))