1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
|
#!/usr/bin/env python3
"""bnw --- headless Bayesian network learning for GeneNetwork
The web application (PHP) builds a handful of files in /var/lib/genenet/bnw
and then runs three C programs over them. This program does the same
computation without the web application: one matrix in, edges out, a private
working directory per invocation, and nothing that needs a session key.
bnw learn --input matrix.tsv [options] # learn a network
bnw version
The matrix is variables × samples: a header line naming the variables, then
one line per sample. That is exactly what BNW's own example datasets carry
(sourcecodes/example_datasets/), and what GeneNetwork4 can build from a set of
traits or probes.
Output (--format tsv, the default) goes to stdout:
# bnw <version> variables=8 samples=200 maxparents=4 k=1 tiers=3 threshold=0.5
from<TAB>to<TAB>posterior<TAB>in_structure
Geno1<TAB>Trait1<TAB>1<TAB>1
...
Every candidate edge is reported, with its posterior, and `in_structure` says
whether it survived the threshold. A posterior is not a p-value: it is the
share of the k-best networks that contain the edge, so it answers "how often
did the search put this arrow there", not "how surprising is it".
Exit codes: 0 ok, 2 bad input (the message names the problem), 3 the
computation failed (the message is the C program's own), 77 the BNW binaries
are not installed -- the conventional "skip" for a test suite.
"""
import argparse
import json
import os
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
VERSION = "0.1.0"
# BNW's own defaults, from sourcecodes/create_tiers_gom_part1.php: the web
# application writes these into <key>parent.txt, <key>k.txt and <key>thr.txt
# unless the user changes them in its settings page.
DEFAULT_MAXPARENTS = 4
DEFAULT_K = 1
DEFAULT_TIERS = 3
DEFAULT_THRESHOLD = 0.5
# Where the C programs live, relative to this file: ../var_lib_genenet_bnw,
# built by var_lib_genenet_bnw/build.sh. --bin-dir overrides it.
HERE = Path(__file__).resolve().parent
DEFAULT_BIN_DIR = HERE.parent / "var_lib_genenet_bnw"
class InputError(Exception):
"""Bad input: exit 2, with a message the caller can act on."""
class ComputeError(Exception):
"""The C programs failed: exit 3, carrying their own message."""
class MissingBinaries(ComputeError):
"""The BNW binaries are not installed: exit 77, the conventional 'skip'."""
# ---------------------------------------------------------------------------
# Input
def read_matrix(path, min_variables=2):
"""Read the variables × samples matrix, and check it before any C runs.
A malformed matrix used to travel all the way into the C programs and come
back as a segmentation fault or an empty result; every check here is one
that failed silently before.
"""
try:
text = Path(path).read_text()
except OSError as error:
raise InputError(f"cannot read {path}: {error.strerror}") from error
lines = [line.rstrip("\r") for line in text.splitlines()]
lines = [line for line in lines if line.strip()]
if not lines:
raise InputError(f"{path} is empty")
header = [cell.strip() for cell in lines[0].split("\t")]
if len(header) == 1:
# A comma-separated header with tab-separated rows is a common mixup.
header = [cell.strip() for cell in lines[0].split(",")]
if len(header) > 1:
raise InputError(
"the header is comma-separated but the rows are not: BNW's "
"matrix is tab-separated throughout (header and data)")
if len(header) < min_variables:
raise InputError(
f"the header names {len(header)} variable(s); a network needs at "
f"least {min_variables}")
seen = {}
for index, name in enumerate(header):
if not name:
raise InputError(f"empty variable name in column {index + 1}")
if name in seen:
raise InputError(
f"variable {name!r} appears twice (columns "
f"{seen[name] + 1} and {index + 1}); names must be unique — "
"rename one copy")
seen[name] = index
rows = []
for number, line in enumerate(lines[1:], start=2):
cells = line.split("\t")
if len(cells) != len(header):
raise InputError(
f"row {number} has {len(cells)} field(s) but the header has "
f"{len(header)}")
row = []
for name, cell in zip(header, cells):
cell = cell.strip()
try:
row.append(float(cell))
except ValueError:
raise InputError(
f"row {number}, variable {name!r}: {cell!r} is not a "
"number (BNW's continuous model wants numeric values)") from None
if row[-1] != row[-1] or row[-1] in (float("inf"), float("-inf")):
raise InputError(
f"row {number}, variable {name!r}: {cell!r} is not finite")
rows.append(row)
if len(rows) < 2:
raise InputError(
f"only {len(rows)} sample row(s): a network cannot be learned from "
"fewer than 2 samples")
return header, rows
def write_input(directory, header, rows):
"""Write continuous_input.txt: names first, then one line per sample.
network_score reads the names with fscanf("%s") and then the numbers, so
the file has to be exactly this shape -- tabs in the header, tabs in the
data, no trailing blank line.
"""
path = Path(directory) / "continuous_input.txt"
with path.open("w") as handle:
handle.write("\t".join(header) + "\n")
# The type line: one integer per variable, 1 = continuous, n = n-level
# discrete. network_score reads it before the data (and counts rows as
# lines - 3, so without it every value shifts by one). This driver is
# continuous-only for now, so it is all 1s; the web application writes
# the level counts because its upload path accepts discrete columns.
handle.write("\t".join("1" for _ in header) + "\n")
for row in rows:
handle.write("\t".join(repr(value) for value in row) + "\n")
return path
def write_constraints(directory, banned, white):
"""Empty ban/white files are what the web application writes by default."""
ban_path = Path(directory) / "ban.txt"
white_path = Path(directory) / "white.txt"
for path, edges, what in ((ban_path, banned, "ban"), (white_path, white, "white")):
if edges is None:
path.write_text("")
continue
lines = []
for edge in edges:
parts = edge.replace("->", ",").split(",")
if len(parts) != 2 or not all(part.strip() for part in parts):
raise InputError(f"--{what} {edge!r} is not 'from,to'")
lines.append(f"{parts[0].strip()},{parts[1].strip()}")
path.write_text("\n".join(lines) + ("\n" if lines else ""))
return ban_path, white_path
# ---------------------------------------------------------------------------
# Computation
def binaries(bin_dir):
"""Locate the four programs the pipeline needs, or say what is missing."""
bin_dir = Path(bin_dir)
wanted = {
"network_score": bin_dir / "network_score",
"structure": bin_dir / "structure",
"get_kbest_parents": bin_dir / "k-best" / "src" / "get_kbest_parents",
"get_kbest_nets": bin_dir / "k-best" / "src" / "get_kbest_nets",
}
# Build output and checkout layout differ: build.sh puts network_score and
# structure at the top and the k-best binaries under k-best/src.
missing = [name for name, path in wanted.items() if not os.access(path, os.X_OK)]
if missing:
raise MissingBinaries(
"missing BNW binaries: " + ", ".join(missing) +
f"\n looked in {bin_dir}" +
"\n build them with var_lib_genenet_bnw/build.sh (needs gcc, g++ and libRmath)")
return wanted
def run(command, what, cwd):
"""Run one stage, and fail with *its* message rather than a traceback."""
completed = subprocess.run(
command, cwd=str(cwd), capture_output=True, text=True, check=False)
if completed.returncode != 0:
message = (completed.stderr or completed.stdout or "").strip()
raise ComputeError(
f"{what} failed (exit {completed.returncode})" +
(f": {message}" if message else ""))
return completed
def learn(args):
header, rows = read_matrix(args.input)
if args.maxparents < 0:
raise InputError("--maxparents must be >= 0")
if args.k < 1:
raise InputError("--k must be >= 1")
if not 0.0 <= args.threshold <= 1.0:
raise InputError("--threshold is an edge posterior in [0, 1]")
workdir = Path(args.workdir) if args.workdir else Path(tempfile.mkdtemp(prefix="bnw-"))
workdir.mkdir(parents=True, exist_ok=True)
bin_dir = Path(args.bin_dir) if args.bin_dir else DEFAULT_BIN_DIR
programs = binaries(bin_dir)
try:
input_path = write_input(workdir, header, rows)
ban_path, white_path = write_constraints(workdir, args.ban, args.white)
# Stage 1: local scores. argv[5] is a directory prefix -- the program
# writes one binary file of doubles per node into it (0, 1, 2, ...).
run([str(programs["network_score"]), str(input_path), str(ban_path),
str(white_path), str(args.maxparents), str(workdir)],
"network_score", workdir)
# Stage 2: the k-best search, which writes postProbEachEdge.txt.
run([str(programs["get_kbest_parents"]), str(len(header)), str(workdir),
str(args.k)], "get_kbest_parents", workdir)
run([str(programs["get_kbest_nets"]), str(len(header)), str(workdir),
str(args.k)], "get_kbest_nets", workdir)
# Stage 3: threshold the posteriors into an adjacency.
structure_tmp = workdir / "structure_input_temp.txt"
structure_out = workdir / "structure_input.txt"
# `structure' exits with whatever its uninitialised return value
# happens to be -- the web application ignores that and reads the file,
# so do the same, and only fail if the file is missing or empty.
subprocess.run([str(programs["structure"]), str(input_path),
str(workdir / "postProbEachEdge.txt"), str(structure_tmp),
str(structure_out), str(args.threshold)],
cwd=str(workdir), capture_output=True, text=True, check=False)
if not structure_out.exists() or structure_out.stat().st_size == 0:
raise ComputeError(
f"structure wrote no {structure_out.name}; the network could not "
"be thresholded (is the posterior table empty?)")
edges = read_edges(workdir / "postProbEachEdge.txt", header)
structure = read_structure(structure_out, header)
for edge in edges:
edge["in_structure"] = (edge["from"], edge["to"]) in structure
if args.svg:
write_svg(args.svg, header, edges, args.threshold)
emit(edges, header, len(rows), args)
finally:
if args.keep:
print(f"work directory kept: {workdir}", file=sys.stderr)
elif not args.workdir:
shutil.rmtree(workdir, ignore_errors=True)
def read_edges(path, header):
"""postProbEachEdge.txt: lines 'j -> i <TAB> posterior', node numbers."""
edges = []
try:
lines = Path(path).read_text().splitlines()
except OSError as error:
raise ComputeError(f"{path}: {error.strerror}") from error
for line in lines:
line = line.strip()
if not line:
continue
try:
arrow, posterior = line.split("\t")
source, target = arrow.split("->")
from_index, to_index = int(source), int(target)
except ValueError:
raise ComputeError(
f"unexpected line in {path.name}: {line!r}") from None
if not (0 <= from_index < len(header) and 0 <= to_index < len(header)):
raise ComputeError(
f"{path.name} names node {from_index}->{to_index}, outside "
f"the {len(header)} variables")
edges.append({
"from": header[from_index],
"to": header[to_index],
"posterior": float(posterior),
})
edges = [edge for edge in edges if edge["from"] != edge["to"]]
edges.sort(key=lambda edge: (-edge["posterior"], edge["from"], edge["to"]))
return edges
def read_structure(path, header):
"""structure_input.txt: a header of names, then a 0/1 adjacency matrix."""
try:
lines = [line.rstrip("\r") for line in Path(path).read_text().splitlines() if line.strip()]
except OSError as error:
raise ComputeError(f"{path}: {error.strerror}") from error
if len(lines) != len(header) + 1:
raise ComputeError(
f"{path.name} has {len(lines)} line(s); expected {len(header) + 1}")
present = set()
for row_index, line in enumerate(lines[1:]):
cells = [cell for cell in line.split("\t") if cell != ""]
if len(cells) != len(header):
raise ComputeError(
f"{path.name} row {row_index + 1} has {len(cells)} column(s); "
f"expected {len(header)}")
for col_index, cell in enumerate(cells):
if cell.strip() == "1":
present.add((header[row_index], header[col_index]))
return present
# ---------------------------------------------------------------------------
# Output
def emit(edges, header, samples, args):
parameters = {
"maxparents": args.maxparents,
"k": args.k,
"tiers": args.tiers,
"threshold": args.threshold,
}
header_line = (
f"# bnw {VERSION} variables={len(header)} samples={samples} "
f"maxparents={args.maxparents} k={args.k} tiers={args.tiers} "
f"threshold={args.threshold}")
if args.format == "tsv":
out = [header_line, "from\tto\tposterior\tin_structure"]
for edge in edges:
out.append(f"{edge['from']}\t{edge['to']}\t"
f"{edge['posterior']:.10g}\t{1 if edge['in_structure'] else 0}")
print("\n".join(out))
else:
json.dump({
"version": VERSION,
"parameters": parameters,
"variables": header,
"samples": samples,
"edges": [
{"from": edge["from"], "to": edge["to"],
"posterior": edge["posterior"],
"in_structure": edge["in_structure"]}
for edge in edges
],
"structure": sorted(
[edge["from"], edge["to"]] for edge in edges if edge["in_structure"]),
}, sys.stdout, indent=1)
print()
def write_svg(path, header, edges, threshold):
"""Optional: a dot drawing of the thresholded network, if graphviz exists."""
dot = shutil.which("dot")
if not dot:
raise ComputeError("--svg needs graphviz's `dot` on PATH")
lines = ["digraph bnw {", ' rankdir=LR;', ' node [shape=ellipse, fontsize=10];']
for name in header:
lines.append(f' "{name}";')
for edge in edges:
if edge["in_structure"]:
weight = max(0.2, min(1.0, edge["posterior"]))
lines.append(
f' "{edge["from"]}" -> "{edge["to"]}" '
f'[label="{edge["posterior"]:.2f}", weight="{weight:.2f}"];')
lines.append("}")
source = "\n".join(lines) + "\n"
completed = subprocess.run([dot, "-Tsvg", "-o", str(path)],
input=source, text=True, capture_output=True, check=False)
if completed.returncode != 0:
raise ComputeError(f"dot failed: {(completed.stderr or '').strip()}")
# ---------------------------------------------------------------------------
def main(argv):
parser = argparse.ArgumentParser(
prog="bnw", description="Headless Bayesian network learning for GeneNetwork.")
sub = parser.add_subparsers(dest="command", required=True)
learn_parser = sub.add_parser(
"learn", help="learn a network from a variables x samples matrix")
learn_parser.add_argument("--input", required=True,
help="matrix: header of variable names, then one row per sample")
learn_parser.add_argument("--maxparents", type=int, default=DEFAULT_MAXPARENTS,
help=f"maximum parents per node (default {DEFAULT_MAXPARENTS})")
learn_parser.add_argument("--k", type=int, default=DEFAULT_K,
help=f"networks to average over (default {DEFAULT_K})")
learn_parser.add_argument("--tiers", type=int, default=DEFAULT_TIERS,
help=f"variable ordering tiers (default {DEFAULT_TIERS})")
learn_parser.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD,
help=f"edge-posterior threshold (default {DEFAULT_THRESHOLD})")
learn_parser.add_argument("--ban", action="append", metavar="FROM,TO",
help="banned edge; repeatable")
learn_parser.add_argument("--white", action="append", metavar="FROM,TO",
help="forced edge; repeatable")
learn_parser.add_argument("--format", choices=("tsv", "json"), default="tsv")
learn_parser.add_argument("--svg", metavar="PATH",
help="also write an SVG of the thresholded network (needs graphviz)")
learn_parser.add_argument("--workdir", metavar="DIR",
help="work here instead of a fresh temporary directory")
learn_parser.add_argument("--keep", action="store_true",
help="keep the work directory (it is removed otherwise)")
learn_parser.add_argument("--bin-dir", metavar="DIR",
help="where the BNW C binaries are "
"(default: ../var_lib_genenet_bnw beside this script)")
sub.add_parser("version", help="print the version and exit")
args = parser.parse_args(argv)
if args.command == "version":
print(f"bnw {VERSION}")
return 0
try:
learn(args)
except InputError as error:
print(f"bnw: {error}", file=sys.stderr)
return 2
except MissingBinaries as error:
print(f"bnw: {error}", file=sys.stderr)
return 77
except ComputeError as error:
print(f"bnw: {error}", file=sys.stderr)
return 3
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
|