1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
|
#!/usr/bin/env python3
"""HTTP functional tests for the BNW system container.
These tests talk to the web front-end over plain HTTP only -- no
headless browser is needed. They assume the container is running with
its nginx on http://127.0.0.1:8881 (protected by the Anubis AI-bot
firewall) and php-fpm executing the BNW php pages.
Usage:
BASE_URL=http://127.0.0.1:8881 python3 bnw_http_tests.py
"""
import os
import sys
import urllib.error
import urllib.parse
import urllib.request
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8881")
TESTS = []
def test(name):
def wrap(fn):
TESTS.append((name, fn))
return fn
return wrap
class Response:
def __init__(self, status, body, headers):
self.status = status
self.body = body
self.headers = headers
def has(self, needle):
return needle in self.body
def request(path, data=None, method=None, timeout=20, follow=True):
"""Request with a non-browser user agent: Anubis lets these
through to the php back-end, which is what the tests exercise."""
return request_ua(path, "bnw-test/1.0", data=data, method=method,
timeout=timeout, follow=follow)
def request_ua(path, ua, data=None, method=None, timeout=20, follow=True):
url = BASE_URL + path
req = urllib.request.Request(url, data=data, method=method,
headers={"User-Agent": ua})
try:
opener = urllib.request.build_opener() if follow else _no_redirect_opener()
with opener.open(req, timeout=timeout) as r:
return Response(r.status,
r.read().decode("utf-8", "replace"),
dict(r.headers))
except urllib.error.HTTPError as e:
return Response(e.code,
e.read().decode("utf-8", "replace"),
dict(e.headers))
class _NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
def _no_redirect_opener():
return urllib.request.build_opener(_NoRedirect)
# --- static front-end ---------------------------------------------------
@test("t01 front page returns 200 with a non-empty body")
def t01():
r = request("/")
return r.status == 200 and r.has("<html")
@test("t02 front page shows the BNW title")
def t02():
r = request("/")
return r.status == 200 and r.has("Bayesian Network Webserver")
@test("t03 stylesheet served with text/css")
def t03():
r = request("/my_new_style.css")
return (r.status == 200
and r.headers.get("Content-Type", "").startswith("text/css"))
@test("t04 bundled javascript library is served")
def t04():
r = request("/scripts/jquery.min.js")
return r.status == 200 and len(r.body) > 1000
# --- php execution --------------------------------------------------------
@test("t05 diag.php executes php and reports the request method")
def t05():
r = request("/diag.php")
return (r.status == 200
and r.has("cwd=")
and r.has("method=GET"))
@test("t06 POST reaches the php back-end through Anubis")
def t06():
r = request("/diag.php", data=b"", method="POST")
return r.status == 200 and r.has("method=POST")
@test("t07 unknown php script gives a clean 404, not a 5xx")
def t07():
r = request("/no-such-script.php")
return r.status == 404
# --- Anubis AI-bot firewall ----------------------------------------------
# A browser-like user agent: Anubis challenges browsers, while other
# agents pass through to php untouched.
BROWSER_UA = ("Mozilla/5.0 (X11; Linux x86_64; rv:130.0) "
"Gecko/20100101 Firefox/130.0")
@test("t08 browser user agent is challenged by Anubis")
def t08():
r = request_ua("/", BROWSER_UA)
return (r.status == 200
and r.has("Making sure")
and r.has("not a bot"))
@test("t09 non-browser user agent passes Anubis through to BNW")
def t09():
r = request_ua("/", "bnw-anubis-check/1.0")
return r.status == 200 and r.has("Bayesian Network Webserver")
# --- GeneNetwork (GN2) server-to-server endpoints -----------------------
#
# GN2 posts My_Gennenet to bn_genenet.php, links straight to
# executionprogress_default.php?My_key=... and fetches rendered
# networks from /bnw-data/. These endpoints are EXEMPT from Anubis in
# the nginx public block -- the tests use a BROWSER user agent to
# prove Anubis really is out of the path (a browser UA on any other
# path gets the challenge).
GN2_KEY = "gna" # keys must be exactly 3 letters
GN2_DATA = ("Geno1,Geno2,Trait1,Trait2;"
"1,2,0.1,0.2;"
"2,1,0.3,0.4;"
"1,2,0.5,0.6")
@test("t10 GN2 handshake POST to bn_genenet.php bypasses Anubis")
def t10():
r = request_ua("/bn_genenet.php", BROWSER_UA,
data=urllib.parse.urlencode(
{"My_Gennenet": GN2_DATA,
"My_key": GN2_KEY}).encode(),
method="POST")
return (r.status == 200
and r.has("Perform Bayesian network modeling")
and r.has("var data1 = ")
and "Making sure" not in r.body)
@test("t11 GN2 data-load POST to bn_genenet.php bypasses Anubis")
def t11():
tsv = GN2_DATA.replace(",", "\t").replace(";", "\n")
r = request_ua("/bn_genenet.php", BROWSER_UA,
data=urllib.parse.urlencode(
{"searchkey": tsv,
"My_key": GN2_KEY,
"MyUpload": "NO"}).encode(),
method="POST")
return (r.status == 200
and r.has("Perform Bayesian network modeling")
and "Making sure" not in r.body)
@test("t12 executionprogress_default.php bypasses Anubis")
def t12():
r = request_ua(f"/executionprogress_default.php?My_key={GN2_KEY}",
BROWSER_UA)
return r.status == 200 and "Making sure" not in r.body
@test("t13 /bnw-data/ bypasses Anubis (404 without challenge)")
def t13():
r = request_ua("/bnw-data/no-such-file.svg", BROWSER_UA)
return (r.status == 404
and "Making sure" not in r.body)
@test("t14 legacy /sourcecodes/ URLs redirect to the web root")
def t14():
# Old bookmarks use the upstream layout /sourcecodes/...; the web
# root is already .../sourcecodes, so the prefix is stripped with
# a 301. Query strings must survive.
r = request("/sourcecodes/BNW_workflow_net1.htm", follow=False)
if r.status != 301:
return False
loc = r.headers.get("Location", "")
r2 = request("/sourcecodes/foo.php?My_key=abc", follow=False)
# Location must be RELATIVE: an absolute Location would carry the
# container's own listen port (e.g. http://host:8881/...), which
# is unreachable through the TLS-terminating host proxy.
return (loc == "/BNW_workflow_net1.htm"
and r2.status == 301
and r2.headers.get("Location", "") == "/foo.php?My_key=abc")
@test("t15 tutorial .htm pages bypass Anubis (browser UA gets content)")
def t15():
# The .htm tutorial pages are fetched server-to-server by external
# GeneNetwork webservices, so they must be exempt from the Anubis
# challenge: a browser UA must get the real page, not the challenge
# (a plain server-to-server GET would hang on the javascript
# proof-of-work).
r = request_ua("/BNW_workflow_net1.htm", BROWSER_UA)
return (r.status == 200
and r.has("<html")
and "Making sure" not in r.body)
@test("t16 GN2 legacy-path POST is served, not redirected")
def t16():
# GN2's bnw_page.html posts to the LEGACY path
# /sourcecodes/bn_genenet.php. A 301 there would make the browser
# resend as GET and silently drop the payload (the classic
# "Loading data from GeneNetwork" hang). The endpoint must be
# served directly, with a browser UA (proving Anubis is out of the
# path), and the handshake must complete.
r = request_ua("/sourcecodes/bn_genenet.php", BROWSER_UA,
data=urllib.parse.urlencode(
{"My_Gennenet": GN2_DATA,
"My_key": "gnb"}).encode(),
method="POST")
return (r.status == 200
and r.has("Perform Bayesian network modeling")
and r.has("var data1 = ")
and "Making sure" not in r.body)
@test("t17 default modeling flow runs browser-UA end to end, no challenge")
def t17():
# The default flow after the prepared page is a chain of php pages
# forwarding via window.open: executionprogress_default.php
# (progress bar) -> execute_bn_gom.php (structure learning runner,
# near-empty body by design) -> layout_svg_no.php (rendered
# network). A challenge mid-chain turns into a permanent blank
# page. Every hop must serve real content to a browser UA.
key = "gnb"
r1 = request_ua(f"/executionprogress_default.php?My_key={key}",
BROWSER_UA)
r2 = request_ua(f"/execute_bn_gom.php?My_key={key}", BROWSER_UA)
r3 = request_ua(f"/layout_svg_no.php?My_key={key}", BROWSER_UA)
return (r1.status == 200 and "Making sure" not in r1.body
and r2.status == 200 and "Making sure" not in r2.body
and r3.status == 200 and "Making sure" not in r3.body
and r3.has("svg"))
@test("t18 graphviz_structure.php (rendering endpoint) bypasses Anubis")
def t18():
r = request_ua("/graphviz_structure.php?My_key=gnb", BROWSER_UA)
return (r.status == 200
and ("digraph" in r.body or "svg" in r.body)
and "Making sure" not in r.body)
@test("t19 static assets bypass Anubis with real MIME types")
def t19():
# The pages load their scripts/styles from /scripts/ and /fonts/;
# a challenge response is text/html, which browsers reject with
# MIME-type errors, breaking the page ("expected expression, got
# '<'"). Every static asset must come back with its real type,
# even for a browser UA.
checks = [
("/scripts/jquery.min.js", "javascript"),
("/scripts/d3.v4.min.js", "javascript"),
("/scripts/accordion.js", "javascript"),
("/scripts/font-awesome.css", "css"),
("/BNW_overview.png", "image"),
]
for path, kind in checks:
r = request_ua(path, BROWSER_UA)
ctype = r.headers.get("Content-Type", "")
if r.status != 200 or kind not in ctype:
return False
if "Making sure" in r.body:
return False
return True
def main():
passed = failed = 0
for name, fn in TESTS:
try:
ok = fn()
except Exception as e: # noqa: BLE001
ok = False
print(f"FAIL {name}: exception {e!r}")
if ok:
passed += 1
print(f"PASS {name}")
else:
failed += 1
print(f"FAIL {name}")
print(f"== {passed} passed, {failed} failed ==")
return 1 if failed else 0
if __name__ == "__main__":
sys.exit(main())
|