about summary refs log tree commit diff
path: root/genenetwork/services/bnw.scm
blob: fc05e7488fc44b44ca17da702ac9213f82ae49df (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
;;; genenetwork-machines --- Guix configuration for genenetwork machines
;;; Copyright © 2021 Bonface K. M. <bonfacemunyuki@gmail.com>
;;; Copyright © 2024 Pjotr Prins <pjotr.prins@thebird.nl>
;;;
;;; This file is part of genenetwork-machines.  See the GPL header in
;;; bnw-container.scm for the full license text.

;;; BNW (Bayesian Network Webserver) services, shared by the BNW
;;; system container (bnw-container.scm) and the BNW VM (vms/bnw-vm.scm).
;;;
;;; NOTE: we do not use nginx-service-type.  Its shepherd services
;;; daemonize nginx and drive it through a one-shot 'invoke' service
;;; (nginx-init) that never completes inside a system container
;;; (shepherd 1.0.9 wedges the nginx service).  Instead nginx runs in
;;; the FOREGROUND via make-forkexec-constructor -- exactly like
;;; php-fpm, which works fine.

(define-module (genenetwork services bnw)
  #:export (bnw-configuration
            bnw-service-type
            bnw-nginx-configuration
            bnw-nginx-service-type)
  #:use-module (gnu)
  #:use-module (gn packages bnw)
  #:use-module ((forge anubis) #:select (anubis-ai-firewall))
  #:use-module ((genenetwork services anubis)
                #:select (%anubis-challenge-all-policy
                          anubis-command))
  #:use-module (guix gexp)
  #:use-module (guix records)
  #:use-module (ice-9 match)
  #:use-module (guix build utils)
  #:use-module (gnu packages graphviz)
  #:use-module (gnu packages web)
  #:use-module (gnu packages admin)
  #:use-module (gnu services base)
  #:use-module (gnu services shepherd)
  #:use-module (gnu services web))

(define-record-type* <bnw-configuration>
  bnw-configuration
  make-bnw-configuration
  bnw-configuration?
  (package          bnw-configuration-package       ; package
                    (default bnw))
  (deploy-directory bnw-deploy-directory            ; string
                    (default "/srv/http"))
  (port             bnw-configuration-port          ; list of strings
                    (default '("8881"))))

(define bnw-activation
  (match-lambda
    (($ <bnw-configuration> package _ port)
     #~(begin
         (let ((genenet "/var/lib/genenet/bnw")
               (php-fpm-uid (passwd:uid (getpw "php-fpm")))
               (php-fpm-gid (passwd:gid (getpw "php-fpm"))))
           (mkdir-p genenet)
           (copy-recursively #$(file-append package "/var_lib_genenet_bnw") genenet)
           (for-each (lambda (file-name)
                       (make-file-writable file-name)
                       (chown file-name php-fpm-uid php-fpm-gid))
                     (find-files genenet
                                 #:directories? #t)))))))

(define bnw-service-type
  (service-type
    (name 'bnw)
    (extensions
      (list
        (service-extension activation-service-type
                           bnw-activation)
        ;; Make sure BNW doesn't get garbage collected.
        (service-extension profile-service-type
                           (compose list bnw-configuration-package))
        ;; Make sure php-fpm is instantiated.
        (service-extension php-fpm-service-type
                           (const #t))))
    (default-value (bnw-configuration))
    (description
     "Run a Bayesian Network Webserver.")))

;;; The BNW nginx front-end.  Runs nginx in the foreground under
;;; shepherd; see the file header comment for why we do not use
;;; nginx-service-type.

(define-record-type* <bnw-nginx-configuration>
  bnw-nginx-configuration
  make-bnw-nginx-configuration
  bnw-nginx-configuration?
  (package bnw-nginx-configuration-package    ; package (the web root)
           (default bnw))
  (port    bnw-nginx-configuration-port       ; string
           (default "8881"))
  ;; Protect the web UI behind the Anubis AI bot firewall (see
  ;; anubis-service-type below).  Off by default (the VM does not
  ;; need it).
  (anubis? bnw-nginx-configuration-anubis?   ; boolean
           (default #f))
  ;; Passed through to the instantiated Anubis service (see the
  ;; cookie-secure? notes there).  Only relevant when anubis? is #t.
  (anubis-cookie-secure? bnw-nginx-configuration-anubis-cookie-secure? ; boolean
                         (default #t))
  ;; Anubis policy document; the default challenges every client
  ;; (%anubis-challenge-all-policy).  The GeneNetwork workflow
  ;; endpoints and /bnw-data/ bypass Anubis at the nginx layer.
  (anubis-policy bnw-nginx-configuration-anubis-policy ; file-like or #f
                 (default %anubis-challenge-all-policy))
  ;; Proof-of-work difficulty; each increment doubles the work a
  ;; client must burn per challenge.
  (anubis-difficulty bnw-nginx-configuration-anubis-difficulty ; integer
                     (default 4)))

(define %bnw-nginx-accounts
  (list (user-group (name "nginx") (system? #t))
        (user-account
         (name "nginx")
         (group "nginx")
         (system? #t)
         (comment "nginx server user")
         (home-directory "/var/empty")
         (shell (file-append shadow "/sbin/nologin")))))

;;; Anubis, the AI bot firewall -- adapted from (forge anubis) via
;;; cgit.scm, without the least-authority wrapper (which does not work
;;; inside system containers).  Instantiated by bnw-nginx-service-type
;;; when the anubis? field is set.  Keep in sync with the anubis
;;; section in cgit.scm.

;;; XXX: keep in sync with (forge anubis) -- it hard-codes this too.
(define %anubis-unix-socket "/var/run/anubis/socket")

;; The internal nginx listener Anubis proxies to after a successful
;; challenge.  Distinct from cgit's 8081 so the two front-ends can
;; never collide within one network namespace.
(define %bnw-anubis-target-port "8082")

(define-record-type* <anubis-configuration>
  anubis-configuration
  make-anubis-configuration
  anubis-configuration?
  this-anubis-configuration
  (package anubis-configuration-package
           (default anubis-ai-firewall))
  ;; Upstream URL Anubis proxies to after a successful challenge; the
  ;; BNW nginx front-end exposes this as an internal listener.
  (target anubis-configuration-target
          (default (string-append "http://127.0.0.1:"
                                  %bnw-anubis-target-port)))
  ;; Anubis marks its challenge cookies "Secure" by default, and
  ;; browsers drop Secure cookies over plain HTTP -- the challenge
  ;; never completes and the browser hangs.  Set to #f for HTTP-only
  ;; test deployments (production is behind a TLS-terminating proxy
  ;; and should keep the default).
  (cookie-secure? anubis-configuration-cookie-secure? ; boolean
                  (default #t))
  ;; Anubis policy document; the default challenges every client
  ;; (%anubis-challenge-all-policy).  The GeneNetwork workflow
  ;; endpoints and /bnw-data/ never reach Anubis at all -- nginx
  ;; serves them directly -- so GN2's server-to-server calls keep
  ;; working for non-browser clients.
  (policy anubis-configuration-policy ; file-like or #f
          (default %anubis-challenge-all-policy))
  ;; Proof-of-work difficulty; each increment doubles the work a
  ;; client must burn per challenge.
  (difficulty anubis-configuration-difficulty ; integer
              (default 4)))

(define anubis-activation
  #~(begin
      (let ((user (getpw "nginx")))
        (mkdir-p (dirname #$%anubis-unix-socket))
        (chown (dirname #$%anubis-unix-socket)
               (passwd:uid user)
               (group:gid user)))))

(define (anubis-shepherd-service config)
  (match-record config <anubis-configuration>
    (package target cookie-secure? policy difficulty)
  (shepherd-service
   (documentation "Run the Anubis AI firewall.")
   (provision '(anubis))
   (requirement '(user-processes loopback))
   (start #~(make-forkexec-constructor
             (list #$@(anubis-command
                       (anubis-configuration-package config)
                       %anubis-unix-socket target policy difficulty
                       #:extra-arguments
                       (if cookie-secure?
                           '()
                           '("-cookie-secure=false"))))
             #:user "nginx" #:group "nginx"
             #:log-file "/var/log/anubis.log"))
   (stop #~(make-kill-destructor)))))

(define anubis-service-type
  (service-type
   (name 'anubis)
   (description "Run the Anubis AI bot firewall, listening on a unix
socket that the BNW nginx front-end proxies to.")
   (extensions
    (list (service-extension activation-service-type
                             (const anubis-activation))
          (service-extension shepherd-root-service-type
                             (compose list anubis-shepherd-service))
          ;; Keep anubis in the system profile for inspection.
          (service-extension profile-service-type
                             (lambda (config)
                               (list (anubis-configuration-package config))))))
   (default-value (anubis-configuration))))

(define (bnw-nginx-php-fastcgi-body)
  "The fastcgi statements that hand a php request to php-fpm."
  (list
   "      fastcgi_split_path_info ^(.+\\.php)(/.+)$;\n"
   ;; Mitigate https://httpoxy.org/ vulnerabilities
   "      fastcgi_param HTTP_PROXY \"\";\n"
   ;; Only pass existing php files to the backend.
   "      if (!-f $document_root$fastcgi_script_name) { return 404; }\n"
   "      fastcgi_pass unix:/var/run/php8-fpm.sock;\n"
   "      fastcgi_index index.php;\n"
   "      include " (file-append nginx "/share/nginx/conf/fastcgi.conf") ";\n"))

(define (bnw-nginx-php-block location)
  (append (list "    " location " {\n")
          (bnw-nginx-php-fastcgi-body)
          (list "    }\n")))

(define (bnw-nginx-bnw-data-block)
  ;; ^~ so the prefix wins over the static-asset regex locations below
  ;; (the rendered-network SVG downloads live here).
  (list "    location ^~ /bnw-data/ {\n"
        "      alias /var/lib/genenet/bnw/;\n"
        "    }\n"))

(define (bnw-nginx-legacy-sourcecodes-block)
  "Legacy deployment layout: upstream BNW was served under
/sourcecodes/... and old bookmarks and search-engine links still use
it.  The web root is already .../sourcecodes, so the /sourcecodes
prefix must be stripped -- otherwise nginx looks for
.../sourcecodes/sourcecodes/x and 404s.  The redirect is answered
directly (before the Anubis proxy in the public block)."
  (list
   "    # legacy upstream layout: /sourcecodes/x -> /x.  ^~ suppresses\n"
   "    # regex locations (.htm/.php) for everything below /sourcecodes/\n"
   "    location ^~ /sourcecodes/ {\n"
   "      rewrite ^/sourcecodes/(.*)$ /$1 permanent;\n"
   "    }\n"))

(define (bnw-nginx-static-block)
  "Static assets (scripts, stylesheets, fonts, images and the .htm
tutorial pages) are served directly WITHOUT the Anubis challenge in
the public block.  Two reasons: (1) the .htm pages are fetched
server-to-server by external GeneNetwork webservices -- a challenge
needs a javascript browser, so a plain GET would hang; (2) the pages
load their scripts and styles from /scripts/ and /fonts/, and a
challenge response is text/html, which browsers reject with MIME-type
errors, breaking the page.  In the plain/Anubis-less config this
block is a no-op (everything is already direct)."
  (list
   "    # static assets: no Anubis (external webservice access; a\n"
   "    # challenge response is text/html and breaks script/css loads)\n"
   "    location ~* \\.(js|css|htm|html|png|gif|jpe?g|svg|ico|ttf|otf|woff2?|eot|txt)$ {\n"
   "      try_files $uri =404;\n"
   "    }\n"))

(define (bnw-nginx-location-blocks)
  "The application locations served by the plain server -- and, with
Anubis enabled, by the internal server behind the challenge."
  (append
   (bnw-nginx-bnw-data-block)
   (bnw-nginx-legacy-sourcecodes-block)
   (list
    "    location / {\n"
    "      try_files $uri $uri/ =404;\n"
    "    }\n")
   (bnw-nginx-php-block "location ~ \\.php$")))

;; The GeneNetwork workflow endpoints that stay DIRECT in the public
;; server block -- Anubis must not sit between GN2 and BNW, and the
;; default modeling flow is a chain of php pages that forwards via
;; window.open; a challenge mid-chain turns into a blank page (the
;; runner pages output almost nothing -- they just forward):
;;
;;   bn_genenet.php (handshake) -> executionprogress_default.php
;;   (progress bar) -> execute_bn_gom.php (structure learning runner)
;;   -> layout_svg_no.php / layout_svg_wt.php (rendered network)
;;
;; graphviz_structure.php is the rendering endpoint external services
;; call.  GN2's form posts to the LEGACY path /sourcecodes/bn_genenet.php
;; (upstream deployment layout), so each endpoint is served direct
;; under BOTH prefixes.
(define %bnw-direct-php-endpoints
  '("bn_genenet.php"
    "executionprogress_default.php"
    "execute_bn_gom.php"
    "graphviz_structure.php"
    "layout_svg_no.php"
    "layout_svg_wt.php"))

(define (bnw-nginx-direct-endpoint-blocks)
  "For each workflow endpoint serve it DIRECT under the root prefix
(generic php body) and under the legacy /sourcecodes/ prefix (explicit
SCRIPT_FILENAME -- the generic body's if(!-f ...) check and
SCRIPT_FILENAME are relative to the web root, which already IS
.../sourcecodes, so the legacy prefix would double up and 404;
fastcgi_params is used instead of fastcgi.conf because the latter
would override SCRIPT_FILENAME).  Exact-match locations win over the
^~ /sourcecodes/ redirect."
  (apply append
   (append
    (map (lambda (script)
           (bnw-nginx-php-block (string-append "location = /" script)))
         %bnw-direct-php-endpoints)
    (map (lambda (script)
           (list
            "    location = /sourcecodes/" script " {\n"
            "      include " (file-append nginx "/share/nginx/conf/fastcgi_params") ";\n"
            "      fastcgi_param SCRIPT_FILENAME $document_root/" script ";\n"
            ;; Mitigate https://httpoxy.org/ vulnerabilities
            "      fastcgi_param HTTP_PROXY \"\";\n"
            "      fastcgi_pass unix:/var/run/php8-fpm.sock;\n"
            "    }\n"))
         %bnw-direct-php-endpoints))))

(define (bnw-nginx-conf-file config)
  "Render the whole nginx.conf.  Without Anubis: one server block with
the php-fpm back-end.  With Anubis: the PUBLIC server block proxies to
Anubis, which challenges AI bots and forwards to the INTERNAL server
block (localhost only); the GeneNetwork server-to-server endpoints
(bn_genenet.php, executionprogress_default.php and /bnw-data/) stay
direct so GN2 is never challenged."
  (match-record config <bnw-nginx-configuration> (package port anubis?)
    (apply mixed-text-file "nginx.conf"
      (append
       (list
        "user nginx nginx;\n"
        "daemon off;\n"
        "pid /var/run/nginx.pid;\n"
        "error_log /var/log/nginx/error.log;\n"
        "events { worker_connections 64; }\n"
        "http {\n"
        "  include " (file-append nginx "/share/nginx/conf/mime.types") ";\n"
        "  default_type application/octet-stream;\n"
        "  access_log /var/log/nginx/access.log;\n"
        "  sendfile on;\n"
        ;; Redirects (the legacy /sourcecodes/ rewrite) must be
        ;; relative: with absolute_redirect on (the default) nginx
        ;; builds Location from its own listen port (8881), which is
        ;; not reachable through the TLS-terminating host proxy.
        "  absolute_redirect off;\n"
        "  client_body_temp_path /var/run/nginx/client_body_temp;\n"
        "  proxy_temp_path /var/run/nginx/proxy_temp;\n"
        "  fastcgi_temp_path /var/run/nginx/fastcgi_temp;\n"
        "  uwsgi_temp_path /var/run/nginx/uwsgi_temp;\n"
        "  scgi_temp_path /var/run/nginx/scgi_temp;\n"
        "  server {\n"
        "    listen " port ";\n"
        "    server_name _;\n"
        "    root " (file-append package "/sourcecodes") ";\n"
        "    index bn_genenet.php index.html;\n")
       (if anubis?
           (append
            (list
             ;; PUBLIC: the GeneNetwork workflow endpoints stay direct
             ;; (no Anubis in between -- GN2 is not a browser, and the
             ;; modeling flow forwards through php runner pages);
             ;; everything else goes through Anubis.
             "    # direct workflow endpoints, root prefix\n")
            (bnw-nginx-direct-endpoint-blocks)
            (list
             "    # end direct workflow endpoints\n")
            (bnw-nginx-bnw-data-block)
            (bnw-nginx-legacy-sourcecodes-block)
            (bnw-nginx-static-block)
            ;; The legacy /sourcecodes/ prefix is handled inside
            ;; bnw-nginx-direct-endpoint-blocks (exact-match locations
            ;; win over the redirect below).
            (list
             "    location / {\n"
             "      proxy_pass http://unix:" %anubis-unix-socket ";\n"
             "      proxy_set_header X-Real-IP $remote_addr;\n"
             "      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"
             "      proxy_set_header X-Forwarded-Proto $scheme;\n"
             "      proxy_set_header Host $http_host;\n"
             "    }\n"
             "  }\n"
             ;; INTERNAL: the plain BNW server, reachable only via
             ;; Anubis and from localhost.
             "  server {\n"
             "    listen 127.0.0.1:" %bnw-anubis-target-port ";\n"
             "    server_name _;\n"
             "    root " (file-append package "/sourcecodes") ";\n"
             "    index bn_genenet.php index.html;\n")
            (bnw-nginx-location-blocks)
            (list
             "  }\n"
             "}\n"))
           (append
            (bnw-nginx-location-blocks)
            (list
             "  }\n"
             "}\n")))))))

(define bnw-nginx-activation
  #~(begin
      (use-modules (guix build utils))
      ;; The network layout php pages call /usr/bin/dot with an
      ;; absolute path; expose the container's graphviz there.
      (mkdir-p "/usr/bin")
      (false-if-exception
       (symlink #$(file-append graphviz "/bin/dot") "/usr/bin/dot"))
      (mkdir-p "/var/log/nginx")
      ;; nginx needs writable temp directories; give them a home
      ;; outside the read-only store.
      (mkdir-p "/var/run/nginx/client_body_temp")
      (mkdir-p "/var/run/nginx/proxy_temp")
      (mkdir-p "/var/run/nginx/fastcgi_temp")
      (mkdir-p "/var/run/nginx/uwsgi_temp")
      (mkdir-p "/var/run/nginx/scgi_temp")))

(define (bnw-nginx-shepherd-service config)
  "Return the nginx shepherd service -- plus, when anubis? is set, the
Anubis shepherd service bundled with it.\n\nBundling (instead of
routing Anubis through a service extension) is deliberate: for a
service type without compose/extend procedures, fold-services DISCARDS
the values contributed by extensions and instantiate-missing-services
creates a default-valued instance instead -- an anubis-configuration
passed through an extension would silently be replaced by the default
(and a default Anubis would even run when anubis? is #f)."
  (append
   (list (shepherd-service
          (provision '(nginx))
          (documentation "Run the BNW nginx front-end in the foreground.")
          (requirement '(user-processes loopback))
          (start #~(make-forkexec-constructor
                    (list #$(file-append nginx "/sbin/nginx")
                          "-c" #$(bnw-nginx-conf-file config))
                    #:pid-file "/var/run/nginx.pid"))
          (stop #~(make-kill-destructor))
          (actions (list (shepherd-configuration-action
                          (bnw-nginx-conf-file config))))))
   (if (bnw-nginx-configuration-anubis? config)
       (list (anubis-shepherd-service
              (anubis-configuration
               (cookie-secure?
                (bnw-nginx-configuration-anubis-cookie-secure? config))
               (policy (bnw-nginx-configuration-anubis-policy config))
               (difficulty
                (bnw-nginx-configuration-anubis-difficulty config)))))
       '())))

(define bnw-nginx-service-type
  (service-type
   (name 'bnw-nginx)
   (extensions
    (list
     (service-extension shepherd-root-service-type
                        bnw-nginx-shepherd-service)
     (service-extension activation-service-type
                        (lambda (config)
                          (if (bnw-nginx-configuration-anubis? config)
                              #~(begin #$bnw-nginx-activation
                                      #$anubis-activation)
                              bnw-nginx-activation)))
     (service-extension account-service-type
                        (const %bnw-nginx-accounts))
     ;; The network-layout php pages call /usr/bin/dot; keep graphviz
     ;; in the system profile so the closure carries it.  With Anubis
     ;; also keep the anubis binary in the profile for inspection.
     (service-extension profile-service-type
                        (lambda (config)
                          (if (bnw-nginx-configuration-anubis? config)
                              (list graphviz anubis-ai-firewall)
                              (list graphviz))))))
   (default-value (bnw-nginx-configuration))
   (description
    "Run nginx in the foreground as BNW's web front-end.")))