about summary refs log tree commit diff
path: root/genenetwork/services/anubis.scm
blob: af261ff007ee784c237cd249f87573a71fb59819 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
;;; genenetwork-machines --- Guix configuration for genenetwork machines
;;; Copyright © 2026 the GeneNetwork team
;;;
;;; This file is part of genenetwork-machines.
;;;
;;; genenetwork-machines is free software: you can redistribute it
;;; and/or modify it under the terms of the GNU General Public License
;;; as published by the Free Software Foundation, either version 3 of
;;; the License, or (at your option) any later version.
;;;
;;; genenetwork-machines is distributed in the hope that it will be
;;; useful, but WITHOUT ANY WARRANTY; without even the implied
;;; warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
;;; See the GNU General Public License for more details.
;;;
;;; You should have received a copy of the GNU General Public License
;;; along with genenetwork-machines.  If not, see
;;; <https://www.gnu.org/licenses/>.

;;; Anubis policy shared by the genenetwork front-ends (cgit, BNW,
;;; GeneCup).

(define-module (genenetwork services anubis)
  #:export (%anubis-challenge-all-policy
            anubis-command
            %anubis-unix-socket
            anubis-configuration
            anubis-configuration?
            anubis-configuration-target
            anubis-service-type)
  #:use-module ((forge anubis) #:select (anubis-ai-firewall))
  #:use-module (gnu services)
  #:use-module (gnu services shepherd)
  #:use-module (guix gexp)
  #:use-module (guix records)
  #:use-module (ice-9 match))

(define %anubis-challenge-all-policy
  ;; Challenge EVERYTHING that reaches Anubis -- browsers solve the
  ;; proof-of-work challenge and get a week-long cookie, every other
  ;; client (curl, wget, feed readers, AI crawlers, ...) stays blocked
  ;; behind the challenge page.  Server-to-server endpoints that must
  ;; keep working for non-browser clients (git clone, the GeneNetwork
  ;; workflow posts) are exempted at the nginx layer instead and never
  ;; reach Anubis at all.  Without this policy Anubis uses its
  ;; built-in default, which passes non-browser user agents through
  ;; unchallenged: in its weight-based threshold system they carry
  ;; zero request weight.
  ;;
  ;; The one exception is raw file downloads through a cgit instance
  ;; (`/<repo>/plain/<path>`): tools fetch those and cannot solve a
  ;; proof-of-work challenge.  See the rule below, which still
  ;; challenges anything that looks like a browser.
  (plain-file "anubis-challenge-all-policy.yaml"
              "\
bots:
  # Known-pathological scrapers and aggressive AI bots are denied
  # outright (rules embedded in the Anubis binary).
  - import: (data)/bots/_deny-pathological.yaml
  - import: (data)/meta/ai-block-aggressive.yaml
  # Raw file downloads (cgit's /plain/ view) are fetched by tools --
  # curl, wget, git archive -- which cannot solve the challenge: let
  # them through.  The Sec-Fetch-* conditions keep browsers out of the
  # exemption, including a headless browser that claims a curl user
  # agent; browsers get the challenge and can solve it.
  #
  # NOTE: header names are looked up case-sensitively against Go's
  # canonical form, so it must be Sec-Fetch-Mode, not sec-fetch-mode.
  - name: plain-download-tools
    expression:
      all:
        - 'userAgent.matches(\"^(curl|Wget)/\")'
        - 'path.matches(\"^/[^/]+/plain/\")'
        - 'missingHeader(headers, \"Sec-Fetch-Mode\")'
        - 'missingHeader(headers, \"Sec-Fetch-Dest\")'
    action: ALLOW
  # Everything else is challenged; browsers solve the
  # proof-of-work, anything that cannot is blocked.
  - name: challenge-everything
    user_agent_regex: \".*\"
    action: CHALLENGE
"))

(define* (anubis-command package socket target policy difficulty
                         #:key (extra-arguments '()))
  "Return the Anubis command line (a list of strings and file-likes):
run PACKAGE listening on unix SOCKET, proxying verified requests to
TARGET, under POLICY (a YAML policy document file-like, or #f for
Anubis's built-in default) with proof-of-work DIFFICULTY (4 is the
Anubis default; each increment doubles the work a client must burn
per challenge, so raising it prices solves out of a per-IP
scraper's budget), plus EXTRA-ARGUMENTS.  Splice the result into a
gexp with (list #$@(anubis-command ...))."
  (append
   (list (file-append package "/bin/anubis")
         "-bind" socket
         "-bind-network" "unix")
   (if policy (list "-policy-fname" policy) (list))
   (list "-difficulty" (number->string difficulty)
         "-target" target)
   extra-arguments))

;;; The unix socket Anubis listens on, and the service running it.
;;; XXX: keep in sync with (forge anubis) -- it hard-codes this too.
(define %anubis-unix-socket "/var/run/anubis/socket")

(define-record-type* <anubis-configuration>
  anubis-configuration
  make-anubis-configuration
  anubis-configuration?
  this-anubis-configuration
  (package anubis-configuration-package
           (default anubis-ai-firewall))
  ;; Upstream URL Anubis proxies to after a successful challenge; the
  ;; front-end's nginx exposes this as an internal listener.
  (target anubis-configuration-target
          (default "http://127.0.0.1:8081"))
  ;; Anubis policy document (YAML file-like), or #f for Anubis's
  ;; built-in default policy (which passes non-browser user agents
  ;; through unchallenged).
  (policy anubis-configuration-policy
          (default %anubis-challenge-all-policy))
  ;; Difficulty of the proof-of-work challenge; 4 is the Anubis
  ;; default.  Each increment doubles the work a client must burn
  ;; to solve the challenge (per IP, before it gets a week-long
  ;; cookie).  Raise it when headless browsers start solving their
  ;; way through -- e.g. 10-12 costs real browsers a few seconds
  ;; once a week, and bots an infeasible amount of compute per IP.
  (difficulty anubis-configuration-difficulty
              (default 4)))

(define anubis-activation
  #~(begin
      (let ((user (getpw "nginx")))
        (mkdir-p (dirname #$%anubis-unix-socket))
        (chown (dirname #$%anubis-unix-socket)
               (passwd:uid user)
               (group:gid user)))))

(define (anubis-shepherd-service config)
  (match-record config <anubis-configuration>
    (package target policy difficulty)
  (shepherd-service
   (documentation "Run the Anubis AI firewall.")
   (provision '(anubis))
   (requirement '(user-processes loopback))
   (start #~(make-forkexec-constructor
             (list #$@(anubis-command
                       package
                       %anubis-unix-socket target policy difficulty))
             #:user "nginx" #:group "nginx"
             #:log-file "/var/log/anubis.log"))
   (stop #~(make-kill-destructor)))))

(define anubis-service-type
  (service-type
   (name 'anubis)
   (description "Run the Anubis AI bot firewall, listening on a unix
socket that the front-end's nginx proxies to.")
   (extensions
    (list (service-extension activation-service-type
                             (const anubis-activation))
          (service-extension shepherd-root-service-type
                             (compose list anubis-shepherd-service))
          ;; Keep the anubis binary in the system profile for
          ;; inspection.
          (service-extension profile-service-type
                             (lambda (config)
                               (list (anubis-configuration-package
                                      config))))))
   ;; Service types without `extend' IGNORE the values contributed by
   ;; other services' extensions (fold-services keeps the instance's own
   ;; value), and the front-ends contribute their Anubis configuration --
   ;; target, policy, difficulty -- through exactly such an extension
   ;; (cgit-nginx-service-type's `anubis-service-type' extension).  With
   ;; compose/extend below, a contributed configuration wins over the
   ;; default value; with none, the default is used.
   (compose (lambda (contributions) (apply append contributions)))
   (extend (lambda (default contributions)
             (match contributions
               (() default)
               ((config _ ...) config))))
   (default-value (anubis-configuration))))