#!/usr/bin/env python3 """HTTP functional tests for the cgit system container. These tests talk to the web front-end over plain HTTP only -- no headless browser is needed. They assume the container is running with its nginx on http://127.0.0.1 (port 80) and the demo repository (seeded at first boot) in /srv/git. The rate-limit tests (t19-t22) rely on the listener trusting 127.0.0.1 for the client address (the `trusted-proxies` field of cgit-web-configuration, i.e. set_real_ip_from/real_ip_header in nginx), so they can send X-Forwarded-For and claim a client bucket of their own. They need the plain front-end: behind Anubis the browsing URIs never reach cgit (t17/t18 cover that variant). The /plain/ tests (t23-t26) cover the reverse case: raw downloads are ALLOWed to curl/wget by the Anubis policy (%anubis-challenge-all-policy), while browsers -- and anything carrying Sec-Fetch-* -- are challenged. Usage: BASE_URL=http://127.0.0.1 python3 cgit_http_tests.py """ import os import re import sys import urllib.error import urllib.request BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1") TESTS = [] def test(name): def wrap(fn): TESTS.append((name, fn)) return fn return wrap class Response: def __init__(self, status, body, headers): self.status = status self.body = body self.headers = headers def has(self, needle): return needle in self.body def request(path, timeout=20, headers=None): url = BASE_URL + path req = urllib.request.Request( url, headers={"User-Agent": "cgit-test/1.0", **(headers or {})}) try: with urllib.request.urlopen(req, timeout=timeout) as r: return Response(r.status, r.read().decode("utf-8", "replace"), dict(r.headers)) except urllib.error.HTTPError as e: return Response(e.code, e.read().decode("utf-8", "replace"), dict(e.headers)) # --- static front-end --------------------------------------------------- @test("t01 front page returns 200 with a non-empty body") def t01(): r = request("/") return r.status == 200 and r.has("")) # --- Anubis AI-bot firewall ----------------------------------------------- # A browser-like user agent: Anubis challenges browsers, while git # clients and other non-browser agents pass through untouched. BROWSER_UA = ("Mozilla/5.0 (X11; Linux x86_64; rv:130.0) " "Gecko/20100101 Firefox/130.0") def request_ua(path, ua, timeout=20): url = BASE_URL + path req = urllib.request.Request(url, headers={"User-Agent": ua}) try: with urllib.request.urlopen(req, timeout=timeout) as r: return Response(r.status, r.read().decode("utf-8", "replace"), dict(r.headers)) except urllib.error.HTTPError as e: return Response(e.code, e.read().decode("utf-8", "replace"), dict(e.headers)) @test("t17 browser user agent is challenged by Anubis") def t17(): r = request_ua("/", BROWSER_UA) return (r.status == 200 and r.has("Making sure") and r.has("not a bot")) @test("t18 non-browser user agents are challenged too (nothing passes unsolved)") def t18(): # The challenge-all Anubis policy blocks every client that cannot # solve the proof-of-work challenge -- curl and other non-browser # user agents included. Only the git smart-HTTP endpoints bypass # Anubis (see t13/t14). t17 and t18 apply to anubis-enabled # deployments; against an anubis-less deployment they fail by # design. r = request_ua("/", "cgit-anubis-check/1.0") return (r.status == 200 and r.has("Making sure") and r.has("not a bot")) @test("t19 expensive endpoints rate limited to 429 after burst") def t19(): # The nginx config limits diff/commit/log to 60 requests/minute per # client IP with burst=20 nodelay (shared zone). Rapid-firing more # requests than 1 + burst within the refill window must produce # HTTP 429. The listener trusts 127.0.0.1 for the client address # (see trusted-proxies in cgit.scm), so the test claims a forwarded # address of its own: it owns that budget instead of spending the # one the earlier tests share, and 429s cannot be somebody else's # traffic. headers = {"X-Forwarded-For": "198.51.100.19"} codes = [request("/demo/log/", headers=headers).status for _ in range(40)] # earlier requests cannot have spent this bucket, so the shape is # sharp: 1 + burst=20 pass, the rest are 429. Allow for a couple # of tokens refilled while the 40 requests were in flight. return (codes[-1] == 429 # burst exhausted -> 429 and codes.count(200) <= 25) @test("t20 every request is rate limited per client IP (429 after burst)") def t20(): # The server-level limit_req zone=perip is 5 requests/second per # client IP with burst=30 nodelay, so at most 1 + 30 requests pass # back-to-back and the rest are answered 429. Own forwarded # address again: the bucket starts full and is nobody else's. headers = {"X-Forwarded-For": "198.51.100.20"} codes = [request("/", headers=headers).status for _ in range(60)] return (codes.count(429) >= 1 # over the burst -> 429 and codes.count(200) <= 31) # 1 + burst=30 pass, no more @test("t21 forwarded clients get separate expensive-endpoint budgets") def t21(): # Regression for the shared-bucket 429 storm. The deployment # proxies every request into the container from 127.0.0.1; before # set_real_ip_from/real_ip_header existed, the limit_req zones # keyed on that proxy address, so the whole internet shared (and # exhausted) one budget. Two forwarded client addresses must # behave like two clients: exhausting A's expensive budget must # leave B's untouched. Requires the plain (anubis? #f) variant, # like t19: behind Anubis /demo/log/ never reaches cgit. a = {"X-Forwarded-For": "198.51.100.21"} b = {"X-Forwarded-For": "198.51.100.22"} for _ in range(40): request("/demo/log/", headers=a) rejected_a = request("/demo/log/", headers=a).status allowed_b = request("/demo/log/", headers=b).status return rejected_a == 429 and allowed_b == 200 @test("t22 exhausting one client's general budget spares the next") def t22(): # Same regression for the server-level general zone (perip), which # is what produced the 429s on the repository summary page. a = {"X-Forwarded-For": "198.51.100.23"} b = {"X-Forwarded-For": "198.51.100.24"} for _ in range(60): request("/", headers=a) rejected_a = request("/", headers=a).status allowed_b = request("/", headers=b).status return rejected_a == 429 and allowed_b == 200 @test("t23 plain view accepts wget as well as curl") def t23(): r = request("/demo/plain/README.md", headers={"User-Agent": "Wget/1.21"}) return (r.status == 200 and r.has("Hello from the cgit container!")) @test("t24 a browser is challenged on /plain/ (not served, not 403)") def t24(): # The Anubis policy allows raw downloads to curl/wget only; a browser # gets the proof-of-work challenge for the same URL, so the "plain" # link keeps working for humans after one solve. Anubis-enabled # deployments only, like t17/t18. r = request_ua("/demo/plain/README.md", BROWSER_UA) return (r.status == 200 and r.has("Making sure") and not r.has("Hello from the cgit container!")) @test("t25 plain view is not challenged by Anubis") def t25(): # The point of plain-anubis-bypass?: tools fetch raw content and # cannot solve a proof-of-work challenge. Applies to anubis-enabled # deployments (like t17/t18 it fails against an anubis-less one, by # design). r = request("/demo/plain/README.md", headers={"User-Agent": "Wget/1.21"}) return (r.status == 200 and r.has("Hello from the cgit container!") and not r.has("Making sure")) @test("t26 a browser claiming a curl user agent is still challenged") def t26(): # The exemption requires the absence of Sec-Fetch-*, which browsers # always send: dressing a browser up as curl must not skip Anubis. r = request("/demo/plain/README.md", headers={"User-Agent": "curl/8.7.1", "Sec-Fetch-Mode": "navigate", "Sec-Fetch-Dest": "document"}) return (r.status == 200 and r.has("Making sure") and not r.has("Hello from the cgit container!")) def main(): passed = failed = 0 for name, fn in TESTS: try: ok = fn() except Exception as e: # noqa: BLE001 ok = False print(f"FAIL {name}: exception {e!r}") if ok: passed += 1 print(f"PASS {name}") else: failed += 1 print(f"FAIL {name}") print(f"== {passed} passed, {failed} failed ==") return 1 if failed else 0 if __name__ == "__main__": sys.exit(main())