about summary refs log tree commit diff
path: root/genenetwork/services/test/run_container_tests.sh
blob: 3f419b410c8fa6ccd22b77a803231d9ee4deaeaa (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
#!/bin/bash
# Build the cgit system container, start it through the
# run-guix-system-container daemon, wait for the web server, and run
# the HTTP functional test suite.
#
# Usage (from anywhere):
#   test/run_container_tests.sh [--keep]
#
#   --keep   leave the container running after the run (default:
#            killed on success, kept on failure for debugging)
#
# Environment overrides:
#   base      checkout root (default: derived from this script; set
#             to a directory holding the checkouts side by side)
#   DAEMON     daemon address (default 127.0.0.1:8900)
#   BASE_URL   cgit base url (default http://127.0.0.1; port 80)
#   SSH_PORT   container ssh port (default 2224)
#   CLIENT_KEY ssh client key (default $HOME/hello-container-key)
#   DAEMON_FINGERPRINT  daemon fingerprint (default: auto-detected --
#              the daemon generates a random fingerprint at startup,
#              so we query it instead of pinning a stale value)
#
# NOTE: tests t01-t16 expect the host-side git repositories to be
# provisioned (see the bind-mount section of
# genenetwork/containers/cgit-container.scm); without them the front page is a
# 404 and only the Anubis tests (t17, t18) and static assets pass.

set -u

HERE="$(cd "$(dirname "$0")" && pwd)"
# Layout note: the default base assumes the factory layout
# (<root>/code/<checkout>).  For a different layout (e.g. bare
# ~/guix/<checkout> on balg01) set the *_DIR variables directly:
#   GUIX_BIOINF, GN_MACHINES, GUIX_FORGE, RUN_GUIX_CONTAINER
base="${base:-$(cd "$HERE/../../../../.." && pwd)}"
DAEMON="${DAEMON:-127.0.0.1:8900}"

GUIX_BIOINF="${GUIX_BIOINF:-$base/code/guix-bioinformatics}"
GN_MACHINES="${GN_MACHINES:-$base/code/gn-machines}"
GUIX_FORGE="${GUIX_FORGE:-$base/code/contrib/guix-forge/guix}"
CONTAINER_SCM="$GN_MACHINES/genenetwork/containers/cgit-container.scm"
HTTP_TESTS="$HERE/cgit_http_tests.py"
RUN_GUIX_CONTAINER="${RUN_GUIX_CONTAINER:-$base/code/run-guix-container/bin/run-guix-container.rb}"
# The containers bake in the keys from the building user's $HOME (the
# container definitions use (getenv "HOME")), so the client key MUST
# be the same file -- a stale copy elsewhere fails with "Permission
# denied (publickey)".
CLIENT_KEY="${CLIENT_KEY:-$HOME/hello-container-key}"
SSH_PORT="${SSH_PORT:-2224}"
SSH_BIN="${SSH_BIN:-$(command -v ssh || echo /gnu/store/by018qdrlqk5akqkh7c9l6g1j2nijm2p-openssh-10.3p1/bin/ssh)}"
export BASE_URL="${BASE_URL:-http://127.0.0.1}"

KEEP=0
[ "${1:-}" = "--keep" ] && KEEP=1

die() { echo "ERROR: $*" >&2; exit 1; }

client() { "$RUN_GUIX_CONTAINER" --daemon "$DAEMON" --fingerprint "$DAEMON_FINGERPRINT" "$@"; }

# The client verifies the daemon's fingerprint before every command, so
# a runner never talks to (and never starts containers under) an
# unexpected daemon generation.  The daemon generates a random
# fingerprint at startup unless --fingerprint is given, so auto-detect
# it by asking the daemon directly.
detect_fingerprint() {
    python3 - "$DAEMON" <<'PYEOF'
import socket, sys
host, port = sys.argv[1].rsplit(":", 1)
s = socket.create_connection((host, int(port)), 5)
s.sendall(b"FINGERPRINT\n")
print(s.recv(4096).decode().strip().split()[1])
PYEOF
}
DAEMON_FINGERPRINT="${DAEMON_FINGERPRINT:-$(detect_fingerprint)}"
[ -n "$DAEMON_FINGERPRINT" ] || die "could not detect daemon fingerprint"
echo "== daemon fingerprint: $DAEMON_FINGERPRINT"

# Harvest the container's logs over ssh into test/logs/<timestamp>/
# BEFORE the container is killed, so post-mortem debugging is possible.
harvest_logs() {
    local DEST="$HERE/logs/$(date +%Y%m%d-%H%M%S)-cgit"
    mkdir -p "$DEST"
    echo "== harvesting container logs into $DEST =="
    "$SSH_BIN" -p "$SSH_PORT" -o StrictHostKeyChecking=no \
        -o UserKnownHostsFile=/dev/null -o ConnectTimeout=10 \
        -i "$CLIENT_KEY" root@127.0.0.1 \
        'for f in /var/log/messages /var/log/nginx/*.log /var/log/fcgiwrap/*.log \
                  /var/log/anubis.log /var/log/cgit-seed.log /srv/git/seed-log.txt; \
         do echo "===== $f"; cat "$f" 2>/dev/null; done; true' \
        > "$DEST/varlog.txt" 2>/dev/null || echo "(log harvest failed; container unreachable?)"
    client --log "$(guix system container --network \
              -L "$GUIX_BIOINF" -L "$GN_MACHINES" -L "$GUIX_FORGE" \
              "$CONTAINER_SCM" 2>/dev/null | tail -1)" \
        > "$DEST/boot.log" 2>/dev/null || true
    echo "logs saved: $DEST"
}

# --- 1. stop any container the daemon is currently running ------------
echo "== stopping any running container =="
for pid in $(client --list 2>/dev/null | awk '$1 ~ /^[0-9]+$/ {print $1}'); do
    echo "killing $pid"
    client --kill "$pid" >/dev/null 2>&1
done
sleep 2

# --- 2. build the container -------------------------------------------
echo "== building cgit container =="
HASH=$(guix system container --network \
       -L "$GUIX_BIOINF" -L "$GN_MACHINES" -L "$GUIX_FORGE" \
       "$CONTAINER_SCM" 2>/dev/null | tail -1)
[ -n "$HASH" ] || die "container build failed"
echo "built: $HASH"

# --- 3. start it via the daemon ---------------------------------------
echo "== starting container =="
OUT=$(client "$HASH" 2>&1) || true
PID=$(echo "$OUT" | awk '/^OK/ {print $2}')
if [ -z "$PID" ]; then
    # a container may still be running (stray); try once more
    for pid in $(client --list 2>/dev/null | awk '$1 ~ /^[0-9]+$/ {print $1}'); do
        client --kill "$pid" >/dev/null 2>&1
    done
    sleep 3
    OUT=$(client "$HASH" 2>&1) || true
    PID=$(echo "$OUT" | awk '/^OK/ {print $2}')
fi
[ -n "$PID" ] || die "could not start container: $OUT"
echo "running as PID $PID"

cleanup() {
    harvest_logs
    if [ "$KEEP" = 1 ] || [ "${FAILED:-0}" -gt 0 ]; then
        echo "container $PID left running (inspect via ssh -p $SSH_PORT
  -i $CLIENT_KEY root@127.0.0.1 or $BASE_URL)"
    else
        echo "== stopping container =="
        client --kill "$PID" >/dev/null 2>&1
    fi
}
trap cleanup EXIT

# --- 4. wait for the web server ---------------------------------------
echo "== waiting for the web server =="
UP=0
for i in $(seq 1 30); do
    code=$(curl -s -m 3 -o /dev/null -w '%{http_code}' "$BASE_URL/" 2>/dev/null)
    if [ "$code" = 200 ]; then UP=1; break; fi
    sleep 4
done
if [ "$UP" = 1 ]; then
    echo "web server is up"
else
    echo "WARNING: front page not 200 (repos provisioned?); continuing with the tests"
fi

# --- 5. run the tests ---------------------------------------------------
echo "== HTTP functional tests =="
python3 "$HTTP_TESTS"
TEST_RC=$?

echo "== container tool check =="
"$SSH_BIN" -p "$SSH_PORT" -o StrictHostKeyChecking=no \
    -o UserKnownHostsFile=/dev/null -i "$CLIENT_KEY" -o ConnectTimeout=10 \
    root@127.0.0.1 'set -e
for tool in git gzip cgit.cgi nginx herd; do
    case $tool in
        cgit.cgi) test -x /run/current-system/profile/lib/cgit/cgit.cgi && echo "OK  cgit.cgi";;
        nginx)    test -x /run/current-system/profile/sbin/nginx && echo "OK  nginx";;
        *)        command -v "$tool" >/dev/null && echo "OK  $tool";;
    esac
done
# fcgiwrap runs from its store path via the shepherd service; check
# the running process instead of the profile
pgrep -x fcgiwrap >/dev/null && echo "OK  fcgiwrap (running)"
pgrep -x anubis   >/dev/null && echo "OK  anubis (running)"
echo "--- services:"
/run/current-system/profile/bin/herd status nginx 2>&1 | head -2
/run/current-system/profile/bin/herd status fcgiwrap 2>&1 | head -2
/run/current-system/profile/bin/herd status anubis 2>&1 | head -2
echo "--- repos:"
ls /srv/git 2>/dev/null || echo "(no /srv/git: host repositories not mounted)"'
TOOLS_RC=$?

echo
if [ "$TEST_RC" = 0 ] && [ "$TOOLS_RC" = 0 ]; then
    echo "== ALL CGIT CONTAINER TESTS PASSED =="
    FAILED=0
    exit 0
else
    echo "== CGIT CONTAINER TESTS FAILED (tests=$TEST_RC tools=$TOOLS_RC) =="
    FAILED=1
    exit 1
fi